1 /* $OpenBSD: subr_hibernate.c,v 1.134 2022/02/19 23:57:09 deraadt Exp $ */ 2 3 /* 4 * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl> 5 * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/hibernate.h> 21 #include <sys/malloc.h> 22 #include <sys/param.h> 23 #include <sys/tree.h> 24 #include <sys/systm.h> 25 #include <sys/disklabel.h> 26 #include <sys/disk.h> 27 #include <sys/conf.h> 28 #include <sys/buf.h> 29 #include <sys/fcntl.h> 30 #include <sys/stat.h> 31 #include <sys/atomic.h> 32 33 #include <uvm/uvm.h> 34 #include <uvm/uvm_swap.h> 35 36 #include <machine/hibernate.h> 37 38 /* Make sure the signature can fit in one block */ 39 CTASSERT(sizeof(union hibernate_info) <= DEV_BSIZE); 40 41 /* 42 * Hibernate piglet layout information 43 * 44 * The piglet is a scratch area of memory allocated by the suspending kernel. 45 * Its phys and virt addrs are recorded in the signature block. The piglet is 46 * used to guarantee an unused area of memory that can be used by the resuming 47 * kernel for various things. The piglet is excluded during unpack operations. 48 * The piglet size is presently 4*HIBERNATE_CHUNK_SIZE (typically 4*4MB). 49 * 50 * Offset from piglet_base Purpose 51 * ---------------------------------------------------------------------------- 52 * 0 Private page for suspend I/O write functions 53 * 1*PAGE_SIZE I/O page used during hibernate suspend 54 * 2*PAGE_SIZE I/O page used during hibernate suspend 55 * 3*PAGE_SIZE copy page used during hibernate suspend 56 * 4*PAGE_SIZE final chunk ordering list (24 pages) 57 * 28*PAGE_SIZE RLE utility page 58 * 29*PAGE_SIZE start of hiballoc area 59 * 30*PAGE_SIZE preserved entropy 60 * 110*PAGE_SIZE end of hiballoc area (80 pages) 61 * 366*PAGE_SIZE end of retguard preservation region (256 pages) 62 * ... unused 63 * HIBERNATE_CHUNK_SIZE start of hibernate chunk table 64 * 2*HIBERNATE_CHUNK_SIZE bounce area for chunks being unpacked 65 * 4*HIBERNATE_CHUNK_SIZE end of piglet 66 */ 67 68 /* Temporary vaddr ranges used during hibernate */ 69 vaddr_t hibernate_temp_page; 70 vaddr_t hibernate_copy_page; 71 vaddr_t hibernate_rle_page; 72 73 /* Hibernate info as read from disk during resume */ 74 union hibernate_info disk_hib; 75 76 /* 77 * Global copy of the pig start address. This needs to be a global as we 78 * switch stacks after computing it - it can't be stored on the stack. 79 */ 80 paddr_t global_pig_start; 81 82 /* 83 * Global copies of the piglet start addresses (PA/VA). We store these 84 * as globals to avoid having to carry them around as parameters, as the 85 * piglet is allocated early and freed late - its lifecycle extends beyond 86 * that of the hibernate info union which is calculated on suspend/resume. 87 */ 88 vaddr_t global_piglet_va; 89 paddr_t global_piglet_pa; 90 91 /* #define HIB_DEBUG */ 92 #ifdef HIB_DEBUG 93 int hib_debug = 99; 94 #define DPRINTF(x...) do { if (hib_debug) printf(x); } while (0) 95 #define DNPRINTF(n,x...) do { if (hib_debug > (n)) printf(x); } while (0) 96 #else 97 #define DPRINTF(x...) 98 #define DNPRINTF(n,x...) 99 #endif 100 101 #ifndef NO_PROPOLICE 102 extern long __guard_local; 103 #endif /* ! NO_PROPOLICE */ 104 105 void hibernate_copy_chunk_to_piglet(paddr_t, vaddr_t, size_t); 106 int hibernate_calc_rle(paddr_t, paddr_t); 107 int hibernate_write_rle(union hibernate_info *, paddr_t, paddr_t, daddr_t *, 108 size_t *); 109 110 #define MAX_RLE (HIBERNATE_CHUNK_SIZE / PAGE_SIZE) 111 112 /* 113 * Hib alloc enforced alignment. 114 */ 115 #define HIB_ALIGN 8 /* bytes alignment */ 116 117 /* 118 * sizeof builtin operation, but with alignment constraint. 119 */ 120 #define HIB_SIZEOF(_type) roundup(sizeof(_type), HIB_ALIGN) 121 122 struct hiballoc_entry { 123 size_t hibe_use; 124 size_t hibe_space; 125 RBT_ENTRY(hiballoc_entry) hibe_entry; 126 }; 127 128 /* 129 * Sort hibernate memory ranges by ascending PA 130 */ 131 void 132 hibernate_sort_ranges(union hibernate_info *hib_info) 133 { 134 int i, j; 135 struct hibernate_memory_range *ranges; 136 paddr_t base, end; 137 138 ranges = hib_info->ranges; 139 140 for (i = 1; i < hib_info->nranges; i++) { 141 j = i; 142 while (j > 0 && ranges[j - 1].base > ranges[j].base) { 143 base = ranges[j].base; 144 end = ranges[j].end; 145 ranges[j].base = ranges[j - 1].base; 146 ranges[j].end = ranges[j - 1].end; 147 ranges[j - 1].base = base; 148 ranges[j - 1].end = end; 149 j--; 150 } 151 } 152 } 153 154 /* 155 * Compare hiballoc entries based on the address they manage. 156 * 157 * Since the address is fixed, relative to struct hiballoc_entry, 158 * we just compare the hiballoc_entry pointers. 159 */ 160 static __inline int 161 hibe_cmp(const struct hiballoc_entry *l, const struct hiballoc_entry *r) 162 { 163 vaddr_t vl = (vaddr_t)l; 164 vaddr_t vr = (vaddr_t)r; 165 166 return vl < vr ? -1 : (vl > vr); 167 } 168 169 RBT_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp) 170 171 /* 172 * Given a hiballoc entry, return the address it manages. 173 */ 174 static __inline void * 175 hib_entry_to_addr(struct hiballoc_entry *entry) 176 { 177 caddr_t addr; 178 179 addr = (caddr_t)entry; 180 addr += HIB_SIZEOF(struct hiballoc_entry); 181 return addr; 182 } 183 184 /* 185 * Given an address, find the hiballoc that corresponds. 186 */ 187 static __inline struct hiballoc_entry* 188 hib_addr_to_entry(void *addr_param) 189 { 190 caddr_t addr; 191 192 addr = (caddr_t)addr_param; 193 addr -= HIB_SIZEOF(struct hiballoc_entry); 194 return (struct hiballoc_entry*)addr; 195 } 196 197 RBT_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp); 198 199 /* 200 * Allocate memory from the arena. 201 * 202 * Returns NULL if no memory is available. 203 */ 204 void * 205 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz) 206 { 207 struct hiballoc_entry *entry, *new_entry; 208 size_t find_sz; 209 210 /* 211 * Enforce alignment of HIB_ALIGN bytes. 212 * 213 * Note that, because the entry is put in front of the allocation, 214 * 0-byte allocations are guaranteed a unique address. 215 */ 216 alloc_sz = roundup(alloc_sz, HIB_ALIGN); 217 218 /* 219 * Find an entry with hibe_space >= find_sz. 220 * 221 * If the root node is not large enough, we switch to tree traversal. 222 * Because all entries are made at the bottom of the free space, 223 * traversal from the end has a slightly better chance of yielding 224 * a sufficiently large space. 225 */ 226 find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry); 227 entry = RBT_ROOT(hiballoc_addr, &arena->hib_addrs); 228 if (entry != NULL && entry->hibe_space < find_sz) { 229 RBT_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) { 230 if (entry->hibe_space >= find_sz) 231 break; 232 } 233 } 234 235 /* 236 * Insufficient or too fragmented memory. 237 */ 238 if (entry == NULL) 239 return NULL; 240 241 /* 242 * Create new entry in allocated space. 243 */ 244 new_entry = (struct hiballoc_entry*)( 245 (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use); 246 new_entry->hibe_space = entry->hibe_space - find_sz; 247 new_entry->hibe_use = alloc_sz; 248 249 /* 250 * Insert entry. 251 */ 252 if (RBT_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL) 253 panic("hib_alloc: insert failure"); 254 entry->hibe_space = 0; 255 256 /* Return address managed by entry. */ 257 return hib_entry_to_addr(new_entry); 258 } 259 260 void 261 hib_getentropy(char **bufp, size_t *bufplen) 262 { 263 if (!bufp || !bufplen) 264 return; 265 266 *bufp = (char *)(global_piglet_va + (29 * PAGE_SIZE)); 267 *bufplen = PAGE_SIZE; 268 } 269 270 /* 271 * Free a pointer previously allocated from this arena. 272 * 273 * If addr is NULL, this will be silently accepted. 274 */ 275 void 276 hib_free(struct hiballoc_arena *arena, void *addr) 277 { 278 struct hiballoc_entry *entry, *prev; 279 280 if (addr == NULL) 281 return; 282 283 /* 284 * Derive entry from addr and check it is really in this arena. 285 */ 286 entry = hib_addr_to_entry(addr); 287 if (RBT_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry) 288 panic("hib_free: freed item %p not in hib arena", addr); 289 290 /* 291 * Give the space in entry to its predecessor. 292 * 293 * If entry has no predecessor, change its used space into free space 294 * instead. 295 */ 296 prev = RBT_PREV(hiballoc_addr, entry); 297 if (prev != NULL && 298 (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) + 299 prev->hibe_use + prev->hibe_space) == entry) { 300 /* Merge entry. */ 301 RBT_REMOVE(hiballoc_addr, &arena->hib_addrs, entry); 302 prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) + 303 entry->hibe_use + entry->hibe_space; 304 } else { 305 /* Flip used memory to free space. */ 306 entry->hibe_space += entry->hibe_use; 307 entry->hibe_use = 0; 308 } 309 } 310 311 /* 312 * Initialize hiballoc. 313 * 314 * The allocator will manage memory at ptr, which is len bytes. 315 */ 316 int 317 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len) 318 { 319 struct hiballoc_entry *entry; 320 caddr_t ptr; 321 size_t len; 322 323 RBT_INIT(hiballoc_addr, &arena->hib_addrs); 324 325 /* 326 * Hib allocator enforces HIB_ALIGN alignment. 327 * Fixup ptr and len. 328 */ 329 ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN); 330 len = p_len - ((size_t)ptr - (size_t)p_ptr); 331 len &= ~((size_t)HIB_ALIGN - 1); 332 333 /* 334 * Insufficient memory to be able to allocate and also do bookkeeping. 335 */ 336 if (len <= HIB_SIZEOF(struct hiballoc_entry)) 337 return ENOMEM; 338 339 /* 340 * Create entry describing space. 341 */ 342 entry = (struct hiballoc_entry*)ptr; 343 entry->hibe_use = 0; 344 entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry); 345 RBT_INSERT(hiballoc_addr, &arena->hib_addrs, entry); 346 347 return 0; 348 } 349 350 /* 351 * Zero all free memory. 352 */ 353 void 354 uvm_pmr_zero_everything(void) 355 { 356 struct uvm_pmemrange *pmr; 357 struct vm_page *pg; 358 int i; 359 360 uvm_lock_fpageq(); 361 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 362 /* Zero single pages. */ 363 while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY])) 364 != NULL) { 365 uvm_pmr_remove(pmr, pg); 366 uvm_pagezero(pg); 367 atomic_setbits_int(&pg->pg_flags, PG_ZERO); 368 uvmexp.zeropages++; 369 uvm_pmr_insert(pmr, pg, 0); 370 } 371 372 /* Zero multi page ranges. */ 373 while ((pg = RBT_ROOT(uvm_pmr_size, 374 &pmr->size[UVM_PMR_MEMTYPE_DIRTY])) != NULL) { 375 pg--; /* Size tree always has second page. */ 376 uvm_pmr_remove(pmr, pg); 377 for (i = 0; i < pg->fpgsz; i++) { 378 uvm_pagezero(&pg[i]); 379 atomic_setbits_int(&pg[i].pg_flags, PG_ZERO); 380 uvmexp.zeropages++; 381 } 382 uvm_pmr_insert(pmr, pg, 0); 383 } 384 } 385 uvm_unlock_fpageq(); 386 } 387 388 /* 389 * Mark all memory as dirty. 390 * 391 * Used to inform the system that the clean memory isn't clean for some 392 * reason, for example because we just came back from hibernate. 393 */ 394 void 395 uvm_pmr_dirty_everything(void) 396 { 397 struct uvm_pmemrange *pmr; 398 struct vm_page *pg; 399 int i; 400 401 uvm_lock_fpageq(); 402 TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) { 403 /* Dirty single pages. */ 404 while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO])) 405 != NULL) { 406 uvm_pmr_remove(pmr, pg); 407 atomic_clearbits_int(&pg->pg_flags, PG_ZERO); 408 uvm_pmr_insert(pmr, pg, 0); 409 } 410 411 /* Dirty multi page ranges. */ 412 while ((pg = RBT_ROOT(uvm_pmr_size, 413 &pmr->size[UVM_PMR_MEMTYPE_ZERO])) != NULL) { 414 pg--; /* Size tree always has second page. */ 415 uvm_pmr_remove(pmr, pg); 416 for (i = 0; i < pg->fpgsz; i++) 417 atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO); 418 uvm_pmr_insert(pmr, pg, 0); 419 } 420 } 421 422 uvmexp.zeropages = 0; 423 uvm_unlock_fpageq(); 424 } 425 426 /* 427 * Allocate an area that can hold sz bytes and doesn't overlap with 428 * the piglet at piglet_pa. 429 */ 430 int 431 uvm_pmr_alloc_pig(paddr_t *pa, psize_t sz, paddr_t piglet_pa) 432 { 433 struct uvm_constraint_range pig_constraint; 434 struct kmem_pa_mode kp_pig = { 435 .kp_constraint = &pig_constraint, 436 .kp_maxseg = 1 437 }; 438 vaddr_t va; 439 440 sz = round_page(sz); 441 442 pig_constraint.ucr_low = piglet_pa + 4 * HIBERNATE_CHUNK_SIZE; 443 pig_constraint.ucr_high = -1; 444 445 va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait); 446 if (va == 0) { 447 pig_constraint.ucr_low = 0; 448 pig_constraint.ucr_high = piglet_pa - 1; 449 450 va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait); 451 if (va == 0) 452 return ENOMEM; 453 } 454 455 pmap_extract(pmap_kernel(), va, pa); 456 return 0; 457 } 458 459 /* 460 * Allocate a piglet area. 461 * 462 * This needs to be in DMA-safe memory. 463 * Piglets are aligned. 464 * 465 * sz and align in bytes. 466 * 467 * The call will sleep for the pagedaemon to attempt to free memory. 468 * The pagedaemon may decide its not possible to free enough memory, causing 469 * the allocation to fail. 470 */ 471 int 472 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align) 473 { 474 struct kmem_pa_mode kp_piglet = { 475 .kp_constraint = &dma_constraint, 476 .kp_align = align, 477 .kp_maxseg = 1 478 }; 479 480 /* Ensure align is a power of 2 */ 481 KASSERT((align & (align - 1)) == 0); 482 483 /* 484 * Fixup arguments: align must be at least PAGE_SIZE, 485 * sz will be converted to pagecount, since that is what 486 * pmemrange uses internally. 487 */ 488 if (align < PAGE_SIZE) 489 kp_piglet.kp_align = PAGE_SIZE; 490 491 sz = round_page(sz); 492 493 *va = (vaddr_t)km_alloc(sz, &kv_any, &kp_piglet, &kd_nowait); 494 if (*va == 0) 495 return ENOMEM; 496 497 pmap_extract(pmap_kernel(), *va, pa); 498 return 0; 499 } 500 501 /* 502 * Free a piglet area. 503 */ 504 void 505 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz) 506 { 507 /* 508 * Fix parameters. 509 */ 510 sz = round_page(sz); 511 512 /* 513 * Free the physical and virtual memory. 514 */ 515 km_free((void *)va, sz, &kv_any, &kp_dma_contig); 516 } 517 518 /* 519 * Physmem RLE compression support. 520 * 521 * Given a physical page address, return the number of pages starting at the 522 * address that are free. Clamps to the number of pages in 523 * HIBERNATE_CHUNK_SIZE. Returns 0 if the page at addr is not free. 524 */ 525 int 526 uvm_page_rle(paddr_t addr) 527 { 528 struct vm_page *pg, *pg_end; 529 struct vm_physseg *vmp; 530 int pseg_idx, off_idx; 531 532 pseg_idx = vm_physseg_find(atop(addr), &off_idx); 533 if (pseg_idx == -1) 534 return 0; 535 536 vmp = &vm_physmem[pseg_idx]; 537 pg = &vmp->pgs[off_idx]; 538 if (!(pg->pg_flags & PQ_FREE)) 539 return 0; 540 541 /* 542 * Search for the first non-free page after pg. 543 * Note that the page may not be the first page in a free pmemrange, 544 * therefore pg->fpgsz cannot be used. 545 */ 546 for (pg_end = pg; pg_end <= vmp->lastpg && 547 (pg_end->pg_flags & PQ_FREE) == PQ_FREE && 548 (pg_end - pg) < HIBERNATE_CHUNK_SIZE/PAGE_SIZE; pg_end++) 549 ; 550 return pg_end - pg; 551 } 552 553 /* 554 * Fills out the hibernate_info union pointed to by hib 555 * with information about this machine (swap signature block 556 * offsets, number of memory ranges, kernel in use, etc) 557 */ 558 int 559 get_hibernate_info(union hibernate_info *hib, int suspend) 560 { 561 struct disklabel dl; 562 char err_string[128], *dl_ret; 563 int part; 564 SHA2_CTX ctx; 565 void *fn; 566 567 #ifndef NO_PROPOLICE 568 /* Save propolice guard */ 569 hib->guard = __guard_local; 570 #endif /* ! NO_PROPOLICE */ 571 572 /* Determine I/O function to use */ 573 hib->io_func = get_hibernate_io_function(swdevt[0].sw_dev); 574 if (hib->io_func == NULL) 575 return (1); 576 577 /* Calculate hibernate device */ 578 hib->dev = swdevt[0].sw_dev; 579 580 /* Read disklabel (used to calculate signature and image offsets) */ 581 dl_ret = disk_readlabel(&dl, hib->dev, err_string, sizeof(err_string)); 582 583 if (dl_ret) { 584 printf("Hibernate error reading disklabel: %s\n", dl_ret); 585 return (1); 586 } 587 588 /* Make sure we have a swap partition. */ 589 part = DISKPART(hib->dev); 590 if (dl.d_npartitions <= part || 591 dl.d_partitions[part].p_fstype != FS_SWAP || 592 DL_GETPSIZE(&dl.d_partitions[part]) == 0) 593 return (1); 594 595 /* Magic number */ 596 hib->magic = HIBERNATE_MAGIC; 597 598 /* Calculate signature block location */ 599 hib->sig_offset = DL_GETPSIZE(&dl.d_partitions[part]) - 600 sizeof(union hibernate_info)/DEV_BSIZE; 601 602 SHA256Init(&ctx); 603 SHA256Update(&ctx, version, strlen(version)); 604 fn = printf; 605 SHA256Update(&ctx, &fn, sizeof(fn)); 606 fn = malloc; 607 SHA256Update(&ctx, &fn, sizeof(fn)); 608 fn = km_alloc; 609 SHA256Update(&ctx, &fn, sizeof(fn)); 610 fn = strlen; 611 SHA256Update(&ctx, &fn, sizeof(fn)); 612 SHA256Final((u_int8_t *)&hib->kern_hash, &ctx); 613 614 if (suspend) { 615 /* Grab the previously-allocated piglet addresses */ 616 hib->piglet_va = global_piglet_va; 617 hib->piglet_pa = global_piglet_pa; 618 hib->io_page = (void *)hib->piglet_va; 619 620 /* 621 * Initialization of the hibernate IO function for drivers 622 * that need to do prep work (such as allocating memory or 623 * setting up data structures that cannot safely be done 624 * during suspend without causing side effects). There is 625 * a matching HIB_DONE call performed after the write is 626 * completed. 627 */ 628 if (hib->io_func(hib->dev, DL_GETPOFFSET(&dl.d_partitions[part]), 629 (vaddr_t)NULL, DL_GETPSIZE(&dl.d_partitions[part]), 630 HIB_INIT, hib->io_page)) 631 goto fail; 632 633 } else { 634 /* 635 * Resuming kernels use a regular private page for the driver 636 * No need to free this I/O page as it will vanish as part of 637 * the resume. 638 */ 639 hib->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT); 640 if (!hib->io_page) 641 goto fail; 642 } 643 644 if (get_hibernate_info_md(hib)) 645 goto fail; 646 647 return (0); 648 649 fail: 650 return (1); 651 } 652 653 /* 654 * Allocate nitems*size bytes from the hiballoc area presently in use 655 */ 656 void * 657 hibernate_zlib_alloc(void *unused, int nitems, int size) 658 { 659 struct hibernate_zlib_state *hibernate_state; 660 661 hibernate_state = 662 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 663 664 return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size); 665 } 666 667 /* 668 * Free the memory pointed to by addr in the hiballoc area presently in 669 * use 670 */ 671 void 672 hibernate_zlib_free(void *unused, void *addr) 673 { 674 struct hibernate_zlib_state *hibernate_state; 675 676 hibernate_state = 677 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 678 679 hib_free(&hibernate_state->hiballoc_arena, addr); 680 } 681 682 /* 683 * Inflate next page of data from the image stream. 684 * The rle parameter is modified on exit to contain the number of pages to 685 * skip in the output stream (or 0 if this page was inflated into). 686 * 687 * Returns 0 if the stream contains additional data, or 1 if the stream is 688 * finished. 689 */ 690 int 691 hibernate_inflate_page(int *rle) 692 { 693 struct hibernate_zlib_state *hibernate_state; 694 int i; 695 696 hibernate_state = 697 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 698 699 /* Set up the stream for RLE code inflate */ 700 hibernate_state->hib_stream.next_out = (unsigned char *)rle; 701 hibernate_state->hib_stream.avail_out = sizeof(*rle); 702 703 /* Inflate RLE code */ 704 i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH); 705 if (i != Z_OK && i != Z_STREAM_END) { 706 /* 707 * XXX - this will likely reboot/hang most machines 708 * since the console output buffer will be unmapped, 709 * but there's not much else we can do here. 710 */ 711 panic("rle inflate stream error"); 712 } 713 714 if (hibernate_state->hib_stream.avail_out != 0) { 715 /* 716 * XXX - this will likely reboot/hang most machines 717 * since the console output buffer will be unmapped, 718 * but there's not much else we can do here. 719 */ 720 panic("rle short inflate error"); 721 } 722 723 if (*rle < 0 || *rle > 1024) { 724 /* 725 * XXX - this will likely reboot/hang most machines 726 * since the console output buffer will be unmapped, 727 * but there's not much else we can do here. 728 */ 729 panic("invalid rle count"); 730 } 731 732 if (i == Z_STREAM_END) 733 return (1); 734 735 if (*rle != 0) 736 return (0); 737 738 /* Set up the stream for page inflate */ 739 hibernate_state->hib_stream.next_out = 740 (unsigned char *)HIBERNATE_INFLATE_PAGE; 741 hibernate_state->hib_stream.avail_out = PAGE_SIZE; 742 743 /* Process next block of data */ 744 i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH); 745 if (i != Z_OK && i != Z_STREAM_END) { 746 /* 747 * XXX - this will likely reboot/hang most machines 748 * since the console output buffer will be unmapped, 749 * but there's not much else we can do here. 750 */ 751 panic("inflate error"); 752 } 753 754 /* We should always have extracted a full page ... */ 755 if (hibernate_state->hib_stream.avail_out != 0) { 756 /* 757 * XXX - this will likely reboot/hang most machines 758 * since the console output buffer will be unmapped, 759 * but there's not much else we can do here. 760 */ 761 panic("incomplete page"); 762 } 763 764 return (i == Z_STREAM_END); 765 } 766 767 /* 768 * Inflate size bytes from src into dest, skipping any pages in 769 * [src..dest] that are special (see hibernate_inflate_skip) 770 * 771 * This function executes while using the resume-time stack 772 * and pmap, and therefore cannot use ddb/printf/etc. Doing so 773 * will likely hang or reset the machine since the console output buffer 774 * will be unmapped. 775 */ 776 void 777 hibernate_inflate_region(union hibernate_info *hib, paddr_t dest, 778 paddr_t src, size_t size) 779 { 780 int end_stream = 0, rle, skip; 781 struct hibernate_zlib_state *hibernate_state; 782 783 hibernate_state = 784 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 785 786 hibernate_state->hib_stream.next_in = (unsigned char *)src; 787 hibernate_state->hib_stream.avail_in = size; 788 789 do { 790 /* 791 * Is this a special page? If yes, redirect the 792 * inflate output to a scratch page (eg, discard it) 793 */ 794 skip = hibernate_inflate_skip(hib, dest); 795 if (skip == HIB_SKIP) { 796 hibernate_enter_resume_mapping( 797 HIBERNATE_INFLATE_PAGE, 798 HIBERNATE_INFLATE_PAGE, 0); 799 } else if (skip == HIB_MOVE) { 800 /* 801 * Special case : retguard region. This gets moved 802 * temporarily into the piglet region and copied into 803 * place immediately before resume 804 */ 805 hibernate_enter_resume_mapping( 806 HIBERNATE_INFLATE_PAGE, 807 hib->piglet_pa + (110 * PAGE_SIZE) + 808 hib->retguard_ofs, 0); 809 hib->retguard_ofs += PAGE_SIZE; 810 if (hib->retguard_ofs > 255 * PAGE_SIZE) { 811 /* 812 * XXX - this will likely reboot/hang most 813 * machines since the console output 814 * buffer will be unmapped, but there's 815 * not much else we can do here. 816 */ 817 panic("retguard move error, out of space"); 818 } 819 } else { 820 hibernate_enter_resume_mapping( 821 HIBERNATE_INFLATE_PAGE, dest, 0); 822 } 823 824 hibernate_flush(); 825 end_stream = hibernate_inflate_page(&rle); 826 827 if (rle == 0) 828 dest += PAGE_SIZE; 829 else 830 dest += (rle * PAGE_SIZE); 831 } while (!end_stream); 832 } 833 834 /* 835 * deflate from src into the I/O page, up to 'remaining' bytes 836 * 837 * Returns number of input bytes consumed, and may reset 838 * the 'remaining' parameter if not all the output space was consumed 839 * (this information is needed to know how much to write to disk) 840 */ 841 size_t 842 hibernate_deflate(union hibernate_info *hib, paddr_t src, 843 size_t *remaining) 844 { 845 vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE; 846 struct hibernate_zlib_state *hibernate_state; 847 848 hibernate_state = 849 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 850 851 /* Set up the stream for deflate */ 852 hibernate_state->hib_stream.next_in = (unsigned char *)src; 853 hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK); 854 hibernate_state->hib_stream.next_out = 855 (unsigned char *)hibernate_io_page + (PAGE_SIZE - *remaining); 856 hibernate_state->hib_stream.avail_out = *remaining; 857 858 /* Process next block of data */ 859 if (deflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH) != Z_OK) 860 panic("hibernate zlib deflate error"); 861 862 /* Update pointers and return number of bytes consumed */ 863 *remaining = hibernate_state->hib_stream.avail_out; 864 return (PAGE_SIZE - (src & PAGE_MASK)) - 865 hibernate_state->hib_stream.avail_in; 866 } 867 868 /* 869 * Write the hibernation information specified in hiber_info 870 * to the location in swap previously calculated (last block of 871 * swap), called the "signature block". 872 */ 873 int 874 hibernate_write_signature(union hibernate_info *hib) 875 { 876 /* Write hibernate info to disk */ 877 return (hib->io_func(hib->dev, hib->sig_offset, 878 (vaddr_t)hib, DEV_BSIZE, HIB_W, 879 hib->io_page)); 880 } 881 882 /* 883 * Write the memory chunk table to the area in swap immediately 884 * preceding the signature block. The chunk table is stored 885 * in the piglet when this function is called. Returns errno. 886 */ 887 int 888 hibernate_write_chunktable(union hibernate_info *hib) 889 { 890 vaddr_t hibernate_chunk_table_start; 891 size_t hibernate_chunk_table_size; 892 int i, err; 893 894 hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE; 895 896 hibernate_chunk_table_start = hib->piglet_va + 897 HIBERNATE_CHUNK_SIZE; 898 899 /* Write chunk table */ 900 for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) { 901 if ((err = hib->io_func(hib->dev, 902 hib->chunktable_offset + (i/DEV_BSIZE), 903 (vaddr_t)(hibernate_chunk_table_start + i), 904 MAXPHYS, HIB_W, hib->io_page))) { 905 DPRINTF("chunktable write error: %d\n", err); 906 return (err); 907 } 908 } 909 910 return (0); 911 } 912 913 /* 914 * Write an empty hiber_info to the swap signature block, which is 915 * guaranteed to not match any valid hib. 916 */ 917 int 918 hibernate_clear_signature(union hibernate_info *hib) 919 { 920 union hibernate_info blank_hiber_info; 921 922 /* Zero out a blank hiber_info */ 923 memset(&blank_hiber_info, 0, sizeof(union hibernate_info)); 924 925 /* Write (zeroed) hibernate info to disk */ 926 DPRINTF("clearing hibernate signature block location: %lld\n", 927 hib->sig_offset); 928 if (hibernate_block_io(hib, 929 hib->sig_offset, 930 DEV_BSIZE, (vaddr_t)&blank_hiber_info, 1)) 931 printf("Warning: could not clear hibernate signature\n"); 932 933 return (0); 934 } 935 936 /* 937 * Compare two hibernate_infos to determine if they are the same (eg, 938 * we should be performing a hibernate resume on this machine. 939 * Not all fields are checked - just enough to verify that the machine 940 * has the same memory configuration and kernel as the one that 941 * wrote the signature previously. 942 */ 943 int 944 hibernate_compare_signature(union hibernate_info *mine, 945 union hibernate_info *disk) 946 { 947 u_int i; 948 949 if (mine->nranges != disk->nranges) { 950 printf("unhibernate failed: memory layout changed\n"); 951 return (1); 952 } 953 954 if (bcmp(mine->kern_hash, disk->kern_hash, SHA256_DIGEST_LENGTH) != 0) { 955 printf("unhibernate failed: original kernel changed\n"); 956 return (1); 957 } 958 959 for (i = 0; i < mine->nranges; i++) { 960 if ((mine->ranges[i].base != disk->ranges[i].base) || 961 (mine->ranges[i].end != disk->ranges[i].end) ) { 962 DPRINTF("hib range %d mismatch [%p-%p != %p-%p]\n", 963 i, 964 (void *)mine->ranges[i].base, 965 (void *)mine->ranges[i].end, 966 (void *)disk->ranges[i].base, 967 (void *)disk->ranges[i].end); 968 printf("unhibernate failed: memory size changed\n"); 969 return (1); 970 } 971 } 972 973 return (0); 974 } 975 976 /* 977 * Transfers xfer_size bytes between the hibernate device specified in 978 * hib_info at offset blkctr and the vaddr specified at dest. 979 * 980 * Separate offsets and pages are used to handle misaligned reads (reads 981 * that span a page boundary). 982 * 983 * blkctr specifies a relative offset (relative to the start of swap), 984 * not an absolute disk offset 985 * 986 */ 987 int 988 hibernate_block_io(union hibernate_info *hib, daddr_t blkctr, 989 size_t xfer_size, vaddr_t dest, int iswrite) 990 { 991 struct buf *bp; 992 struct bdevsw *bdsw; 993 int error; 994 995 bp = geteblk(xfer_size); 996 bdsw = &bdevsw[major(hib->dev)]; 997 998 error = (*bdsw->d_open)(hib->dev, FREAD, S_IFCHR, curproc); 999 if (error) { 1000 printf("hibernate_block_io open failed\n"); 1001 return (1); 1002 } 1003 1004 if (iswrite) 1005 bcopy((caddr_t)dest, bp->b_data, xfer_size); 1006 1007 bp->b_bcount = xfer_size; 1008 bp->b_blkno = blkctr; 1009 CLR(bp->b_flags, B_READ | B_WRITE | B_DONE); 1010 SET(bp->b_flags, B_BUSY | (iswrite ? B_WRITE : B_READ) | B_RAW); 1011 bp->b_dev = hib->dev; 1012 (*bdsw->d_strategy)(bp); 1013 1014 error = biowait(bp); 1015 if (error) { 1016 printf("hib block_io biowait error %d blk %lld size %zu\n", 1017 error, (long long)blkctr, xfer_size); 1018 error = (*bdsw->d_close)(hib->dev, 0, S_IFCHR, 1019 curproc); 1020 if (error) 1021 printf("hibernate_block_io error close failed\n"); 1022 return (1); 1023 } 1024 1025 error = (*bdsw->d_close)(hib->dev, FREAD, S_IFCHR, curproc); 1026 if (error) { 1027 printf("hibernate_block_io close failed\n"); 1028 return (1); 1029 } 1030 1031 if (!iswrite) 1032 bcopy(bp->b_data, (caddr_t)dest, xfer_size); 1033 1034 bp->b_flags |= B_INVAL; 1035 brelse(bp); 1036 1037 return (0); 1038 } 1039 1040 /* 1041 * Preserve one page worth of random data, generated from the resuming 1042 * kernel's arc4random. After resume, this preserved entropy can be used 1043 * to further improve the un-hibernated machine's entropy pool. This 1044 * random data is stored in the piglet, which is preserved across the 1045 * unpack operation, and is restored later in the resume process (see 1046 * hib_getentropy) 1047 */ 1048 void 1049 hibernate_preserve_entropy(union hibernate_info *hib) 1050 { 1051 void *entropy; 1052 1053 entropy = km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_nowait); 1054 1055 if (!entropy) 1056 return; 1057 1058 pmap_activate(curproc); 1059 pmap_kenter_pa((vaddr_t)entropy, 1060 (paddr_t)(hib->piglet_pa + (29 * PAGE_SIZE)), 1061 PROT_READ | PROT_WRITE); 1062 1063 arc4random_buf((void *)entropy, PAGE_SIZE); 1064 pmap_kremove((vaddr_t)entropy, PAGE_SIZE); 1065 km_free(entropy, PAGE_SIZE, &kv_any, &kp_none); 1066 } 1067 1068 #ifndef NO_PROPOLICE 1069 vaddr_t 1070 hibernate_unprotect_ssp(void) 1071 { 1072 struct kmem_dyn_mode kd_avoidalias; 1073 vaddr_t va = trunc_page((vaddr_t)&__guard_local); 1074 paddr_t pa; 1075 1076 pmap_extract(pmap_kernel(), va, &pa); 1077 1078 memset(&kd_avoidalias, 0, sizeof kd_avoidalias); 1079 kd_avoidalias.kd_prefer = pa; 1080 kd_avoidalias.kd_waitok = 1; 1081 va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_avoidalias); 1082 if (!va) 1083 panic("hibernate_unprotect_ssp"); 1084 1085 pmap_kenter_pa(va, pa, PROT_READ | PROT_WRITE); 1086 pmap_update(pmap_kernel()); 1087 1088 return va; 1089 } 1090 1091 void 1092 hibernate_reprotect_ssp(vaddr_t va) 1093 { 1094 pmap_kremove(va, PAGE_SIZE); 1095 km_free((void *)va, PAGE_SIZE, &kv_any, &kp_none); 1096 } 1097 #endif /* NO_PROPOLICE */ 1098 1099 /* 1100 * Reads the signature block from swap, checks against the current machine's 1101 * information. If the information matches, perform a resume by reading the 1102 * saved image into the pig area, and unpacking. 1103 * 1104 * Must be called with interrupts enabled. 1105 */ 1106 void 1107 hibernate_resume(void) 1108 { 1109 union hibernate_info hib; 1110 int s; 1111 #ifndef NO_PROPOLICE 1112 vsize_t off = (vaddr_t)&__guard_local - 1113 trunc_page((vaddr_t)&__guard_local); 1114 vaddr_t guard_va; 1115 #endif 1116 1117 /* Get current running machine's hibernate info */ 1118 memset(&hib, 0, sizeof(hib)); 1119 if (get_hibernate_info(&hib, 0)) { 1120 DPRINTF("couldn't retrieve machine's hibernate info\n"); 1121 return; 1122 } 1123 1124 /* Read hibernate info from disk */ 1125 s = splbio(); 1126 1127 DPRINTF("reading hibernate signature block location: %lld\n", 1128 hib.sig_offset); 1129 1130 if (hibernate_block_io(&hib, 1131 hib.sig_offset, 1132 DEV_BSIZE, (vaddr_t)&disk_hib, 0)) { 1133 DPRINTF("error in hibernate read"); 1134 splx(s); 1135 return; 1136 } 1137 1138 /* Check magic number */ 1139 if (disk_hib.magic != HIBERNATE_MAGIC) { 1140 DPRINTF("wrong magic number in hibernate signature: %x\n", 1141 disk_hib.magic); 1142 splx(s); 1143 return; 1144 } 1145 1146 /* 1147 * We (possibly) found a hibernate signature. Clear signature first, 1148 * to prevent accidental resume or endless resume cycles later. 1149 */ 1150 if (hibernate_clear_signature(&hib)) { 1151 DPRINTF("error clearing hibernate signature block\n"); 1152 splx(s); 1153 return; 1154 } 1155 1156 /* 1157 * If on-disk and in-memory hibernate signatures match, 1158 * this means we should do a resume from hibernate. 1159 */ 1160 if (hibernate_compare_signature(&hib, &disk_hib)) { 1161 DPRINTF("mismatched hibernate signature block\n"); 1162 splx(s); 1163 return; 1164 } 1165 disk_hib.dev = hib.dev; 1166 1167 #ifdef MULTIPROCESSOR 1168 /* XXX - if we fail later, we may need to rehatch APs on some archs */ 1169 DPRINTF("hibernate: quiescing APs\n"); 1170 hibernate_quiesce_cpus(); 1171 #endif /* MULTIPROCESSOR */ 1172 1173 /* Read the image from disk into the image (pig) area */ 1174 if (hibernate_read_image(&disk_hib)) 1175 goto fail; 1176 1177 DPRINTF("hibernate: quiescing devices\n"); 1178 if (config_suspend_all(DVACT_QUIESCE) != 0) 1179 goto fail; 1180 1181 #ifndef NO_PROPOLICE 1182 guard_va = hibernate_unprotect_ssp(); 1183 #endif /* NO_PROPOLICE */ 1184 1185 (void) splhigh(); 1186 hibernate_disable_intr_machdep(); 1187 cold = 2; 1188 1189 DPRINTF("hibernate: suspending devices\n"); 1190 if (config_suspend_all(DVACT_SUSPEND) != 0) { 1191 cold = 0; 1192 hibernate_enable_intr_machdep(); 1193 #ifndef NO_PROPOLICE 1194 hibernate_reprotect_ssp(guard_va); 1195 #endif /* ! NO_PROPOLICE */ 1196 goto fail; 1197 } 1198 1199 hibernate_preserve_entropy(&disk_hib); 1200 1201 printf("Unpacking image...\n"); 1202 1203 /* Switch stacks */ 1204 DPRINTF("hibernate: switching stacks\n"); 1205 hibernate_switch_stack_machdep(); 1206 1207 #ifndef NO_PROPOLICE 1208 /* Start using suspended kernel's propolice guard */ 1209 *(long *)(guard_va + off) = disk_hib.guard; 1210 hibernate_reprotect_ssp(guard_va); 1211 #endif /* ! NO_PROPOLICE */ 1212 1213 /* Unpack and resume */ 1214 hibernate_unpack_image(&disk_hib); 1215 1216 fail: 1217 splx(s); 1218 printf("\nUnable to resume hibernated image\n"); 1219 } 1220 1221 /* 1222 * Unpack image from pig area to original location by looping through the 1223 * list of output chunks in the order they should be restored (fchunks). 1224 * 1225 * Note that due to the stack smash protector and the fact that we have 1226 * switched stacks, it is not permitted to return from this function. 1227 */ 1228 void 1229 hibernate_unpack_image(union hibernate_info *hib) 1230 { 1231 struct hibernate_disk_chunk *chunks; 1232 union hibernate_info local_hib; 1233 paddr_t image_cur = global_pig_start; 1234 short i, *fchunks; 1235 char *pva; 1236 1237 /* Piglet will be identity mapped (VA == PA) */ 1238 pva = (char *)hib->piglet_pa; 1239 1240 fchunks = (short *)(pva + (4 * PAGE_SIZE)); 1241 1242 chunks = (struct hibernate_disk_chunk *)(pva + HIBERNATE_CHUNK_SIZE); 1243 1244 /* Can't use hiber_info that's passed in after this point */ 1245 bcopy(hib, &local_hib, sizeof(union hibernate_info)); 1246 local_hib.retguard_ofs = 0; 1247 1248 /* VA == PA */ 1249 local_hib.piglet_va = local_hib.piglet_pa; 1250 1251 /* 1252 * Point of no return. Once we pass this point, only kernel code can 1253 * be accessed. No global variables or other kernel data structures 1254 * are guaranteed to be coherent after unpack starts. 1255 * 1256 * The image is now in high memory (pig area), we unpack from the pig 1257 * to the correct location in memory. We'll eventually end up copying 1258 * on top of ourself, but we are assured the kernel code here is the 1259 * same between the hibernated and resuming kernel, and we are running 1260 * on our own stack, so the overwrite is ok. 1261 */ 1262 DPRINTF("hibernate: activating alt. pagetable and starting unpack\n"); 1263 hibernate_activate_resume_pt_machdep(); 1264 1265 for (i = 0; i < local_hib.chunk_ctr; i++) { 1266 /* Reset zlib for inflate */ 1267 if (hibernate_zlib_reset(&local_hib, 0) != Z_OK) 1268 panic("hibernate failed to reset zlib for inflate"); 1269 1270 hibernate_process_chunk(&local_hib, &chunks[fchunks[i]], 1271 image_cur); 1272 1273 image_cur += chunks[fchunks[i]].compressed_size; 1274 1275 } 1276 1277 /* 1278 * Resume the loaded kernel by jumping to the MD resume vector. 1279 * We won't be returning from this call. We pass the location of 1280 * the retguard save area so the MD code can replace it before 1281 * resuming. See the piglet layout at the top of this file for 1282 * more information on the layout of the piglet area. 1283 * 1284 * We use 'global_piglet_va' here since by the time we are at 1285 * this point, we have already unpacked the image, and we want 1286 * the suspended kernel's view of what the piglet was, before 1287 * suspend occurred (since we will need to use that in the retguard 1288 * copy code in hibernate_resume_machdep.) 1289 */ 1290 hibernate_resume_machdep(global_piglet_va + (110 * PAGE_SIZE)); 1291 } 1292 1293 /* 1294 * Bounce a compressed image chunk to the piglet, entering mappings for the 1295 * copied pages as needed 1296 */ 1297 void 1298 hibernate_copy_chunk_to_piglet(paddr_t img_cur, vaddr_t piglet, size_t size) 1299 { 1300 size_t ct, ofs; 1301 paddr_t src = img_cur; 1302 vaddr_t dest = piglet; 1303 1304 /* Copy first partial page */ 1305 ct = (PAGE_SIZE) - (src & PAGE_MASK); 1306 ofs = (src & PAGE_MASK); 1307 1308 if (ct < PAGE_SIZE) { 1309 hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, 1310 (src - ofs), 0); 1311 hibernate_flush(); 1312 bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE + ofs), (caddr_t)dest, ct); 1313 src += ct; 1314 dest += ct; 1315 } 1316 1317 /* Copy remaining pages */ 1318 while (src < size + img_cur) { 1319 hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, src, 0); 1320 hibernate_flush(); 1321 ct = PAGE_SIZE; 1322 bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE), (caddr_t)dest, ct); 1323 hibernate_flush(); 1324 src += ct; 1325 dest += ct; 1326 } 1327 } 1328 1329 /* 1330 * Process a chunk by bouncing it to the piglet, followed by unpacking 1331 */ 1332 void 1333 hibernate_process_chunk(union hibernate_info *hib, 1334 struct hibernate_disk_chunk *chunk, paddr_t img_cur) 1335 { 1336 char *pva = (char *)hib->piglet_va; 1337 1338 hibernate_copy_chunk_to_piglet(img_cur, 1339 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), chunk->compressed_size); 1340 hibernate_inflate_region(hib, chunk->base, 1341 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), 1342 chunk->compressed_size); 1343 } 1344 1345 /* 1346 * Calculate RLE component for 'inaddr'. Clamps to max RLE pages between 1347 * inaddr and range_end. 1348 */ 1349 int 1350 hibernate_calc_rle(paddr_t inaddr, paddr_t range_end) 1351 { 1352 int rle; 1353 1354 rle = uvm_page_rle(inaddr); 1355 KASSERT(rle >= 0 && rle <= MAX_RLE); 1356 1357 /* Clamp RLE to range end */ 1358 if (rle > 0 && inaddr + (rle * PAGE_SIZE) > range_end) 1359 rle = (range_end - inaddr) / PAGE_SIZE; 1360 1361 return (rle); 1362 } 1363 1364 /* 1365 * Write the RLE byte for page at 'inaddr' to the output stream. 1366 * Returns the number of pages to be skipped at 'inaddr'. 1367 */ 1368 int 1369 hibernate_write_rle(union hibernate_info *hib, paddr_t inaddr, 1370 paddr_t range_end, daddr_t *blkctr, 1371 size_t *out_remaining) 1372 { 1373 int rle, err, *rleloc; 1374 struct hibernate_zlib_state *hibernate_state; 1375 vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE; 1376 1377 hibernate_state = 1378 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 1379 1380 rle = hibernate_calc_rle(inaddr, range_end); 1381 1382 rleloc = (int *)hibernate_rle_page + MAX_RLE - 1; 1383 *rleloc = rle; 1384 1385 /* Deflate the RLE byte into the stream */ 1386 hibernate_deflate(hib, (paddr_t)rleloc, out_remaining); 1387 1388 /* Did we fill the output page? If so, flush to disk */ 1389 if (*out_remaining == 0) { 1390 if ((err = hib->io_func(hib->dev, *blkctr + hib->image_offset, 1391 (vaddr_t)hibernate_io_page, PAGE_SIZE, HIB_W, 1392 hib->io_page))) { 1393 DPRINTF("hib write error %d\n", err); 1394 return (err); 1395 } 1396 1397 *blkctr += PAGE_SIZE / DEV_BSIZE; 1398 *out_remaining = PAGE_SIZE; 1399 1400 /* If we didn't deflate the entire RLE byte, finish it now */ 1401 if (hibernate_state->hib_stream.avail_in != 0) 1402 hibernate_deflate(hib, 1403 (vaddr_t)hibernate_state->hib_stream.next_in, 1404 out_remaining); 1405 } 1406 1407 return (rle); 1408 } 1409 1410 /* 1411 * Write a compressed version of this machine's memory to disk, at the 1412 * precalculated swap offset: 1413 * 1414 * end of swap - signature block size - chunk table size - memory size 1415 * 1416 * The function begins by looping through each phys mem range, cutting each 1417 * one into MD sized chunks. These chunks are then compressed individually 1418 * and written out to disk, in phys mem order. Some chunks might compress 1419 * more than others, and for this reason, each chunk's size is recorded 1420 * in the chunk table, which is written to disk after the image has 1421 * properly been compressed and written (in hibernate_write_chunktable). 1422 * 1423 * When this function is called, the machine is nearly suspended - most 1424 * devices are quiesced/suspended, interrupts are off, and cold has 1425 * been set. This means that there can be no side effects once the 1426 * write has started, and the write function itself can also have no 1427 * side effects. This also means no printfs are permitted (since printf 1428 * has side effects.) 1429 * 1430 * Return values : 1431 * 1432 * 0 - success 1433 * EIO - I/O error occurred writing the chunks 1434 * EINVAL - Failed to write a complete range 1435 * ENOMEM - Memory allocation failure during preparation of the zlib arena 1436 */ 1437 int 1438 hibernate_write_chunks(union hibernate_info *hib) 1439 { 1440 paddr_t range_base, range_end, inaddr, temp_inaddr; 1441 size_t nblocks, out_remaining, used; 1442 struct hibernate_disk_chunk *chunks; 1443 vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE; 1444 daddr_t blkctr = 0; 1445 int i, rle, err; 1446 struct hibernate_zlib_state *hibernate_state; 1447 1448 hibernate_state = 1449 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 1450 1451 hib->chunk_ctr = 0; 1452 1453 /* 1454 * Map the utility VAs to the piglet. See the piglet map at the 1455 * top of this file for piglet layout information. 1456 */ 1457 hibernate_copy_page = hib->piglet_va + 3 * PAGE_SIZE; 1458 hibernate_rle_page = hib->piglet_va + 28 * PAGE_SIZE; 1459 1460 chunks = (struct hibernate_disk_chunk *)(hib->piglet_va + 1461 HIBERNATE_CHUNK_SIZE); 1462 1463 /* Calculate the chunk regions */ 1464 for (i = 0; i < hib->nranges; i++) { 1465 range_base = hib->ranges[i].base; 1466 range_end = hib->ranges[i].end; 1467 1468 inaddr = range_base; 1469 1470 while (inaddr < range_end) { 1471 chunks[hib->chunk_ctr].base = inaddr; 1472 if (inaddr + HIBERNATE_CHUNK_SIZE < range_end) 1473 chunks[hib->chunk_ctr].end = inaddr + 1474 HIBERNATE_CHUNK_SIZE; 1475 else 1476 chunks[hib->chunk_ctr].end = range_end; 1477 1478 inaddr += HIBERNATE_CHUNK_SIZE; 1479 hib->chunk_ctr ++; 1480 } 1481 } 1482 1483 uvm_pmr_dirty_everything(); 1484 uvm_pmr_zero_everything(); 1485 1486 /* Compress and write the chunks in the chunktable */ 1487 for (i = 0; i < hib->chunk_ctr; i++) { 1488 range_base = chunks[i].base; 1489 range_end = chunks[i].end; 1490 1491 chunks[i].offset = blkctr + hib->image_offset; 1492 1493 /* Reset zlib for deflate */ 1494 if (hibernate_zlib_reset(hib, 1) != Z_OK) { 1495 DPRINTF("hibernate_zlib_reset failed for deflate\n"); 1496 return (ENOMEM); 1497 } 1498 1499 inaddr = range_base; 1500 1501 /* 1502 * For each range, loop through its phys mem region 1503 * and write out the chunks (the last chunk might be 1504 * smaller than the chunk size). 1505 */ 1506 while (inaddr < range_end) { 1507 out_remaining = PAGE_SIZE; 1508 while (out_remaining > 0 && inaddr < range_end) { 1509 /* 1510 * Adjust for regions that are not evenly 1511 * divisible by PAGE_SIZE or overflowed 1512 * pages from the previous iteration. 1513 */ 1514 temp_inaddr = (inaddr & PAGE_MASK) + 1515 hibernate_copy_page; 1516 1517 /* Deflate from temp_inaddr to IO page */ 1518 if (inaddr != range_end) { 1519 if (inaddr % PAGE_SIZE == 0) { 1520 rle = hibernate_write_rle(hib, 1521 inaddr, 1522 range_end, 1523 &blkctr, 1524 &out_remaining); 1525 } 1526 1527 if (rle == 0) { 1528 pmap_kenter_pa(hibernate_temp_page, 1529 inaddr & PMAP_PA_MASK, 1530 PROT_READ); 1531 1532 bcopy((caddr_t)hibernate_temp_page, 1533 (caddr_t)hibernate_copy_page, 1534 PAGE_SIZE); 1535 inaddr += hibernate_deflate(hib, 1536 temp_inaddr, 1537 &out_remaining); 1538 } else { 1539 inaddr += rle * PAGE_SIZE; 1540 if (inaddr > range_end) 1541 inaddr = range_end; 1542 } 1543 1544 } 1545 1546 if (out_remaining == 0) { 1547 /* Filled up the page */ 1548 nblocks = PAGE_SIZE / DEV_BSIZE; 1549 1550 if ((err = hib->io_func(hib->dev, 1551 blkctr + hib->image_offset, 1552 (vaddr_t)hibernate_io_page, 1553 PAGE_SIZE, HIB_W, hib->io_page))) { 1554 DPRINTF("hib write error %d\n", 1555 err); 1556 return (err); 1557 } 1558 1559 blkctr += nblocks; 1560 } 1561 } 1562 } 1563 1564 if (inaddr != range_end) { 1565 DPRINTF("deflate range ended prematurely\n"); 1566 return (EINVAL); 1567 } 1568 1569 /* 1570 * End of range. Round up to next secsize bytes 1571 * after finishing compress 1572 */ 1573 if (out_remaining == 0) 1574 out_remaining = PAGE_SIZE; 1575 1576 /* Finish compress */ 1577 hibernate_state->hib_stream.next_in = (unsigned char *)inaddr; 1578 hibernate_state->hib_stream.avail_in = 0; 1579 hibernate_state->hib_stream.next_out = 1580 (unsigned char *)hibernate_io_page + 1581 (PAGE_SIZE - out_remaining); 1582 1583 /* We have an extra output page available for finalize */ 1584 hibernate_state->hib_stream.avail_out = 1585 out_remaining + PAGE_SIZE; 1586 1587 if ((err = deflate(&hibernate_state->hib_stream, Z_FINISH)) != 1588 Z_STREAM_END) { 1589 DPRINTF("deflate error in output stream: %d\n", err); 1590 return (err); 1591 } 1592 1593 out_remaining = hibernate_state->hib_stream.avail_out; 1594 1595 used = 2 * PAGE_SIZE - out_remaining; 1596 nblocks = used / DEV_BSIZE; 1597 1598 /* Round up to next block if needed */ 1599 if (used % DEV_BSIZE != 0) 1600 nblocks ++; 1601 1602 /* Write final block(s) for this chunk */ 1603 if ((err = hib->io_func(hib->dev, blkctr + hib->image_offset, 1604 (vaddr_t)hibernate_io_page, nblocks*DEV_BSIZE, 1605 HIB_W, hib->io_page))) { 1606 DPRINTF("hib final write error %d\n", err); 1607 return (err); 1608 } 1609 1610 blkctr += nblocks; 1611 1612 chunks[i].compressed_size = (blkctr + hib->image_offset - 1613 chunks[i].offset) * DEV_BSIZE; 1614 } 1615 1616 hib->chunktable_offset = hib->image_offset + blkctr; 1617 return (0); 1618 } 1619 1620 /* 1621 * Reset the zlib stream state and allocate a new hiballoc area for either 1622 * inflate or deflate. This function is called once for each hibernate chunk. 1623 * Calling hiballoc_init multiple times is acceptable since the memory it is 1624 * provided is unmanaged memory (stolen). We use the memory provided to us 1625 * by the piglet allocated via the supplied hib. 1626 */ 1627 int 1628 hibernate_zlib_reset(union hibernate_info *hib, int deflate) 1629 { 1630 vaddr_t hibernate_zlib_start; 1631 size_t hibernate_zlib_size; 1632 char *pva = (char *)hib->piglet_va; 1633 struct hibernate_zlib_state *hibernate_state; 1634 1635 hibernate_state = 1636 (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE; 1637 1638 if (!deflate) 1639 pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK)); 1640 1641 /* 1642 * See piglet layout information at the start of this file for 1643 * information on the zlib page assignments. 1644 */ 1645 hibernate_zlib_start = (vaddr_t)(pva + (30 * PAGE_SIZE)); 1646 hibernate_zlib_size = 80 * PAGE_SIZE; 1647 1648 memset((void *)hibernate_zlib_start, 0, hibernate_zlib_size); 1649 memset(hibernate_state, 0, PAGE_SIZE); 1650 1651 /* Set up stream structure */ 1652 hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc; 1653 hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free; 1654 1655 /* Initialize the hiballoc arena for zlib allocs/frees */ 1656 hiballoc_init(&hibernate_state->hiballoc_arena, 1657 (caddr_t)hibernate_zlib_start, hibernate_zlib_size); 1658 1659 if (deflate) { 1660 return deflateInit(&hibernate_state->hib_stream, 1661 Z_BEST_SPEED); 1662 } else 1663 return inflateInit(&hibernate_state->hib_stream); 1664 } 1665 1666 /* 1667 * Reads the hibernated memory image from disk, whose location and 1668 * size are recorded in hib. Begin by reading the persisted 1669 * chunk table, which records the original chunk placement location 1670 * and compressed size for each. Next, allocate a pig region of 1671 * sufficient size to hold the compressed image. Next, read the 1672 * chunks into the pig area (calling hibernate_read_chunks to do this), 1673 * and finally, if all of the above succeeds, clear the hibernate signature. 1674 * The function will then return to hibernate_resume, which will proceed 1675 * to unpack the pig image to the correct place in memory. 1676 */ 1677 int 1678 hibernate_read_image(union hibernate_info *hib) 1679 { 1680 size_t compressed_size, disk_size, chunktable_size, pig_sz; 1681 paddr_t image_start, image_end, pig_start, pig_end; 1682 struct hibernate_disk_chunk *chunks; 1683 daddr_t blkctr; 1684 vaddr_t chunktable = (vaddr_t)NULL; 1685 paddr_t piglet_chunktable = hib->piglet_pa + 1686 HIBERNATE_CHUNK_SIZE; 1687 int i, status; 1688 1689 status = 0; 1690 pmap_activate(curproc); 1691 1692 /* Calculate total chunk table size in disk blocks */ 1693 chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / DEV_BSIZE; 1694 1695 blkctr = hib->chunktable_offset; 1696 1697 chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any, 1698 &kp_none, &kd_nowait); 1699 1700 if (!chunktable) 1701 return (1); 1702 1703 /* Map chunktable pages */ 1704 for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; i += PAGE_SIZE) 1705 pmap_kenter_pa(chunktable + i, piglet_chunktable + i, 1706 PROT_READ | PROT_WRITE); 1707 pmap_update(pmap_kernel()); 1708 1709 /* Read the chunktable from disk into the piglet chunktable */ 1710 for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; 1711 i += MAXPHYS, blkctr += MAXPHYS/DEV_BSIZE) 1712 hibernate_block_io(hib, blkctr, MAXPHYS, 1713 chunktable + i, 0); 1714 1715 blkctr = hib->image_offset; 1716 compressed_size = 0; 1717 1718 chunks = (struct hibernate_disk_chunk *)chunktable; 1719 1720 for (i = 0; i < hib->chunk_ctr; i++) 1721 compressed_size += chunks[i].compressed_size; 1722 1723 disk_size = compressed_size; 1724 1725 printf("unhibernating @ block %lld length %luMB\n", 1726 hib->sig_offset - chunktable_size, 1727 compressed_size / (1024 * 1024)); 1728 1729 /* Allocate the pig area */ 1730 pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE; 1731 if (uvm_pmr_alloc_pig(&pig_start, pig_sz, hib->piglet_pa) == ENOMEM) { 1732 status = 1; 1733 goto unmap; 1734 } 1735 1736 pig_end = pig_start + pig_sz; 1737 1738 /* Calculate image extents. Pig image must end on a chunk boundary. */ 1739 image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1); 1740 image_start = image_end - disk_size; 1741 1742 hibernate_read_chunks(hib, image_start, image_end, disk_size, 1743 chunks); 1744 1745 /* Prepare the resume time pmap/page table */ 1746 hibernate_populate_resume_pt(hib, image_start, image_end); 1747 1748 unmap: 1749 /* Unmap chunktable pages */ 1750 pmap_kremove(chunktable, HIBERNATE_CHUNK_TABLE_SIZE); 1751 pmap_update(pmap_kernel()); 1752 1753 return (status); 1754 } 1755 1756 /* 1757 * Read the hibernated memory chunks from disk (chunk information at this 1758 * point is stored in the piglet) into the pig area specified by 1759 * [pig_start .. pig_end]. Order the chunks so that the final chunk is the 1760 * only chunk with overlap possibilities. 1761 */ 1762 int 1763 hibernate_read_chunks(union hibernate_info *hib, paddr_t pig_start, 1764 paddr_t pig_end, size_t image_compr_size, 1765 struct hibernate_disk_chunk *chunks) 1766 { 1767 paddr_t img_cur, piglet_base; 1768 daddr_t blkctr; 1769 size_t processed, compressed_size, read_size; 1770 int nchunks, nfchunks, num_io_pages; 1771 vaddr_t tempva, hibernate_fchunk_area; 1772 short *fchunks, i, j; 1773 1774 tempva = (vaddr_t)NULL; 1775 hibernate_fchunk_area = (vaddr_t)NULL; 1776 nfchunks = 0; 1777 piglet_base = hib->piglet_pa; 1778 global_pig_start = pig_start; 1779 1780 /* 1781 * These mappings go into the resuming kernel's page table, and are 1782 * used only during image read. They disappear from existence 1783 * when the suspended kernel is unpacked on top of us. 1784 */ 1785 tempva = (vaddr_t)km_alloc(MAXPHYS + PAGE_SIZE, &kv_any, &kp_none, 1786 &kd_nowait); 1787 if (!tempva) 1788 return (1); 1789 hibernate_fchunk_area = (vaddr_t)km_alloc(24 * PAGE_SIZE, &kv_any, 1790 &kp_none, &kd_nowait); 1791 if (!hibernate_fchunk_area) 1792 return (1); 1793 1794 /* Final output chunk ordering VA */ 1795 fchunks = (short *)hibernate_fchunk_area; 1796 1797 /* Map the chunk ordering region */ 1798 for(i = 0; i < 24 ; i++) 1799 pmap_kenter_pa(hibernate_fchunk_area + (i * PAGE_SIZE), 1800 piglet_base + ((4 + i) * PAGE_SIZE), 1801 PROT_READ | PROT_WRITE); 1802 pmap_update(pmap_kernel()); 1803 1804 nchunks = hib->chunk_ctr; 1805 1806 /* Initially start all chunks as unplaced */ 1807 for (i = 0; i < nchunks; i++) 1808 chunks[i].flags = 0; 1809 1810 /* 1811 * Search the list for chunks that are outside the pig area. These 1812 * can be placed first in the final output list. 1813 */ 1814 for (i = 0; i < nchunks; i++) { 1815 if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) { 1816 fchunks[nfchunks] = i; 1817 nfchunks++; 1818 chunks[i].flags |= HIBERNATE_CHUNK_PLACED; 1819 } 1820 } 1821 1822 /* 1823 * Walk the ordering, place the chunks in ascending memory order. 1824 */ 1825 for (i = 0; i < nchunks; i++) { 1826 if (chunks[i].flags != HIBERNATE_CHUNK_PLACED) { 1827 fchunks[nfchunks] = i; 1828 nfchunks++; 1829 chunks[i].flags = HIBERNATE_CHUNK_PLACED; 1830 } 1831 } 1832 1833 img_cur = pig_start; 1834 1835 for (i = 0; i < nfchunks; i++) { 1836 blkctr = chunks[fchunks[i]].offset; 1837 processed = 0; 1838 compressed_size = chunks[fchunks[i]].compressed_size; 1839 1840 while (processed < compressed_size) { 1841 if (compressed_size - processed >= MAXPHYS) 1842 read_size = MAXPHYS; 1843 else 1844 read_size = compressed_size - processed; 1845 1846 /* 1847 * We're reading read_size bytes, offset from the 1848 * start of a page by img_cur % PAGE_SIZE, so the 1849 * end will be read_size + (img_cur % PAGE_SIZE) 1850 * from the start of the first page. Round that 1851 * up to the next page size. 1852 */ 1853 num_io_pages = (read_size + (img_cur % PAGE_SIZE) 1854 + PAGE_SIZE - 1) / PAGE_SIZE; 1855 1856 KASSERT(num_io_pages <= MAXPHYS/PAGE_SIZE + 1); 1857 1858 /* Map pages for this read */ 1859 for (j = 0; j < num_io_pages; j ++) 1860 pmap_kenter_pa(tempva + j * PAGE_SIZE, 1861 img_cur + j * PAGE_SIZE, 1862 PROT_READ | PROT_WRITE); 1863 1864 pmap_update(pmap_kernel()); 1865 1866 hibernate_block_io(hib, blkctr, read_size, 1867 tempva + (img_cur & PAGE_MASK), 0); 1868 1869 blkctr += (read_size / DEV_BSIZE); 1870 1871 pmap_kremove(tempva, num_io_pages * PAGE_SIZE); 1872 pmap_update(pmap_kernel()); 1873 1874 processed += read_size; 1875 img_cur += read_size; 1876 } 1877 } 1878 1879 pmap_kremove(hibernate_fchunk_area, 24 * PAGE_SIZE); 1880 pmap_update(pmap_kernel()); 1881 1882 return (0); 1883 } 1884 1885 /* 1886 * Hibernating a machine comprises the following operations: 1887 * 1. Calculating this machine's hibernate_info information 1888 * 2. Allocating a piglet and saving the piglet's physaddr 1889 * 3. Calculating the memory chunks 1890 * 4. Writing the compressed chunks to disk 1891 * 5. Writing the chunk table 1892 * 6. Writing the signature block (hibernate_info) 1893 * 1894 * On most architectures, the function calling hibernate_suspend would 1895 * then power off the machine using some MD-specific implementation. 1896 */ 1897 int 1898 hibernate_suspend(void) 1899 { 1900 union hibernate_info hib; 1901 u_long start, end; 1902 1903 /* 1904 * Calculate memory ranges, swap offsets, etc. 1905 * This also allocates a piglet whose physaddr is stored in 1906 * hib->piglet_pa and vaddr stored in hib->piglet_va 1907 */ 1908 if (get_hibernate_info(&hib, 1)) { 1909 DPRINTF("failed to obtain hibernate info\n"); 1910 return (1); 1911 } 1912 1913 /* Find a page-addressed region in swap [start,end] */ 1914 if (uvm_hibswap(hib.dev, &start, &end)) { 1915 printf("hibernate: cannot find any swap\n"); 1916 return (1); 1917 } 1918 1919 if (end - start < 1000) { 1920 printf("hibernate: insufficient swap (%lu is too small)\n", 1921 end - start); 1922 return (1); 1923 } 1924 1925 /* Calculate block offsets in swap */ 1926 hib.image_offset = ctod(start); 1927 1928 DPRINTF("hibernate @ block %lld max-length %lu blocks\n", 1929 hib.image_offset, ctod(end) - ctod(start)); 1930 1931 pmap_activate(curproc); 1932 DPRINTF("hibernate: writing chunks\n"); 1933 if (hibernate_write_chunks(&hib)) { 1934 DPRINTF("hibernate_write_chunks failed\n"); 1935 return (1); 1936 } 1937 1938 DPRINTF("hibernate: writing chunktable\n"); 1939 if (hibernate_write_chunktable(&hib)) { 1940 DPRINTF("hibernate_write_chunktable failed\n"); 1941 return (1); 1942 } 1943 1944 DPRINTF("hibernate: writing signature\n"); 1945 if (hibernate_write_signature(&hib)) { 1946 DPRINTF("hibernate_write_signature failed\n"); 1947 return (1); 1948 } 1949 1950 /* Allow the disk to settle */ 1951 delay(500000); 1952 1953 /* 1954 * Give the device-specific I/O function a notification that we're 1955 * done, and that it can clean up or shutdown as needed. 1956 */ 1957 hib.io_func(hib.dev, 0, (vaddr_t)NULL, 0, HIB_DONE, hib.io_page); 1958 return (0); 1959 } 1960 1961 int 1962 hibernate_alloc(void) 1963 { 1964 KASSERT(global_piglet_va == 0); 1965 KASSERT(hibernate_temp_page == 0); 1966 1967 pmap_activate(curproc); 1968 pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE, 1969 PROT_READ | PROT_WRITE); 1970 1971 /* Allocate a piglet, store its addresses in the supplied globals */ 1972 if (uvm_pmr_alloc_piglet(&global_piglet_va, &global_piglet_pa, 1973 HIBERNATE_CHUNK_SIZE * 4, HIBERNATE_CHUNK_SIZE)) 1974 goto unmap; 1975 1976 /* 1977 * Allocate VA for the temp page. 1978 * 1979 * This will become part of the suspended kernel and will 1980 * be freed in hibernate_free, upon resume (or hibernate 1981 * failure) 1982 */ 1983 hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, 1984 &kp_none, &kd_nowait); 1985 if (!hibernate_temp_page) { 1986 uvm_pmr_free_piglet(global_piglet_va, 1987 4 * HIBERNATE_CHUNK_SIZE); 1988 global_piglet_va = 0; 1989 goto unmap; 1990 } 1991 return (0); 1992 unmap: 1993 pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE); 1994 pmap_update(pmap_kernel()); 1995 return (ENOMEM); 1996 } 1997 1998 /* 1999 * Free items allocated by hibernate_alloc() 2000 */ 2001 void 2002 hibernate_free(void) 2003 { 2004 pmap_activate(curproc); 2005 2006 if (global_piglet_va) 2007 uvm_pmr_free_piglet(global_piglet_va, 2008 4 * HIBERNATE_CHUNK_SIZE); 2009 2010 if (hibernate_temp_page) { 2011 pmap_kremove(hibernate_temp_page, PAGE_SIZE); 2012 km_free((void *)hibernate_temp_page, PAGE_SIZE, 2013 &kv_any, &kp_none); 2014 } 2015 2016 global_piglet_va = 0; 2017 hibernate_temp_page = 0; 2018 pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE); 2019 pmap_update(pmap_kernel()); 2020 } 2021