xref: /openbsd-src/sys/kern/subr_hibernate.c (revision 02de433d8c4801712cfa33b1092e7f2c411d534e)
1 /*	$OpenBSD: subr_hibernate.c,v 1.135 2022/07/29 17:47:12 semarie Exp $	*/
2 
3 /*
4  * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl>
5  * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/hibernate.h>
21 #include <sys/malloc.h>
22 #include <sys/param.h>
23 #include <sys/tree.h>
24 #include <sys/systm.h>
25 #include <sys/disklabel.h>
26 #include <sys/disk.h>
27 #include <sys/conf.h>
28 #include <sys/buf.h>
29 #include <sys/fcntl.h>
30 #include <sys/stat.h>
31 #include <sys/atomic.h>
32 
33 #include <uvm/uvm.h>
34 #include <uvm/uvm_swap.h>
35 
36 #include <machine/hibernate.h>
37 
38 /* Make sure the signature can fit in one block */
39 CTASSERT(sizeof(union hibernate_info) <= DEV_BSIZE);
40 
41 /*
42  * Hibernate piglet layout information
43  *
44  * The piglet is a scratch area of memory allocated by the suspending kernel.
45  * Its phys and virt addrs are recorded in the signature block. The piglet is
46  * used to guarantee an unused area of memory that can be used by the resuming
47  * kernel for various things. The piglet is excluded during unpack operations.
48  * The piglet size is presently 4*HIBERNATE_CHUNK_SIZE (typically 4*4MB).
49  *
50  * Offset from piglet_base	Purpose
51  * ----------------------------------------------------------------------------
52  * 0				Private page for suspend I/O write functions
53  * 1*PAGE_SIZE			I/O page used during hibernate suspend
54  * 2*PAGE_SIZE			I/O page used during hibernate suspend
55  * 3*PAGE_SIZE			copy page used during hibernate suspend
56  * 4*PAGE_SIZE			final chunk ordering list (24 pages)
57  * 28*PAGE_SIZE			RLE utility page
58  * 29*PAGE_SIZE			start of hiballoc area
59  * 30*PAGE_SIZE			preserved entropy
60  * 110*PAGE_SIZE		end of hiballoc area (80 pages)
61  * 366*PAGE_SIZE		end of retguard preservation region (256 pages)
62  * ...				unused
63  * HIBERNATE_CHUNK_SIZE		start of hibernate chunk table
64  * 2*HIBERNATE_CHUNK_SIZE	bounce area for chunks being unpacked
65  * 4*HIBERNATE_CHUNK_SIZE	end of piglet
66  */
67 
68 /* Temporary vaddr ranges used during hibernate */
69 vaddr_t hibernate_temp_page;
70 vaddr_t hibernate_copy_page;
71 vaddr_t hibernate_rle_page;
72 
73 /* Hibernate info as read from disk during resume */
74 union hibernate_info disk_hib;
75 
76 /*
77  * Global copy of the pig start address. This needs to be a global as we
78  * switch stacks after computing it - it can't be stored on the stack.
79  */
80 paddr_t global_pig_start;
81 
82 /*
83  * Global copies of the piglet start addresses (PA/VA). We store these
84  * as globals to avoid having to carry them around as parameters, as the
85  * piglet is allocated early and freed late - its lifecycle extends beyond
86  * that of the hibernate info union which is calculated on suspend/resume.
87  */
88 vaddr_t global_piglet_va;
89 paddr_t global_piglet_pa;
90 
91 /* #define HIB_DEBUG */
92 #ifdef HIB_DEBUG
93 int	hib_debug = 99;
94 #define DPRINTF(x...)     do { if (hib_debug) printf(x); } while (0)
95 #define DNPRINTF(n,x...)  do { if (hib_debug > (n)) printf(x); } while (0)
96 #else
97 #define DPRINTF(x...)
98 #define DNPRINTF(n,x...)
99 #endif
100 
101 #ifndef NO_PROPOLICE
102 extern long __guard_local;
103 #endif /* ! NO_PROPOLICE */
104 
105 void hibernate_copy_chunk_to_piglet(paddr_t, vaddr_t, size_t);
106 int hibernate_calc_rle(paddr_t, paddr_t);
107 int hibernate_write_rle(union hibernate_info *, paddr_t, paddr_t, daddr_t *,
108 	size_t *);
109 
110 #define MAX_RLE (HIBERNATE_CHUNK_SIZE / PAGE_SIZE)
111 
112 /*
113  * Hib alloc enforced alignment.
114  */
115 #define HIB_ALIGN		8 /* bytes alignment */
116 
117 /*
118  * sizeof builtin operation, but with alignment constraint.
119  */
120 #define HIB_SIZEOF(_type)	roundup(sizeof(_type), HIB_ALIGN)
121 
122 struct hiballoc_entry {
123 	size_t			hibe_use;
124 	size_t			hibe_space;
125 	RBT_ENTRY(hiballoc_entry) hibe_entry;
126 };
127 
128 /*
129  * Sort hibernate memory ranges by ascending PA
130  */
131 void
132 hibernate_sort_ranges(union hibernate_info *hib_info)
133 {
134 	int i, j;
135 	struct hibernate_memory_range *ranges;
136 	paddr_t base, end;
137 
138 	ranges = hib_info->ranges;
139 
140 	for (i = 1; i < hib_info->nranges; i++) {
141 		j = i;
142 		while (j > 0 && ranges[j - 1].base > ranges[j].base) {
143 			base = ranges[j].base;
144 			end = ranges[j].end;
145 			ranges[j].base = ranges[j - 1].base;
146 			ranges[j].end = ranges[j - 1].end;
147 			ranges[j - 1].base = base;
148 			ranges[j - 1].end = end;
149 			j--;
150 		}
151 	}
152 }
153 
154 /*
155  * Compare hiballoc entries based on the address they manage.
156  *
157  * Since the address is fixed, relative to struct hiballoc_entry,
158  * we just compare the hiballoc_entry pointers.
159  */
160 static __inline int
161 hibe_cmp(const struct hiballoc_entry *l, const struct hiballoc_entry *r)
162 {
163 	vaddr_t vl = (vaddr_t)l;
164 	vaddr_t vr = (vaddr_t)r;
165 
166 	return vl < vr ? -1 : (vl > vr);
167 }
168 
169 RBT_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
170 
171 /*
172  * Given a hiballoc entry, return the address it manages.
173  */
174 static __inline void *
175 hib_entry_to_addr(struct hiballoc_entry *entry)
176 {
177 	caddr_t addr;
178 
179 	addr = (caddr_t)entry;
180 	addr += HIB_SIZEOF(struct hiballoc_entry);
181 	return addr;
182 }
183 
184 /*
185  * Given an address, find the hiballoc that corresponds.
186  */
187 static __inline struct hiballoc_entry*
188 hib_addr_to_entry(void *addr_param)
189 {
190 	caddr_t addr;
191 
192 	addr = (caddr_t)addr_param;
193 	addr -= HIB_SIZEOF(struct hiballoc_entry);
194 	return (struct hiballoc_entry*)addr;
195 }
196 
197 RBT_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp);
198 
199 /*
200  * Allocate memory from the arena.
201  *
202  * Returns NULL if no memory is available.
203  */
204 void *
205 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz)
206 {
207 	struct hiballoc_entry *entry, *new_entry;
208 	size_t find_sz;
209 
210 	/*
211 	 * Enforce alignment of HIB_ALIGN bytes.
212 	 *
213 	 * Note that, because the entry is put in front of the allocation,
214 	 * 0-byte allocations are guaranteed a unique address.
215 	 */
216 	alloc_sz = roundup(alloc_sz, HIB_ALIGN);
217 
218 	/*
219 	 * Find an entry with hibe_space >= find_sz.
220 	 *
221 	 * If the root node is not large enough, we switch to tree traversal.
222 	 * Because all entries are made at the bottom of the free space,
223 	 * traversal from the end has a slightly better chance of yielding
224 	 * a sufficiently large space.
225 	 */
226 	find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry);
227 	entry = RBT_ROOT(hiballoc_addr, &arena->hib_addrs);
228 	if (entry != NULL && entry->hibe_space < find_sz) {
229 		RBT_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) {
230 			if (entry->hibe_space >= find_sz)
231 				break;
232 		}
233 	}
234 
235 	/*
236 	 * Insufficient or too fragmented memory.
237 	 */
238 	if (entry == NULL)
239 		return NULL;
240 
241 	/*
242 	 * Create new entry in allocated space.
243 	 */
244 	new_entry = (struct hiballoc_entry*)(
245 	    (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use);
246 	new_entry->hibe_space = entry->hibe_space - find_sz;
247 	new_entry->hibe_use = alloc_sz;
248 
249 	/*
250 	 * Insert entry.
251 	 */
252 	if (RBT_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL)
253 		panic("hib_alloc: insert failure");
254 	entry->hibe_space = 0;
255 
256 	/* Return address managed by entry. */
257 	return hib_entry_to_addr(new_entry);
258 }
259 
260 void
261 hib_getentropy(char **bufp, size_t *bufplen)
262 {
263 	if (!bufp || !bufplen)
264 		return;
265 
266 	*bufp = (char *)(global_piglet_va + (29 * PAGE_SIZE));
267 	*bufplen = PAGE_SIZE;
268 }
269 
270 /*
271  * Free a pointer previously allocated from this arena.
272  *
273  * If addr is NULL, this will be silently accepted.
274  */
275 void
276 hib_free(struct hiballoc_arena *arena, void *addr)
277 {
278 	struct hiballoc_entry *entry, *prev;
279 
280 	if (addr == NULL)
281 		return;
282 
283 	/*
284 	 * Derive entry from addr and check it is really in this arena.
285 	 */
286 	entry = hib_addr_to_entry(addr);
287 	if (RBT_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry)
288 		panic("hib_free: freed item %p not in hib arena", addr);
289 
290 	/*
291 	 * Give the space in entry to its predecessor.
292 	 *
293 	 * If entry has no predecessor, change its used space into free space
294 	 * instead.
295 	 */
296 	prev = RBT_PREV(hiballoc_addr, entry);
297 	if (prev != NULL &&
298 	    (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) +
299 	    prev->hibe_use + prev->hibe_space) == entry) {
300 		/* Merge entry. */
301 		RBT_REMOVE(hiballoc_addr, &arena->hib_addrs, entry);
302 		prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) +
303 		    entry->hibe_use + entry->hibe_space;
304 	} else {
305 		/* Flip used memory to free space. */
306 		entry->hibe_space += entry->hibe_use;
307 		entry->hibe_use = 0;
308 	}
309 }
310 
311 /*
312  * Initialize hiballoc.
313  *
314  * The allocator will manage memory at ptr, which is len bytes.
315  */
316 int
317 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len)
318 {
319 	struct hiballoc_entry *entry;
320 	caddr_t ptr;
321 	size_t len;
322 
323 	RBT_INIT(hiballoc_addr, &arena->hib_addrs);
324 
325 	/*
326 	 * Hib allocator enforces HIB_ALIGN alignment.
327 	 * Fixup ptr and len.
328 	 */
329 	ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN);
330 	len = p_len - ((size_t)ptr - (size_t)p_ptr);
331 	len &= ~((size_t)HIB_ALIGN - 1);
332 
333 	/*
334 	 * Insufficient memory to be able to allocate and also do bookkeeping.
335 	 */
336 	if (len <= HIB_SIZEOF(struct hiballoc_entry))
337 		return ENOMEM;
338 
339 	/*
340 	 * Create entry describing space.
341 	 */
342 	entry = (struct hiballoc_entry*)ptr;
343 	entry->hibe_use = 0;
344 	entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry);
345 	RBT_INSERT(hiballoc_addr, &arena->hib_addrs, entry);
346 
347 	return 0;
348 }
349 
350 /*
351  * Zero all free memory.
352  */
353 void
354 uvm_pmr_zero_everything(void)
355 {
356 	struct uvm_pmemrange	*pmr;
357 	struct vm_page		*pg;
358 	int			 i;
359 
360 	uvm_lock_fpageq();
361 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
362 		/* Zero single pages. */
363 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY]))
364 		    != NULL) {
365 			uvm_pmr_remove(pmr, pg);
366 			uvm_pagezero(pg);
367 			atomic_setbits_int(&pg->pg_flags, PG_ZERO);
368 			uvmexp.zeropages++;
369 			uvm_pmr_insert(pmr, pg, 0);
370 		}
371 
372 		/* Zero multi page ranges. */
373 		while ((pg = RBT_ROOT(uvm_pmr_size,
374 		    &pmr->size[UVM_PMR_MEMTYPE_DIRTY])) != NULL) {
375 			pg--; /* Size tree always has second page. */
376 			uvm_pmr_remove(pmr, pg);
377 			for (i = 0; i < pg->fpgsz; i++) {
378 				uvm_pagezero(&pg[i]);
379 				atomic_setbits_int(&pg[i].pg_flags, PG_ZERO);
380 				uvmexp.zeropages++;
381 			}
382 			uvm_pmr_insert(pmr, pg, 0);
383 		}
384 	}
385 	uvm_unlock_fpageq();
386 }
387 
388 /*
389  * Mark all memory as dirty.
390  *
391  * Used to inform the system that the clean memory isn't clean for some
392  * reason, for example because we just came back from hibernate.
393  */
394 void
395 uvm_pmr_dirty_everything(void)
396 {
397 	struct uvm_pmemrange	*pmr;
398 	struct vm_page		*pg;
399 	int			 i;
400 
401 	uvm_lock_fpageq();
402 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
403 		/* Dirty single pages. */
404 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO]))
405 		    != NULL) {
406 			uvm_pmr_remove(pmr, pg);
407 			atomic_clearbits_int(&pg->pg_flags, PG_ZERO);
408 			uvm_pmr_insert(pmr, pg, 0);
409 		}
410 
411 		/* Dirty multi page ranges. */
412 		while ((pg = RBT_ROOT(uvm_pmr_size,
413 		    &pmr->size[UVM_PMR_MEMTYPE_ZERO])) != NULL) {
414 			pg--; /* Size tree always has second page. */
415 			uvm_pmr_remove(pmr, pg);
416 			for (i = 0; i < pg->fpgsz; i++)
417 				atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO);
418 			uvm_pmr_insert(pmr, pg, 0);
419 		}
420 	}
421 
422 	uvmexp.zeropages = 0;
423 	uvm_unlock_fpageq();
424 }
425 
426 /*
427  * Allocate an area that can hold sz bytes and doesn't overlap with
428  * the piglet at piglet_pa.
429  */
430 int
431 uvm_pmr_alloc_pig(paddr_t *pa, psize_t sz, paddr_t piglet_pa)
432 {
433 	struct uvm_constraint_range pig_constraint;
434 	struct kmem_pa_mode kp_pig = {
435 		.kp_constraint = &pig_constraint,
436 		.kp_maxseg = 1
437 	};
438 	vaddr_t va;
439 
440 	sz = round_page(sz);
441 
442 	pig_constraint.ucr_low = piglet_pa + 4 * HIBERNATE_CHUNK_SIZE;
443 	pig_constraint.ucr_high = -1;
444 
445 	va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait);
446 	if (va == 0) {
447 		pig_constraint.ucr_low = 0;
448 		pig_constraint.ucr_high = piglet_pa - 1;
449 
450 		va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait);
451 		if (va == 0)
452 			return ENOMEM;
453 	}
454 
455 	pmap_extract(pmap_kernel(), va, pa);
456 	return 0;
457 }
458 
459 /*
460  * Allocate a piglet area.
461  *
462  * This needs to be in DMA-safe memory.
463  * Piglets are aligned.
464  *
465  * sz and align in bytes.
466  *
467  * The call will sleep for the pagedaemon to attempt to free memory.
468  * The pagedaemon may decide its not possible to free enough memory, causing
469  * the allocation to fail.
470  */
471 int
472 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align)
473 {
474 	struct kmem_pa_mode kp_piglet = {
475 		.kp_constraint = &dma_constraint,
476 		.kp_align = align,
477 		.kp_maxseg = 1
478 	};
479 
480 	/* Ensure align is a power of 2 */
481 	KASSERT((align & (align - 1)) == 0);
482 
483 	/*
484 	 * Fixup arguments: align must be at least PAGE_SIZE,
485 	 * sz will be converted to pagecount, since that is what
486 	 * pmemrange uses internally.
487 	 */
488 	if (align < PAGE_SIZE)
489 		kp_piglet.kp_align = PAGE_SIZE;
490 
491 	sz = round_page(sz);
492 
493 	*va = (vaddr_t)km_alloc(sz, &kv_any, &kp_piglet, &kd_nowait);
494 	if (*va == 0)
495 		return ENOMEM;
496 
497 	pmap_extract(pmap_kernel(), *va, pa);
498 	return 0;
499 }
500 
501 /*
502  * Free a piglet area.
503  */
504 void
505 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz)
506 {
507 	/*
508 	 * Fix parameters.
509 	 */
510 	sz = round_page(sz);
511 
512 	/*
513 	 * Free the physical and virtual memory.
514 	 */
515 	km_free((void *)va, sz, &kv_any, &kp_dma_contig);
516 }
517 
518 /*
519  * Physmem RLE compression support.
520  *
521  * Given a physical page address, return the number of pages starting at the
522  * address that are free.  Clamps to the number of pages in
523  * HIBERNATE_CHUNK_SIZE. Returns 0 if the page at addr is not free.
524  */
525 int
526 uvm_page_rle(paddr_t addr)
527 {
528 	struct vm_page		*pg, *pg_end;
529 	struct vm_physseg	*vmp;
530 	int			 pseg_idx, off_idx;
531 
532 	pseg_idx = vm_physseg_find(atop(addr), &off_idx);
533 	if (pseg_idx == -1)
534 		return 0;
535 
536 	vmp = &vm_physmem[pseg_idx];
537 	pg = &vmp->pgs[off_idx];
538 	if (!(pg->pg_flags & PQ_FREE))
539 		return 0;
540 
541 	/*
542 	 * Search for the first non-free page after pg.
543 	 * Note that the page may not be the first page in a free pmemrange,
544 	 * therefore pg->fpgsz cannot be used.
545 	 */
546 	for (pg_end = pg; pg_end <= vmp->lastpg &&
547 	    (pg_end->pg_flags & PQ_FREE) == PQ_FREE &&
548 	    (pg_end - pg) < HIBERNATE_CHUNK_SIZE/PAGE_SIZE; pg_end++)
549 		;
550 	return pg_end - pg;
551 }
552 
553 /*
554  * Fills out the hibernate_info union pointed to by hib
555  * with information about this machine (swap signature block
556  * offsets, number of memory ranges, kernel in use, etc)
557  */
558 int
559 get_hibernate_info(union hibernate_info *hib, int suspend)
560 {
561 	struct disklabel dl;
562 	char err_string[128], *dl_ret;
563 	int part;
564 	SHA2_CTX ctx;
565 	void *fn;
566 
567 #ifndef NO_PROPOLICE
568 	/* Save propolice guard */
569 	hib->guard = __guard_local;
570 #endif /* ! NO_PROPOLICE */
571 
572 	/* Determine I/O function to use */
573 	hib->io_func = get_hibernate_io_function(swdevt[0].sw_dev);
574 	if (hib->io_func == NULL)
575 		return (1);
576 
577 	/* Calculate hibernate device */
578 	hib->dev = swdevt[0].sw_dev;
579 
580 	/* Read disklabel (used to calculate signature and image offsets) */
581 	dl_ret = disk_readlabel(&dl, hib->dev, err_string, sizeof(err_string));
582 
583 	if (dl_ret) {
584 		printf("Hibernate error reading disklabel: %s\n", dl_ret);
585 		return (1);
586 	}
587 
588 	/* Make sure we have a swap partition. */
589 	part = DISKPART(hib->dev);
590 	if (dl.d_npartitions <= part ||
591 	    dl.d_partitions[part].p_fstype != FS_SWAP ||
592 	    DL_GETPSIZE(&dl.d_partitions[part]) == 0)
593 		return (1);
594 
595 	/* Magic number */
596 	hib->magic = HIBERNATE_MAGIC;
597 
598 	/* Calculate signature block location */
599 	hib->sig_offset = DL_GETPSIZE(&dl.d_partitions[part]) -
600 	    sizeof(union hibernate_info)/DEV_BSIZE;
601 
602 	SHA256Init(&ctx);
603 	SHA256Update(&ctx, version, strlen(version));
604 	fn = printf;
605 	SHA256Update(&ctx, &fn, sizeof(fn));
606 	fn = malloc;
607 	SHA256Update(&ctx, &fn, sizeof(fn));
608 	fn = km_alloc;
609 	SHA256Update(&ctx, &fn, sizeof(fn));
610 	fn = strlen;
611 	SHA256Update(&ctx, &fn, sizeof(fn));
612 	SHA256Final((u_int8_t *)&hib->kern_hash, &ctx);
613 
614 	if (suspend) {
615 		/* Grab the previously-allocated piglet addresses */
616 		hib->piglet_va = global_piglet_va;
617 		hib->piglet_pa = global_piglet_pa;
618 		hib->io_page = (void *)hib->piglet_va;
619 
620 		/*
621 		 * Initialization of the hibernate IO function for drivers
622 		 * that need to do prep work (such as allocating memory or
623 		 * setting up data structures that cannot safely be done
624 		 * during suspend without causing side effects). There is
625 		 * a matching HIB_DONE call performed after the write is
626 		 * completed.
627 		 */
628 		if (hib->io_func(hib->dev, DL_GETPOFFSET(&dl.d_partitions[part]),
629 		    (vaddr_t)NULL, DL_GETPSIZE(&dl.d_partitions[part]),
630 		    HIB_INIT, hib->io_page))
631 			goto fail;
632 
633 	} else {
634 		/*
635 		 * Resuming kernels use a regular private page for the driver
636 		 * No need to free this I/O page as it will vanish as part of
637 		 * the resume.
638 		 */
639 		hib->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
640 		if (!hib->io_page)
641 			goto fail;
642 	}
643 
644 	if (get_hibernate_info_md(hib))
645 		goto fail;
646 
647 	return (0);
648 
649 fail:
650 	return (1);
651 }
652 
653 /*
654  * Allocate nitems*size bytes from the hiballoc area presently in use
655  */
656 void *
657 hibernate_zlib_alloc(void *unused, int nitems, int size)
658 {
659 	struct hibernate_zlib_state *hibernate_state;
660 
661 	hibernate_state =
662 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
663 
664 	return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size);
665 }
666 
667 /*
668  * Free the memory pointed to by addr in the hiballoc area presently in
669  * use
670  */
671 void
672 hibernate_zlib_free(void *unused, void *addr)
673 {
674 	struct hibernate_zlib_state *hibernate_state;
675 
676 	hibernate_state =
677 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
678 
679 	hib_free(&hibernate_state->hiballoc_arena, addr);
680 }
681 
682 /*
683  * Inflate next page of data from the image stream.
684  * The rle parameter is modified on exit to contain the number of pages to
685  * skip in the output stream (or 0 if this page was inflated into).
686  *
687  * Returns 0 if the stream contains additional data, or 1 if the stream is
688  * finished.
689  */
690 int
691 hibernate_inflate_page(int *rle)
692 {
693 	struct hibernate_zlib_state *hibernate_state;
694 	int i;
695 
696 	hibernate_state =
697 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
698 
699 	/* Set up the stream for RLE code inflate */
700 	hibernate_state->hib_stream.next_out = (unsigned char *)rle;
701 	hibernate_state->hib_stream.avail_out = sizeof(*rle);
702 
703 	/* Inflate RLE code */
704 	i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH);
705 	if (i != Z_OK && i != Z_STREAM_END) {
706 		/*
707 		 * XXX - this will likely reboot/hang most machines
708 		 *       since the console output buffer will be unmapped,
709 		 *       but there's not much else we can do here.
710 		 */
711 		panic("rle inflate stream error");
712 	}
713 
714 	if (hibernate_state->hib_stream.avail_out != 0) {
715 		/*
716 		 * XXX - this will likely reboot/hang most machines
717 		 *       since the console output buffer will be unmapped,
718 		 *       but there's not much else we can do here.
719 		 */
720 		panic("rle short inflate error");
721 	}
722 
723 	if (*rle < 0 || *rle > 1024) {
724 		/*
725 		 * XXX - this will likely reboot/hang most machines
726 		 *       since the console output buffer will be unmapped,
727 		 *       but there's not much else we can do here.
728 		 */
729 		panic("invalid rle count");
730 	}
731 
732 	if (i == Z_STREAM_END)
733 		return (1);
734 
735 	if (*rle != 0)
736 		return (0);
737 
738 	/* Set up the stream for page inflate */
739 	hibernate_state->hib_stream.next_out =
740 		(unsigned char *)HIBERNATE_INFLATE_PAGE;
741 	hibernate_state->hib_stream.avail_out = PAGE_SIZE;
742 
743 	/* Process next block of data */
744 	i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH);
745 	if (i != Z_OK && i != Z_STREAM_END) {
746 		/*
747 		 * XXX - this will likely reboot/hang most machines
748 		 *       since the console output buffer will be unmapped,
749 		 *       but there's not much else we can do here.
750 		 */
751 		panic("inflate error");
752 	}
753 
754 	/* We should always have extracted a full page ... */
755 	if (hibernate_state->hib_stream.avail_out != 0) {
756 		/*
757 		 * XXX - this will likely reboot/hang most machines
758 		 *       since the console output buffer will be unmapped,
759 		 *       but there's not much else we can do here.
760 		 */
761 		panic("incomplete page");
762 	}
763 
764 	return (i == Z_STREAM_END);
765 }
766 
767 /*
768  * Inflate size bytes from src into dest, skipping any pages in
769  * [src..dest] that are special (see hibernate_inflate_skip)
770  *
771  * This function executes while using the resume-time stack
772  * and pmap, and therefore cannot use ddb/printf/etc. Doing so
773  * will likely hang or reset the machine since the console output buffer
774  * will be unmapped.
775  */
776 void
777 hibernate_inflate_region(union hibernate_info *hib, paddr_t dest,
778     paddr_t src, size_t size)
779 {
780 	int end_stream = 0, rle, skip;
781 	struct hibernate_zlib_state *hibernate_state;
782 
783 	hibernate_state =
784 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
785 
786 	hibernate_state->hib_stream.next_in = (unsigned char *)src;
787 	hibernate_state->hib_stream.avail_in = size;
788 
789 	do {
790 		/*
791 		 * Is this a special page? If yes, redirect the
792 		 * inflate output to a scratch page (eg, discard it)
793 		 */
794 		skip = hibernate_inflate_skip(hib, dest);
795 		if (skip == HIB_SKIP) {
796 			hibernate_enter_resume_mapping(
797 			    HIBERNATE_INFLATE_PAGE,
798 			    HIBERNATE_INFLATE_PAGE, 0);
799 		} else if (skip == HIB_MOVE) {
800 			/*
801 			 * Special case : retguard region. This gets moved
802 			 * temporarily into the piglet region and copied into
803 			 * place immediately before resume
804 			 */
805 			hibernate_enter_resume_mapping(
806 			    HIBERNATE_INFLATE_PAGE,
807 			    hib->piglet_pa + (110 * PAGE_SIZE) +
808 			    hib->retguard_ofs, 0);
809 			hib->retguard_ofs += PAGE_SIZE;
810 			if (hib->retguard_ofs > 255 * PAGE_SIZE) {
811 				/*
812 				 * XXX - this will likely reboot/hang most
813 				 *       machines since the console output
814 				 *       buffer will be unmapped, but there's
815 				 *       not much else we can do here.
816 				 */
817 				panic("retguard move error, out of space");
818 			}
819 		} else {
820 			hibernate_enter_resume_mapping(
821 			    HIBERNATE_INFLATE_PAGE, dest, 0);
822 		}
823 
824 		hibernate_flush();
825 		end_stream = hibernate_inflate_page(&rle);
826 
827 		if (rle == 0)
828 			dest += PAGE_SIZE;
829 		else
830 			dest += (rle * PAGE_SIZE);
831 	} while (!end_stream);
832 }
833 
834 /*
835  * deflate from src into the I/O page, up to 'remaining' bytes
836  *
837  * Returns number of input bytes consumed, and may reset
838  * the 'remaining' parameter if not all the output space was consumed
839  * (this information is needed to know how much to write to disk)
840  */
841 size_t
842 hibernate_deflate(union hibernate_info *hib, paddr_t src,
843     size_t *remaining)
844 {
845 	vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
846 	struct hibernate_zlib_state *hibernate_state;
847 
848 	hibernate_state =
849 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
850 
851 	/* Set up the stream for deflate */
852 	hibernate_state->hib_stream.next_in = (unsigned char *)src;
853 	hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK);
854 	hibernate_state->hib_stream.next_out =
855 		(unsigned char *)hibernate_io_page + (PAGE_SIZE - *remaining);
856 	hibernate_state->hib_stream.avail_out = *remaining;
857 
858 	/* Process next block of data */
859 	if (deflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH) != Z_OK)
860 		panic("hibernate zlib deflate error");
861 
862 	/* Update pointers and return number of bytes consumed */
863 	*remaining = hibernate_state->hib_stream.avail_out;
864 	return (PAGE_SIZE - (src & PAGE_MASK)) -
865 	    hibernate_state->hib_stream.avail_in;
866 }
867 
868 /*
869  * Write the hibernation information specified in hiber_info
870  * to the location in swap previously calculated (last block of
871  * swap), called the "signature block".
872  */
873 int
874 hibernate_write_signature(union hibernate_info *hib)
875 {
876 	/* Write hibernate info to disk */
877 	return (hib->io_func(hib->dev, hib->sig_offset,
878 	    (vaddr_t)hib, DEV_BSIZE, HIB_W,
879 	    hib->io_page));
880 }
881 
882 /*
883  * Write the memory chunk table to the area in swap immediately
884  * preceding the signature block. The chunk table is stored
885  * in the piglet when this function is called.  Returns errno.
886  */
887 int
888 hibernate_write_chunktable(union hibernate_info *hib)
889 {
890 	vaddr_t hibernate_chunk_table_start;
891 	size_t hibernate_chunk_table_size;
892 	int i, err;
893 
894 	hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE;
895 
896 	hibernate_chunk_table_start = hib->piglet_va +
897 	    HIBERNATE_CHUNK_SIZE;
898 
899 	/* Write chunk table */
900 	for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) {
901 		if ((err = hib->io_func(hib->dev,
902 		    hib->chunktable_offset + (i/DEV_BSIZE),
903 		    (vaddr_t)(hibernate_chunk_table_start + i),
904 		    MAXPHYS, HIB_W, hib->io_page))) {
905 			DPRINTF("chunktable write error: %d\n", err);
906 			return (err);
907 		}
908 	}
909 
910 	return (0);
911 }
912 
913 /*
914  * Write an empty hiber_info to the swap signature block, which is
915  * guaranteed to not match any valid hib.
916  */
917 int
918 hibernate_clear_signature(union hibernate_info *hib)
919 {
920 	union hibernate_info blank_hiber_info;
921 
922 	/* Zero out a blank hiber_info */
923 	memset(&blank_hiber_info, 0, sizeof(union hibernate_info));
924 
925 	/* Write (zeroed) hibernate info to disk */
926 	DPRINTF("clearing hibernate signature block location: %lld\n",
927 		hib->sig_offset);
928 	if (hibernate_block_io(hib,
929 	    hib->sig_offset,
930 	    DEV_BSIZE, (vaddr_t)&blank_hiber_info, 1))
931 		printf("Warning: could not clear hibernate signature\n");
932 
933 	return (0);
934 }
935 
936 /*
937  * Compare two hibernate_infos to determine if they are the same (eg,
938  * we should be performing a hibernate resume on this machine.
939  * Not all fields are checked - just enough to verify that the machine
940  * has the same memory configuration and kernel as the one that
941  * wrote the signature previously.
942  */
943 int
944 hibernate_compare_signature(union hibernate_info *mine,
945     union hibernate_info *disk)
946 {
947 	u_int i;
948 
949 	if (mine->nranges != disk->nranges) {
950 		printf("unhibernate failed: memory layout changed\n");
951 		return (1);
952 	}
953 
954 	if (bcmp(mine->kern_hash, disk->kern_hash, SHA256_DIGEST_LENGTH) != 0) {
955 		printf("unhibernate failed: original kernel changed\n");
956 		return (1);
957 	}
958 
959 	for (i = 0; i < mine->nranges; i++) {
960 		if ((mine->ranges[i].base != disk->ranges[i].base) ||
961 		    (mine->ranges[i].end != disk->ranges[i].end) ) {
962 			DPRINTF("hib range %d mismatch [%p-%p != %p-%p]\n",
963 				i,
964 				(void *)mine->ranges[i].base,
965 				(void *)mine->ranges[i].end,
966 				(void *)disk->ranges[i].base,
967 				(void *)disk->ranges[i].end);
968 			printf("unhibernate failed: memory size changed\n");
969 			return (1);
970 		}
971 	}
972 
973 	return (0);
974 }
975 
976 /*
977  * Transfers xfer_size bytes between the hibernate device specified in
978  * hib_info at offset blkctr and the vaddr specified at dest.
979  *
980  * Separate offsets and pages are used to handle misaligned reads (reads
981  * that span a page boundary).
982  *
983  * blkctr specifies a relative offset (relative to the start of swap),
984  * not an absolute disk offset
985  *
986  */
987 int
988 hibernate_block_io(union hibernate_info *hib, daddr_t blkctr,
989     size_t xfer_size, vaddr_t dest, int iswrite)
990 {
991 	struct buf *bp;
992 	struct bdevsw *bdsw;
993 	int error;
994 
995 	bp = geteblk(xfer_size);
996 	bdsw = &bdevsw[major(hib->dev)];
997 
998 	error = (*bdsw->d_open)(hib->dev, FREAD, S_IFCHR, curproc);
999 	if (error) {
1000 		printf("hibernate_block_io open failed\n");
1001 		return (1);
1002 	}
1003 
1004 	if (iswrite)
1005 		bcopy((caddr_t)dest, bp->b_data, xfer_size);
1006 
1007 	bp->b_bcount = xfer_size;
1008 	bp->b_blkno = blkctr;
1009 	CLR(bp->b_flags, B_READ | B_WRITE | B_DONE);
1010 	SET(bp->b_flags, B_BUSY | (iswrite ? B_WRITE : B_READ) | B_RAW);
1011 	bp->b_dev = hib->dev;
1012 	(*bdsw->d_strategy)(bp);
1013 
1014 	error = biowait(bp);
1015 	if (error) {
1016 		printf("hib block_io biowait error %d blk %lld size %zu\n",
1017 			error, (long long)blkctr, xfer_size);
1018 		error = (*bdsw->d_close)(hib->dev, 0, S_IFCHR,
1019 		    curproc);
1020 		if (error)
1021 			printf("hibernate_block_io error close failed\n");
1022 		return (1);
1023 	}
1024 
1025 	error = (*bdsw->d_close)(hib->dev, FREAD, S_IFCHR, curproc);
1026 	if (error) {
1027 		printf("hibernate_block_io close failed\n");
1028 		return (1);
1029 	}
1030 
1031 	if (!iswrite)
1032 		bcopy(bp->b_data, (caddr_t)dest, xfer_size);
1033 
1034 	bp->b_flags |= B_INVAL;
1035 	brelse(bp);
1036 
1037 	return (0);
1038 }
1039 
1040 /*
1041  * Preserve one page worth of random data, generated from the resuming
1042  * kernel's arc4random. After resume, this preserved entropy can be used
1043  * to further improve the un-hibernated machine's entropy pool. This
1044  * random data is stored in the piglet, which is preserved across the
1045  * unpack operation, and is restored later in the resume process (see
1046  * hib_getentropy)
1047  */
1048 void
1049 hibernate_preserve_entropy(union hibernate_info *hib)
1050 {
1051 	void *entropy;
1052 
1053 	entropy = km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_nowait);
1054 
1055 	if (!entropy)
1056 		return;
1057 
1058 	pmap_activate(curproc);
1059 	pmap_kenter_pa((vaddr_t)entropy,
1060 	    (paddr_t)(hib->piglet_pa + (29 * PAGE_SIZE)),
1061 	    PROT_READ | PROT_WRITE);
1062 
1063 	arc4random_buf((void *)entropy, PAGE_SIZE);
1064 	pmap_kremove((vaddr_t)entropy, PAGE_SIZE);
1065 	km_free(entropy, PAGE_SIZE, &kv_any, &kp_none);
1066 }
1067 
1068 #ifndef NO_PROPOLICE
1069 vaddr_t
1070 hibernate_unprotect_ssp(void)
1071 {
1072 	struct kmem_dyn_mode kd_avoidalias;
1073 	vaddr_t va = trunc_page((vaddr_t)&__guard_local);
1074 	paddr_t pa;
1075 
1076 	pmap_extract(pmap_kernel(), va, &pa);
1077 
1078 	memset(&kd_avoidalias, 0, sizeof kd_avoidalias);
1079 	kd_avoidalias.kd_prefer = pa;
1080 	kd_avoidalias.kd_waitok = 1;
1081 	va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_avoidalias);
1082 	if (!va)
1083 		panic("hibernate_unprotect_ssp");
1084 
1085 	pmap_kenter_pa(va, pa, PROT_READ | PROT_WRITE);
1086 	pmap_update(pmap_kernel());
1087 
1088 	return va;
1089 }
1090 
1091 void
1092 hibernate_reprotect_ssp(vaddr_t va)
1093 {
1094 	pmap_kremove(va, PAGE_SIZE);
1095 	km_free((void *)va, PAGE_SIZE, &kv_any, &kp_none);
1096 }
1097 #endif /* NO_PROPOLICE */
1098 
1099 /*
1100  * Reads the signature block from swap, checks against the current machine's
1101  * information. If the information matches, perform a resume by reading the
1102  * saved image into the pig area, and unpacking.
1103  *
1104  * Must be called with interrupts enabled.
1105  */
1106 void
1107 hibernate_resume(void)
1108 {
1109 	union hibernate_info hib;
1110 	int s;
1111 #ifndef NO_PROPOLICE
1112 	vsize_t off = (vaddr_t)&__guard_local -
1113 	    trunc_page((vaddr_t)&__guard_local);
1114 	vaddr_t guard_va;
1115 #endif
1116 
1117 	/* Get current running machine's hibernate info */
1118 	memset(&hib, 0, sizeof(hib));
1119 	if (get_hibernate_info(&hib, 0)) {
1120 		DPRINTF("couldn't retrieve machine's hibernate info\n");
1121 		return;
1122 	}
1123 
1124 	/* Read hibernate info from disk */
1125 	s = splbio();
1126 
1127 	DPRINTF("reading hibernate signature block location: %lld\n",
1128 		hib.sig_offset);
1129 
1130 	if (hibernate_block_io(&hib,
1131 	    hib.sig_offset,
1132 	    DEV_BSIZE, (vaddr_t)&disk_hib, 0)) {
1133 		DPRINTF("error in hibernate read");
1134 		splx(s);
1135 		return;
1136 	}
1137 
1138 	/* Check magic number */
1139 	if (disk_hib.magic != HIBERNATE_MAGIC) {
1140 		DPRINTF("wrong magic number in hibernate signature: %x\n",
1141 			disk_hib.magic);
1142 		splx(s);
1143 		return;
1144 	}
1145 
1146 	/*
1147 	 * We (possibly) found a hibernate signature. Clear signature first,
1148 	 * to prevent accidental resume or endless resume cycles later.
1149 	 */
1150 	if (hibernate_clear_signature(&hib)) {
1151 		DPRINTF("error clearing hibernate signature block\n");
1152 		splx(s);
1153 		return;
1154 	}
1155 
1156 	/*
1157 	 * If on-disk and in-memory hibernate signatures match,
1158 	 * this means we should do a resume from hibernate.
1159 	 */
1160 	if (hibernate_compare_signature(&hib, &disk_hib)) {
1161 		DPRINTF("mismatched hibernate signature block\n");
1162 		splx(s);
1163 		return;
1164 	}
1165 	disk_hib.dev = hib.dev;
1166 
1167 #ifdef MULTIPROCESSOR
1168 	/* XXX - if we fail later, we may need to rehatch APs on some archs */
1169 	DPRINTF("hibernate: quiescing APs\n");
1170 	hibernate_quiesce_cpus();
1171 #endif /* MULTIPROCESSOR */
1172 
1173 	/* Read the image from disk into the image (pig) area */
1174 	if (hibernate_read_image(&disk_hib))
1175 		goto fail;
1176 
1177 	DPRINTF("hibernate: quiescing devices\n");
1178 	if (config_suspend_all(DVACT_QUIESCE) != 0)
1179 		goto fail;
1180 
1181 #ifndef NO_PROPOLICE
1182 	guard_va = hibernate_unprotect_ssp();
1183 #endif /* NO_PROPOLICE */
1184 
1185 	(void) splhigh();
1186 	hibernate_disable_intr_machdep();
1187 	cold = 2;
1188 
1189 	DPRINTF("hibernate: suspending devices\n");
1190 	if (config_suspend_all(DVACT_SUSPEND) != 0) {
1191 		cold = 0;
1192 		hibernate_enable_intr_machdep();
1193 #ifndef NO_PROPOLICE
1194 		hibernate_reprotect_ssp(guard_va);
1195 #endif /* ! NO_PROPOLICE */
1196 		goto fail;
1197 	}
1198 
1199 	hibernate_preserve_entropy(&disk_hib);
1200 
1201 	printf("Unpacking image...\n");
1202 
1203 	/* Switch stacks */
1204 	DPRINTF("hibernate: switching stacks\n");
1205 	hibernate_switch_stack_machdep();
1206 
1207 #ifndef NO_PROPOLICE
1208 	/* Start using suspended kernel's propolice guard */
1209 	*(long *)(guard_va + off) = disk_hib.guard;
1210 	hibernate_reprotect_ssp(guard_va);
1211 #endif /* ! NO_PROPOLICE */
1212 
1213 	/* Unpack and resume */
1214 	hibernate_unpack_image(&disk_hib);
1215 
1216 fail:
1217 	splx(s);
1218 	printf("\nUnable to resume hibernated image\n");
1219 }
1220 
1221 /*
1222  * Unpack image from pig area to original location by looping through the
1223  * list of output chunks in the order they should be restored (fchunks).
1224  *
1225  * Note that due to the stack smash protector and the fact that we have
1226  * switched stacks, it is not permitted to return from this function.
1227  */
1228 void
1229 hibernate_unpack_image(union hibernate_info *hib)
1230 {
1231 	struct hibernate_disk_chunk *chunks;
1232 	union hibernate_info local_hib;
1233 	paddr_t image_cur = global_pig_start;
1234 	short i, *fchunks;
1235 	char *pva;
1236 
1237 	/* Piglet will be identity mapped (VA == PA) */
1238 	pva = (char *)hib->piglet_pa;
1239 
1240 	fchunks = (short *)(pva + (4 * PAGE_SIZE));
1241 
1242 	chunks = (struct hibernate_disk_chunk *)(pva + HIBERNATE_CHUNK_SIZE);
1243 
1244 	/* Can't use hiber_info that's passed in after this point */
1245 	bcopy(hib, &local_hib, sizeof(union hibernate_info));
1246 	local_hib.retguard_ofs = 0;
1247 
1248 	/* VA == PA */
1249 	local_hib.piglet_va = local_hib.piglet_pa;
1250 
1251 	/*
1252 	 * Point of no return. Once we pass this point, only kernel code can
1253 	 * be accessed. No global variables or other kernel data structures
1254 	 * are guaranteed to be coherent after unpack starts.
1255 	 *
1256 	 * The image is now in high memory (pig area), we unpack from the pig
1257 	 * to the correct location in memory. We'll eventually end up copying
1258 	 * on top of ourself, but we are assured the kernel code here is the
1259 	 * same between the hibernated and resuming kernel, and we are running
1260 	 * on our own stack, so the overwrite is ok.
1261 	 */
1262 	DPRINTF("hibernate: activating alt. pagetable and starting unpack\n");
1263 	hibernate_activate_resume_pt_machdep();
1264 
1265 	for (i = 0; i < local_hib.chunk_ctr; i++) {
1266 		/* Reset zlib for inflate */
1267 		if (hibernate_zlib_reset(&local_hib, 0) != Z_OK)
1268 			panic("hibernate failed to reset zlib for inflate");
1269 
1270 		hibernate_process_chunk(&local_hib, &chunks[fchunks[i]],
1271 		    image_cur);
1272 
1273 		image_cur += chunks[fchunks[i]].compressed_size;
1274 
1275 	}
1276 
1277 	/*
1278 	 * Resume the loaded kernel by jumping to the MD resume vector.
1279 	 * We won't be returning from this call. We pass the location of
1280 	 * the retguard save area so the MD code can replace it before
1281 	 * resuming. See the piglet layout at the top of this file for
1282 	 * more information on the layout of the piglet area.
1283 	 *
1284 	 * We use 'global_piglet_va' here since by the time we are at
1285 	 * this point, we have already unpacked the image, and we want
1286 	 * the suspended kernel's view of what the piglet was, before
1287 	 * suspend occurred (since we will need to use that in the retguard
1288 	 * copy code in hibernate_resume_machdep.)
1289 	 */
1290 	hibernate_resume_machdep(global_piglet_va + (110 * PAGE_SIZE));
1291 }
1292 
1293 /*
1294  * Bounce a compressed image chunk to the piglet, entering mappings for the
1295  * copied pages as needed
1296  */
1297 void
1298 hibernate_copy_chunk_to_piglet(paddr_t img_cur, vaddr_t piglet, size_t size)
1299 {
1300 	size_t ct, ofs;
1301 	paddr_t src = img_cur;
1302 	vaddr_t dest = piglet;
1303 
1304 	/* Copy first partial page */
1305 	ct = (PAGE_SIZE) - (src & PAGE_MASK);
1306 	ofs = (src & PAGE_MASK);
1307 
1308 	if (ct < PAGE_SIZE) {
1309 		hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE,
1310 			(src - ofs), 0);
1311 		hibernate_flush();
1312 		bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE + ofs), (caddr_t)dest, ct);
1313 		src += ct;
1314 		dest += ct;
1315 	}
1316 
1317 	/* Copy remaining pages */
1318 	while (src < size + img_cur) {
1319 		hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, src, 0);
1320 		hibernate_flush();
1321 		ct = PAGE_SIZE;
1322 		bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE), (caddr_t)dest, ct);
1323 		hibernate_flush();
1324 		src += ct;
1325 		dest += ct;
1326 	}
1327 }
1328 
1329 /*
1330  * Process a chunk by bouncing it to the piglet, followed by unpacking
1331  */
1332 void
1333 hibernate_process_chunk(union hibernate_info *hib,
1334     struct hibernate_disk_chunk *chunk, paddr_t img_cur)
1335 {
1336 	char *pva = (char *)hib->piglet_va;
1337 
1338 	hibernate_copy_chunk_to_piglet(img_cur,
1339 	 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), chunk->compressed_size);
1340 	hibernate_inflate_region(hib, chunk->base,
1341 	    (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)),
1342 	    chunk->compressed_size);
1343 }
1344 
1345 /*
1346  * Calculate RLE component for 'inaddr'. Clamps to max RLE pages between
1347  * inaddr and range_end.
1348  */
1349 int
1350 hibernate_calc_rle(paddr_t inaddr, paddr_t range_end)
1351 {
1352 	int rle;
1353 
1354 	rle = uvm_page_rle(inaddr);
1355 	KASSERT(rle >= 0 && rle <= MAX_RLE);
1356 
1357 	/* Clamp RLE to range end */
1358 	if (rle > 0 && inaddr + (rle * PAGE_SIZE) > range_end)
1359 		rle = (range_end - inaddr) / PAGE_SIZE;
1360 
1361 	return (rle);
1362 }
1363 
1364 /*
1365  * Write the RLE byte for page at 'inaddr' to the output stream.
1366  * Returns the number of pages to be skipped at 'inaddr'.
1367  */
1368 int
1369 hibernate_write_rle(union hibernate_info *hib, paddr_t inaddr,
1370 	paddr_t range_end, daddr_t *blkctr,
1371 	size_t *out_remaining)
1372 {
1373 	int rle, err, *rleloc;
1374 	struct hibernate_zlib_state *hibernate_state;
1375 	vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
1376 
1377 	hibernate_state =
1378 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1379 
1380 	rle = hibernate_calc_rle(inaddr, range_end);
1381 
1382 	rleloc = (int *)hibernate_rle_page + MAX_RLE - 1;
1383 	*rleloc = rle;
1384 
1385 	/* Deflate the RLE byte into the stream */
1386 	hibernate_deflate(hib, (paddr_t)rleloc, out_remaining);
1387 
1388 	/* Did we fill the output page? If so, flush to disk */
1389 	if (*out_remaining == 0) {
1390 		if ((err = hib->io_func(hib->dev, *blkctr + hib->image_offset,
1391 			(vaddr_t)hibernate_io_page, PAGE_SIZE, HIB_W,
1392 			hib->io_page))) {
1393 				DPRINTF("hib write error %d\n", err);
1394 				return (err);
1395 		}
1396 
1397 		*blkctr += PAGE_SIZE / DEV_BSIZE;
1398 		*out_remaining = PAGE_SIZE;
1399 
1400 		/* If we didn't deflate the entire RLE byte, finish it now */
1401 		if (hibernate_state->hib_stream.avail_in != 0)
1402 			hibernate_deflate(hib,
1403 				(vaddr_t)hibernate_state->hib_stream.next_in,
1404 				out_remaining);
1405 	}
1406 
1407 	return (rle);
1408 }
1409 
1410 /*
1411  * Write a compressed version of this machine's memory to disk, at the
1412  * precalculated swap offset:
1413  *
1414  * end of swap - signature block size - chunk table size - memory size
1415  *
1416  * The function begins by looping through each phys mem range, cutting each
1417  * one into MD sized chunks. These chunks are then compressed individually
1418  * and written out to disk, in phys mem order. Some chunks might compress
1419  * more than others, and for this reason, each chunk's size is recorded
1420  * in the chunk table, which is written to disk after the image has
1421  * properly been compressed and written (in hibernate_write_chunktable).
1422  *
1423  * When this function is called, the machine is nearly suspended - most
1424  * devices are quiesced/suspended, interrupts are off, and cold has
1425  * been set. This means that there can be no side effects once the
1426  * write has started, and the write function itself can also have no
1427  * side effects. This also means no printfs are permitted (since printf
1428  * has side effects.)
1429  *
1430  * Return values :
1431  *
1432  * 0      - success
1433  * EIO    - I/O error occurred writing the chunks
1434  * EINVAL - Failed to write a complete range
1435  * ENOMEM - Memory allocation failure during preparation of the zlib arena
1436  */
1437 int
1438 hibernate_write_chunks(union hibernate_info *hib)
1439 {
1440 	paddr_t range_base, range_end, inaddr, temp_inaddr;
1441 	size_t nblocks, out_remaining, used;
1442 	struct hibernate_disk_chunk *chunks;
1443 	vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
1444 	daddr_t blkctr = 0;
1445 	int i, rle, err;
1446 	struct hibernate_zlib_state *hibernate_state;
1447 
1448 	hibernate_state =
1449 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1450 
1451 	hib->chunk_ctr = 0;
1452 
1453 	/*
1454 	 * Map the utility VAs to the piglet. See the piglet map at the
1455 	 * top of this file for piglet layout information.
1456 	 */
1457 	hibernate_copy_page = hib->piglet_va + 3 * PAGE_SIZE;
1458 	hibernate_rle_page = hib->piglet_va + 28 * PAGE_SIZE;
1459 
1460 	chunks = (struct hibernate_disk_chunk *)(hib->piglet_va +
1461 	    HIBERNATE_CHUNK_SIZE);
1462 
1463 	/* Calculate the chunk regions */
1464 	for (i = 0; i < hib->nranges; i++) {
1465 		range_base = hib->ranges[i].base;
1466 		range_end = hib->ranges[i].end;
1467 
1468 		inaddr = range_base;
1469 
1470 		while (inaddr < range_end) {
1471 			chunks[hib->chunk_ctr].base = inaddr;
1472 			if (inaddr + HIBERNATE_CHUNK_SIZE < range_end)
1473 				chunks[hib->chunk_ctr].end = inaddr +
1474 				    HIBERNATE_CHUNK_SIZE;
1475 			else
1476 				chunks[hib->chunk_ctr].end = range_end;
1477 
1478 			inaddr += HIBERNATE_CHUNK_SIZE;
1479 			hib->chunk_ctr ++;
1480 		}
1481 	}
1482 
1483 	uvm_pmr_dirty_everything();
1484 	uvm_pmr_zero_everything();
1485 
1486 	/* Compress and write the chunks in the chunktable */
1487 	for (i = 0; i < hib->chunk_ctr; i++) {
1488 		range_base = chunks[i].base;
1489 		range_end = chunks[i].end;
1490 
1491 		chunks[i].offset = blkctr + hib->image_offset;
1492 
1493 		/* Reset zlib for deflate */
1494 		if (hibernate_zlib_reset(hib, 1) != Z_OK) {
1495 			DPRINTF("hibernate_zlib_reset failed for deflate\n");
1496 			return (ENOMEM);
1497 		}
1498 
1499 		inaddr = range_base;
1500 
1501 		/*
1502 		 * For each range, loop through its phys mem region
1503 		 * and write out the chunks (the last chunk might be
1504 		 * smaller than the chunk size).
1505 		 */
1506 		while (inaddr < range_end) {
1507 			out_remaining = PAGE_SIZE;
1508 			while (out_remaining > 0 && inaddr < range_end) {
1509 				/*
1510 				 * Adjust for regions that are not evenly
1511 				 * divisible by PAGE_SIZE or overflowed
1512 				 * pages from the previous iteration.
1513 				 */
1514 				temp_inaddr = (inaddr & PAGE_MASK) +
1515 				    hibernate_copy_page;
1516 
1517 				/* Deflate from temp_inaddr to IO page */
1518 				if (inaddr != range_end) {
1519 					if (inaddr % PAGE_SIZE == 0) {
1520 						rle = hibernate_write_rle(hib,
1521 							inaddr,
1522 							range_end,
1523 							&blkctr,
1524 							&out_remaining);
1525 					}
1526 
1527 					if (rle == 0) {
1528 						pmap_kenter_pa(hibernate_temp_page,
1529 							inaddr & PMAP_PA_MASK,
1530 							PROT_READ);
1531 
1532 						bcopy((caddr_t)hibernate_temp_page,
1533 							(caddr_t)hibernate_copy_page,
1534 							PAGE_SIZE);
1535 						inaddr += hibernate_deflate(hib,
1536 							temp_inaddr,
1537 							&out_remaining);
1538 					} else {
1539 						inaddr += rle * PAGE_SIZE;
1540 						if (inaddr > range_end)
1541 							inaddr = range_end;
1542 					}
1543 
1544 				}
1545 
1546 				if (out_remaining == 0) {
1547 					/* Filled up the page */
1548 					nblocks = PAGE_SIZE / DEV_BSIZE;
1549 
1550 					if ((err = hib->io_func(hib->dev,
1551 					    blkctr + hib->image_offset,
1552 					    (vaddr_t)hibernate_io_page,
1553 					    PAGE_SIZE, HIB_W, hib->io_page))) {
1554 						DPRINTF("hib write error %d\n",
1555 						    err);
1556 						return (err);
1557 					}
1558 
1559 					blkctr += nblocks;
1560 				}
1561 			}
1562 		}
1563 
1564 		if (inaddr != range_end) {
1565 			DPRINTF("deflate range ended prematurely\n");
1566 			return (EINVAL);
1567 		}
1568 
1569 		/*
1570 		 * End of range. Round up to next secsize bytes
1571 		 * after finishing compress
1572 		 */
1573 		if (out_remaining == 0)
1574 			out_remaining = PAGE_SIZE;
1575 
1576 		/* Finish compress */
1577 		hibernate_state->hib_stream.next_in = (unsigned char *)inaddr;
1578 		hibernate_state->hib_stream.avail_in = 0;
1579 		hibernate_state->hib_stream.next_out =
1580 		    (unsigned char *)hibernate_io_page +
1581 			(PAGE_SIZE - out_remaining);
1582 
1583 		/* We have an extra output page available for finalize */
1584 		hibernate_state->hib_stream.avail_out =
1585 			out_remaining + PAGE_SIZE;
1586 
1587 		if ((err = deflate(&hibernate_state->hib_stream, Z_FINISH)) !=
1588 		    Z_STREAM_END) {
1589 			DPRINTF("deflate error in output stream: %d\n", err);
1590 			return (err);
1591 		}
1592 
1593 		out_remaining = hibernate_state->hib_stream.avail_out;
1594 
1595 		used = 2 * PAGE_SIZE - out_remaining;
1596 		nblocks = used / DEV_BSIZE;
1597 
1598 		/* Round up to next block if needed */
1599 		if (used % DEV_BSIZE != 0)
1600 			nblocks ++;
1601 
1602 		/* Write final block(s) for this chunk */
1603 		if ((err = hib->io_func(hib->dev, blkctr + hib->image_offset,
1604 		    (vaddr_t)hibernate_io_page, nblocks*DEV_BSIZE,
1605 		    HIB_W, hib->io_page))) {
1606 			DPRINTF("hib final write error %d\n", err);
1607 			return (err);
1608 		}
1609 
1610 		blkctr += nblocks;
1611 
1612 		chunks[i].compressed_size = (blkctr + hib->image_offset -
1613 		    chunks[i].offset) * DEV_BSIZE;
1614 	}
1615 
1616 	hib->chunktable_offset = hib->image_offset + blkctr;
1617 	return (0);
1618 }
1619 
1620 /*
1621  * Reset the zlib stream state and allocate a new hiballoc area for either
1622  * inflate or deflate. This function is called once for each hibernate chunk.
1623  * Calling hiballoc_init multiple times is acceptable since the memory it is
1624  * provided is unmanaged memory (stolen). We use the memory provided to us
1625  * by the piglet allocated via the supplied hib.
1626  */
1627 int
1628 hibernate_zlib_reset(union hibernate_info *hib, int deflate)
1629 {
1630 	vaddr_t hibernate_zlib_start;
1631 	size_t hibernate_zlib_size;
1632 	char *pva = (char *)hib->piglet_va;
1633 	struct hibernate_zlib_state *hibernate_state;
1634 
1635 	hibernate_state =
1636 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1637 
1638 	if (!deflate)
1639 		pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK));
1640 
1641 	/*
1642 	 * See piglet layout information at the start of this file for
1643 	 * information on the zlib page assignments.
1644 	 */
1645 	hibernate_zlib_start = (vaddr_t)(pva + (30 * PAGE_SIZE));
1646 	hibernate_zlib_size = 80 * PAGE_SIZE;
1647 
1648 	memset((void *)hibernate_zlib_start, 0, hibernate_zlib_size);
1649 	memset(hibernate_state, 0, PAGE_SIZE);
1650 
1651 	/* Set up stream structure */
1652 	hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc;
1653 	hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free;
1654 
1655 	/* Initialize the hiballoc arena for zlib allocs/frees */
1656 	hiballoc_init(&hibernate_state->hiballoc_arena,
1657 	    (caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1658 
1659 	if (deflate) {
1660 		return deflateInit(&hibernate_state->hib_stream,
1661 		    Z_BEST_SPEED);
1662 	} else
1663 		return inflateInit(&hibernate_state->hib_stream);
1664 }
1665 
1666 /*
1667  * Reads the hibernated memory image from disk, whose location and
1668  * size are recorded in hib. Begin by reading the persisted
1669  * chunk table, which records the original chunk placement location
1670  * and compressed size for each. Next, allocate a pig region of
1671  * sufficient size to hold the compressed image. Next, read the
1672  * chunks into the pig area (calling hibernate_read_chunks to do this),
1673  * and finally, if all of the above succeeds, clear the hibernate signature.
1674  * The function will then return to hibernate_resume, which will proceed
1675  * to unpack the pig image to the correct place in memory.
1676  */
1677 int
1678 hibernate_read_image(union hibernate_info *hib)
1679 {
1680 	size_t compressed_size, disk_size, chunktable_size, pig_sz;
1681 	paddr_t image_start, image_end, pig_start, pig_end;
1682 	struct hibernate_disk_chunk *chunks;
1683 	daddr_t blkctr;
1684 	vaddr_t chunktable = (vaddr_t)NULL;
1685 	paddr_t piglet_chunktable = hib->piglet_pa +
1686 	    HIBERNATE_CHUNK_SIZE;
1687 	int i, status;
1688 
1689 	status = 0;
1690 	pmap_activate(curproc);
1691 
1692 	/* Calculate total chunk table size in disk blocks */
1693 	chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / DEV_BSIZE;
1694 
1695 	blkctr = hib->chunktable_offset;
1696 
1697 	chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any,
1698 	    &kp_none, &kd_nowait);
1699 
1700 	if (!chunktable)
1701 		return (1);
1702 
1703 	/* Map chunktable pages */
1704 	for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; i += PAGE_SIZE)
1705 		pmap_kenter_pa(chunktable + i, piglet_chunktable + i,
1706 		    PROT_READ | PROT_WRITE);
1707 	pmap_update(pmap_kernel());
1708 
1709 	/* Read the chunktable from disk into the piglet chunktable */
1710 	for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE;
1711 	    i += MAXPHYS, blkctr += MAXPHYS/DEV_BSIZE)
1712 		hibernate_block_io(hib, blkctr, MAXPHYS,
1713 		    chunktable + i, 0);
1714 
1715 	blkctr = hib->image_offset;
1716 	compressed_size = 0;
1717 
1718 	chunks = (struct hibernate_disk_chunk *)chunktable;
1719 
1720 	for (i = 0; i < hib->chunk_ctr; i++)
1721 		compressed_size += chunks[i].compressed_size;
1722 
1723 	disk_size = compressed_size;
1724 
1725 	printf("unhibernating @ block %lld length %luMB\n",
1726 	    hib->sig_offset - chunktable_size,
1727 	    compressed_size / (1024 * 1024));
1728 
1729 	/* Allocate the pig area */
1730 	pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE;
1731 	if (uvm_pmr_alloc_pig(&pig_start, pig_sz, hib->piglet_pa) == ENOMEM) {
1732 		status = 1;
1733 		goto unmap;
1734 	}
1735 
1736 	pig_end = pig_start + pig_sz;
1737 
1738 	/* Calculate image extents. Pig image must end on a chunk boundary. */
1739 	image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1);
1740 	image_start = image_end - disk_size;
1741 
1742 	hibernate_read_chunks(hib, image_start, image_end, disk_size,
1743 	    chunks);
1744 
1745 	/* Prepare the resume time pmap/page table */
1746 	hibernate_populate_resume_pt(hib, image_start, image_end);
1747 
1748 unmap:
1749 	/* Unmap chunktable pages */
1750 	pmap_kremove(chunktable, HIBERNATE_CHUNK_TABLE_SIZE);
1751 	pmap_update(pmap_kernel());
1752 
1753 	return (status);
1754 }
1755 
1756 /*
1757  * Read the hibernated memory chunks from disk (chunk information at this
1758  * point is stored in the piglet) into the pig area specified by
1759  * [pig_start .. pig_end]. Order the chunks so that the final chunk is the
1760  * only chunk with overlap possibilities.
1761  */
1762 int
1763 hibernate_read_chunks(union hibernate_info *hib, paddr_t pig_start,
1764     paddr_t pig_end, size_t image_compr_size,
1765     struct hibernate_disk_chunk *chunks)
1766 {
1767 	paddr_t img_cur, piglet_base;
1768 	daddr_t blkctr;
1769 	size_t processed, compressed_size, read_size;
1770 	int nchunks, nfchunks, num_io_pages;
1771 	vaddr_t tempva, hibernate_fchunk_area;
1772 	short *fchunks, i, j;
1773 
1774 	tempva = (vaddr_t)NULL;
1775 	hibernate_fchunk_area = (vaddr_t)NULL;
1776 	nfchunks = 0;
1777 	piglet_base = hib->piglet_pa;
1778 	global_pig_start = pig_start;
1779 
1780 	/*
1781 	 * These mappings go into the resuming kernel's page table, and are
1782 	 * used only during image read. They disappear from existence
1783 	 * when the suspended kernel is unpacked on top of us.
1784 	 */
1785 	tempva = (vaddr_t)km_alloc(MAXPHYS + PAGE_SIZE, &kv_any, &kp_none,
1786 		&kd_nowait);
1787 	if (!tempva)
1788 		return (1);
1789 	hibernate_fchunk_area = (vaddr_t)km_alloc(24 * PAGE_SIZE, &kv_any,
1790 	    &kp_none, &kd_nowait);
1791 	if (!hibernate_fchunk_area)
1792 		return (1);
1793 
1794 	/* Final output chunk ordering VA */
1795 	fchunks = (short *)hibernate_fchunk_area;
1796 
1797 	/* Map the chunk ordering region */
1798 	for(i = 0; i < 24 ; i++)
1799 		pmap_kenter_pa(hibernate_fchunk_area + (i * PAGE_SIZE),
1800 			piglet_base + ((4 + i) * PAGE_SIZE),
1801 			PROT_READ | PROT_WRITE);
1802 	pmap_update(pmap_kernel());
1803 
1804 	nchunks = hib->chunk_ctr;
1805 
1806 	/* Initially start all chunks as unplaced */
1807 	for (i = 0; i < nchunks; i++)
1808 		chunks[i].flags = 0;
1809 
1810 	/*
1811 	 * Search the list for chunks that are outside the pig area. These
1812 	 * can be placed first in the final output list.
1813 	 */
1814 	for (i = 0; i < nchunks; i++) {
1815 		if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) {
1816 			fchunks[nfchunks] = i;
1817 			nfchunks++;
1818 			chunks[i].flags |= HIBERNATE_CHUNK_PLACED;
1819 		}
1820 	}
1821 
1822 	/*
1823 	 * Walk the ordering, place the chunks in ascending memory order.
1824 	 */
1825 	for (i = 0; i < nchunks; i++) {
1826 		if (chunks[i].flags != HIBERNATE_CHUNK_PLACED) {
1827 			fchunks[nfchunks] = i;
1828 			nfchunks++;
1829 			chunks[i].flags = HIBERNATE_CHUNK_PLACED;
1830 		}
1831 	}
1832 
1833 	img_cur = pig_start;
1834 
1835 	for (i = 0; i < nfchunks; i++) {
1836 		blkctr = chunks[fchunks[i]].offset;
1837 		processed = 0;
1838 		compressed_size = chunks[fchunks[i]].compressed_size;
1839 
1840 		while (processed < compressed_size) {
1841 			if (compressed_size - processed >= MAXPHYS)
1842 				read_size = MAXPHYS;
1843 			else
1844 				read_size = compressed_size - processed;
1845 
1846 			/*
1847 			 * We're reading read_size bytes, offset from the
1848 			 * start of a page by img_cur % PAGE_SIZE, so the
1849 			 * end will be read_size + (img_cur % PAGE_SIZE)
1850 			 * from the start of the first page.  Round that
1851 			 * up to the next page size.
1852 			 */
1853 			num_io_pages = (read_size + (img_cur % PAGE_SIZE)
1854 				+ PAGE_SIZE - 1) / PAGE_SIZE;
1855 
1856 			KASSERT(num_io_pages <= MAXPHYS/PAGE_SIZE + 1);
1857 
1858 			/* Map pages for this read */
1859 			for (j = 0; j < num_io_pages; j ++)
1860 				pmap_kenter_pa(tempva + j * PAGE_SIZE,
1861 				    img_cur + j * PAGE_SIZE,
1862 				    PROT_READ | PROT_WRITE);
1863 
1864 			pmap_update(pmap_kernel());
1865 
1866 			hibernate_block_io(hib, blkctr, read_size,
1867 			    tempva + (img_cur & PAGE_MASK), 0);
1868 
1869 			blkctr += (read_size / DEV_BSIZE);
1870 
1871 			pmap_kremove(tempva, num_io_pages * PAGE_SIZE);
1872 			pmap_update(pmap_kernel());
1873 
1874 			processed += read_size;
1875 			img_cur += read_size;
1876 		}
1877 	}
1878 
1879 	pmap_kremove(hibernate_fchunk_area, 24 * PAGE_SIZE);
1880 	pmap_update(pmap_kernel());
1881 
1882 	return (0);
1883 }
1884 
1885 /*
1886  * Hibernating a machine comprises the following operations:
1887  *  1. Calculating this machine's hibernate_info information
1888  *  2. Allocating a piglet and saving the piglet's physaddr
1889  *  3. Calculating the memory chunks
1890  *  4. Writing the compressed chunks to disk
1891  *  5. Writing the chunk table
1892  *  6. Writing the signature block (hibernate_info)
1893  *
1894  * On most architectures, the function calling hibernate_suspend would
1895  * then power off the machine using some MD-specific implementation.
1896  */
1897 int
1898 hibernate_suspend(void)
1899 {
1900 	union hibernate_info hib;
1901 	u_long start, end;
1902 
1903 	/*
1904 	 * Calculate memory ranges, swap offsets, etc.
1905 	 * This also allocates a piglet whose physaddr is stored in
1906 	 * hib->piglet_pa and vaddr stored in hib->piglet_va
1907 	 */
1908 	if (get_hibernate_info(&hib, 1)) {
1909 		DPRINTF("failed to obtain hibernate info\n");
1910 		return (1);
1911 	}
1912 
1913 	/* Find a page-addressed region in swap [start,end] */
1914 	if (uvm_hibswap(hib.dev, &start, &end)) {
1915 		printf("hibernate: cannot find any swap\n");
1916 		return (1);
1917 	}
1918 
1919 	if (end - start < 1000) {
1920 		printf("hibernate: insufficient swap (%lu is too small)\n",
1921 			end - start + 1);
1922 		return (1);
1923 	}
1924 
1925 	/* Calculate block offsets in swap */
1926 	hib.image_offset = ctod(start);
1927 
1928 	DPRINTF("hibernate @ block %lld max-length %lu blocks\n",
1929 	    hib.image_offset, ctod(end) - ctod(start) + 1);
1930 
1931 	pmap_activate(curproc);
1932 	DPRINTF("hibernate: writing chunks\n");
1933 	if (hibernate_write_chunks(&hib)) {
1934 		DPRINTF("hibernate_write_chunks failed\n");
1935 		return (1);
1936 	}
1937 
1938 	DPRINTF("hibernate: writing chunktable\n");
1939 	if (hibernate_write_chunktable(&hib)) {
1940 		DPRINTF("hibernate_write_chunktable failed\n");
1941 		return (1);
1942 	}
1943 
1944 	DPRINTF("hibernate: writing signature\n");
1945 	if (hibernate_write_signature(&hib)) {
1946 		DPRINTF("hibernate_write_signature failed\n");
1947 		return (1);
1948 	}
1949 
1950 	/* Allow the disk to settle */
1951 	delay(500000);
1952 
1953 	/*
1954 	 * Give the device-specific I/O function a notification that we're
1955 	 * done, and that it can clean up or shutdown as needed.
1956 	 */
1957 	hib.io_func(hib.dev, 0, (vaddr_t)NULL, 0, HIB_DONE, hib.io_page);
1958 	return (0);
1959 }
1960 
1961 int
1962 hibernate_alloc(void)
1963 {
1964 	KASSERT(global_piglet_va == 0);
1965 	KASSERT(hibernate_temp_page == 0);
1966 
1967 	pmap_activate(curproc);
1968 	pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE,
1969 	    PROT_READ | PROT_WRITE);
1970 
1971 	/* Allocate a piglet, store its addresses in the supplied globals */
1972 	if (uvm_pmr_alloc_piglet(&global_piglet_va, &global_piglet_pa,
1973 	    HIBERNATE_CHUNK_SIZE * 4, HIBERNATE_CHUNK_SIZE))
1974 		goto unmap;
1975 
1976 	/*
1977 	 * Allocate VA for the temp page.
1978 	 *
1979 	 * This will become part of the suspended kernel and will
1980 	 * be freed in hibernate_free, upon resume (or hibernate
1981 	 * failure)
1982 	 */
1983 	hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1984 	    &kp_none, &kd_nowait);
1985 	if (!hibernate_temp_page) {
1986 		uvm_pmr_free_piglet(global_piglet_va,
1987 		    4 * HIBERNATE_CHUNK_SIZE);
1988 		global_piglet_va = 0;
1989 		goto unmap;
1990 	}
1991 	return (0);
1992 unmap:
1993 	pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE);
1994 	pmap_update(pmap_kernel());
1995 	return (ENOMEM);
1996 }
1997 
1998 /*
1999  * Free items allocated by hibernate_alloc()
2000  */
2001 void
2002 hibernate_free(void)
2003 {
2004 	pmap_activate(curproc);
2005 
2006 	if (global_piglet_va)
2007 		uvm_pmr_free_piglet(global_piglet_va,
2008 		    4 * HIBERNATE_CHUNK_SIZE);
2009 
2010 	if (hibernate_temp_page) {
2011 		pmap_kremove(hibernate_temp_page, PAGE_SIZE);
2012 		km_free((void *)hibernate_temp_page, PAGE_SIZE,
2013 		    &kv_any, &kp_none);
2014 	}
2015 
2016 	global_piglet_va = 0;
2017 	hibernate_temp_page = 0;
2018 	pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE);
2019 	pmap_update(pmap_kernel());
2020 }
2021