xref: /openbsd-src/sys/kern/subr_hibernate.c (revision b237021b0416d290b9edfa7670aa825182b0a00a)
1 /*	$OpenBSD: subr_hibernate.c,v 1.121 2017/03/27 20:26:39 deraadt Exp $	*/
2 
3 /*
4  * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl>
5  * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/hibernate.h>
21 #include <sys/malloc.h>
22 #include <sys/param.h>
23 #include <sys/tree.h>
24 #include <sys/systm.h>
25 #include <sys/disklabel.h>
26 #include <sys/disk.h>
27 #include <sys/conf.h>
28 #include <sys/buf.h>
29 #include <sys/fcntl.h>
30 #include <sys/stat.h>
31 #include <sys/atomic.h>
32 
33 #include <uvm/uvm.h>
34 #include <uvm/uvm_swap.h>
35 
36 #include <machine/hibernate.h>
37 
38 /*
39  * Hibernate piglet layout information
40  *
41  * The piglet is a scratch area of memory allocated by the suspending kernel.
42  * Its phys and virt addrs are recorded in the signature block. The piglet is
43  * used to guarantee an unused area of memory that can be used by the resuming
44  * kernel for various things. The piglet is excluded during unpack operations.
45  * The piglet size is presently 4*HIBERNATE_CHUNK_SIZE (typically 4*4MB).
46  *
47  * Offset from piglet_base	Purpose
48  * ----------------------------------------------------------------------------
49  * 0				Private page for suspend I/O write functions
50  * 1*PAGE_SIZE			I/O page used during hibernate suspend
51  * 2*PAGE_SIZE			I/O page used during hibernate suspend
52  * 3*PAGE_SIZE			copy page used during hibernate suspend
53  * 4*PAGE_SIZE			final chunk ordering list (24 pages)
54  * 28*PAGE_SIZE			RLE utility page
55  * 29*PAGE_SIZE			start of hiballoc area
56  * 30*PAGE_SIZE			preserved entropy
57  * 110*PAGE_SIZE		end of hiballoc area (80 pages)
58  * ...				unused
59  * HIBERNATE_CHUNK_SIZE		start of hibernate chunk table
60  * 2*HIBERNATE_CHUNK_SIZE	bounce area for chunks being unpacked
61  * 4*HIBERNATE_CHUNK_SIZE	end of piglet
62  */
63 
64 /* Temporary vaddr ranges used during hibernate */
65 vaddr_t hibernate_temp_page;
66 vaddr_t hibernate_copy_page;
67 vaddr_t hibernate_rle_page;
68 
69 /* Hibernate info as read from disk during resume */
70 union hibernate_info disk_hib;
71 
72 /*
73  * Global copy of the pig start address. This needs to be a global as we
74  * switch stacks after computing it - it can't be stored on the stack.
75  */
76 paddr_t global_pig_start;
77 
78 /*
79  * Global copies of the piglet start addresses (PA/VA). We store these
80  * as globals to avoid having to carry them around as parameters, as the
81  * piglet is allocated early and freed late - its lifecycle extends beyond
82  * that of the hibernate info union which is calculated on suspend/resume.
83  */
84 vaddr_t global_piglet_va;
85 paddr_t global_piglet_pa;
86 
87 /* #define HIB_DEBUG */
88 #ifdef HIB_DEBUG
89 int	hib_debug = 99;
90 #define DPRINTF(x...)     do { if (hib_debug) printf(x); } while (0)
91 #define DNPRINTF(n,x...)  do { if (hib_debug > (n)) printf(x); } while (0)
92 #else
93 #define DPRINTF(x...)
94 #define DNPRINTF(n,x...)
95 #endif
96 
97 #ifndef NO_PROPOLICE
98 extern long __guard_local;
99 #endif /* ! NO_PROPOLICE */
100 
101 void hibernate_copy_chunk_to_piglet(paddr_t, vaddr_t, size_t);
102 int hibernate_calc_rle(paddr_t, paddr_t);
103 int hibernate_write_rle(union hibernate_info *, paddr_t, paddr_t, daddr_t *,
104 	size_t *);
105 
106 #define MAX_RLE (HIBERNATE_CHUNK_SIZE / PAGE_SIZE)
107 
108 /*
109  * Hib alloc enforced alignment.
110  */
111 #define HIB_ALIGN		8 /* bytes alignment */
112 
113 /*
114  * sizeof builtin operation, but with alignment constraint.
115  */
116 #define HIB_SIZEOF(_type)	roundup(sizeof(_type), HIB_ALIGN)
117 
118 struct hiballoc_entry {
119 	size_t			hibe_use;
120 	size_t			hibe_space;
121 	RBT_ENTRY(hiballoc_entry) hibe_entry;
122 };
123 
124 /*
125  * Sort hibernate memory ranges by ascending PA
126  */
127 void
128 hibernate_sort_ranges(union hibernate_info *hib_info)
129 {
130 	int i, j;
131 	struct hibernate_memory_range *ranges;
132 	paddr_t base, end;
133 
134 	ranges = hib_info->ranges;
135 
136 	for (i = 1; i < hib_info->nranges; i++) {
137 		j = i;
138 		while (j > 0 && ranges[j - 1].base > ranges[j].base) {
139 			base = ranges[j].base;
140 			end = ranges[j].end;
141 			ranges[j].base = ranges[j - 1].base;
142 			ranges[j].end = ranges[j - 1].end;
143 			ranges[j - 1].base = base;
144 			ranges[j - 1].end = end;
145 			j--;
146 		}
147 	}
148 }
149 
150 /*
151  * Compare hiballoc entries based on the address they manage.
152  *
153  * Since the address is fixed, relative to struct hiballoc_entry,
154  * we just compare the hiballoc_entry pointers.
155  */
156 static __inline int
157 hibe_cmp(const struct hiballoc_entry *l, const struct hiballoc_entry *r)
158 {
159 	vaddr_t vl = (vaddr_t)l;
160 	vaddr_t vr = (vaddr_t)r;
161 
162 	return vl < vr ? -1 : (vl > vr);
163 }
164 
165 RBT_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
166 
167 /*
168  * Given a hiballoc entry, return the address it manages.
169  */
170 static __inline void *
171 hib_entry_to_addr(struct hiballoc_entry *entry)
172 {
173 	caddr_t addr;
174 
175 	addr = (caddr_t)entry;
176 	addr += HIB_SIZEOF(struct hiballoc_entry);
177 	return addr;
178 }
179 
180 /*
181  * Given an address, find the hiballoc that corresponds.
182  */
183 static __inline struct hiballoc_entry*
184 hib_addr_to_entry(void *addr_param)
185 {
186 	caddr_t addr;
187 
188 	addr = (caddr_t)addr_param;
189 	addr -= HIB_SIZEOF(struct hiballoc_entry);
190 	return (struct hiballoc_entry*)addr;
191 }
192 
193 RBT_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp);
194 
195 /*
196  * Allocate memory from the arena.
197  *
198  * Returns NULL if no memory is available.
199  */
200 void *
201 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz)
202 {
203 	struct hiballoc_entry *entry, *new_entry;
204 	size_t find_sz;
205 
206 	/*
207 	 * Enforce alignment of HIB_ALIGN bytes.
208 	 *
209 	 * Note that, because the entry is put in front of the allocation,
210 	 * 0-byte allocations are guaranteed a unique address.
211 	 */
212 	alloc_sz = roundup(alloc_sz, HIB_ALIGN);
213 
214 	/*
215 	 * Find an entry with hibe_space >= find_sz.
216 	 *
217 	 * If the root node is not large enough, we switch to tree traversal.
218 	 * Because all entries are made at the bottom of the free space,
219 	 * traversal from the end has a slightly better chance of yielding
220 	 * a sufficiently large space.
221 	 */
222 	find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry);
223 	entry = RBT_ROOT(hiballoc_addr, &arena->hib_addrs);
224 	if (entry != NULL && entry->hibe_space < find_sz) {
225 		RBT_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) {
226 			if (entry->hibe_space >= find_sz)
227 				break;
228 		}
229 	}
230 
231 	/*
232 	 * Insufficient or too fragmented memory.
233 	 */
234 	if (entry == NULL)
235 		return NULL;
236 
237 	/*
238 	 * Create new entry in allocated space.
239 	 */
240 	new_entry = (struct hiballoc_entry*)(
241 	    (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use);
242 	new_entry->hibe_space = entry->hibe_space - find_sz;
243 	new_entry->hibe_use = alloc_sz;
244 
245 	/*
246 	 * Insert entry.
247 	 */
248 	if (RBT_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL)
249 		panic("hib_alloc: insert failure");
250 	entry->hibe_space = 0;
251 
252 	/* Return address managed by entry. */
253 	return hib_entry_to_addr(new_entry);
254 }
255 
256 void
257 hib_getentropy(char **bufp, size_t *bufplen)
258 {
259 	if (!bufp || !bufplen)
260 		return;
261 
262 	*bufp = (char *)(global_piglet_va + (29 * PAGE_SIZE));
263 	*bufplen = PAGE_SIZE;
264 }
265 
266 /*
267  * Free a pointer previously allocated from this arena.
268  *
269  * If addr is NULL, this will be silently accepted.
270  */
271 void
272 hib_free(struct hiballoc_arena *arena, void *addr)
273 {
274 	struct hiballoc_entry *entry, *prev;
275 
276 	if (addr == NULL)
277 		return;
278 
279 	/*
280 	 * Derive entry from addr and check it is really in this arena.
281 	 */
282 	entry = hib_addr_to_entry(addr);
283 	if (RBT_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry)
284 		panic("hib_free: freed item %p not in hib arena", addr);
285 
286 	/*
287 	 * Give the space in entry to its predecessor.
288 	 *
289 	 * If entry has no predecessor, change its used space into free space
290 	 * instead.
291 	 */
292 	prev = RBT_PREV(hiballoc_addr, entry);
293 	if (prev != NULL &&
294 	    (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) +
295 	    prev->hibe_use + prev->hibe_space) == entry) {
296 		/* Merge entry. */
297 		RBT_REMOVE(hiballoc_addr, &arena->hib_addrs, entry);
298 		prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) +
299 		    entry->hibe_use + entry->hibe_space;
300 	} else {
301 		/* Flip used memory to free space. */
302 		entry->hibe_space += entry->hibe_use;
303 		entry->hibe_use = 0;
304 	}
305 }
306 
307 /*
308  * Initialize hiballoc.
309  *
310  * The allocator will manage memmory at ptr, which is len bytes.
311  */
312 int
313 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len)
314 {
315 	struct hiballoc_entry *entry;
316 	caddr_t ptr;
317 	size_t len;
318 
319 	RBT_INIT(hiballoc_addr, &arena->hib_addrs);
320 
321 	/*
322 	 * Hib allocator enforces HIB_ALIGN alignment.
323 	 * Fixup ptr and len.
324 	 */
325 	ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN);
326 	len = p_len - ((size_t)ptr - (size_t)p_ptr);
327 	len &= ~((size_t)HIB_ALIGN - 1);
328 
329 	/*
330 	 * Insufficient memory to be able to allocate and also do bookkeeping.
331 	 */
332 	if (len <= HIB_SIZEOF(struct hiballoc_entry))
333 		return ENOMEM;
334 
335 	/*
336 	 * Create entry describing space.
337 	 */
338 	entry = (struct hiballoc_entry*)ptr;
339 	entry->hibe_use = 0;
340 	entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry);
341 	RBT_INSERT(hiballoc_addr, &arena->hib_addrs, entry);
342 
343 	return 0;
344 }
345 
346 /*
347  * Zero all free memory.
348  */
349 void
350 uvm_pmr_zero_everything(void)
351 {
352 	struct uvm_pmemrange	*pmr;
353 	struct vm_page		*pg;
354 	int			 i;
355 
356 	uvm_lock_fpageq();
357 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
358 		/* Zero single pages. */
359 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY]))
360 		    != NULL) {
361 			uvm_pmr_remove(pmr, pg);
362 			uvm_pagezero(pg);
363 			atomic_setbits_int(&pg->pg_flags, PG_ZERO);
364 			uvmexp.zeropages++;
365 			uvm_pmr_insert(pmr, pg, 0);
366 		}
367 
368 		/* Zero multi page ranges. */
369 		while ((pg = RBT_ROOT(uvm_pmr_size,
370 		    &pmr->size[UVM_PMR_MEMTYPE_DIRTY])) != NULL) {
371 			pg--; /* Size tree always has second page. */
372 			uvm_pmr_remove(pmr, pg);
373 			for (i = 0; i < pg->fpgsz; i++) {
374 				uvm_pagezero(&pg[i]);
375 				atomic_setbits_int(&pg[i].pg_flags, PG_ZERO);
376 				uvmexp.zeropages++;
377 			}
378 			uvm_pmr_insert(pmr, pg, 0);
379 		}
380 	}
381 	uvm_unlock_fpageq();
382 }
383 
384 /*
385  * Mark all memory as dirty.
386  *
387  * Used to inform the system that the clean memory isn't clean for some
388  * reason, for example because we just came back from hibernate.
389  */
390 void
391 uvm_pmr_dirty_everything(void)
392 {
393 	struct uvm_pmemrange	*pmr;
394 	struct vm_page		*pg;
395 	int			 i;
396 
397 	uvm_lock_fpageq();
398 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
399 		/* Dirty single pages. */
400 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO]))
401 		    != NULL) {
402 			uvm_pmr_remove(pmr, pg);
403 			atomic_clearbits_int(&pg->pg_flags, PG_ZERO);
404 			uvm_pmr_insert(pmr, pg, 0);
405 		}
406 
407 		/* Dirty multi page ranges. */
408 		while ((pg = RBT_ROOT(uvm_pmr_size,
409 		    &pmr->size[UVM_PMR_MEMTYPE_ZERO])) != NULL) {
410 			pg--; /* Size tree always has second page. */
411 			uvm_pmr_remove(pmr, pg);
412 			for (i = 0; i < pg->fpgsz; i++)
413 				atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO);
414 			uvm_pmr_insert(pmr, pg, 0);
415 		}
416 	}
417 
418 	uvmexp.zeropages = 0;
419 	uvm_unlock_fpageq();
420 }
421 
422 /*
423  * Allocate an area that can hold sz bytes and doesn't overlap with
424  * the piglet at piglet_pa.
425  */
426 int
427 uvm_pmr_alloc_pig(paddr_t *pa, psize_t sz, paddr_t piglet_pa)
428 {
429 	struct uvm_constraint_range pig_constraint;
430 	struct kmem_pa_mode kp_pig = {
431 		.kp_constraint = &pig_constraint,
432 		.kp_maxseg = 1
433 	};
434 	vaddr_t va;
435 
436 	sz = round_page(sz);
437 
438 	pig_constraint.ucr_low = piglet_pa + 4 * HIBERNATE_CHUNK_SIZE;
439 	pig_constraint.ucr_high = -1;
440 
441 	va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait);
442 	if (va == 0) {
443 		pig_constraint.ucr_low = 0;
444 		pig_constraint.ucr_high = piglet_pa - 1;
445 
446 		va = (vaddr_t)km_alloc(sz, &kv_any, &kp_pig, &kd_nowait);
447 		if (va == 0)
448 			return ENOMEM;
449 	}
450 
451 	pmap_extract(pmap_kernel(), va, pa);
452 	return 0;
453 }
454 
455 /*
456  * Allocate a piglet area.
457  *
458  * This needs to be in DMA-safe memory.
459  * Piglets are aligned.
460  *
461  * sz and align in bytes.
462  *
463  * The call will sleep for the pagedaemon to attempt to free memory.
464  * The pagedaemon may decide its not possible to free enough memory, causing
465  * the allocation to fail.
466  */
467 int
468 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align)
469 {
470 	struct kmem_pa_mode kp_piglet = {
471 		.kp_constraint = &dma_constraint,
472 		.kp_align = align,
473 		.kp_maxseg = 1
474 	};
475 
476 	/* Ensure align is a power of 2 */
477 	KASSERT((align & (align - 1)) == 0);
478 
479 	/*
480 	 * Fixup arguments: align must be at least PAGE_SIZE,
481 	 * sz will be converted to pagecount, since that is what
482 	 * pmemrange uses internally.
483 	 */
484 	if (align < PAGE_SIZE)
485 		kp_piglet.kp_align = PAGE_SIZE;
486 
487 	sz = round_page(sz);
488 
489 	*va = (vaddr_t)km_alloc(sz, &kv_any, &kp_piglet, &kd_nowait);
490 	if (*va == 0)
491 		return ENOMEM;
492 
493 	pmap_extract(pmap_kernel(), *va, pa);
494 	return 0;
495 }
496 
497 /*
498  * Free a piglet area.
499  */
500 void
501 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz)
502 {
503 	/*
504 	 * Fix parameters.
505 	 */
506 	sz = round_page(sz);
507 
508 	/*
509 	 * Free the physical and virtual memory.
510 	 */
511 	km_free((void *)va, sz, &kv_any, &kp_dma_contig);
512 }
513 
514 /*
515  * Physmem RLE compression support.
516  *
517  * Given a physical page address, return the number of pages starting at the
518  * address that are free.  Clamps to the number of pages in
519  * HIBERNATE_CHUNK_SIZE. Returns 0 if the page at addr is not free.
520  */
521 int
522 uvm_page_rle(paddr_t addr)
523 {
524 	struct vm_page		*pg, *pg_end;
525 	struct vm_physseg	*vmp;
526 	int			 pseg_idx, off_idx;
527 
528 	pseg_idx = vm_physseg_find(atop(addr), &off_idx);
529 	if (pseg_idx == -1)
530 		return 0;
531 
532 	vmp = &vm_physmem[pseg_idx];
533 	pg = &vmp->pgs[off_idx];
534 	if (!(pg->pg_flags & PQ_FREE))
535 		return 0;
536 
537 	/*
538 	 * Search for the first non-free page after pg.
539 	 * Note that the page may not be the first page in a free pmemrange,
540 	 * therefore pg->fpgsz cannot be used.
541 	 */
542 	for (pg_end = pg; pg_end <= vmp->lastpg &&
543 	    (pg_end->pg_flags & PQ_FREE) == PQ_FREE; pg_end++)
544 		;
545 	return min((pg_end - pg), HIBERNATE_CHUNK_SIZE/PAGE_SIZE);
546 }
547 
548 /*
549  * Fills out the hibernate_info union pointed to by hib
550  * with information about this machine (swap signature block
551  * offsets, number of memory ranges, kernel in use, etc)
552  */
553 int
554 get_hibernate_info(union hibernate_info *hib, int suspend)
555 {
556 	struct disklabel dl;
557 	char err_string[128], *dl_ret;
558 
559 #ifndef NO_PROPOLICE
560 	/* Save propolice guard */
561 	hib->guard = __guard_local;
562 #endif /* ! NO_PROPOLICE */
563 
564 	/* Determine I/O function to use */
565 	hib->io_func = get_hibernate_io_function(swdevt[0].sw_dev);
566 	if (hib->io_func == NULL)
567 		return (1);
568 
569 	/* Calculate hibernate device */
570 	hib->dev = swdevt[0].sw_dev;
571 
572 	/* Read disklabel (used to calculate signature and image offsets) */
573 	dl_ret = disk_readlabel(&dl, hib->dev, err_string, sizeof(err_string));
574 
575 	if (dl_ret) {
576 		printf("Hibernate error reading disklabel: %s\n", dl_ret);
577 		return (1);
578 	}
579 
580 	/* Make sure we have a swap partition. */
581 	if (dl.d_partitions[1].p_fstype != FS_SWAP ||
582 	    DL_GETPSIZE(&dl.d_partitions[1]) == 0)
583 		return (1);
584 
585 	/* Make sure the signature can fit in one block */
586 	if (sizeof(union hibernate_info) > DEV_BSIZE)
587 		return (1);
588 
589 	/* Magic number */
590 	hib->magic = HIBERNATE_MAGIC;
591 
592 	/* Calculate signature block location */
593 	hib->sig_offset = DL_GETPSIZE(&dl.d_partitions[1]) -
594 	    sizeof(union hibernate_info)/DEV_BSIZE;
595 
596 	/* Stash kernel version information */
597 	memset(&hib->kernel_version, 0, 128);
598 	bcopy(version, &hib->kernel_version,
599 	    min(strlen(version), sizeof(hib->kernel_version)-1));
600 
601 	if (suspend) {
602 		/* Grab the previously-allocated piglet addresses */
603 		hib->piglet_va = global_piglet_va;
604 		hib->piglet_pa = global_piglet_pa;
605 		hib->io_page = (void *)hib->piglet_va;
606 
607 		/*
608 		 * Initialization of the hibernate IO function for drivers
609 		 * that need to do prep work (such as allocating memory or
610 		 * setting up data structures that cannot safely be done
611 		 * during suspend without causing side effects). There is
612 		 * a matching HIB_DONE call performed after the write is
613 		 * completed.
614 		 */
615 		if (hib->io_func(hib->dev, DL_GETPOFFSET(&dl.d_partitions[1]),
616 		    (vaddr_t)NULL, DL_GETPSIZE(&dl.d_partitions[1]),
617 		    HIB_INIT, hib->io_page))
618 			goto fail;
619 
620 	} else {
621 		/*
622 		 * Resuming kernels use a regular private page for the driver
623 		 * No need to free this I/O page as it will vanish as part of
624 		 * the resume.
625 		 */
626 		hib->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
627 		if (!hib->io_page)
628 			goto fail;
629 	}
630 
631 	if (get_hibernate_info_md(hib))
632 		goto fail;
633 
634 	return (0);
635 
636 fail:
637 	return (1);
638 }
639 
640 /*
641  * Allocate nitems*size bytes from the hiballoc area presently in use
642  */
643 void *
644 hibernate_zlib_alloc(void *unused, int nitems, int size)
645 {
646 	struct hibernate_zlib_state *hibernate_state;
647 
648 	hibernate_state =
649 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
650 
651 	return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size);
652 }
653 
654 /*
655  * Free the memory pointed to by addr in the hiballoc area presently in
656  * use
657  */
658 void
659 hibernate_zlib_free(void *unused, void *addr)
660 {
661 	struct hibernate_zlib_state *hibernate_state;
662 
663 	hibernate_state =
664 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
665 
666 	hib_free(&hibernate_state->hiballoc_arena, addr);
667 }
668 
669 /*
670  * Inflate next page of data from the image stream.
671  * The rle parameter is modified on exit to contain the number of pages to
672  * skip in the output stream (or 0 if this page was inflated into).
673  *
674  * Returns 0 if the stream contains additional data, or 1 if the stream is
675  * finished.
676  */
677 int
678 hibernate_inflate_page(int *rle)
679 {
680 	struct hibernate_zlib_state *hibernate_state;
681 	int i;
682 
683 	hibernate_state =
684 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
685 
686 	/* Set up the stream for RLE code inflate */
687 	hibernate_state->hib_stream.next_out = (unsigned char *)rle;
688 	hibernate_state->hib_stream.avail_out = sizeof(*rle);
689 
690 	/* Inflate RLE code */
691 	i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH);
692 	if (i != Z_OK && i != Z_STREAM_END) {
693 		/*
694 		 * XXX - this will likely reboot/hang most machines
695 		 *       since the console output buffer will be unmapped,
696 		 *       but there's not much else we can do here.
697 		 */
698 		panic("rle inflate stream error");
699 	}
700 
701 	if (hibernate_state->hib_stream.avail_out != 0) {
702 		/*
703 		 * XXX - this will likely reboot/hang most machines
704 		 *       since the console output buffer will be unmapped,
705 		 *       but there's not much else we can do here.
706 		 */
707 		panic("rle short inflate error");
708 	}
709 
710 	if (*rle < 0 || *rle > 1024) {
711 		/*
712 		 * XXX - this will likely reboot/hang most machines
713 		 *       since the console output buffer will be unmapped,
714 		 *       but there's not much else we can do here.
715 		 */
716 		panic("invalid rle count");
717 	}
718 
719 	if (i == Z_STREAM_END)
720 		return (1);
721 
722 	if (*rle != 0)
723 		return (0);
724 
725 	/* Set up the stream for page inflate */
726 	hibernate_state->hib_stream.next_out =
727 		(unsigned char *)HIBERNATE_INFLATE_PAGE;
728 	hibernate_state->hib_stream.avail_out = PAGE_SIZE;
729 
730 	/* Process next block of data */
731 	i = inflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH);
732 	if (i != Z_OK && i != Z_STREAM_END) {
733 		/*
734 		 * XXX - this will likely reboot/hang most machines
735 		 *       since the console output buffer will be unmapped,
736 		 *       but there's not much else we can do here.
737 		 */
738 		panic("inflate error");
739 	}
740 
741 	/* We should always have extracted a full page ... */
742 	if (hibernate_state->hib_stream.avail_out != 0) {
743 		/*
744 		 * XXX - this will likely reboot/hang most machines
745 		 *       since the console output buffer will be unmapped,
746 		 *       but there's not much else we can do here.
747 		 */
748 		panic("incomplete page");
749 	}
750 
751 	return (i == Z_STREAM_END);
752 }
753 
754 /*
755  * Inflate size bytes from src into dest, skipping any pages in
756  * [src..dest] that are special (see hibernate_inflate_skip)
757  *
758  * This function executes while using the resume-time stack
759  * and pmap, and therefore cannot use ddb/printf/etc. Doing so
760  * will likely hang or reset the machine since the console output buffer
761  * will be unmapped.
762  */
763 void
764 hibernate_inflate_region(union hibernate_info *hib, paddr_t dest,
765     paddr_t src, size_t size)
766 {
767 	int end_stream = 0, rle;
768 	struct hibernate_zlib_state *hibernate_state;
769 
770 	hibernate_state =
771 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
772 
773 	hibernate_state->hib_stream.next_in = (unsigned char *)src;
774 	hibernate_state->hib_stream.avail_in = size;
775 
776 	do {
777 		/*
778 		 * Is this a special page? If yes, redirect the
779 		 * inflate output to a scratch page (eg, discard it)
780 		 */
781 		if (hibernate_inflate_skip(hib, dest)) {
782 			hibernate_enter_resume_mapping(
783 			    HIBERNATE_INFLATE_PAGE,
784 			    HIBERNATE_INFLATE_PAGE, 0);
785 		} else {
786 			hibernate_enter_resume_mapping(
787 			    HIBERNATE_INFLATE_PAGE, dest, 0);
788 		}
789 
790 		hibernate_flush();
791 		end_stream = hibernate_inflate_page(&rle);
792 
793 		if (rle == 0)
794 			dest += PAGE_SIZE;
795 		else
796 			dest += (rle * PAGE_SIZE);
797 	} while (!end_stream);
798 }
799 
800 /*
801  * deflate from src into the I/O page, up to 'remaining' bytes
802  *
803  * Returns number of input bytes consumed, and may reset
804  * the 'remaining' parameter if not all the output space was consumed
805  * (this information is needed to know how much to write to disk
806  */
807 size_t
808 hibernate_deflate(union hibernate_info *hib, paddr_t src,
809     size_t *remaining)
810 {
811 	vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
812 	struct hibernate_zlib_state *hibernate_state;
813 
814 	hibernate_state =
815 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
816 
817 	/* Set up the stream for deflate */
818 	hibernate_state->hib_stream.next_in = (unsigned char *)src;
819 	hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK);
820 	hibernate_state->hib_stream.next_out =
821 		(unsigned char *)hibernate_io_page + (PAGE_SIZE - *remaining);
822 	hibernate_state->hib_stream.avail_out = *remaining;
823 
824 	/* Process next block of data */
825 	if (deflate(&hibernate_state->hib_stream, Z_SYNC_FLUSH) != Z_OK)
826 		panic("hibernate zlib deflate error");
827 
828 	/* Update pointers and return number of bytes consumed */
829 	*remaining = hibernate_state->hib_stream.avail_out;
830 	return (PAGE_SIZE - (src & PAGE_MASK)) -
831 	    hibernate_state->hib_stream.avail_in;
832 }
833 
834 /*
835  * Write the hibernation information specified in hiber_info
836  * to the location in swap previously calculated (last block of
837  * swap), called the "signature block".
838  */
839 int
840 hibernate_write_signature(union hibernate_info *hib)
841 {
842 	/* Write hibernate info to disk */
843 	return (hib->io_func(hib->dev, hib->sig_offset,
844 	    (vaddr_t)hib, DEV_BSIZE, HIB_W,
845 	    hib->io_page));
846 }
847 
848 /*
849  * Write the memory chunk table to the area in swap immediately
850  * preceding the signature block. The chunk table is stored
851  * in the piglet when this function is called.  Returns errno.
852  */
853 int
854 hibernate_write_chunktable(union hibernate_info *hib)
855 {
856 	vaddr_t hibernate_chunk_table_start;
857 	size_t hibernate_chunk_table_size;
858 	int i, err;
859 
860 	hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE;
861 
862 	hibernate_chunk_table_start = hib->piglet_va +
863 	    HIBERNATE_CHUNK_SIZE;
864 
865 	/* Write chunk table */
866 	for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) {
867 		if ((err = hib->io_func(hib->dev,
868 		    hib->chunktable_offset + (i/DEV_BSIZE),
869 		    (vaddr_t)(hibernate_chunk_table_start + i),
870 		    MAXPHYS, HIB_W, hib->io_page))) {
871 			DPRINTF("chunktable write error: %d\n", err);
872 			return (err);
873 		}
874 	}
875 
876 	return (0);
877 }
878 
879 /*
880  * Write an empty hiber_info to the swap signature block, which is
881  * guaranteed to not match any valid hib.
882  */
883 int
884 hibernate_clear_signature(void)
885 {
886 	union hibernate_info blank_hiber_info;
887 	union hibernate_info hib;
888 
889 	/* Zero out a blank hiber_info */
890 	memset(&blank_hiber_info, 0, sizeof(union hibernate_info));
891 
892 	/* Get the signature block location */
893 	if (get_hibernate_info(&hib, 0))
894 		return (1);
895 
896 	/* Write (zeroed) hibernate info to disk */
897 	DPRINTF("clearing hibernate signature block location: %lld\n",
898 		hib.sig_offset);
899 	if (hibernate_block_io(&hib,
900 	    hib.sig_offset,
901 	    DEV_BSIZE, (vaddr_t)&blank_hiber_info, 1))
902 		printf("Warning: could not clear hibernate signature\n");
903 
904 	return (0);
905 }
906 
907 /*
908  * Compare two hibernate_infos to determine if they are the same (eg,
909  * we should be performing a hibernate resume on this machine.
910  * Not all fields are checked - just enough to verify that the machine
911  * has the same memory configuration and kernel as the one that
912  * wrote the signature previously.
913  */
914 int
915 hibernate_compare_signature(union hibernate_info *mine,
916     union hibernate_info *disk)
917 {
918 	u_int i;
919 
920 	if (mine->nranges != disk->nranges) {
921 		DPRINTF("hibernate memory range count mismatch\n");
922 		return (1);
923 	}
924 
925 	if (strcmp(mine->kernel_version, disk->kernel_version) != 0) {
926 		DPRINTF("hibernate kernel version mismatch\n");
927 		return (1);
928 	}
929 
930 	for (i = 0; i < mine->nranges; i++) {
931 		if ((mine->ranges[i].base != disk->ranges[i].base) ||
932 		    (mine->ranges[i].end != disk->ranges[i].end) ) {
933 			DPRINTF("hib range %d mismatch [%p-%p != %p-%p]\n",
934 				i,
935 				(void *)mine->ranges[i].base,
936 				(void *)mine->ranges[i].end,
937 				(void *)disk->ranges[i].base,
938 				(void *)disk->ranges[i].end);
939 			return (1);
940 		}
941 	}
942 
943 	return (0);
944 }
945 
946 /*
947  * Transfers xfer_size bytes between the hibernate device specified in
948  * hib_info at offset blkctr and the vaddr specified at dest.
949  *
950  * Separate offsets and pages are used to handle misaligned reads (reads
951  * that span a page boundary).
952  *
953  * blkctr specifies a relative offset (relative to the start of swap),
954  * not an absolute disk offset
955  *
956  */
957 int
958 hibernate_block_io(union hibernate_info *hib, daddr_t blkctr,
959     size_t xfer_size, vaddr_t dest, int iswrite)
960 {
961 	struct buf *bp;
962 	struct bdevsw *bdsw;
963 	int error;
964 
965 	bp = geteblk(xfer_size);
966 	bdsw = &bdevsw[major(hib->dev)];
967 
968 	error = (*bdsw->d_open)(hib->dev, FREAD, S_IFCHR, curproc);
969 	if (error) {
970 		printf("hibernate_block_io open failed\n");
971 		return (1);
972 	}
973 
974 	if (iswrite)
975 		bcopy((caddr_t)dest, bp->b_data, xfer_size);
976 
977 	bp->b_bcount = xfer_size;
978 	bp->b_blkno = blkctr;
979 	CLR(bp->b_flags, B_READ | B_WRITE | B_DONE);
980 	SET(bp->b_flags, B_BUSY | (iswrite ? B_WRITE : B_READ) | B_RAW);
981 	bp->b_dev = hib->dev;
982 	(*bdsw->d_strategy)(bp);
983 
984 	error = biowait(bp);
985 	if (error) {
986 		printf("hib block_io biowait error %d blk %lld size %zu\n",
987 			error, (long long)blkctr, xfer_size);
988 		error = (*bdsw->d_close)(hib->dev, 0, S_IFCHR,
989 		    curproc);
990 		if (error)
991 			printf("hibernate_block_io error close failed\n");
992 		return (1);
993 	}
994 
995 	error = (*bdsw->d_close)(hib->dev, FREAD, S_IFCHR, curproc);
996 	if (error) {
997 		printf("hibernate_block_io close failed\n");
998 		return (1);
999 	}
1000 
1001 	if (!iswrite)
1002 		bcopy(bp->b_data, (caddr_t)dest, xfer_size);
1003 
1004 	bp->b_flags |= B_INVAL;
1005 	brelse(bp);
1006 
1007 	return (0);
1008 }
1009 
1010 /*
1011  * Preserve one page worth of random data, generated from the resuming
1012  * kernel's arc4random. After resume, this preserved entropy can be used
1013  * to further improve the un-hibernated machine's entropy pool. This
1014  * random data is stored in the piglet, which is preserved across the
1015  * unpack operation, and is restored later in the resume process (see
1016  * hib_getentropy)
1017  */
1018 void
1019 hibernate_preserve_entropy(union hibernate_info *hib)
1020 {
1021 	void *entropy;
1022 
1023 	entropy = km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_nowait);
1024 
1025 	if (!entropy)
1026 		return;
1027 
1028 	pmap_activate(curproc);
1029 	pmap_kenter_pa((vaddr_t)entropy,
1030 	    (paddr_t)(hib->piglet_pa + (29 * PAGE_SIZE)),
1031 	    PROT_READ | PROT_WRITE);
1032 
1033 	arc4random_buf((void *)entropy, PAGE_SIZE);
1034 	pmap_kremove((vaddr_t)entropy, PAGE_SIZE);
1035 	km_free(entropy, PAGE_SIZE, &kv_any, &kp_none);
1036 }
1037 
1038 #ifndef NO_PROPOLICE
1039 vaddr_t
1040 hibernate_unprotect_ssp(void)
1041 {
1042 	struct kmem_dyn_mode kd_avoidalias;
1043 	vaddr_t va = trunc_page((vaddr_t)&__guard_local);
1044 	paddr_t pa;
1045 
1046 	pmap_extract(pmap_kernel(), va, &pa);
1047 
1048 	memset(&kd_avoidalias, 0, sizeof kd_avoidalias);
1049 	kd_avoidalias.kd_prefer = pa;
1050 	kd_avoidalias.kd_waitok = 1;
1051 	va = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any, &kp_none, &kd_avoidalias);
1052 	if (!va)
1053 		panic("hibernate_unprotect_ssp");
1054 
1055 	pmap_kenter_pa(va, pa, PROT_READ | PROT_WRITE);
1056 	pmap_update(pmap_kernel());
1057 
1058 	return va;
1059 }
1060 
1061 void
1062 hibernate_reprotect_ssp(vaddr_t va)
1063 {
1064 	pmap_kremove(va, PAGE_SIZE);
1065 	km_free((void *)va, PAGE_SIZE, &kv_any, &kp_none);
1066 }
1067 #endif /* NO_PROPOLICE */
1068 
1069 /*
1070  * Reads the signature block from swap, checks against the current machine's
1071  * information. If the information matches, perform a resume by reading the
1072  * saved image into the pig area, and unpacking.
1073  *
1074  * Must be called with interrupts enabled.
1075  */
1076 void
1077 hibernate_resume(void)
1078 {
1079 	union hibernate_info hib;
1080 	int s;
1081 #ifndef NO_PROPOLICE
1082 	vsize_t off = (vaddr_t)&__guard_local -
1083 	    trunc_page((vaddr_t)&__guard_local);
1084 	vaddr_t guard_va;
1085 #endif
1086 
1087 	/* Get current running machine's hibernate info */
1088 	memset(&hib, 0, sizeof(hib));
1089 	if (get_hibernate_info(&hib, 0)) {
1090 		DPRINTF("couldn't retrieve machine's hibernate info\n");
1091 		return;
1092 	}
1093 
1094 	/* Read hibernate info from disk */
1095 	s = splbio();
1096 
1097 	DPRINTF("reading hibernate signature block location: %lld\n",
1098 		hib.sig_offset);
1099 
1100 	if (hibernate_block_io(&hib,
1101 	    hib.sig_offset,
1102 	    DEV_BSIZE, (vaddr_t)&disk_hib, 0)) {
1103 		DPRINTF("error in hibernate read");
1104 		splx(s);
1105 		return;
1106 	}
1107 
1108 	/* Check magic number */
1109 	if (disk_hib.magic != HIBERNATE_MAGIC) {
1110 		DPRINTF("wrong magic number in hibernate signature: %x\n",
1111 			disk_hib.magic);
1112 		splx(s);
1113 		return;
1114 	}
1115 
1116 	/*
1117 	 * We (possibly) found a hibernate signature. Clear signature first,
1118 	 * to prevent accidental resume or endless resume cycles later.
1119 	 */
1120 	if (hibernate_clear_signature()) {
1121 		DPRINTF("error clearing hibernate signature block\n");
1122 		splx(s);
1123 		return;
1124 	}
1125 
1126 	/*
1127 	 * If on-disk and in-memory hibernate signatures match,
1128 	 * this means we should do a resume from hibernate.
1129 	 */
1130 	if (hibernate_compare_signature(&hib, &disk_hib)) {
1131 		DPRINTF("mismatched hibernate signature block\n");
1132 		splx(s);
1133 		return;
1134 	}
1135 
1136 #ifdef MULTIPROCESSOR
1137 	/* XXX - if we fail later, we may need to rehatch APs on some archs */
1138 	DPRINTF("hibernate: quiescing APs\n");
1139 	hibernate_quiesce_cpus();
1140 #endif /* MULTIPROCESSOR */
1141 
1142 	/* Read the image from disk into the image (pig) area */
1143 	if (hibernate_read_image(&disk_hib))
1144 		goto fail;
1145 
1146 	DPRINTF("hibernate: quiescing devices\n");
1147 	if (config_suspend_all(DVACT_QUIESCE) != 0)
1148 		goto fail;
1149 
1150 #ifndef NO_PROPOLICE
1151 	guard_va = hibernate_unprotect_ssp();
1152 #endif /* NO_PROPOLICE */
1153 
1154 	(void) splhigh();
1155 	hibernate_disable_intr_machdep();
1156 	cold = 1;
1157 
1158 	DPRINTF("hibernate: suspending devices\n");
1159 	if (config_suspend_all(DVACT_SUSPEND) != 0) {
1160 		cold = 0;
1161 		hibernate_enable_intr_machdep();
1162 #ifndef NO_PROPOLICE
1163 		hibernate_reprotect_ssp(guard_va);
1164 #endif /* ! NO_PROPOLICE */
1165 		goto fail;
1166 	}
1167 
1168 	hibernate_preserve_entropy(&disk_hib);
1169 
1170 	printf("Unpacking image...\n");
1171 
1172 	/* Switch stacks */
1173 	DPRINTF("hibernate: switching stacks\n");
1174 	hibernate_switch_stack_machdep();
1175 
1176 #ifndef NO_PROPOLICE
1177 	/* Start using suspended kernel's propolice guard */
1178 	*(long *)(guard_va + off) = disk_hib.guard;
1179 	hibernate_reprotect_ssp(guard_va);
1180 #endif /* ! NO_PROPOLICE */
1181 
1182 	/* Unpack and resume */
1183 	hibernate_unpack_image(&disk_hib);
1184 
1185 fail:
1186 	splx(s);
1187 	printf("\nUnable to resume hibernated image\n");
1188 }
1189 
1190 /*
1191  * Unpack image from pig area to original location by looping through the
1192  * list of output chunks in the order they should be restored (fchunks).
1193  *
1194  * Note that due to the stack smash protector and the fact that we have
1195  * switched stacks, it is not permitted to return from this function.
1196  */
1197 void
1198 hibernate_unpack_image(union hibernate_info *hib)
1199 {
1200 	struct hibernate_disk_chunk *chunks;
1201 	union hibernate_info local_hib;
1202 	paddr_t image_cur = global_pig_start;
1203 	short i, *fchunks;
1204 	char *pva;
1205 
1206 	/* Piglet will be identity mapped (VA == PA) */
1207 	pva = (char *)hib->piglet_pa;
1208 
1209 	fchunks = (short *)(pva + (4 * PAGE_SIZE));
1210 
1211 	chunks = (struct hibernate_disk_chunk *)(pva + HIBERNATE_CHUNK_SIZE);
1212 
1213 	/* Can't use hiber_info that's passed in after this point */
1214 	bcopy(hib, &local_hib, sizeof(union hibernate_info));
1215 
1216 	/* VA == PA */
1217 	local_hib.piglet_va = local_hib.piglet_pa;
1218 
1219 	/*
1220 	 * Point of no return. Once we pass this point, only kernel code can
1221 	 * be accessed. No global variables or other kernel data structures
1222 	 * are guaranteed to be coherent after unpack starts.
1223 	 *
1224 	 * The image is now in high memory (pig area), we unpack from the pig
1225 	 * to the correct location in memory. We'll eventually end up copying
1226 	 * on top of ourself, but we are assured the kernel code here is the
1227 	 * same between the hibernated and resuming kernel, and we are running
1228 	 * on our own stack, so the overwrite is ok.
1229 	 */
1230 	DPRINTF("hibernate: activating alt. pagetable and starting unpack\n");
1231 	hibernate_activate_resume_pt_machdep();
1232 
1233 	for (i = 0; i < local_hib.chunk_ctr; i++) {
1234 		/* Reset zlib for inflate */
1235 		if (hibernate_zlib_reset(&local_hib, 0) != Z_OK)
1236 			panic("hibernate failed to reset zlib for inflate");
1237 
1238 		hibernate_process_chunk(&local_hib, &chunks[fchunks[i]],
1239 		    image_cur);
1240 
1241 		image_cur += chunks[fchunks[i]].compressed_size;
1242 
1243 	}
1244 
1245 	/*
1246 	 * Resume the loaded kernel by jumping to the MD resume vector.
1247 	 * We won't be returning from this call.
1248 	 */
1249 	hibernate_resume_machdep();
1250 }
1251 
1252 /*
1253  * Bounce a compressed image chunk to the piglet, entering mappings for the
1254  * copied pages as needed
1255  */
1256 void
1257 hibernate_copy_chunk_to_piglet(paddr_t img_cur, vaddr_t piglet, size_t size)
1258 {
1259 	size_t ct, ofs;
1260 	paddr_t src = img_cur;
1261 	vaddr_t dest = piglet;
1262 
1263 	/* Copy first partial page */
1264 	ct = (PAGE_SIZE) - (src & PAGE_MASK);
1265 	ofs = (src & PAGE_MASK);
1266 
1267 	if (ct < PAGE_SIZE) {
1268 		hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE,
1269 			(src - ofs), 0);
1270 		hibernate_flush();
1271 		bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE + ofs), (caddr_t)dest, ct);
1272 		src += ct;
1273 		dest += ct;
1274 	}
1275 
1276 	/* Copy remaining pages */
1277 	while (src < size + img_cur) {
1278 		hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, src, 0);
1279 		hibernate_flush();
1280 		ct = PAGE_SIZE;
1281 		bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE), (caddr_t)dest, ct);
1282 		hibernate_flush();
1283 		src += ct;
1284 		dest += ct;
1285 	}
1286 }
1287 
1288 /*
1289  * Process a chunk by bouncing it to the piglet, followed by unpacking
1290  */
1291 void
1292 hibernate_process_chunk(union hibernate_info *hib,
1293     struct hibernate_disk_chunk *chunk, paddr_t img_cur)
1294 {
1295 	char *pva = (char *)hib->piglet_va;
1296 
1297 	hibernate_copy_chunk_to_piglet(img_cur,
1298 	 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), chunk->compressed_size);
1299 	hibernate_inflate_region(hib, chunk->base,
1300 	    (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)),
1301 	    chunk->compressed_size);
1302 }
1303 
1304 /*
1305  * Calculate RLE component for 'inaddr'. Clamps to max RLE pages between
1306  * inaddr and range_end.
1307  */
1308 int
1309 hibernate_calc_rle(paddr_t inaddr, paddr_t range_end)
1310 {
1311 	int rle;
1312 
1313 	rle = uvm_page_rle(inaddr);
1314 	KASSERT(rle >= 0 && rle <= MAX_RLE);
1315 
1316 	/* Clamp RLE to range end */
1317 	if (rle > 0 && inaddr + (rle * PAGE_SIZE) > range_end)
1318 		rle = (range_end - inaddr) / PAGE_SIZE;
1319 
1320 	return (rle);
1321 }
1322 
1323 /*
1324  * Write the RLE byte for page at 'inaddr' to the output stream.
1325  * Returns the number of pages to be skipped at 'inaddr'.
1326  */
1327 int
1328 hibernate_write_rle(union hibernate_info *hib, paddr_t inaddr,
1329 	paddr_t range_end, daddr_t *blkctr,
1330 	size_t *out_remaining)
1331 {
1332 	int rle, err, *rleloc;
1333 	struct hibernate_zlib_state *hibernate_state;
1334 	vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
1335 
1336 	hibernate_state =
1337 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1338 
1339 	rle = hibernate_calc_rle(inaddr, range_end);
1340 
1341 	rleloc = (int *)hibernate_rle_page + MAX_RLE - 1;
1342 	*rleloc = rle;
1343 
1344 	/* Deflate the RLE byte into the stream */
1345 	hibernate_deflate(hib, (paddr_t)rleloc, out_remaining);
1346 
1347 	/* Did we fill the output page? If so, flush to disk */
1348 	if (*out_remaining == 0) {
1349 		if ((err = hib->io_func(hib->dev, *blkctr + hib->image_offset,
1350 			(vaddr_t)hibernate_io_page, PAGE_SIZE, HIB_W,
1351 			hib->io_page))) {
1352 				DPRINTF("hib write error %d\n", err);
1353 				return (err);
1354 		}
1355 
1356 		*blkctr += PAGE_SIZE / DEV_BSIZE;
1357 		*out_remaining = PAGE_SIZE;
1358 
1359 		/* If we didn't deflate the entire RLE byte, finish it now */
1360 		if (hibernate_state->hib_stream.avail_in != 0)
1361 			hibernate_deflate(hib,
1362 				(vaddr_t)hibernate_state->hib_stream.next_in,
1363 				out_remaining);
1364 	}
1365 
1366 	return (rle);
1367 }
1368 
1369 /*
1370  * Write a compressed version of this machine's memory to disk, at the
1371  * precalculated swap offset:
1372  *
1373  * end of swap - signature block size - chunk table size - memory size
1374  *
1375  * The function begins by looping through each phys mem range, cutting each
1376  * one into MD sized chunks. These chunks are then compressed individually
1377  * and written out to disk, in phys mem order. Some chunks might compress
1378  * more than others, and for this reason, each chunk's size is recorded
1379  * in the chunk table, which is written to disk after the image has
1380  * properly been compressed and written (in hibernate_write_chunktable).
1381  *
1382  * When this function is called, the machine is nearly suspended - most
1383  * devices are quiesced/suspended, interrupts are off, and cold has
1384  * been set. This means that there can be no side effects once the
1385  * write has started, and the write function itself can also have no
1386  * side effects. This also means no printfs are permitted (since printf
1387  * has side effects.)
1388  *
1389  * Return values :
1390  *
1391  * 0      - success
1392  * EIO    - I/O error occurred writing the chunks
1393  * EINVAL - Failed to write a complete range
1394  * ENOMEM - Memory allocation failure during preparation of the zlib arena
1395  */
1396 int
1397 hibernate_write_chunks(union hibernate_info *hib)
1398 {
1399 	paddr_t range_base, range_end, inaddr, temp_inaddr;
1400 	size_t nblocks, out_remaining, used;
1401 	struct hibernate_disk_chunk *chunks;
1402 	vaddr_t hibernate_io_page = hib->piglet_va + PAGE_SIZE;
1403 	daddr_t blkctr = 0;
1404 	int i, rle, err;
1405 	struct hibernate_zlib_state *hibernate_state;
1406 
1407 	hibernate_state =
1408 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1409 
1410 	hib->chunk_ctr = 0;
1411 
1412 	/*
1413 	 * Map the utility VAs to the piglet. See the piglet map at the
1414 	 * top of this file for piglet layout information.
1415 	 */
1416 	hibernate_copy_page = hib->piglet_va + 3 * PAGE_SIZE;
1417 	hibernate_rle_page = hib->piglet_va + 28 * PAGE_SIZE;
1418 
1419 	chunks = (struct hibernate_disk_chunk *)(hib->piglet_va +
1420 	    HIBERNATE_CHUNK_SIZE);
1421 
1422 	/* Calculate the chunk regions */
1423 	for (i = 0; i < hib->nranges; i++) {
1424 		range_base = hib->ranges[i].base;
1425 		range_end = hib->ranges[i].end;
1426 
1427 		inaddr = range_base;
1428 
1429 		while (inaddr < range_end) {
1430 			chunks[hib->chunk_ctr].base = inaddr;
1431 			if (inaddr + HIBERNATE_CHUNK_SIZE < range_end)
1432 				chunks[hib->chunk_ctr].end = inaddr +
1433 				    HIBERNATE_CHUNK_SIZE;
1434 			else
1435 				chunks[hib->chunk_ctr].end = range_end;
1436 
1437 			inaddr += HIBERNATE_CHUNK_SIZE;
1438 			hib->chunk_ctr ++;
1439 		}
1440 	}
1441 
1442 	uvm_pmr_dirty_everything();
1443 	uvm_pmr_zero_everything();
1444 
1445 	/* Compress and write the chunks in the chunktable */
1446 	for (i = 0; i < hib->chunk_ctr; i++) {
1447 		range_base = chunks[i].base;
1448 		range_end = chunks[i].end;
1449 
1450 		chunks[i].offset = blkctr + hib->image_offset;
1451 
1452 		/* Reset zlib for deflate */
1453 		if (hibernate_zlib_reset(hib, 1) != Z_OK) {
1454 			DPRINTF("hibernate_zlib_reset failed for deflate\n");
1455 			return (ENOMEM);
1456 		}
1457 
1458 		inaddr = range_base;
1459 
1460 		/*
1461 		 * For each range, loop through its phys mem region
1462 		 * and write out the chunks (the last chunk might be
1463 		 * smaller than the chunk size).
1464 		 */
1465 		while (inaddr < range_end) {
1466 			out_remaining = PAGE_SIZE;
1467 			while (out_remaining > 0 && inaddr < range_end) {
1468 				/*
1469 				 * Adjust for regions that are not evenly
1470 				 * divisible by PAGE_SIZE or overflowed
1471 				 * pages from the previous iteration.
1472 				 */
1473 				temp_inaddr = (inaddr & PAGE_MASK) +
1474 				    hibernate_copy_page;
1475 
1476 				/* Deflate from temp_inaddr to IO page */
1477 				if (inaddr != range_end) {
1478 					if (inaddr % PAGE_SIZE == 0) {
1479 						rle = hibernate_write_rle(hib,
1480 							inaddr,
1481 							range_end,
1482 							&blkctr,
1483 							&out_remaining);
1484 					}
1485 
1486 					if (rle == 0) {
1487 						pmap_kenter_pa(hibernate_temp_page,
1488 							inaddr & PMAP_PA_MASK,
1489 							PROT_READ);
1490 
1491 						bcopy((caddr_t)hibernate_temp_page,
1492 							(caddr_t)hibernate_copy_page,
1493 							PAGE_SIZE);
1494 						inaddr += hibernate_deflate(hib,
1495 							temp_inaddr,
1496 							&out_remaining);
1497 					} else {
1498 						inaddr += rle * PAGE_SIZE;
1499 						if (inaddr > range_end)
1500 							inaddr = range_end;
1501 					}
1502 
1503 				}
1504 
1505 				if (out_remaining == 0) {
1506 					/* Filled up the page */
1507 					nblocks = PAGE_SIZE / DEV_BSIZE;
1508 
1509 					if ((err = hib->io_func(hib->dev,
1510 					    blkctr + hib->image_offset,
1511 					    (vaddr_t)hibernate_io_page,
1512 					    PAGE_SIZE, HIB_W, hib->io_page))) {
1513 						DPRINTF("hib write error %d\n",
1514 						    err);
1515 						return (err);
1516 					}
1517 
1518 					blkctr += nblocks;
1519 				}
1520 			}
1521 		}
1522 
1523 		if (inaddr != range_end) {
1524 			DPRINTF("deflate range ended prematurely\n");
1525 			return (EINVAL);
1526 		}
1527 
1528 		/*
1529 		 * End of range. Round up to next secsize bytes
1530 		 * after finishing compress
1531 		 */
1532 		if (out_remaining == 0)
1533 			out_remaining = PAGE_SIZE;
1534 
1535 		/* Finish compress */
1536 		hibernate_state->hib_stream.next_in = (unsigned char *)inaddr;
1537 		hibernate_state->hib_stream.avail_in = 0;
1538 		hibernate_state->hib_stream.next_out =
1539 		    (unsigned char *)hibernate_io_page +
1540 			(PAGE_SIZE - out_remaining);
1541 
1542 		/* We have an extra output page available for finalize */
1543 		hibernate_state->hib_stream.avail_out =
1544 			out_remaining + PAGE_SIZE;
1545 
1546 		if ((err = deflate(&hibernate_state->hib_stream, Z_FINISH)) !=
1547 		    Z_STREAM_END) {
1548 			DPRINTF("deflate error in output stream: %d\n", err);
1549 			return (err);
1550 		}
1551 
1552 		out_remaining = hibernate_state->hib_stream.avail_out;
1553 
1554 		used = 2 * PAGE_SIZE - out_remaining;
1555 		nblocks = used / DEV_BSIZE;
1556 
1557 		/* Round up to next block if needed */
1558 		if (used % DEV_BSIZE != 0)
1559 			nblocks ++;
1560 
1561 		/* Write final block(s) for this chunk */
1562 		if ((err = hib->io_func(hib->dev, blkctr + hib->image_offset,
1563 		    (vaddr_t)hibernate_io_page, nblocks*DEV_BSIZE,
1564 		    HIB_W, hib->io_page))) {
1565 			DPRINTF("hib final write error %d\n", err);
1566 			return (err);
1567 		}
1568 
1569 		blkctr += nblocks;
1570 
1571 		chunks[i].compressed_size = (blkctr + hib->image_offset -
1572 		    chunks[i].offset) * DEV_BSIZE;
1573 	}
1574 
1575 	hib->chunktable_offset = hib->image_offset + blkctr;
1576 	return (0);
1577 }
1578 
1579 /*
1580  * Reset the zlib stream state and allocate a new hiballoc area for either
1581  * inflate or deflate. This function is called once for each hibernate chunk.
1582  * Calling hiballoc_init multiple times is acceptable since the memory it is
1583  * provided is unmanaged memory (stolen). We use the memory provided to us
1584  * by the piglet allocated via the supplied hib.
1585  */
1586 int
1587 hibernate_zlib_reset(union hibernate_info *hib, int deflate)
1588 {
1589 	vaddr_t hibernate_zlib_start;
1590 	size_t hibernate_zlib_size;
1591 	char *pva = (char *)hib->piglet_va;
1592 	struct hibernate_zlib_state *hibernate_state;
1593 
1594 	hibernate_state =
1595 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1596 
1597 	if (!deflate)
1598 		pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK));
1599 
1600 	/*
1601 	 * See piglet layout information at the start of this file for
1602 	 * information on the zlib page assignments.
1603 	 */
1604 	hibernate_zlib_start = (vaddr_t)(pva + (30 * PAGE_SIZE));
1605 	hibernate_zlib_size = 80 * PAGE_SIZE;
1606 
1607 	memset((void *)hibernate_zlib_start, 0, hibernate_zlib_size);
1608 	memset(hibernate_state, 0, PAGE_SIZE);
1609 
1610 	/* Set up stream structure */
1611 	hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc;
1612 	hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free;
1613 
1614 	/* Initialize the hiballoc arena for zlib allocs/frees */
1615 	hiballoc_init(&hibernate_state->hiballoc_arena,
1616 	    (caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1617 
1618 	if (deflate) {
1619 		return deflateInit(&hibernate_state->hib_stream,
1620 		    Z_BEST_SPEED);
1621 	} else
1622 		return inflateInit(&hibernate_state->hib_stream);
1623 }
1624 
1625 /*
1626  * Reads the hibernated memory image from disk, whose location and
1627  * size are recorded in hib. Begin by reading the persisted
1628  * chunk table, which records the original chunk placement location
1629  * and compressed size for each. Next, allocate a pig region of
1630  * sufficient size to hold the compressed image. Next, read the
1631  * chunks into the pig area (calling hibernate_read_chunks to do this),
1632  * and finally, if all of the above succeeds, clear the hibernate signature.
1633  * The function will then return to hibernate_resume, which will proceed
1634  * to unpack the pig image to the correct place in memory.
1635  */
1636 int
1637 hibernate_read_image(union hibernate_info *hib)
1638 {
1639 	size_t compressed_size, disk_size, chunktable_size, pig_sz;
1640 	paddr_t image_start, image_end, pig_start, pig_end;
1641 	struct hibernate_disk_chunk *chunks;
1642 	daddr_t blkctr;
1643 	vaddr_t chunktable = (vaddr_t)NULL;
1644 	paddr_t piglet_chunktable = hib->piglet_pa +
1645 	    HIBERNATE_CHUNK_SIZE;
1646 	int i, status;
1647 
1648 	status = 0;
1649 	pmap_activate(curproc);
1650 
1651 	/* Calculate total chunk table size in disk blocks */
1652 	chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / DEV_BSIZE;
1653 
1654 	blkctr = hib->chunktable_offset;
1655 
1656 	chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any,
1657 	    &kp_none, &kd_nowait);
1658 
1659 	if (!chunktable)
1660 		return (1);
1661 
1662 	/* Map chunktable pages */
1663 	for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE; i += PAGE_SIZE)
1664 		pmap_kenter_pa(chunktable + i, piglet_chunktable + i,
1665 		    PROT_READ | PROT_WRITE);
1666 	pmap_update(pmap_kernel());
1667 
1668 	/* Read the chunktable from disk into the piglet chunktable */
1669 	for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE;
1670 	    i += MAXPHYS, blkctr += MAXPHYS/DEV_BSIZE)
1671 		hibernate_block_io(hib, blkctr, MAXPHYS,
1672 		    chunktable + i, 0);
1673 
1674 	blkctr = hib->image_offset;
1675 	compressed_size = 0;
1676 
1677 	chunks = (struct hibernate_disk_chunk *)chunktable;
1678 
1679 	for (i = 0; i < hib->chunk_ctr; i++)
1680 		compressed_size += chunks[i].compressed_size;
1681 
1682 	disk_size = compressed_size;
1683 
1684 	printf("unhibernating @ block %lld length %lu bytes\n",
1685 	    hib->sig_offset - chunktable_size,
1686 	    compressed_size);
1687 
1688 	/* Allocate the pig area */
1689 	pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE;
1690 	if (uvm_pmr_alloc_pig(&pig_start, pig_sz, hib->piglet_pa) == ENOMEM) {
1691 		status = 1;
1692 		goto unmap;
1693 	}
1694 
1695 	pig_end = pig_start + pig_sz;
1696 
1697 	/* Calculate image extents. Pig image must end on a chunk boundary. */
1698 	image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1);
1699 	image_start = image_end - disk_size;
1700 
1701 	hibernate_read_chunks(hib, image_start, image_end, disk_size,
1702 	    chunks);
1703 
1704 	/* Prepare the resume time pmap/page table */
1705 	hibernate_populate_resume_pt(hib, image_start, image_end);
1706 
1707 unmap:
1708 	/* Unmap chunktable pages */
1709 	pmap_kremove(chunktable, HIBERNATE_CHUNK_TABLE_SIZE);
1710 	pmap_update(pmap_kernel());
1711 
1712 	return (status);
1713 }
1714 
1715 /*
1716  * Read the hibernated memory chunks from disk (chunk information at this
1717  * point is stored in the piglet) into the pig area specified by
1718  * [pig_start .. pig_end]. Order the chunks so that the final chunk is the
1719  * only chunk with overlap possibilities.
1720  */
1721 int
1722 hibernate_read_chunks(union hibernate_info *hib, paddr_t pig_start,
1723     paddr_t pig_end, size_t image_compr_size,
1724     struct hibernate_disk_chunk *chunks)
1725 {
1726 	paddr_t img_cur, piglet_base;
1727 	daddr_t blkctr;
1728 	size_t processed, compressed_size, read_size;
1729 	int nchunks, nfchunks, num_io_pages;
1730 	vaddr_t tempva, hibernate_fchunk_area;
1731 	short *fchunks, i, j;
1732 
1733 	tempva = (vaddr_t)NULL;
1734 	hibernate_fchunk_area = (vaddr_t)NULL;
1735 	nfchunks = 0;
1736 	piglet_base = hib->piglet_pa;
1737 	global_pig_start = pig_start;
1738 
1739 	/*
1740 	 * These mappings go into the resuming kernel's page table, and are
1741 	 * used only during image read. They dissappear from existence
1742 	 * when the suspended kernel is unpacked on top of us.
1743 	 */
1744 	tempva = (vaddr_t)km_alloc(MAXPHYS + PAGE_SIZE, &kv_any, &kp_none,
1745 		&kd_nowait);
1746 	if (!tempva)
1747 		return (1);
1748 	hibernate_fchunk_area = (vaddr_t)km_alloc(24 * PAGE_SIZE, &kv_any,
1749 	    &kp_none, &kd_nowait);
1750 	if (!hibernate_fchunk_area)
1751 		return (1);
1752 
1753 	/* Final output chunk ordering VA */
1754 	fchunks = (short *)hibernate_fchunk_area;
1755 
1756 	/* Map the chunk ordering region */
1757 	for(i = 0; i < 24 ; i++)
1758 		pmap_kenter_pa(hibernate_fchunk_area + (i * PAGE_SIZE),
1759 			piglet_base + ((4 + i) * PAGE_SIZE),
1760 			PROT_READ | PROT_WRITE);
1761 	pmap_update(pmap_kernel());
1762 
1763 	nchunks = hib->chunk_ctr;
1764 
1765 	/* Initially start all chunks as unplaced */
1766 	for (i = 0; i < nchunks; i++)
1767 		chunks[i].flags = 0;
1768 
1769 	/*
1770 	 * Search the list for chunks that are outside the pig area. These
1771 	 * can be placed first in the final output list.
1772 	 */
1773 	for (i = 0; i < nchunks; i++) {
1774 		if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) {
1775 			fchunks[nfchunks] = i;
1776 			nfchunks++;
1777 			chunks[i].flags |= HIBERNATE_CHUNK_PLACED;
1778 		}
1779 	}
1780 
1781 	/*
1782 	 * Walk the ordering, place the chunks in ascending memory order.
1783 	 */
1784 	for (i = 0; i < nchunks; i++) {
1785 		if (chunks[i].flags != HIBERNATE_CHUNK_PLACED) {
1786 			fchunks[nfchunks] = i;
1787 			nfchunks++;
1788 			chunks[i].flags = HIBERNATE_CHUNK_PLACED;
1789 		}
1790 	}
1791 
1792 	img_cur = pig_start;
1793 
1794 	for (i = 0; i < nfchunks; i++) {
1795 		blkctr = chunks[fchunks[i]].offset;
1796 		processed = 0;
1797 		compressed_size = chunks[fchunks[i]].compressed_size;
1798 
1799 		while (processed < compressed_size) {
1800 			if (compressed_size - processed >= MAXPHYS)
1801 				read_size = MAXPHYS;
1802 			else
1803 				read_size = compressed_size - processed;
1804 
1805 			/*
1806 			 * We're reading read_size bytes, offset from the
1807 			 * start of a page by img_cur % PAGE_SIZE, so the
1808 			 * end will be read_size + (img_cur % PAGE_SIZE)
1809 			 * from the start of the first page.  Round that
1810 			 * up to the next page size.
1811 			 */
1812 			num_io_pages = (read_size + (img_cur % PAGE_SIZE)
1813 				+ PAGE_SIZE - 1) / PAGE_SIZE;
1814 
1815 			KASSERT(num_io_pages <= MAXPHYS/PAGE_SIZE + 1);
1816 
1817 			/* Map pages for this read */
1818 			for (j = 0; j < num_io_pages; j ++)
1819 				pmap_kenter_pa(tempva + j * PAGE_SIZE,
1820 				    img_cur + j * PAGE_SIZE,
1821 				    PROT_READ | PROT_WRITE);
1822 
1823 			pmap_update(pmap_kernel());
1824 
1825 			hibernate_block_io(hib, blkctr, read_size,
1826 			    tempva + (img_cur & PAGE_MASK), 0);
1827 
1828 			blkctr += (read_size / DEV_BSIZE);
1829 
1830 			pmap_kremove(tempva, num_io_pages * PAGE_SIZE);
1831 			pmap_update(pmap_kernel());
1832 
1833 			processed += read_size;
1834 			img_cur += read_size;
1835 		}
1836 	}
1837 
1838 	pmap_kremove(hibernate_fchunk_area, 24 * PAGE_SIZE);
1839 	pmap_update(pmap_kernel());
1840 
1841 	return (0);
1842 }
1843 
1844 /*
1845  * Hibernating a machine comprises the following operations:
1846  *  1. Calculating this machine's hibernate_info information
1847  *  2. Allocating a piglet and saving the piglet's physaddr
1848  *  3. Calculating the memory chunks
1849  *  4. Writing the compressed chunks to disk
1850  *  5. Writing the chunk table
1851  *  6. Writing the signature block (hibernate_info)
1852  *
1853  * On most architectures, the function calling hibernate_suspend would
1854  * then power off the machine using some MD-specific implementation.
1855  */
1856 int
1857 hibernate_suspend(void)
1858 {
1859 	union hibernate_info hib;
1860 	u_long start, end;
1861 
1862 	/*
1863 	 * Calculate memory ranges, swap offsets, etc.
1864 	 * This also allocates a piglet whose physaddr is stored in
1865 	 * hib->piglet_pa and vaddr stored in hib->piglet_va
1866 	 */
1867 	if (get_hibernate_info(&hib, 1)) {
1868 		DPRINTF("failed to obtain hibernate info\n");
1869 		return (1);
1870 	}
1871 
1872 	/* Find a page-addressed region in swap [start,end] */
1873 	if (uvm_hibswap(hib.dev, &start, &end)) {
1874 		printf("hibernate: cannot find any swap\n");
1875 		return (1);
1876 	}
1877 
1878 	if (end - start < 1000) {
1879 		printf("hibernate: insufficient swap (%lu is too small)\n",
1880 			end - start);
1881 		return (1);
1882 	}
1883 
1884 	/* Calculate block offsets in swap */
1885 	hib.image_offset = ctod(start);
1886 
1887 	DPRINTF("hibernate @ block %lld max-length %lu blocks\n",
1888 	    hib.image_offset, ctod(end) - ctod(start));
1889 
1890 	pmap_activate(curproc);
1891 	DPRINTF("hibernate: writing chunks\n");
1892 	if (hibernate_write_chunks(&hib)) {
1893 		DPRINTF("hibernate_write_chunks failed\n");
1894 		return (1);
1895 	}
1896 
1897 	DPRINTF("hibernate: writing chunktable\n");
1898 	if (hibernate_write_chunktable(&hib)) {
1899 		DPRINTF("hibernate_write_chunktable failed\n");
1900 		return (1);
1901 	}
1902 
1903 	DPRINTF("hibernate: writing signature\n");
1904 	if (hibernate_write_signature(&hib)) {
1905 		DPRINTF("hibernate_write_signature failed\n");
1906 		return (1);
1907 	}
1908 
1909 	/* Allow the disk to settle */
1910 	delay(500000);
1911 
1912 	/*
1913 	 * Give the device-specific I/O function a notification that we're
1914 	 * done, and that it can clean up or shutdown as needed.
1915 	 */
1916 	hib.io_func(hib.dev, 0, (vaddr_t)NULL, 0, HIB_DONE, hib.io_page);
1917 	return (0);
1918 }
1919 
1920 int
1921 hibernate_alloc(void)
1922 {
1923 	KASSERT(global_piglet_va == 0);
1924 	KASSERT(hibernate_temp_page == 0);
1925 
1926 	pmap_activate(curproc);
1927 	pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE,
1928 	    PROT_READ | PROT_WRITE);
1929 
1930 	/* Allocate a piglet, store its addresses in the supplied globals */
1931 	if (uvm_pmr_alloc_piglet(&global_piglet_va, &global_piglet_pa,
1932 	    HIBERNATE_CHUNK_SIZE * 4, HIBERNATE_CHUNK_SIZE))
1933 		goto unmap;
1934 
1935 	/*
1936 	 * Allocate VA for the temp page.
1937 	 *
1938 	 * This will become part of the suspended kernel and will
1939 	 * be freed in hibernate_free, upon resume (or hibernate
1940 	 * failure)
1941 	 */
1942 	hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1943 	    &kp_none, &kd_nowait);
1944 	if (!hibernate_temp_page) {
1945 		uvm_pmr_free_piglet(global_piglet_va,
1946 		    4 * HIBERNATE_CHUNK_SIZE);
1947 		global_piglet_va = 0;
1948 		goto unmap;
1949 	}
1950 	return (0);
1951 unmap:
1952 	pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE);
1953 	pmap_update(pmap_kernel());
1954 	return (ENOMEM);
1955 }
1956 
1957 /*
1958  * Free items allocated by hibernate_alloc()
1959  */
1960 void
1961 hibernate_free(void)
1962 {
1963 	pmap_activate(curproc);
1964 
1965 	if (global_piglet_va)
1966 		uvm_pmr_free_piglet(global_piglet_va,
1967 		    4 * HIBERNATE_CHUNK_SIZE);
1968 
1969 	if (hibernate_temp_page) {
1970 		pmap_kremove(hibernate_temp_page, PAGE_SIZE);
1971 		km_free((void *)hibernate_temp_page, PAGE_SIZE,
1972 		    &kv_any, &kp_none);
1973 	}
1974 
1975 	global_piglet_va = 0;
1976 	hibernate_temp_page = 0;
1977 	pmap_kremove(HIBERNATE_HIBALLOC_PAGE, PAGE_SIZE);
1978 	pmap_update(pmap_kernel());
1979 }
1980