xref: /openbsd-src/sys/kern/subr_hibernate.c (revision e39a80c3e1654497d2e022a4e9a0c27ea26e820a)
1 /*	$OpenBSD: subr_hibernate.c,v 1.51 2013/03/06 08:34:05 mlarkin Exp $	*/
2 
3 /*
4  * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl>
5  * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/hibernate.h>
21 #include <sys/malloc.h>
22 #include <sys/param.h>
23 #include <sys/tree.h>
24 #include <sys/types.h>
25 #include <sys/systm.h>
26 #include <sys/disklabel.h>
27 #include <sys/disk.h>
28 #include <sys/conf.h>
29 #include <sys/buf.h>
30 #include <sys/fcntl.h>
31 #include <sys/stat.h>
32 #include <uvm/uvm.h>
33 #include <uvm/uvm_swap.h>
34 #include <machine/hibernate.h>
35 
36 /* Temporary vaddr ranges used during hibernate */
37 vaddr_t hibernate_temp_page;
38 vaddr_t hibernate_copy_page;
39 
40 /* Hibernate info as read from disk during resume */
41 union hibernate_info disk_hiber_info;
42 paddr_t global_pig_start;
43 vaddr_t global_piglet_va;
44 
45 void hibernate_copy_chunk_to_piglet(paddr_t, vaddr_t, size_t);
46 
47 /*
48  * Hib alloc enforced alignment.
49  */
50 #define HIB_ALIGN		8 /* bytes alignment */
51 
52 /*
53  * sizeof builtin operation, but with alignment constraint.
54  */
55 #define HIB_SIZEOF(_type)	roundup(sizeof(_type), HIB_ALIGN)
56 
57 struct hiballoc_entry {
58 	size_t			hibe_use;
59 	size_t			hibe_space;
60 	RB_ENTRY(hiballoc_entry) hibe_entry;
61 };
62 
63 /*
64  * Compare hiballoc entries based on the address they manage.
65  *
66  * Since the address is fixed, relative to struct hiballoc_entry,
67  * we just compare the hiballoc_entry pointers.
68  */
69 static __inline int
70 hibe_cmp(struct hiballoc_entry *l, struct hiballoc_entry *r)
71 {
72 	return l < r ? -1 : (l > r);
73 }
74 
75 RB_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
76 
77 /*
78  * Given a hiballoc entry, return the address it manages.
79  */
80 static __inline void *
81 hib_entry_to_addr(struct hiballoc_entry *entry)
82 {
83 	caddr_t addr;
84 
85 	addr = (caddr_t)entry;
86 	addr += HIB_SIZEOF(struct hiballoc_entry);
87 	return addr;
88 }
89 
90 /*
91  * Given an address, find the hiballoc that corresponds.
92  */
93 static __inline struct hiballoc_entry*
94 hib_addr_to_entry(void *addr_param)
95 {
96 	caddr_t addr;
97 
98 	addr = (caddr_t)addr_param;
99 	addr -= HIB_SIZEOF(struct hiballoc_entry);
100 	return (struct hiballoc_entry*)addr;
101 }
102 
103 RB_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
104 
105 /*
106  * Allocate memory from the arena.
107  *
108  * Returns NULL if no memory is available.
109  */
110 void *
111 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz)
112 {
113 	struct hiballoc_entry *entry, *new_entry;
114 	size_t find_sz;
115 
116 	/*
117 	 * Enforce alignment of HIB_ALIGN bytes.
118 	 *
119 	 * Note that, because the entry is put in front of the allocation,
120 	 * 0-byte allocations are guaranteed a unique address.
121 	 */
122 	alloc_sz = roundup(alloc_sz, HIB_ALIGN);
123 
124 	/*
125 	 * Find an entry with hibe_space >= find_sz.
126 	 *
127 	 * If the root node is not large enough, we switch to tree traversal.
128 	 * Because all entries are made at the bottom of the free space,
129 	 * traversal from the end has a slightly better chance of yielding
130 	 * a sufficiently large space.
131 	 */
132 	find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry);
133 	entry = RB_ROOT(&arena->hib_addrs);
134 	if (entry != NULL && entry->hibe_space < find_sz) {
135 		RB_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) {
136 			if (entry->hibe_space >= find_sz)
137 				break;
138 		}
139 	}
140 
141 	/*
142 	 * Insufficient or too fragmented memory.
143 	 */
144 	if (entry == NULL)
145 		return NULL;
146 
147 	/*
148 	 * Create new entry in allocated space.
149 	 */
150 	new_entry = (struct hiballoc_entry*)(
151 	    (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use);
152 	new_entry->hibe_space = entry->hibe_space - find_sz;
153 	new_entry->hibe_use = alloc_sz;
154 
155 	/*
156 	 * Insert entry.
157 	 */
158 	if (RB_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL)
159 		panic("hib_alloc: insert failure");
160 	entry->hibe_space = 0;
161 
162 	/* Return address managed by entry. */
163 	return hib_entry_to_addr(new_entry);
164 }
165 
166 /*
167  * Free a pointer previously allocated from this arena.
168  *
169  * If addr is NULL, this will be silently accepted.
170  */
171 void
172 hib_free(struct hiballoc_arena *arena, void *addr)
173 {
174 	struct hiballoc_entry *entry, *prev;
175 
176 	if (addr == NULL)
177 		return;
178 
179 	/*
180 	 * Derive entry from addr and check it is really in this arena.
181 	 */
182 	entry = hib_addr_to_entry(addr);
183 	if (RB_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry)
184 		panic("hib_free: freed item %p not in hib arena", addr);
185 
186 	/*
187 	 * Give the space in entry to its predecessor.
188 	 *
189 	 * If entry has no predecessor, change its used space into free space
190 	 * instead.
191 	 */
192 	prev = RB_PREV(hiballoc_addr, &arena->hib_addrs, entry);
193 	if (prev != NULL &&
194 	    (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) +
195 	    prev->hibe_use + prev->hibe_space) == entry) {
196 		/* Merge entry. */
197 		RB_REMOVE(hiballoc_addr, &arena->hib_addrs, entry);
198 		prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) +
199 		    entry->hibe_use + entry->hibe_space;
200 	} else {
201 		/* Flip used memory to free space. */
202 		entry->hibe_space += entry->hibe_use;
203 		entry->hibe_use = 0;
204 	}
205 }
206 
207 /*
208  * Initialize hiballoc.
209  *
210  * The allocator will manage memmory at ptr, which is len bytes.
211  */
212 int
213 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len)
214 {
215 	struct hiballoc_entry *entry;
216 	caddr_t ptr;
217 	size_t len;
218 
219 	RB_INIT(&arena->hib_addrs);
220 
221 	/*
222 	 * Hib allocator enforces HIB_ALIGN alignment.
223 	 * Fixup ptr and len.
224 	 */
225 	ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN);
226 	len = p_len - ((size_t)ptr - (size_t)p_ptr);
227 	len &= ~((size_t)HIB_ALIGN - 1);
228 
229 	/*
230 	 * Insufficient memory to be able to allocate and also do bookkeeping.
231 	 */
232 	if (len <= HIB_SIZEOF(struct hiballoc_entry))
233 		return ENOMEM;
234 
235 	/*
236 	 * Create entry describing space.
237 	 */
238 	entry = (struct hiballoc_entry*)ptr;
239 	entry->hibe_use = 0;
240 	entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry);
241 	RB_INSERT(hiballoc_addr, &arena->hib_addrs, entry);
242 
243 	return 0;
244 }
245 
246 /*
247  * Zero all free memory.
248  */
249 void
250 uvm_pmr_zero_everything(void)
251 {
252 	struct uvm_pmemrange	*pmr;
253 	struct vm_page		*pg;
254 	int			 i;
255 
256 	uvm_lock_fpageq();
257 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
258 		/* Zero single pages. */
259 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY]))
260 		    != NULL) {
261 			uvm_pmr_remove(pmr, pg);
262 			uvm_pagezero(pg);
263 			atomic_setbits_int(&pg->pg_flags, PG_ZERO);
264 			uvmexp.zeropages++;
265 			uvm_pmr_insert(pmr, pg, 0);
266 		}
267 
268 		/* Zero multi page ranges. */
269 		while ((pg = RB_ROOT(&pmr->size[UVM_PMR_MEMTYPE_DIRTY]))
270 		    != NULL) {
271 			pg--; /* Size tree always has second page. */
272 			uvm_pmr_remove(pmr, pg);
273 			for (i = 0; i < pg->fpgsz; i++) {
274 				uvm_pagezero(&pg[i]);
275 				atomic_setbits_int(&pg[i].pg_flags, PG_ZERO);
276 				uvmexp.zeropages++;
277 			}
278 			uvm_pmr_insert(pmr, pg, 0);
279 		}
280 	}
281 	uvm_unlock_fpageq();
282 }
283 
284 /*
285  * Mark all memory as dirty.
286  *
287  * Used to inform the system that the clean memory isn't clean for some
288  * reason, for example because we just came back from hibernate.
289  */
290 void
291 uvm_pmr_dirty_everything(void)
292 {
293 	struct uvm_pmemrange	*pmr;
294 	struct vm_page		*pg;
295 	int			 i;
296 
297 	uvm_lock_fpageq();
298 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
299 		/* Dirty single pages. */
300 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO]))
301 		    != NULL) {
302 			uvm_pmr_remove(pmr, pg);
303 			atomic_clearbits_int(&pg->pg_flags, PG_ZERO);
304 			uvm_pmr_insert(pmr, pg, 0);
305 		}
306 
307 		/* Dirty multi page ranges. */
308 		while ((pg = RB_ROOT(&pmr->size[UVM_PMR_MEMTYPE_ZERO]))
309 		    != NULL) {
310 			pg--; /* Size tree always has second page. */
311 			uvm_pmr_remove(pmr, pg);
312 			for (i = 0; i < pg->fpgsz; i++)
313 				atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO);
314 			uvm_pmr_insert(pmr, pg, 0);
315 		}
316 	}
317 
318 	uvmexp.zeropages = 0;
319 	uvm_unlock_fpageq();
320 }
321 
322 /*
323  * Allocate the highest address that can hold sz.
324  *
325  * sz in bytes.
326  */
327 int
328 uvm_pmr_alloc_pig(paddr_t *addr, psize_t sz)
329 {
330 	struct uvm_pmemrange	*pmr;
331 	struct vm_page		*pig_pg, *pg;
332 
333 	/*
334 	 * Convert sz to pages, since that is what pmemrange uses internally.
335 	 */
336 	sz = atop(round_page(sz));
337 
338 	uvm_lock_fpageq();
339 
340 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
341 		RB_FOREACH_REVERSE(pig_pg, uvm_pmr_addr, &pmr->addr) {
342 			if (pig_pg->fpgsz >= sz) {
343 				goto found;
344 			}
345 		}
346 	}
347 
348 	/*
349 	 * Allocation failure.
350 	 */
351 	uvm_unlock_fpageq();
352 	return ENOMEM;
353 
354 found:
355 	/* Remove page from freelist. */
356 	uvm_pmr_remove_size(pmr, pig_pg);
357 	pig_pg->fpgsz -= sz;
358 	pg = pig_pg + pig_pg->fpgsz;
359 	if (pig_pg->fpgsz == 0)
360 		uvm_pmr_remove_addr(pmr, pig_pg);
361 	else
362 		uvm_pmr_insert_size(pmr, pig_pg);
363 
364 	uvmexp.free -= sz;
365 	*addr = VM_PAGE_TO_PHYS(pg);
366 
367 	/*
368 	 * Update pg flags.
369 	 *
370 	 * Note that we trash the sz argument now.
371 	 */
372 	while (sz > 0) {
373 		KASSERT(pg->pg_flags & PQ_FREE);
374 
375 		atomic_clearbits_int(&pg->pg_flags,
376 		    PG_PMAP0|PG_PMAP1|PG_PMAP2|PG_PMAP3);
377 
378 		if (pg->pg_flags & PG_ZERO)
379 			uvmexp.zeropages -= sz;
380 		atomic_clearbits_int(&pg->pg_flags,
381 		    PG_ZERO|PQ_FREE);
382 
383 		pg->uobject = NULL;
384 		pg->uanon = NULL;
385 		pg->pg_version++;
386 
387 		/*
388 		 * Next.
389 		 */
390 		pg++;
391 		sz--;
392 	}
393 
394 	/* Return. */
395 	uvm_unlock_fpageq();
396 	return 0;
397 }
398 
399 /*
400  * Allocate a piglet area.
401  *
402  * This is as low as possible.
403  * Piglets are aligned.
404  *
405  * sz and align in bytes.
406  *
407  * The call will sleep for the pagedaemon to attempt to free memory.
408  * The pagedaemon may decide its not possible to free enough memory, causing
409  * the allocation to fail.
410  */
411 int
412 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align)
413 {
414 	paddr_t			 pg_addr, piglet_addr;
415 	struct uvm_pmemrange	*pmr;
416 	struct vm_page		*pig_pg, *pg;
417 	struct pglist		 pageq;
418 	int			 pdaemon_woken;
419 	vaddr_t			 piglet_va;
420 
421 	KASSERT((align & (align - 1)) == 0);
422 	pdaemon_woken = 0; /* Didn't wake the pagedaemon. */
423 
424 	/*
425 	 * Fixup arguments: align must be at least PAGE_SIZE,
426 	 * sz will be converted to pagecount, since that is what
427 	 * pmemrange uses internally.
428 	 */
429 	if (align < PAGE_SIZE)
430 		align = PAGE_SIZE;
431 	sz = round_page(sz);
432 
433 	uvm_lock_fpageq();
434 
435 	TAILQ_FOREACH_REVERSE(pmr, &uvm.pmr_control.use, uvm_pmemrange_use,
436 	    pmr_use) {
437 retry:
438 		/*
439 		 * Search for a range with enough space.
440 		 * Use the address tree, to ensure the range is as low as
441 		 * possible.
442 		 */
443 		RB_FOREACH(pig_pg, uvm_pmr_addr, &pmr->addr) {
444 			pg_addr = VM_PAGE_TO_PHYS(pig_pg);
445 			piglet_addr = (pg_addr + (align - 1)) & ~(align - 1);
446 
447 			if (atop(pg_addr) + pig_pg->fpgsz >=
448 			    atop(piglet_addr) + atop(sz))
449 				goto found;
450 		}
451 	}
452 
453 	/*
454 	 * Try to coerse the pagedaemon into freeing memory
455 	 * for the piglet.
456 	 *
457 	 * pdaemon_woken is set to prevent the code from
458 	 * falling into an endless loop.
459 	 */
460 	if (!pdaemon_woken) {
461 		pdaemon_woken = 1;
462 		if (uvm_wait_pla(ptoa(pmr->low), ptoa(pmr->high) - 1,
463 		    sz, UVM_PLA_FAILOK) == 0)
464 			goto retry;
465 	}
466 
467 	/* Return failure. */
468 	uvm_unlock_fpageq();
469 	return ENOMEM;
470 
471 found:
472 	/*
473 	 * Extract piglet from pigpen.
474 	 */
475 	TAILQ_INIT(&pageq);
476 	uvm_pmr_extract_range(pmr, pig_pg,
477 	    atop(piglet_addr), atop(piglet_addr) + atop(sz), &pageq);
478 
479 	*pa = piglet_addr;
480 	uvmexp.free -= atop(sz);
481 
482 	/*
483 	 * Update pg flags.
484 	 *
485 	 * Note that we trash the sz argument now.
486 	 */
487 	TAILQ_FOREACH(pg, &pageq, pageq) {
488 		KASSERT(pg->pg_flags & PQ_FREE);
489 
490 		atomic_clearbits_int(&pg->pg_flags,
491 		    PG_PMAP0|PG_PMAP1|PG_PMAP2|PG_PMAP3);
492 
493 		if (pg->pg_flags & PG_ZERO)
494 			uvmexp.zeropages--;
495 		atomic_clearbits_int(&pg->pg_flags,
496 		    PG_ZERO|PQ_FREE);
497 
498 		pg->uobject = NULL;
499 		pg->uanon = NULL;
500 		pg->pg_version++;
501 	}
502 
503 	uvm_unlock_fpageq();
504 
505 	/*
506 	 * Now allocate a va.
507 	 * Use direct mappings for the pages.
508 	 */
509 
510 	piglet_va = *va = (vaddr_t)km_alloc(sz, &kv_any, &kp_none, &kd_waitok);
511 	if (!piglet_va) {
512 		uvm_pglistfree(&pageq);
513 		return ENOMEM;
514 	}
515 
516 	/*
517 	 * Map piglet to va.
518 	 */
519 	TAILQ_FOREACH(pg, &pageq, pageq) {
520 		pmap_kenter_pa(piglet_va, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW);
521 		piglet_va += PAGE_SIZE;
522 	}
523 	pmap_update(pmap_kernel());
524 
525 	return 0;
526 }
527 
528 /*
529  * Free a piglet area.
530  */
531 void
532 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz)
533 {
534 	paddr_t			 pa;
535 	struct vm_page		*pg;
536 
537 	/*
538 	 * Fix parameters.
539 	 */
540 	sz = round_page(sz);
541 
542 	/*
543 	 * Find the first page in piglet.
544 	 * Since piglets are contiguous, the first pg is all we need.
545 	 */
546 	if (!pmap_extract(pmap_kernel(), va, &pa))
547 		panic("uvm_pmr_free_piglet: piglet 0x%lx has no pages", va);
548 	pg = PHYS_TO_VM_PAGE(pa);
549 	if (pg == NULL)
550 		panic("uvm_pmr_free_piglet: unmanaged page 0x%lx", pa);
551 
552 	/*
553 	 * Unmap.
554 	 */
555 	pmap_kremove(va, sz);
556 	pmap_update(pmap_kernel());
557 
558 	/*
559 	 * Free the physical and virtual memory.
560 	 */
561 	uvm_pmr_freepages(pg, atop(sz));
562 	km_free((void *)va, sz, &kv_any, &kp_none);
563 }
564 
565 /*
566  * Physmem RLE compression support.
567  *
568  * Given a physical page address, it will return the number of pages
569  * starting at the address, that are free.  Clamps to the number of pages in
570  * HIBERNATE_CHUNK_SIZE. Returns 0 if the page at addr is not free.
571  */
572 int
573 uvm_page_rle(paddr_t addr)
574 {
575 	struct vm_page		*pg, *pg_end;
576 	struct vm_physseg	*vmp;
577 	int			 pseg_idx, off_idx;
578 
579 	pseg_idx = vm_physseg_find(atop(addr), &off_idx);
580 	if (pseg_idx == -1)
581 		return 0;
582 
583 	vmp = &vm_physmem[pseg_idx];
584 	pg = &vmp->pgs[off_idx];
585 	if (!(pg->pg_flags & PQ_FREE))
586 		return 0;
587 
588 	/*
589 	 * Search for the first non-free page after pg.
590 	 * Note that the page may not be the first page in a free pmemrange,
591 	 * therefore pg->fpgsz cannot be used.
592 	 */
593 	for (pg_end = pg; pg_end <= vmp->lastpg &&
594 	    (pg_end->pg_flags & PQ_FREE) == PQ_FREE; pg_end++)
595 		;
596 	return min((pg_end - pg), HIBERNATE_CHUNK_SIZE/PAGE_SIZE);
597 }
598 
599 /*
600  * Fills out the hibernate_info union pointed to by hiber_info
601  * with information about this machine (swap signature block
602  * offsets, number of memory ranges, kernel in use, etc)
603  */
604 int
605 get_hibernate_info(union hibernate_info *hiber_info, int suspend)
606 {
607 	int chunktable_size;
608 	struct disklabel dl;
609 	char err_string[128], *dl_ret;
610 
611 	/* Determine I/O function to use */
612 	hiber_info->io_func = get_hibernate_io_function();
613 	if (hiber_info->io_func == NULL)
614 		return (1);
615 
616 	/* Calculate hibernate device */
617 	hiber_info->device = swdevt[0].sw_dev;
618 
619 	/* Read disklabel (used to calculate signature and image offsets) */
620 	dl_ret = disk_readlabel(&dl, hiber_info->device, err_string, 128);
621 
622 	if (dl_ret) {
623 		printf("Hibernate error reading disklabel: %s\n", dl_ret);
624 		return (1);
625 	}
626 
627 	/* Make sure we have a swap partition. */
628 	if (dl.d_partitions[1].p_fstype != FS_SWAP ||
629 	    dl.d_partitions[1].p_size == 0)
630 		return (1);
631 
632 	hiber_info->secsize = dl.d_secsize;
633 
634 	/* Make sure the signature can fit in one block */
635 	KASSERT(sizeof(union hibernate_info) <= hiber_info->secsize);
636 
637 	/* Calculate swap offset from start of disk */
638 	hiber_info->swap_offset = dl.d_partitions[1].p_offset;
639 
640 	/* Calculate signature block location */
641 	hiber_info->sig_offset = dl.d_partitions[1].p_offset +
642 	    dl.d_partitions[1].p_size -
643 	    sizeof(union hibernate_info)/hiber_info->secsize;
644 
645 	chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / hiber_info->secsize;
646 
647 	/* Stash kernel version information */
648 	bzero(&hiber_info->kernel_version, 128);
649 	bcopy(version, &hiber_info->kernel_version,
650 	    min(strlen(version), sizeof(hiber_info->kernel_version)-1));
651 
652 	if (suspend) {
653 		/* Allocate piglet region */
654 		if (uvm_pmr_alloc_piglet(&hiber_info->piglet_va,
655 		    &hiber_info->piglet_pa, HIBERNATE_CHUNK_SIZE*3,
656 		    HIBERNATE_CHUNK_SIZE)) {
657 			printf("Hibernate failed to allocate the piglet\n");
658 			return (1);
659 		}
660 		hiber_info->io_page = (void *)hiber_info->piglet_va;
661 
662 		/*
663 		 * Initialize of the hibernate IO function (for drivers which
664 		 * need that)
665 		 */
666 		if (hiber_info->io_func(hiber_info->device, 0,
667 		    (vaddr_t)NULL, 0, HIB_INIT, hiber_info->io_page))
668 			goto fail;
669 
670 	} else {
671 		/*
672 		 * Resuming kernels use a regular I/O page since we won't
673 		 * have access to the suspended kernel's piglet VA at this
674 		 * point. No need to free this I/O page as it will vanish
675 		 * as part of the resume.
676 		 */
677 		hiber_info->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
678 		if (!hiber_info->io_page)
679 			return (1);
680 	}
681 
682 
683 	if (get_hibernate_info_md(hiber_info))
684 		goto fail;
685 
686 	/* Calculate memory image location */
687 	hiber_info->image_offset = dl.d_partitions[1].p_offset +
688 	    dl.d_partitions[1].p_size -
689 	    (hiber_info->image_size / hiber_info->secsize) -
690 	    sizeof(union hibernate_info)/hiber_info->secsize -
691 	    chunktable_size;
692 
693 	return (0);
694 fail:
695 	if (suspend)
696 		uvm_pmr_free_piglet(hiber_info->piglet_va,
697 		    HIBERNATE_CHUNK_SIZE * 3);
698 
699 	return (1);
700 }
701 
702 /*
703  * Allocate nitems*size bytes from the hiballoc area presently in use
704  */
705 void *
706 hibernate_zlib_alloc(void *unused, int nitems, int size)
707 {
708 	struct hibernate_zlib_state *hibernate_state;
709 
710 	hibernate_state =
711 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
712 
713 	return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size);
714 }
715 
716 /*
717  * Free the memory pointed to by addr in the hiballoc area presently in
718  * use
719  */
720 void
721 hibernate_zlib_free(void *unused, void *addr)
722 {
723 	struct hibernate_zlib_state *hibernate_state;
724 
725 	hibernate_state =
726 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
727 
728 	hib_free(&hibernate_state->hiballoc_arena, addr);
729 }
730 
731 /*
732  * Gets the next RLE value from the image stream
733  */
734 int
735 hibernate_get_next_rle(void)
736 {
737 	int rle, i;
738 	struct hibernate_zlib_state *hibernate_state;
739 
740 	hibernate_state =
741 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
742 
743 	/* Read RLE code */
744 	hibernate_state->hib_stream.next_out = (char *)&rle;
745 	hibernate_state->hib_stream.avail_out = sizeof(rle);
746 
747 	i = inflate(&hibernate_state->hib_stream, Z_FULL_FLUSH);
748 	if (i != Z_OK && i != Z_STREAM_END) {
749 		/*
750 		 * XXX - this will likely reboot/hang most machines,
751 		 *       but there's not much else we can do here.
752 		 */
753 		panic("inflate rle error");
754 	}
755 
756 	/* Sanity check what RLE value we got */
757 	if (rle > HIBERNATE_CHUNK_SIZE/PAGE_SIZE || rle < 0)
758 		panic("invalid RLE code");
759 
760 	if (i == Z_STREAM_END)
761 		rle = -1;
762 
763 	return rle;
764 }
765 
766 /*
767  * Inflate next page of data from the image stream
768  */
769 int
770 hibernate_inflate_page(void)
771 {
772 	struct hibernate_zlib_state *hibernate_state;
773 	int i;
774 
775 	hibernate_state =
776 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
777 
778 	/* Set up the stream for inflate */
779 	hibernate_state->hib_stream.next_out = (char *)HIBERNATE_INFLATE_PAGE;
780 	hibernate_state->hib_stream.avail_out = PAGE_SIZE;
781 
782 	/* Process next block of data */
783 	i = inflate(&hibernate_state->hib_stream, Z_PARTIAL_FLUSH);
784 	if (i != Z_OK && i != Z_STREAM_END) {
785 		/*
786 		 * XXX - this will likely reboot/hang most machines,
787 		 *       but there's not much else we can do here.
788 		 */
789 
790 		panic("inflate error");
791 	}
792 
793 	/* We should always have extracted a full page ... */
794 	if (hibernate_state->hib_stream.avail_out != 0)
795 		panic("incomplete page");
796 
797 	return (i == Z_STREAM_END);
798 }
799 
800 /*
801  * Inflate size bytes from src into dest, skipping any pages in
802  * [src..dest] that are special (see hibernate_inflate_skip)
803  *
804  * This function executes while using the resume-time stack
805  * and pmap, and therefore cannot use ddb/printf/etc. Doing so
806  * will likely hang or reset the machine.
807  */
808 void
809 hibernate_inflate_region(union hibernate_info *hiber_info, paddr_t dest,
810     paddr_t src, size_t size)
811 {
812 	int end_stream = 0 ;
813 	struct hibernate_zlib_state *hibernate_state;
814 
815 	hibernate_state =
816 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
817 
818 	hibernate_state->hib_stream.next_in = (char *)src;
819 	hibernate_state->hib_stream.avail_in = size;
820 
821 	do {
822 		/* Flush cache and TLB */
823 		hibernate_flush();
824 
825 		/*
826 		 * Is this a special page? If yes, redirect the
827 		 * inflate output to a scratch page (eg, discard it)
828 		 */
829 		if (hibernate_inflate_skip(hiber_info, dest)) {
830 			hibernate_enter_resume_mapping(
831 			    HIBERNATE_INFLATE_PAGE,
832 			    HIBERNATE_INFLATE_PAGE, 0);
833 		} else {
834 			hibernate_enter_resume_mapping(
835 			    HIBERNATE_INFLATE_PAGE, dest, 0);
836 		}
837 
838 		hibernate_flush();
839 		end_stream = hibernate_inflate_page();
840 
841 		dest += PAGE_SIZE;
842 	} while (!end_stream);
843 }
844 
845 /*
846  * deflate from src into the I/O page, up to 'remaining' bytes
847  *
848  * Returns number of input bytes consumed, and may reset
849  * the 'remaining' parameter if not all the output space was consumed
850  * (this information is needed to know how much to write to disk
851  */
852 size_t
853 hibernate_deflate(union hibernate_info *hiber_info, paddr_t src,
854     size_t *remaining)
855 {
856 	vaddr_t hibernate_io_page = hiber_info->piglet_va + PAGE_SIZE;
857 	struct hibernate_zlib_state *hibernate_state;
858 
859 	hibernate_state =
860 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
861 
862 	/* Set up the stream for deflate */
863 	hibernate_state->hib_stream.next_in = (caddr_t)src;
864 	hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK);
865 	hibernate_state->hib_stream.next_out = (caddr_t)hibernate_io_page +
866 	    (PAGE_SIZE - *remaining);
867 	hibernate_state->hib_stream.avail_out = *remaining;
868 
869 	/* Process next block of data */
870 	if (deflate(&hibernate_state->hib_stream, Z_PARTIAL_FLUSH) != Z_OK)
871 		panic("hibernate zlib deflate error");
872 
873 	/* Update pointers and return number of bytes consumed */
874 	*remaining = hibernate_state->hib_stream.avail_out;
875 	return (PAGE_SIZE - (src & PAGE_MASK)) -
876 	    hibernate_state->hib_stream.avail_in;
877 }
878 
879 /*
880  * Write the hibernation information specified in hiber_info
881  * to the location in swap previously calculated (last block of
882  * swap), called the "signature block".
883  *
884  * Write the memory chunk table to the area in swap immediately
885  * preceding the signature block.
886  */
887 int
888 hibernate_write_signature(union hibernate_info *hiber_info)
889 {
890 	/* Write hibernate info to disk */
891 	return (hiber_info->io_func(hiber_info->device, hiber_info->sig_offset,
892 	    (vaddr_t)hiber_info, hiber_info->secsize, HIB_W,
893 	    hiber_info->io_page));
894 }
895 
896 /*
897  * Write the memory chunk table to the area in swap immediately
898  * preceding the signature block. The chunk table is stored
899  * in the piglet when this function is called.
900  */
901 int
902 hibernate_write_chunktable(union hibernate_info *hiber_info)
903 {
904 	struct hibernate_disk_chunk *chunks;
905 	vaddr_t hibernate_chunk_table_start;
906 	size_t hibernate_chunk_table_size;
907 	daddr_t chunkbase;
908 	int i;
909 
910 	hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE;
911 
912 	chunkbase = hiber_info->sig_offset -
913 	    (hibernate_chunk_table_size / hiber_info->secsize);
914 
915 	hibernate_chunk_table_start = hiber_info->piglet_va +
916 	    HIBERNATE_CHUNK_SIZE;
917 
918 	chunks = (struct hibernate_disk_chunk *)(hiber_info->piglet_va +
919 	    HIBERNATE_CHUNK_SIZE);
920 
921 	/* Write chunk table */
922 	for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) {
923 		if (hiber_info->io_func(hiber_info->device,
924 		    chunkbase + (i/hiber_info->secsize),
925 		    (vaddr_t)(hibernate_chunk_table_start + i),
926 		    MAXPHYS, HIB_W, hiber_info->io_page))
927 			return (1);
928 	}
929 
930 	return (0);
931 }
932 
933 /*
934  * Write an empty hiber_info to the swap signature block, which is
935  * guaranteed to not match any valid hiber_info.
936  */
937 int
938 hibernate_clear_signature(void)
939 {
940 	union hibernate_info blank_hiber_info;
941 	union hibernate_info hiber_info;
942 
943 	/* Zero out a blank hiber_info */
944 	bzero(&blank_hiber_info, sizeof(hiber_info));
945 
946 	if (get_hibernate_info(&hiber_info, 0))
947 		return (1);
948 
949 	/* Write (zeroed) hibernate info to disk */
950 	if (hibernate_block_io(&hiber_info,
951 	    hiber_info.sig_offset - hiber_info.swap_offset,
952 	    hiber_info.secsize, (vaddr_t)&blank_hiber_info, 1))
953 		panic("error hibernate write 6");
954 
955 	return (0);
956 }
957 
958 /*
959  * Check chunk range overlap when calculating whether or not to copy a
960  * compressed chunk to the piglet area before decompressing.
961  *
962  * returns zero if the ranges do not overlap, non-zero otherwise.
963  */
964 int
965 hibernate_check_overlap(paddr_t r1s, paddr_t r1e, paddr_t r2s, paddr_t r2e)
966 {
967 	/* case A : end of r1 overlaps start of r2 */
968 	if (r1s < r2s && r1e > r2s)
969 		return (1);
970 
971 	/* case B : r1 entirely inside r2 */
972 	if (r1s >= r2s && r1e <= r2e)
973 		return (1);
974 
975 	/* case C : r2 entirely inside r1 */
976 	if (r2s >= r1s && r2e <= r1e)
977 		return (1);
978 
979 	/* case D : end of r2 overlaps start of r1 */
980 	if (r2s < r1s && r2e > r1s)
981 		return (1);
982 
983 	return (0);
984 }
985 
986 /*
987  * Compare two hibernate_infos to determine if they are the same (eg,
988  * we should be performing a hibernate resume on this machine.
989  * Not all fields are checked - just enough to verify that the machine
990  * has the same memory configuration and kernel as the one that
991  * wrote the signature previously.
992  */
993 int
994 hibernate_compare_signature(union hibernate_info *mine,
995     union hibernate_info *disk)
996 {
997 	u_int i;
998 
999 	if (mine->nranges != disk->nranges)
1000 		return (1);
1001 
1002 	if (strcmp(mine->kernel_version, disk->kernel_version) != 0)
1003 		return (1);
1004 
1005 	for (i = 0; i < mine->nranges; i++) {
1006 		if ((mine->ranges[i].base != disk->ranges[i].base) ||
1007 		    (mine->ranges[i].end != disk->ranges[i].end) )
1008 			return (1);
1009 	}
1010 
1011 	return (0);
1012 }
1013 
1014 /*
1015  * Transfers xfer_size bytes between the hibernate device specified in
1016  * hib_info at offset blkctr and the vaddr specified at dest.
1017  *
1018  * Separate offsets and pages are used to handle misaligned reads (reads
1019  * that span a page boundary).
1020  *
1021  * blkctr specifies a relative offset (relative to the start of swap),
1022  * not an absolute disk offset
1023  *
1024  */
1025 int
1026 hibernate_block_io(union hibernate_info *hib_info, daddr_t blkctr,
1027     size_t xfer_size, vaddr_t dest, int iswrite)
1028 {
1029 	struct buf *bp;
1030 	struct bdevsw *bdsw;
1031 	int error;
1032 
1033 	bp = geteblk(xfer_size);
1034 	bdsw = &bdevsw[major(hib_info->device)];
1035 
1036 	error = (*bdsw->d_open)(hib_info->device, FREAD, S_IFCHR, curproc);
1037 	if (error) {
1038 		printf("hibernate_block_io open failed\n");
1039 		return (1);
1040 	}
1041 
1042 	if (iswrite)
1043 		bcopy((caddr_t)dest, bp->b_data, xfer_size);
1044 
1045 	bp->b_bcount = xfer_size;
1046 	bp->b_blkno = blkctr;
1047 	CLR(bp->b_flags, B_READ | B_WRITE | B_DONE);
1048 	SET(bp->b_flags, B_BUSY | (iswrite ? B_WRITE : B_READ) | B_RAW);
1049 	bp->b_dev = hib_info->device;
1050 	bp->b_cylinder = 0;
1051 	(*bdsw->d_strategy)(bp);
1052 
1053 	error = biowait(bp);
1054 	if (error) {
1055 		printf("hibernate_block_io biowait failed %d\n", error);
1056 		error = (*bdsw->d_close)(hib_info->device, 0, S_IFCHR,
1057 		    curproc);
1058 		if (error)
1059 			printf("hibernate_block_io error close failed\n");
1060 		return (1);
1061 	}
1062 
1063 	error = (*bdsw->d_close)(hib_info->device, FREAD, S_IFCHR, curproc);
1064 	if (error) {
1065 		printf("hibernate_block_io close failed\n");
1066 		return (1);
1067 	}
1068 
1069 	if (!iswrite)
1070 		bcopy(bp->b_data, (caddr_t)dest, xfer_size);
1071 
1072 	bp->b_flags |= B_INVAL;
1073 	brelse(bp);
1074 
1075 	return (0);
1076 }
1077 
1078 /*
1079  * Reads the signature block from swap, checks against the current machine's
1080  * information. If the information matches, perform a resume by reading the
1081  * saved image into the pig area, and unpacking.
1082  */
1083 void
1084 hibernate_resume(void)
1085 {
1086 	union hibernate_info hiber_info;
1087 	int s;
1088 
1089 	/* Get current running machine's hibernate info */
1090 	bzero(&hiber_info, sizeof(hiber_info));
1091 	if (get_hibernate_info(&hiber_info, 0))
1092 		return;
1093 
1094 	/* Read hibernate info from disk */
1095 	s = splbio();
1096 
1097 	if (hibernate_block_io(&hiber_info,
1098 	    hiber_info.sig_offset - hiber_info.swap_offset,
1099 	    hiber_info.secsize, (vaddr_t)&disk_hiber_info, 0))
1100 		panic("error in hibernate read");
1101 
1102 	/*
1103 	 * If on-disk and in-memory hibernate signatures match,
1104 	 * this means we should do a resume from hibernate.
1105 	 */
1106 	if (hibernate_compare_signature(&hiber_info, &disk_hiber_info)) {
1107 		splx(s);
1108 		return;
1109 	}
1110 
1111 	printf("Unhibernating...\n");
1112 
1113 	/* Read the image from disk into the image (pig) area */
1114 	if (hibernate_read_image(&disk_hiber_info))
1115 		goto fail;
1116 
1117 	if (config_suspend(TAILQ_FIRST(&alldevs), DVACT_QUIESCE) != 0)
1118 		goto fail;
1119 
1120 	(void) splhigh();
1121 	disable_intr();
1122 	cold = 1;
1123 
1124 	if (config_suspend(TAILQ_FIRST(&alldevs), DVACT_SUSPEND) != 0) {
1125 		cold = 0;
1126 		enable_intr();
1127 		goto fail;
1128 	}
1129 
1130 	/* Point of no return ... */
1131 
1132 	pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE,
1133 	    VM_PROT_ALL);
1134 	pmap_activate(curproc);
1135 
1136 	/* Switch stacks */
1137 	hibernate_switch_stack_machdep();
1138 
1139 	/*
1140 	 * Image is now in high memory (pig area), copy to correct location
1141 	 * in memory. We'll eventually end up copying on top of ourself, but
1142 	 * we are assured the kernel code here is the same between the
1143 	 * hibernated and resuming kernel, and we are running on our own
1144 	 * stack, so the overwrite is ok.
1145 	 */
1146 	hibernate_unpack_image(&disk_hiber_info);
1147 
1148 	/*
1149 	 * Resume the loaded kernel by jumping to the MD resume vector.
1150 	 * We won't be returning from this call.
1151 	 */
1152 	hibernate_resume_machdep();
1153 
1154 fail:
1155 	splx(s);
1156 	printf("Unable to resume hibernated image\n");
1157 }
1158 
1159 /*
1160  * Unpack image from pig area to original location by looping through the
1161  * list of output chunks in the order they should be restored (fchunks).
1162  * This ordering is used to avoid having inflate overwrite a chunk in the
1163  * middle of processing that chunk. This will, of course, happen during the
1164  * final output chunk, where we copy the chunk to the piglet area first,
1165  * before inflating.
1166  */
1167 void
1168 hibernate_unpack_image(union hibernate_info *hiber_info)
1169 {
1170 	struct hibernate_disk_chunk *chunks;
1171 	union hibernate_info local_hiber_info;
1172 	paddr_t image_cur = global_pig_start;
1173 	short i, *fchunks;
1174 	char *pva = (char *)hiber_info->piglet_va;
1175 	struct hibernate_zlib_state *hibernate_state;
1176 
1177 	hibernate_state =
1178 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1179 
1180 	/* Mask off based on arch-specific piglet page size */
1181 	pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK));
1182 	fchunks = (short *)(pva + (4 * PAGE_SIZE));
1183 
1184 	chunks = (struct hibernate_disk_chunk *)(pva +  HIBERNATE_CHUNK_SIZE);
1185 
1186 	/* Can't use hiber_info that's passed in after this point */
1187 	bcopy(hiber_info, &local_hiber_info, sizeof(union hibernate_info));
1188 
1189 	hibernate_activate_resume_pt_machdep();
1190 
1191 	for (i = 0; i < local_hiber_info.chunk_ctr; i++) {
1192 		/* Reset zlib for inflate */
1193 		if (hibernate_zlib_reset(&local_hiber_info, 0) != Z_OK)
1194 			panic("hibernate failed to reset zlib for inflate");
1195 
1196 		hibernate_process_chunk(&local_hiber_info, &chunks[fchunks[i]],
1197 		    image_cur);
1198 
1199 		image_cur += chunks[fchunks[i]].compressed_size;
1200 
1201 	}
1202 }
1203 
1204 /*
1205  * Bounce a compressed image chunk to the piglet, entering mappings for the
1206  * copied pages as needed
1207  */
1208 void
1209 hibernate_copy_chunk_to_piglet(paddr_t img_cur, vaddr_t piglet, size_t size)
1210 {
1211 	size_t ct, ofs;
1212 	paddr_t src = img_cur;
1213 	vaddr_t dest = piglet;
1214 
1215 	/* Copy first partial page */
1216 	ct = (PAGE_SIZE) - (src & PAGE_MASK);
1217 	ofs = (src & PAGE_MASK);
1218 
1219 	if (ct < PAGE_SIZE) {
1220 		hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE,
1221 			(src - ofs), 0);
1222 		hibernate_flush();
1223 		bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE + ofs), (caddr_t)dest, ct);
1224 		src += ct;
1225 		dest += ct;
1226 	}
1227 
1228 	/* Copy remaining pages */
1229 	while (src < size + img_cur) {
1230 		hibernate_enter_resume_mapping(HIBERNATE_INFLATE_PAGE, src, 0);
1231 		hibernate_flush();
1232 		ct = PAGE_SIZE;
1233 		bcopy((caddr_t)(HIBERNATE_INFLATE_PAGE), (caddr_t)dest, ct);
1234 		hibernate_flush();
1235 		src += ct;
1236 		dest += ct;
1237 	}
1238 }
1239 
1240 /*
1241  * Process a chunk by bouncing it to the piglet, followed by unpacking
1242  */
1243 void
1244 hibernate_process_chunk(union hibernate_info *hiber_info,
1245     struct hibernate_disk_chunk *chunk, paddr_t img_cur)
1246 {
1247 	char *pva = (char *)hiber_info->piglet_va;
1248 
1249 	hibernate_copy_chunk_to_piglet(img_cur,
1250 	 (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)), chunk->compressed_size);
1251 
1252 	hibernate_inflate_region(hiber_info, chunk->base,
1253 	    (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)),
1254 	    chunk->compressed_size);
1255 }
1256 
1257 /*
1258  * Write a compressed version of this machine's memory to disk, at the
1259  * precalculated swap offset:
1260  *
1261  * end of swap - signature block size - chunk table size - memory size
1262  *
1263  * The function begins by looping through each phys mem range, cutting each
1264  * one into MD sized chunks. These chunks are then compressed individually
1265  * and written out to disk, in phys mem order. Some chunks might compress
1266  * more than others, and for this reason, each chunk's size is recorded
1267  * in the chunk table, which is written to disk after the image has
1268  * properly been compressed and written (in hibernate_write_chunktable).
1269  *
1270  * When this function is called, the machine is nearly suspended - most
1271  * devices are quiesced/suspended, interrupts are off, and cold has
1272  * been set. This means that there can be no side effects once the
1273  * write has started, and the write function itself can also have no
1274  * side effects. This also means no printfs are permitted (since it
1275  * has side effects.)
1276  */
1277 int
1278 hibernate_write_chunks(union hibernate_info *hiber_info)
1279 {
1280 	paddr_t range_base, range_end, inaddr, temp_inaddr;
1281 	size_t nblocks, out_remaining, used;
1282 	struct hibernate_disk_chunk *chunks;
1283 	vaddr_t hibernate_io_page = hiber_info->piglet_va + PAGE_SIZE;
1284 	daddr_t blkctr = hiber_info->image_offset, offset = 0;
1285 	int i;
1286 	struct hibernate_zlib_state *hibernate_state;
1287 
1288 	hibernate_state =
1289 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1290 
1291 	hiber_info->chunk_ctr = 0;
1292 
1293 	/*
1294 	 * Allocate VA for the temp and copy page.
1295 	 * These will become part of the suspended kernel and will
1296 	 * be freed in hibernate_free, upon resume.
1297 	 */
1298 	hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1299 	    &kp_none, &kd_nowait);
1300 	if (!hibernate_temp_page)
1301 		return (1);
1302 
1303 	hibernate_copy_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1304 	    &kp_none, &kd_nowait);
1305 	if (!hibernate_copy_page)
1306 		return (1);
1307 
1308 	pmap_kenter_pa(hibernate_copy_page,
1309 	    (hiber_info->piglet_pa + 3*PAGE_SIZE), VM_PROT_ALL);
1310 
1311 	/* XXX - not needed on all archs */
1312 	pmap_activate(curproc);
1313 
1314 	chunks = (struct hibernate_disk_chunk *)(hiber_info->piglet_va +
1315 	    HIBERNATE_CHUNK_SIZE);
1316 
1317 	/* Calculate the chunk regions */
1318 	for (i = 0; i < hiber_info->nranges; i++) {
1319 		range_base = hiber_info->ranges[i].base;
1320 		range_end = hiber_info->ranges[i].end;
1321 
1322 		inaddr = range_base;
1323 
1324 		while (inaddr < range_end) {
1325 			chunks[hiber_info->chunk_ctr].base = inaddr;
1326 			if (inaddr + HIBERNATE_CHUNK_SIZE < range_end)
1327 				chunks[hiber_info->chunk_ctr].end = inaddr +
1328 				    HIBERNATE_CHUNK_SIZE;
1329 			else
1330 				chunks[hiber_info->chunk_ctr].end = range_end;
1331 
1332 			inaddr += HIBERNATE_CHUNK_SIZE;
1333 			hiber_info->chunk_ctr ++;
1334 		}
1335 	}
1336 
1337 	/* Compress and write the chunks in the chunktable */
1338 	for (i = 0; i < hiber_info->chunk_ctr; i++) {
1339 		range_base = chunks[i].base;
1340 		range_end = chunks[i].end;
1341 
1342 		chunks[i].offset = blkctr;
1343 
1344 		/* Reset zlib for deflate */
1345 		if (hibernate_zlib_reset(hiber_info, 1) != Z_OK)
1346 			return (1);
1347 
1348 		inaddr = range_base;
1349 
1350 		/*
1351 		 * For each range, loop through its phys mem region
1352 		 * and write out the chunks (the last chunk might be
1353 		 * smaller than the chunk size).
1354 		 */
1355 		while (inaddr < range_end) {
1356 			out_remaining = PAGE_SIZE;
1357 			while (out_remaining > 0 && inaddr < range_end) {
1358 
1359 				/*
1360 				 * Adjust for regions that are not evenly
1361 				 * divisible by PAGE_SIZE or overflowed
1362 				 * pages from the previous iteration.
1363 				 */
1364 				temp_inaddr = (inaddr & PAGE_MASK) +
1365 				    hibernate_copy_page;
1366 
1367 				/* Deflate from temp_inaddr to IO page */
1368 				if (inaddr != range_end) {
1369 					pmap_kenter_pa(hibernate_temp_page,
1370 					    inaddr & PMAP_PA_MASK, VM_PROT_ALL);
1371 
1372 					/* XXX - not needed on all archs */
1373 					pmap_activate(curproc);
1374 
1375 					bcopy((caddr_t)hibernate_temp_page,
1376 					    (caddr_t)hibernate_copy_page,
1377 					    PAGE_SIZE);
1378 					inaddr += hibernate_deflate(hiber_info,
1379 					    temp_inaddr, &out_remaining);
1380 				}
1381 
1382 				if (out_remaining == 0) {
1383 					/* Filled up the page */
1384 					nblocks =
1385 					    PAGE_SIZE / hiber_info->secsize;
1386 
1387 					if (hiber_info->io_func(
1388 					    hiber_info->device,
1389 					    blkctr, (vaddr_t)hibernate_io_page,
1390 					    PAGE_SIZE, HIB_W,
1391 					    hiber_info->io_page))
1392 						return (1);
1393 
1394 					blkctr += nblocks;
1395 				}
1396 			}
1397 		}
1398 
1399 		if (inaddr != range_end)
1400 			return (1);
1401 
1402 		/*
1403 		 * End of range. Round up to next secsize bytes
1404 		 * after finishing compress
1405 		 */
1406 		if (out_remaining == 0)
1407 			out_remaining = PAGE_SIZE;
1408 
1409 		/* Finish compress */
1410 		hibernate_state->hib_stream.next_in = (caddr_t)inaddr;
1411 		hibernate_state->hib_stream.avail_in = 0;
1412 		hibernate_state->hib_stream.next_out =
1413 		    (caddr_t)hibernate_io_page + (PAGE_SIZE - out_remaining);
1414 		hibernate_state->hib_stream.avail_out = out_remaining;
1415 
1416 		if (deflate(&hibernate_state->hib_stream, Z_FINISH) !=
1417 		    Z_STREAM_END)
1418 			return (1);
1419 
1420 		out_remaining = hibernate_state->hib_stream.avail_out;
1421 
1422 		used = PAGE_SIZE - out_remaining;
1423 		nblocks = used / hiber_info->secsize;
1424 
1425 		/* Round up to next block if needed */
1426 		if (used % hiber_info->secsize != 0)
1427 			nblocks ++;
1428 
1429 		/* Write final block(s) for this chunk */
1430 		if (hiber_info->io_func(hiber_info->device, blkctr,
1431 		    (vaddr_t)hibernate_io_page, nblocks*hiber_info->secsize,
1432 		    HIB_W, hiber_info->io_page))
1433 			return (1);
1434 
1435 		blkctr += nblocks;
1436 
1437 		offset = blkctr;
1438 		chunks[i].compressed_size = (offset - chunks[i].offset) *
1439 		    hiber_info->secsize;
1440 	}
1441 
1442 	return (0);
1443 }
1444 
1445 /*
1446  * Reset the zlib stream state and allocate a new hiballoc area for either
1447  * inflate or deflate. This function is called once for each hibernate chunk.
1448  * Calling hiballoc_init multiple times is acceptable since the memory it is
1449  * provided is unmanaged memory (stolen). We use the memory provided to us
1450  * by the piglet allocated via the supplied hiber_info.
1451  */
1452 int
1453 hibernate_zlib_reset(union hibernate_info *hiber_info, int deflate)
1454 {
1455 	vaddr_t hibernate_zlib_start;
1456 	size_t hibernate_zlib_size;
1457 	char *pva = (char *)hiber_info->piglet_va;
1458 	struct hibernate_zlib_state *hibernate_state;
1459 
1460 	hibernate_state =
1461 	    (struct hibernate_zlib_state *)HIBERNATE_HIBALLOC_PAGE;
1462 
1463 	if(!deflate)
1464 		pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK));
1465 
1466 	hibernate_zlib_start = (vaddr_t)(pva + (28 * PAGE_SIZE));
1467 	hibernate_zlib_size = 80 * PAGE_SIZE;
1468 
1469 	bzero((caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1470 	bzero((caddr_t)hibernate_state, PAGE_SIZE);
1471 
1472 	/* Set up stream structure */
1473 	hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc;
1474 	hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free;
1475 
1476 	/* Initialize the hiballoc arena for zlib allocs/frees */
1477 	hiballoc_init(&hibernate_state->hiballoc_arena,
1478 	    (caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1479 
1480 	if (deflate) {
1481 		return deflateInit(&hibernate_state->hib_stream,
1482 		    Z_BEST_SPEED);
1483 	} else
1484 		return inflateInit(&hibernate_state->hib_stream);
1485 }
1486 
1487 /*
1488  * Reads the hibernated memory image from disk, whose location and
1489  * size are recorded in hiber_info. Begin by reading the persisted
1490  * chunk table, which records the original chunk placement location
1491  * and compressed size for each. Next, allocate a pig region of
1492  * sufficient size to hold the compressed image. Next, read the
1493  * chunks into the pig area (calling hibernate_read_chunks to do this),
1494  * and finally, if all of the above succeeds, clear the hibernate signature.
1495  * The function will then return to hibernate_resume, which will proceed
1496  * to unpack the pig image to the correct place in memory.
1497  */
1498 int
1499 hibernate_read_image(union hibernate_info *hiber_info)
1500 {
1501 	size_t compressed_size, disk_size, chunktable_size, pig_sz;
1502 	paddr_t image_start, image_end, pig_start, pig_end;
1503 	struct hibernate_disk_chunk *chunks;
1504 	daddr_t blkctr;
1505 	vaddr_t chunktable = (vaddr_t)NULL;
1506 	paddr_t piglet_chunktable = hiber_info->piglet_pa +
1507 	    HIBERNATE_CHUNK_SIZE;
1508 	int i;
1509 
1510 	pmap_activate(curproc);
1511 
1512 	/* Calculate total chunk table size in disk blocks */
1513 	chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / hiber_info->secsize;
1514 
1515 	blkctr = hiber_info->sig_offset - chunktable_size -
1516 			hiber_info->swap_offset;
1517 
1518 	chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any,
1519 	    &kp_none, &kd_nowait);
1520 
1521 	if (!chunktable)
1522 		return (1);
1523 
1524 	/* Read the chunktable from disk into the piglet chunktable */
1525 	for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE;
1526 	    i += PAGE_SIZE, blkctr += PAGE_SIZE/hiber_info->secsize) {
1527 		pmap_kenter_pa(chunktable + i, piglet_chunktable + i,
1528 		    VM_PROT_ALL);
1529 		pmap_update(pmap_kernel());
1530 		hibernate_block_io(hiber_info, blkctr, PAGE_SIZE,
1531 		    chunktable + i, 0);
1532 	}
1533 
1534 	blkctr = hiber_info->image_offset;
1535 	compressed_size = 0;
1536 
1537 	chunks = (struct hibernate_disk_chunk *)chunktable;
1538 
1539 	for (i = 0; i < hiber_info->chunk_ctr; i++)
1540 		compressed_size += chunks[i].compressed_size;
1541 
1542 	disk_size = compressed_size;
1543 
1544 	/* Allocate the pig area */
1545 	pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE;
1546 	if (uvm_pmr_alloc_pig(&pig_start, pig_sz) == ENOMEM)
1547 		return (1);
1548 
1549 	pig_end = pig_start + pig_sz;
1550 
1551 	/* Calculate image extents. Pig image must end on a chunk boundary. */
1552 	image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1);
1553 	image_start = pig_start;
1554 
1555 	image_start = image_end - disk_size;
1556 
1557 	hibernate_read_chunks(hiber_info, image_start, image_end, disk_size,
1558 	    chunks);
1559 
1560 	pmap_kremove(chunktable, PAGE_SIZE);
1561 	pmap_update(pmap_kernel());
1562 
1563 	/* Prepare the resume time pmap/page table */
1564 	hibernate_populate_resume_pt(hiber_info, image_start, image_end);
1565 
1566 	/* Read complete, clear the signature and return */
1567 	return hibernate_clear_signature();
1568 }
1569 
1570 /*
1571  * Read the hibernated memory chunks from disk (chunk information at this
1572  * point is stored in the piglet) into the pig area specified by
1573  * [pig_start .. pig_end]. Order the chunks so that the final chunk is the
1574  * only chunk with overlap possibilities.
1575  */
1576 int
1577 hibernate_read_chunks(union hibernate_info *hib_info, paddr_t pig_start,
1578     paddr_t pig_end, size_t image_compr_size,
1579     struct hibernate_disk_chunk *chunks)
1580 {
1581 	paddr_t img_index, img_cur, r1s, r1e, r2s, r2e;
1582 	paddr_t copy_start, copy_end, piglet_cur;
1583 	paddr_t piglet_base = hib_info->piglet_pa;
1584 	paddr_t piglet_end = piglet_base + HIBERNATE_CHUNK_SIZE;
1585 	daddr_t blkctr;
1586 	size_t processed, compressed_size, read_size;
1587 	int overlap, found, nchunks, nochunks = 0, nfchunks = 0, npchunks = 0;
1588 	short *ochunks, *pchunks, *fchunks, i, j;
1589 	vaddr_t tempva = (vaddr_t)NULL, hibernate_fchunk_area = (vaddr_t)NULL;
1590 
1591 	global_pig_start = pig_start;
1592 
1593 	/* XXX - dont need this on all archs */
1594 	pmap_activate(curproc);
1595 
1596 	/*
1597 	 * These mappings go into the resuming kernel's page table, and are
1598 	 * used only during image read. They dissappear from existence
1599 	 * when the suspended kernel is unpacked on top of us.
1600 	 */
1601 	tempva = (vaddr_t)km_alloc(2*PAGE_SIZE, &kv_any, &kp_none, &kd_nowait);
1602 	if (!tempva)
1603 		return (1);
1604 	hibernate_fchunk_area = (vaddr_t)km_alloc(24*PAGE_SIZE, &kv_any,
1605 	    &kp_none, &kd_nowait);
1606 	if (!hibernate_fchunk_area)
1607 		return (1);
1608 
1609 	/* Final output chunk ordering VA */
1610 	fchunks = (short *)hibernate_fchunk_area;
1611 
1612 	/* Piglet chunk ordering VA */
1613 	pchunks = (short *)(hibernate_fchunk_area + (8*PAGE_SIZE));
1614 
1615 	/* Final chunk ordering VA */
1616 	ochunks = (short *)(hibernate_fchunk_area + (16*PAGE_SIZE));
1617 
1618 	/* Map the chunk ordering region */
1619 	for(i=0; i<24 ; i++) {
1620 		pmap_kenter_pa(hibernate_fchunk_area + (i*PAGE_SIZE),
1621 			piglet_base + ((4+i)*PAGE_SIZE), VM_PROT_ALL);
1622 		pmap_update(pmap_kernel());
1623 	}
1624 
1625 	nchunks = hib_info->chunk_ctr;
1626 
1627 	/* Initially start all chunks as unplaced */
1628 	for (i = 0; i < nchunks; i++)
1629 		chunks[i].flags = 0;
1630 
1631 	/*
1632 	 * Search the list for chunks that are outside the pig area. These
1633 	 * can be placed first in the final output list.
1634 	 */
1635 	for (i = 0; i < nchunks; i++) {
1636 		if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) {
1637 			ochunks[nochunks] = i;
1638 			fchunks[nfchunks] = i;
1639 			nochunks++;
1640 			nfchunks++;
1641 			chunks[i].flags |= HIBERNATE_CHUNK_USED;
1642 		}
1643 	}
1644 
1645 	/*
1646 	 * Walk the ordering, place the chunks in ascending memory order.
1647 	 * Conflicts might arise, these are handled next.
1648 	 */
1649 	do {
1650 		img_index = -1;
1651 		found = 0;
1652 		j = -1;
1653 		for (i = 0; i < nchunks; i++)
1654 			if (chunks[i].base < img_index &&
1655 			    chunks[i].flags == 0 ) {
1656 				j = i;
1657 				img_index = chunks[i].base;
1658 			}
1659 
1660 		if (j != -1) {
1661 			found = 1;
1662 			ochunks[nochunks] = j;
1663 			nochunks++;
1664 			chunks[j].flags |= HIBERNATE_CHUNK_PLACED;
1665 		}
1666 	} while (found);
1667 
1668 	img_index = pig_start;
1669 
1670 	/*
1671 	 * Identify chunk output conflicts (chunks whose pig load area
1672 	 * corresponds to their original memory placement location)
1673 	 */
1674 	for (i = 0; i < nochunks ; i++) {
1675 		overlap = 0;
1676 		r1s = img_index;
1677 		r1e = img_index + chunks[ochunks[i]].compressed_size;
1678 		r2s = chunks[ochunks[i]].base;
1679 		r2e = chunks[ochunks[i]].end;
1680 
1681 		overlap = hibernate_check_overlap(r1s, r1e, r2s, r2e);
1682 		if (overlap)
1683 			chunks[ochunks[i]].flags |= HIBERNATE_CHUNK_CONFLICT;
1684 		img_index += chunks[ochunks[i]].compressed_size;
1685 	}
1686 
1687 	/*
1688 	 * Prepare the final output chunk list. Calculate an output
1689 	 * inflate strategy for overlapping chunks if needed.
1690 	 */
1691 	img_index = pig_start;
1692 	for (i = 0; i < nochunks ; i++) {
1693 		/*
1694 		 * If a conflict is detected, consume enough compressed
1695 		 * output chunks to fill the piglet
1696 		 */
1697 		if (chunks[ochunks[i]].flags & HIBERNATE_CHUNK_CONFLICT) {
1698 			copy_start = piglet_base;
1699 			copy_end = piglet_end;
1700 			piglet_cur = piglet_base;
1701 			npchunks = 0;
1702 			j = i;
1703 
1704 			while (copy_start < copy_end && j < nochunks) {
1705 				piglet_cur +=
1706 				    chunks[ochunks[j]].compressed_size;
1707 				pchunks[npchunks] = ochunks[j];
1708 				npchunks++;
1709 				copy_start +=
1710 				    chunks[ochunks[j]].compressed_size;
1711 				img_index += chunks[ochunks[j]].compressed_size;
1712 				i++;
1713 				j++;
1714 			}
1715 
1716 			piglet_cur = piglet_base;
1717 			for (j = 0; j < npchunks; j++) {
1718 				piglet_cur +=
1719 				    chunks[pchunks[j]].compressed_size;
1720 				fchunks[nfchunks] = pchunks[j];
1721 				chunks[pchunks[j]].flags |=
1722 				    HIBERNATE_CHUNK_USED;
1723 				nfchunks++;
1724 			}
1725 		} else {
1726 			/*
1727 			 * No conflict, chunk can be added without copying
1728 			 */
1729 			if ((chunks[ochunks[i]].flags &
1730 			    HIBERNATE_CHUNK_USED) == 0) {
1731 				fchunks[nfchunks] = ochunks[i];
1732 				chunks[ochunks[i]].flags |=
1733 				    HIBERNATE_CHUNK_USED;
1734 				nfchunks++;
1735 			}
1736 			img_index += chunks[ochunks[i]].compressed_size;
1737 		}
1738 	}
1739 
1740 	img_index = pig_start;
1741 	for (i = 0; i < nfchunks; i++) {
1742 		piglet_cur = piglet_base;
1743 		img_index += chunks[fchunks[i]].compressed_size;
1744 	}
1745 
1746 	img_cur = pig_start;
1747 
1748 	for (i = 0; i < nfchunks; i++) {
1749 		blkctr = chunks[fchunks[i]].offset - hib_info->swap_offset;
1750 		processed = 0;
1751 		compressed_size = chunks[fchunks[i]].compressed_size;
1752 
1753 		while (processed < compressed_size) {
1754 			pmap_kenter_pa(tempva, img_cur, VM_PROT_ALL);
1755 			pmap_kenter_pa(tempva + PAGE_SIZE, img_cur+PAGE_SIZE,
1756 			    VM_PROT_ALL);
1757 			pmap_update(pmap_kernel());
1758 
1759 			if (compressed_size - processed >= PAGE_SIZE)
1760 				read_size = PAGE_SIZE;
1761 			else
1762 				read_size = compressed_size - processed;
1763 
1764 			hibernate_block_io(hib_info, blkctr, read_size,
1765 			    tempva + (img_cur & PAGE_MASK), 0);
1766 
1767 			blkctr += (read_size / hib_info->secsize);
1768 
1769 			hibernate_flush();
1770 			pmap_kremove(tempva, PAGE_SIZE);
1771 			pmap_kremove(tempva + PAGE_SIZE, PAGE_SIZE);
1772 			processed += read_size;
1773 			img_cur += read_size;
1774 		}
1775 	}
1776 
1777 	pmap_kremove(hibernate_fchunk_area, PAGE_SIZE);
1778 	pmap_kremove((vaddr_t)pchunks, PAGE_SIZE);
1779 	pmap_kremove((vaddr_t)fchunks, PAGE_SIZE);
1780 	pmap_update(pmap_kernel());
1781 
1782 	return (0);
1783 }
1784 
1785 /*
1786  * Hibernating a machine comprises the following operations:
1787  *  1. Calculating this machine's hibernate_info information
1788  *  2. Allocating a piglet and saving the piglet's physaddr
1789  *  3. Calculating the memory chunks
1790  *  4. Writing the compressed chunks to disk
1791  *  5. Writing the chunk table
1792  *  6. Writing the signature block (hibernate_info)
1793  *
1794  * On most architectures, the function calling hibernate_suspend would
1795  * then power off the machine using some MD-specific implementation.
1796  */
1797 int
1798 hibernate_suspend(void)
1799 {
1800 	union hibernate_info hib_info;
1801 	size_t swap_size;
1802 
1803 	/*
1804 	 * Calculate memory ranges, swap offsets, etc.
1805 	 * This also allocates a piglet whose physaddr is stored in
1806 	 * hib_info->piglet_pa and vaddr stored in hib_info->piglet_va
1807 	 */
1808 	if (get_hibernate_info(&hib_info, 1))
1809 		return (1);
1810 
1811 	swap_size = hib_info.image_size + hib_info.secsize +
1812 		HIBERNATE_CHUNK_TABLE_SIZE;
1813 
1814 	if (uvm_swap_check_range(hib_info.device, swap_size)) {
1815 		printf("insufficient swap space for hibernate\n");
1816 		return (1);
1817 	}
1818 
1819 	pmap_kenter_pa(HIBERNATE_HIBALLOC_PAGE, HIBERNATE_HIBALLOC_PAGE,
1820 		VM_PROT_ALL);
1821 	pmap_activate(curproc);
1822 
1823 	/* Stash the piglet VA so we can free it in the resuming kernel */
1824 	global_piglet_va = hib_info.piglet_va;
1825 
1826 	if (hibernate_write_chunks(&hib_info))
1827 		return (1);
1828 
1829 	if (hibernate_write_chunktable(&hib_info))
1830 		return (1);
1831 
1832 	if (hibernate_write_signature(&hib_info))
1833 		return (1);
1834 
1835 	/* Allow the disk to settle */
1836 	delay(500000);
1837 
1838 	hib_info.io_func(hib_info.device, 0, (vaddr_t)NULL, 0,
1839 	    HIB_DONE, hib_info.io_page);
1840 
1841 	return (0);
1842 }
1843 
1844 /*
1845  * Free items allocated by hibernate_suspend()
1846  */
1847 void
1848 hibernate_free(void)
1849 {
1850 	if (global_piglet_va)
1851 		uvm_pmr_free_piglet(global_piglet_va,
1852 		    3*HIBERNATE_CHUNK_SIZE);
1853 
1854 	if (hibernate_copy_page)
1855 		pmap_kremove(hibernate_copy_page, PAGE_SIZE);
1856 	if (hibernate_temp_page)
1857 		pmap_kremove(hibernate_temp_page, PAGE_SIZE);
1858 
1859 	pmap_update(pmap_kernel());
1860 
1861 	if (hibernate_copy_page)
1862 		km_free((void *)hibernate_copy_page, PAGE_SIZE,
1863 		    &kv_any, &kp_none);
1864 	if (hibernate_temp_page)
1865 		km_free((void *)hibernate_temp_page, PAGE_SIZE,
1866 		    &kv_any, &kp_none);
1867 
1868 	global_piglet_va = 0;
1869 	hibernate_copy_page = 0;
1870 	hibernate_temp_page = 0;
1871 }
1872