xref: /openbsd-src/sys/kern/subr_hibernate.c (revision af7dea425a21512b95f36c863ffa579ccf820111)
1 /*	$OpenBSD: subr_hibernate.c,v 1.27 2011/11/18 00:51:27 jasper Exp $	*/
2 
3 /*
4  * Copyright (c) 2011 Ariane van der Steldt <ariane@stack.nl>
5  * Copyright (c) 2011 Mike Larkin <mlarkin@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include <sys/hibernate.h>
21 #include <sys/malloc.h>
22 #include <sys/param.h>
23 #include <sys/tree.h>
24 #include <sys/types.h>
25 #include <sys/systm.h>
26 #include <sys/disklabel.h>
27 #include <sys/disk.h>
28 #include <sys/conf.h>
29 #include <sys/buf.h>
30 #include <sys/fcntl.h>
31 #include <sys/stat.h>
32 #include <uvm/uvm.h>
33 #include <machine/hibernate.h>
34 
35 struct hibernate_zlib_state *hibernate_state;
36 
37 /* Temporary vaddr ranges used during hibernate */
38 vaddr_t hibernate_temp_page;
39 vaddr_t hibernate_copy_page;
40 
41 /* Hibernate info as read from disk during resume */
42 union hibernate_info disk_hiber_info;
43 paddr_t global_pig_start;
44 vaddr_t global_piglet_va;
45 
46 /*
47  * Hib alloc enforced alignment.
48  */
49 #define HIB_ALIGN		8 /* bytes alignment */
50 
51 /*
52  * sizeof builtin operation, but with alignment constraint.
53  */
54 #define HIB_SIZEOF(_type)	roundup(sizeof(_type), HIB_ALIGN)
55 
56 struct hiballoc_entry {
57 	size_t			hibe_use;
58 	size_t			hibe_space;
59 	RB_ENTRY(hiballoc_entry) hibe_entry;
60 };
61 
62 /*
63  * Compare hiballoc entries based on the address they manage.
64  *
65  * Since the address is fixed, relative to struct hiballoc_entry,
66  * we just compare the hiballoc_entry pointers.
67  */
68 static __inline int
69 hibe_cmp(struct hiballoc_entry *l, struct hiballoc_entry *r)
70 {
71 	return l < r ? -1 : (l > r);
72 }
73 
74 RB_PROTOTYPE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
75 
76 /*
77  * Given a hiballoc entry, return the address it manages.
78  */
79 static __inline void *
80 hib_entry_to_addr(struct hiballoc_entry *entry)
81 {
82 	caddr_t addr;
83 
84 	addr = (caddr_t)entry;
85 	addr += HIB_SIZEOF(struct hiballoc_entry);
86 	return addr;
87 }
88 
89 /*
90  * Given an address, find the hiballoc that corresponds.
91  */
92 static __inline struct hiballoc_entry*
93 hib_addr_to_entry(void *addr_param)
94 {
95 	caddr_t addr;
96 
97 	addr = (caddr_t)addr_param;
98 	addr -= HIB_SIZEOF(struct hiballoc_entry);
99 	return (struct hiballoc_entry*)addr;
100 }
101 
102 RB_GENERATE(hiballoc_addr, hiballoc_entry, hibe_entry, hibe_cmp)
103 
104 /*
105  * Allocate memory from the arena.
106  *
107  * Returns NULL if no memory is available.
108  */
109 void *
110 hib_alloc(struct hiballoc_arena *arena, size_t alloc_sz)
111 {
112 	struct hiballoc_entry *entry, *new_entry;
113 	size_t find_sz;
114 
115 	/*
116 	 * Enforce alignment of HIB_ALIGN bytes.
117 	 *
118 	 * Note that, because the entry is put in front of the allocation,
119 	 * 0-byte allocations are guaranteed a unique address.
120 	 */
121 	alloc_sz = roundup(alloc_sz, HIB_ALIGN);
122 
123 	/*
124 	 * Find an entry with hibe_space >= find_sz.
125 	 *
126 	 * If the root node is not large enough, we switch to tree traversal.
127 	 * Because all entries are made at the bottom of the free space,
128 	 * traversal from the end has a slightly better chance of yielding
129 	 * a sufficiently large space.
130 	 */
131 	find_sz = alloc_sz + HIB_SIZEOF(struct hiballoc_entry);
132 	entry = RB_ROOT(&arena->hib_addrs);
133 	if (entry != NULL && entry->hibe_space < find_sz) {
134 		RB_FOREACH_REVERSE(entry, hiballoc_addr, &arena->hib_addrs) {
135 			if (entry->hibe_space >= find_sz)
136 				break;
137 		}
138 	}
139 
140 	/*
141 	 * Insufficient or too fragmented memory.
142 	 */
143 	if (entry == NULL)
144 		return NULL;
145 
146 	/*
147 	 * Create new entry in allocated space.
148 	 */
149 	new_entry = (struct hiballoc_entry*)(
150 	    (caddr_t)hib_entry_to_addr(entry) + entry->hibe_use);
151 	new_entry->hibe_space = entry->hibe_space - find_sz;
152 	new_entry->hibe_use = alloc_sz;
153 
154 	/*
155 	 * Insert entry.
156 	 */
157 	if (RB_INSERT(hiballoc_addr, &arena->hib_addrs, new_entry) != NULL)
158 		panic("hib_alloc: insert failure");
159 	entry->hibe_space = 0;
160 
161 	/* Return address managed by entry. */
162 	return hib_entry_to_addr(new_entry);
163 }
164 
165 /*
166  * Free a pointer previously allocated from this arena.
167  *
168  * If addr is NULL, this will be silently accepted.
169  */
170 void
171 hib_free(struct hiballoc_arena *arena, void *addr)
172 {
173 	struct hiballoc_entry *entry, *prev;
174 
175 	if (addr == NULL)
176 		return;
177 
178 	/*
179 	 * Derive entry from addr and check it is really in this arena.
180 	 */
181 	entry = hib_addr_to_entry(addr);
182 	if (RB_FIND(hiballoc_addr, &arena->hib_addrs, entry) != entry)
183 		panic("hib_free: freed item %p not in hib arena", addr);
184 
185 	/*
186 	 * Give the space in entry to its predecessor.
187 	 *
188 	 * If entry has no predecessor, change its used space into free space
189 	 * instead.
190 	 */
191 	prev = RB_PREV(hiballoc_addr, &arena->hib_addrs, entry);
192 	if (prev != NULL &&
193 	    (void *)((caddr_t)prev + HIB_SIZEOF(struct hiballoc_entry) +
194 	    prev->hibe_use + prev->hibe_space) == entry) {
195 		/* Merge entry. */
196 		RB_REMOVE(hiballoc_addr, &arena->hib_addrs, entry);
197 		prev->hibe_space += HIB_SIZEOF(struct hiballoc_entry) +
198 		    entry->hibe_use + entry->hibe_space;
199 	} else {
200 		/* Flip used memory to free space. */
201 		entry->hibe_space += entry->hibe_use;
202 		entry->hibe_use = 0;
203 	}
204 }
205 
206 /*
207  * Initialize hiballoc.
208  *
209  * The allocator will manage memmory at ptr, which is len bytes.
210  */
211 int
212 hiballoc_init(struct hiballoc_arena *arena, void *p_ptr, size_t p_len)
213 {
214 	struct hiballoc_entry *entry;
215 	caddr_t ptr;
216 	size_t len;
217 
218 	RB_INIT(&arena->hib_addrs);
219 
220 	/*
221 	 * Hib allocator enforces HIB_ALIGN alignment.
222 	 * Fixup ptr and len.
223 	 */
224 	ptr = (caddr_t)roundup((vaddr_t)p_ptr, HIB_ALIGN);
225 	len = p_len - ((size_t)ptr - (size_t)p_ptr);
226 	len &= ~((size_t)HIB_ALIGN - 1);
227 
228 	/*
229 	 * Insufficient memory to be able to allocate and also do bookkeeping.
230 	 */
231 	if (len <= HIB_SIZEOF(struct hiballoc_entry))
232 		return ENOMEM;
233 
234 	/*
235 	 * Create entry describing space.
236 	 */
237 	entry = (struct hiballoc_entry*)ptr;
238 	entry->hibe_use = 0;
239 	entry->hibe_space = len - HIB_SIZEOF(struct hiballoc_entry);
240 	RB_INSERT(hiballoc_addr, &arena->hib_addrs, entry);
241 
242 	return 0;
243 }
244 
245 /*
246  * Zero all free memory.
247  */
248 void
249 uvm_pmr_zero_everything(void)
250 {
251 	struct uvm_pmemrange	*pmr;
252 	struct vm_page		*pg;
253 	int			 i;
254 
255 	uvm_lock_fpageq();
256 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
257 		/* Zero single pages. */
258 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_DIRTY]))
259 		    != NULL) {
260 			uvm_pmr_remove(pmr, pg);
261 			uvm_pagezero(pg);
262 			atomic_setbits_int(&pg->pg_flags, PG_ZERO);
263 			uvmexp.zeropages++;
264 			uvm_pmr_insert(pmr, pg, 0);
265 		}
266 
267 		/* Zero multi page ranges. */
268 		while ((pg = RB_ROOT(&pmr->size[UVM_PMR_MEMTYPE_DIRTY]))
269 		    != NULL) {
270 			pg--; /* Size tree always has second page. */
271 			uvm_pmr_remove(pmr, pg);
272 			for (i = 0; i < pg->fpgsz; i++) {
273 				uvm_pagezero(&pg[i]);
274 				atomic_setbits_int(&pg[i].pg_flags, PG_ZERO);
275 				uvmexp.zeropages++;
276 			}
277 			uvm_pmr_insert(pmr, pg, 0);
278 		}
279 	}
280 	uvm_unlock_fpageq();
281 }
282 
283 /*
284  * Mark all memory as dirty.
285  *
286  * Used to inform the system that the clean memory isn't clean for some
287  * reason, for example because we just came back from hibernate.
288  */
289 void
290 uvm_pmr_dirty_everything(void)
291 {
292 	struct uvm_pmemrange	*pmr;
293 	struct vm_page		*pg;
294 	int			 i;
295 
296 	uvm_lock_fpageq();
297 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
298 		/* Dirty single pages. */
299 		while ((pg = TAILQ_FIRST(&pmr->single[UVM_PMR_MEMTYPE_ZERO]))
300 		    != NULL) {
301 			uvm_pmr_remove(pmr, pg);
302 			atomic_clearbits_int(&pg->pg_flags, PG_ZERO);
303 			uvm_pmr_insert(pmr, pg, 0);
304 		}
305 
306 		/* Dirty multi page ranges. */
307 		while ((pg = RB_ROOT(&pmr->size[UVM_PMR_MEMTYPE_ZERO]))
308 		    != NULL) {
309 			pg--; /* Size tree always has second page. */
310 			uvm_pmr_remove(pmr, pg);
311 			for (i = 0; i < pg->fpgsz; i++)
312 				atomic_clearbits_int(&pg[i].pg_flags, PG_ZERO);
313 			uvm_pmr_insert(pmr, pg, 0);
314 		}
315 	}
316 
317 	uvmexp.zeropages = 0;
318 	uvm_unlock_fpageq();
319 }
320 
321 /*
322  * Allocate the highest address that can hold sz.
323  *
324  * sz in bytes.
325  */
326 int
327 uvm_pmr_alloc_pig(paddr_t *addr, psize_t sz)
328 {
329 	struct uvm_pmemrange	*pmr;
330 	struct vm_page		*pig_pg, *pg;
331 
332 	/*
333 	 * Convert sz to pages, since that is what pmemrange uses internally.
334 	 */
335 	sz = atop(round_page(sz));
336 
337 	uvm_lock_fpageq();
338 
339 	TAILQ_FOREACH(pmr, &uvm.pmr_control.use, pmr_use) {
340 		RB_FOREACH_REVERSE(pig_pg, uvm_pmr_addr, &pmr->addr) {
341 			if (pig_pg->fpgsz >= sz) {
342 				goto found;
343 			}
344 		}
345 	}
346 
347 	/*
348 	 * Allocation failure.
349 	 */
350 	uvm_unlock_pageq();
351 	return ENOMEM;
352 
353 found:
354 	/* Remove page from freelist. */
355 	uvm_pmr_remove_size(pmr, pig_pg);
356 	pig_pg->fpgsz -= sz;
357 	pg = pig_pg + pig_pg->fpgsz;
358 	if (pig_pg->fpgsz == 0)
359 		uvm_pmr_remove_addr(pmr, pig_pg);
360 	else
361 		uvm_pmr_insert_size(pmr, pig_pg);
362 
363 	uvmexp.free -= sz;
364 	*addr = VM_PAGE_TO_PHYS(pg);
365 
366 	/*
367 	 * Update pg flags.
368 	 *
369 	 * Note that we trash the sz argument now.
370 	 */
371 	while (sz > 0) {
372 		KASSERT(pg->pg_flags & PQ_FREE);
373 
374 		atomic_clearbits_int(&pg->pg_flags,
375 		    PG_PMAP0|PG_PMAP1|PG_PMAP2|PG_PMAP3);
376 
377 		if (pg->pg_flags & PG_ZERO)
378 			uvmexp.zeropages -= sz;
379 		atomic_clearbits_int(&pg->pg_flags,
380 		    PG_ZERO|PQ_FREE);
381 
382 		pg->uobject = NULL;
383 		pg->uanon = NULL;
384 		pg->pg_version++;
385 
386 		/*
387 		 * Next.
388 		 */
389 		pg++;
390 		sz--;
391 	}
392 
393 	/* Return. */
394 	uvm_unlock_fpageq();
395 	return 0;
396 }
397 
398 /*
399  * Allocate a piglet area.
400  *
401  * This is as low as possible.
402  * Piglets are aligned.
403  *
404  * sz and align in bytes.
405  *
406  * The call will sleep for the pagedaemon to attempt to free memory.
407  * The pagedaemon may decide its not possible to free enough memory, causing
408  * the allocation to fail.
409  */
410 int
411 uvm_pmr_alloc_piglet(vaddr_t *va, paddr_t *pa, vsize_t sz, paddr_t align)
412 {
413 	paddr_t			 pg_addr, piglet_addr;
414 	struct uvm_pmemrange	*pmr;
415 	struct vm_page		*pig_pg, *pg;
416 	struct pglist		 pageq;
417 	int			 pdaemon_woken;
418 	vaddr_t			 piglet_va;
419 
420 	KASSERT((align & (align - 1)) == 0);
421 	pdaemon_woken = 0; /* Didn't wake the pagedaemon. */
422 
423 	/*
424 	 * Fixup arguments: align must be at least PAGE_SIZE,
425 	 * sz will be converted to pagecount, since that is what
426 	 * pmemrange uses internally.
427 	 */
428 	if (align < PAGE_SIZE)
429 		align = PAGE_SIZE;
430 	sz = round_page(sz);
431 
432 	uvm_lock_fpageq();
433 
434 	TAILQ_FOREACH_REVERSE(pmr, &uvm.pmr_control.use, uvm_pmemrange_use,
435 	    pmr_use) {
436 retry:
437 		/*
438 		 * Search for a range with enough space.
439 		 * Use the address tree, to ensure the range is as low as
440 		 * possible.
441 		 */
442 		RB_FOREACH(pig_pg, uvm_pmr_addr, &pmr->addr) {
443 			pg_addr = VM_PAGE_TO_PHYS(pig_pg);
444 			piglet_addr = (pg_addr + (align - 1)) & ~(align - 1);
445 
446 			if (atop(pg_addr) + pig_pg->fpgsz >=
447 			    atop(piglet_addr) + atop(sz))
448 				goto found;
449 		}
450 	}
451 
452 	/*
453 	 * Try to coerse the pagedaemon into freeing memory
454 	 * for the piglet.
455 	 *
456 	 * pdaemon_woken is set to prevent the code from
457 	 * falling into an endless loop.
458 	 */
459 	if (!pdaemon_woken) {
460 		pdaemon_woken = 1;
461 		if (uvm_wait_pla(ptoa(pmr->low), ptoa(pmr->high) - 1,
462 		    sz, UVM_PLA_FAILOK) == 0)
463 			goto retry;
464 	}
465 
466 	/* Return failure. */
467 	uvm_unlock_fpageq();
468 	return ENOMEM;
469 
470 found:
471 	/*
472 	 * Extract piglet from pigpen.
473 	 */
474 	TAILQ_INIT(&pageq);
475 	uvm_pmr_extract_range(pmr, pig_pg,
476 	    atop(piglet_addr), atop(piglet_addr) + atop(sz), &pageq);
477 
478 	*pa = piglet_addr;
479 	uvmexp.free -= atop(sz);
480 
481 	/*
482 	 * Update pg flags.
483 	 *
484 	 * Note that we trash the sz argument now.
485 	 */
486 	TAILQ_FOREACH(pg, &pageq, pageq) {
487 		KASSERT(pg->pg_flags & PQ_FREE);
488 
489 		atomic_clearbits_int(&pg->pg_flags,
490 		    PG_PMAP0|PG_PMAP1|PG_PMAP2|PG_PMAP3);
491 
492 		if (pg->pg_flags & PG_ZERO)
493 			uvmexp.zeropages--;
494 		atomic_clearbits_int(&pg->pg_flags,
495 		    PG_ZERO|PQ_FREE);
496 
497 		pg->uobject = NULL;
498 		pg->uanon = NULL;
499 		pg->pg_version++;
500 	}
501 
502 	uvm_unlock_fpageq();
503 
504 	/*
505 	 * Now allocate a va.
506 	 * Use direct mappings for the pages.
507 	 */
508 
509 	piglet_va = *va = (vaddr_t)km_alloc(sz, &kv_any, &kp_none, &kd_waitok);
510 	if (!piglet_va) {
511 		uvm_pglistfree(&pageq);
512 		return ENOMEM;
513 	}
514 
515 	/*
516 	 * Map piglet to va.
517 	 */
518 	TAILQ_FOREACH(pg, &pageq, pageq) {
519 		pmap_kenter_pa(piglet_va, VM_PAGE_TO_PHYS(pg), UVM_PROT_RW);
520 		piglet_va += PAGE_SIZE;
521 	}
522 	pmap_update(pmap_kernel());
523 
524 	return 0;
525 }
526 
527 /*
528  * Free a piglet area.
529  */
530 void
531 uvm_pmr_free_piglet(vaddr_t va, vsize_t sz)
532 {
533 	paddr_t			 pa;
534 	struct vm_page		*pg;
535 
536 	/*
537 	 * Fix parameters.
538 	 */
539 	sz = round_page(sz);
540 
541 	/*
542 	 * Find the first page in piglet.
543 	 * Since piglets are contiguous, the first pg is all we need.
544 	 */
545 	if (!pmap_extract(pmap_kernel(), va, &pa))
546 		panic("uvm_pmr_free_piglet: piglet 0x%lx has no pages", va);
547 	pg = PHYS_TO_VM_PAGE(pa);
548 	if (pg == NULL)
549 		panic("uvm_pmr_free_piglet: unmanaged page 0x%lx", pa);
550 
551 	/*
552 	 * Unmap.
553 	 */
554 	pmap_kremove(va, sz);
555 	pmap_update(pmap_kernel());
556 
557 	/*
558 	 * Free the physical and virtual memory.
559 	 */
560 	uvm_pmr_freepages(pg, atop(sz));
561 	km_free((void *)va, sz, &kv_any, &kp_none);
562 }
563 
564 /*
565  * Physmem RLE compression support.
566  *
567  * Given a physical page address, it will return the number of pages
568  * starting at the address, that are free.
569  * Returns 0 if the page at addr is not free.
570  */
571 psize_t
572 uvm_page_rle(paddr_t addr)
573 {
574 	struct vm_page		*pg, *pg_end;
575 	struct vm_physseg	*vmp;
576 	int			 pseg_idx, off_idx;
577 
578 	pseg_idx = vm_physseg_find(atop(addr), &off_idx);
579 	if (pseg_idx == -1)
580 		return 0;
581 
582 	vmp = &vm_physmem[pseg_idx];
583 	pg = &vmp->pgs[off_idx];
584 	if (!(pg->pg_flags & PQ_FREE))
585 		return 0;
586 
587 	/*
588 	 * Search for the first non-free page after pg.
589 	 * Note that the page may not be the first page in a free pmemrange,
590 	 * therefore pg->fpgsz cannot be used.
591 	 */
592 	for (pg_end = pg; pg_end <= vmp->lastpg &&
593 	    (pg_end->pg_flags & PQ_FREE) == PQ_FREE; pg_end++);
594 	return pg_end - pg;
595 }
596 
597 /*
598  * Fills out the hibernate_info union pointed to by hiber_info
599  * with information about this machine (swap signature block
600  * offsets, number of memory ranges, kernel in use, etc)
601  */
602 int
603 get_hibernate_info(union hibernate_info *hiber_info, int suspend)
604 {
605 	int chunktable_size;
606 	struct disklabel dl;
607 	char err_string[128], *dl_ret;
608 
609 	/* Determine I/O function to use */
610 	hiber_info->io_func = get_hibernate_io_function();
611 	if (hiber_info->io_func == NULL)
612 		return (1);
613 
614 	/* Calculate hibernate device */
615 	hiber_info->device = swdevt[0].sw_dev;
616 
617 	/* Read disklabel (used to calculate signature and image offsets) */
618 	dl_ret = disk_readlabel(&dl, hiber_info->device, err_string, 128);
619 
620 	if (dl_ret) {
621 		printf("Hibernate error reading disklabel: %s\n", dl_ret);
622 		return (1);
623 	}
624 
625 	hiber_info->secsize = dl.d_secsize;
626 
627 	/* Make sure the signature can fit in one block */
628 	KASSERT(sizeof(union hibernate_info)/hiber_info->secsize == 1);
629 
630 	/* Calculate swap offset from start of disk */
631 	hiber_info->swap_offset = dl.d_partitions[1].p_offset;
632 
633 	/* Calculate signature block location */
634 	hiber_info->sig_offset = dl.d_partitions[1].p_offset +
635 	    dl.d_partitions[1].p_size -
636 	    sizeof(union hibernate_info)/hiber_info->secsize;
637 
638 	chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / hiber_info->secsize;
639 
640 	/* Stash kernel version information */
641 	bzero(&hiber_info->kernel_version, 128);
642 	bcopy(version, &hiber_info->kernel_version,
643 	    min(strlen(version), sizeof(hiber_info->kernel_version)-1));
644 
645 	if (suspend) {
646 		/* Allocate piglet region */
647 		if (uvm_pmr_alloc_piglet(&hiber_info->piglet_va,
648 		    &hiber_info->piglet_pa, HIBERNATE_CHUNK_SIZE*3,
649 		    HIBERNATE_CHUNK_SIZE)) {
650 			printf("Hibernate failed to allocate the piglet\n");
651 			return (1);
652 		}
653 		hiber_info->io_page = (void *)hiber_info->piglet_va;
654 	} else {
655 		/*
656 		 * Resuming kernels use a regular I/O page since we won't
657 		 * have access to the suspended kernel's piglet VA at this
658 		 * point. No need to free this I/O page as it will vanish
659 		 * as part of the resume.
660 		 */
661 		hiber_info->io_page = malloc(PAGE_SIZE, M_DEVBUF, M_NOWAIT);
662 		if (!hiber_info->io_page)
663 			return (1);
664 	}
665 
666 
667 	/*
668 	 * Initialize of the hibernate IO function (for drivers which
669 	 * need that)
670 	 */
671 	if (hiber_info->io_func(hiber_info->device, 0,
672 	    (vaddr_t)NULL, 0, HIB_INIT, hiber_info->io_page))
673 		goto fail;
674 
675 	if (get_hibernate_info_md(hiber_info))
676 		goto fail;
677 
678 	/* Calculate memory image location */
679 	hiber_info->image_offset = dl.d_partitions[1].p_offset +
680 	    dl.d_partitions[1].p_size -
681 	    (hiber_info->image_size / hiber_info->secsize) -
682 	    sizeof(union hibernate_info)/hiber_info->secsize -
683 	    chunktable_size;
684 
685 	return (0);
686 fail:
687 	uvm_pmr_free_piglet(hiber_info->piglet_va, HIBERNATE_CHUNK_SIZE*3);
688 	return (1);
689 }
690 
691 /*
692  * Allocate nitems*size bytes from the hiballoc area presently in use
693  */
694 void
695 *hibernate_zlib_alloc(void *unused, int nitems, int size)
696 {
697 	return hib_alloc(&hibernate_state->hiballoc_arena, nitems*size);
698 }
699 
700 /*
701  * Free the memory pointed to by addr in the hiballoc area presently in
702  * use
703  */
704 void
705 hibernate_zlib_free(void *unused, void *addr)
706 {
707 	hib_free(&hibernate_state->hiballoc_arena, addr);
708 }
709 
710 /*
711  * Inflate size bytes from src into dest, skipping any pages in
712  * [src..dest] that are special (see hibernate_inflate_skip)
713  *
714  * This function executes while using the resume-time stack
715  * and pmap, and therefore cannot use ddb/printf/etc. Doing so
716  * will likely hang or reset the machine.
717  */
718 void
719 hibernate_inflate(union hibernate_info *hiber_info, paddr_t dest,
720     paddr_t src, size_t size)
721 {
722 	int i;
723 	psize_t rle;
724 
725 	hibernate_state->hib_stream.avail_in = size;
726 	hibernate_state->hib_stream.next_in = (char *)src;
727 
728 	do {
729 		/* Flush cache and TLB */
730 		hibernate_flush();
731 
732 		/*
733 		 * Is this a special page? If yes, redirect the
734 		 * inflate output to a scratch page (eg, discard it)
735 		 */
736 		if (hibernate_inflate_skip(hiber_info, dest))
737 			hibernate_enter_resume_mapping(
738 			    HIBERNATE_INFLATE_PAGE,
739 			    HIBERNATE_INFLATE_PAGE, 0);
740 		else
741 			hibernate_enter_resume_mapping(
742 			    HIBERNATE_INFLATE_PAGE, dest, 0);
743 
744 		/* Read RLE code */
745 		hibernate_state->hib_stream.avail_out = sizeof(psize_t);
746 		hibernate_state->hib_stream.next_out = (char *)&rle;
747 
748 		i = inflate(&hibernate_state->hib_stream, Z_FULL_FLUSH);
749 		if (i != Z_OK && i != Z_STREAM_END) {
750 			/*
751 			 * XXX - this will likely reboot/hang most machines,
752 			 *       but there's not much else we can do here.
753 			 */
754 			panic("inflate rle error");
755 		}
756 
757 		/* Skip while RLE code is != 0 */
758 		while (rle != 0) {
759 			dest += (rle * PAGE_SIZE);
760 			hibernate_state->hib_stream.avail_out =
761 			    sizeof(psize_t);
762 			hibernate_state->hib_stream.next_out = (char *)&rle;
763 
764 			i = inflate(&hibernate_state->hib_stream,
765 			    Z_FULL_FLUSH);
766 			if (i != Z_OK && i != Z_STREAM_END) {
767 				/*
768 				 * XXX - this will likely reboot/hang most
769 				 *       machines but there's not much else
770 				 *       we can do here.
771 				 */
772 				panic("inflate rle error 2");
773 			}
774 		}
775 
776 		/* Set up the stream for inflate */
777 		hibernate_state->hib_stream.avail_out = PAGE_SIZE;
778 		hibernate_state->hib_stream.next_out =
779 		    (char *)HIBERNATE_INFLATE_PAGE;
780 
781 		/* Process next block of data */
782 		i = inflate(&hibernate_state->hib_stream, Z_PARTIAL_FLUSH);
783 		if (i != Z_OK && i != Z_STREAM_END) {
784 			/*
785 			 * XXX - this will likely reboot/hang most machines,
786 			 *       but there's not much else we can do here.
787 			 */
788 			panic("inflate error");
789 		}
790 
791 		dest += PAGE_SIZE - hibernate_state->hib_stream.avail_out;
792 	} while (i != Z_STREAM_END);
793 }
794 
795 /*
796  * deflate from src into the I/O page, up to 'remaining' bytes
797  *
798  * Returns number of input bytes consumed, and may reset
799  * the 'remaining' parameter if not all the output space was consumed
800  * (this information is needed to know how much to write to disk
801  */
802 size_t
803 hibernate_deflate(union hibernate_info *hiber_info, paddr_t src,
804     size_t *remaining)
805 {
806 	vaddr_t hibernate_io_page = hiber_info->piglet_va + PAGE_SIZE;
807 
808 	/* Set up the stream for deflate */
809 	hibernate_state->hib_stream.avail_in = PAGE_SIZE - (src & PAGE_MASK);
810 	hibernate_state->hib_stream.avail_out = *remaining;
811 	hibernate_state->hib_stream.next_in = (caddr_t)src;
812 	hibernate_state->hib_stream.next_out = (caddr_t)hibernate_io_page +
813 	    (PAGE_SIZE - *remaining);
814 
815 	/* Process next block of data */
816 	if (deflate(&hibernate_state->hib_stream, Z_PARTIAL_FLUSH) != Z_OK)
817 		panic("hibernate zlib deflate error");
818 
819 	/* Update pointers and return number of bytes consumed */
820 	*remaining = hibernate_state->hib_stream.avail_out;
821 	return (PAGE_SIZE - (src & PAGE_MASK)) -
822 		hibernate_state->hib_stream.avail_in;
823 }
824 
825 /*
826  * Write the hibernation information specified in hiber_info
827  * to the location in swap previously calculated (last block of
828  * swap), called the "signature block".
829  *
830  * Write the memory chunk table to the area in swap immediately
831  * preceding the signature block.
832  */
833 int
834 hibernate_write_signature(union hibernate_info *hiber_info)
835 {
836 	/* Write hibernate info to disk */
837 	return (hiber_info->io_func(hiber_info->device, hiber_info->sig_offset,
838 	    (vaddr_t)hiber_info, hiber_info->secsize, HIB_W,
839 	    hiber_info->io_page));
840 }
841 
842 /*
843  * Write the memory chunk table to the area in swap immediately
844  * preceding the signature block. The chunk table is stored
845  * in the piglet when this function is called.
846  */
847 int
848 hibernate_write_chunktable(union hibernate_info *hiber_info)
849 {
850 	struct hibernate_disk_chunk *chunks;
851 	vaddr_t hibernate_chunk_table_start;
852 	size_t hibernate_chunk_table_size;
853 	daddr_t chunkbase;
854 	int i;
855 
856 	hibernate_chunk_table_size = HIBERNATE_CHUNK_TABLE_SIZE;
857 
858 	chunkbase = hiber_info->sig_offset -
859 	    (hibernate_chunk_table_size / hiber_info->secsize);
860 
861 	hibernate_chunk_table_start = hiber_info->piglet_va +
862 	    HIBERNATE_CHUNK_SIZE;
863 
864 	chunks = (struct hibernate_disk_chunk *)(hiber_info->piglet_va +
865 	    HIBERNATE_CHUNK_SIZE);
866 
867 	/* Write chunk table */
868 	for (i = 0; i < hibernate_chunk_table_size; i += MAXPHYS) {
869 		if (hiber_info->io_func(hiber_info->device,
870 		    chunkbase + (i/hiber_info->secsize),
871 		    (vaddr_t)(hibernate_chunk_table_start + i),
872 		    MAXPHYS, HIB_W, hiber_info->io_page))
873 			return (1);
874 	}
875 
876 	return (0);
877 }
878 
879 /*
880  * Write an empty hiber_info to the swap signature block, which is
881  * guaranteed to not match any valid hiber_info.
882  */
883 int
884 hibernate_clear_signature(void)
885 {
886 	union hibernate_info blank_hiber_info;
887 	union hibernate_info hiber_info;
888 
889 	/* Zero out a blank hiber_info */
890 	bzero(&blank_hiber_info, sizeof(hiber_info));
891 
892 	if (get_hibernate_info(&hiber_info, 0))
893 		return (1);
894 
895 	/* Write (zeroed) hibernate info to disk */
896 	/* XXX - use regular kernel write routine for this */
897 	if (hiber_info.io_func(hiber_info.device, hiber_info.sig_offset,
898 	    (vaddr_t)&blank_hiber_info, hiber_info.secsize, HIB_W,
899 	    hiber_info.io_page))
900 		panic("error hibernate write 6");
901 
902 	return (0);
903 }
904 
905 /*
906  * Check chunk range overlap when calculating whether or not to copy a
907  * compressed chunk to the piglet area before decompressing.
908  *
909  * returns zero if the ranges do not overlap, non-zero otherwise.
910  */
911 int
912 hibernate_check_overlap(paddr_t r1s, paddr_t r1e, paddr_t r2s, paddr_t r2e)
913 {
914 	/* case A : end of r1 overlaps start of r2 */
915 	if (r1s < r2s && r1e > r2s)
916 		return (1);
917 
918 	/* case B : r1 entirely inside r2 */
919 	if (r1s >= r2s && r1e <= r2e)
920 		return (1);
921 
922 	/* case C : r2 entirely inside r1 */
923 	if (r2s >= r1s && r2e <= r1e)
924 		return (1);
925 
926 	/* case D : end of r2 overlaps start of r1 */
927 	if (r2s < r1s && r2e > r1s)
928 		return (1);
929 
930 	return (0);
931 }
932 
933 /*
934  * Compare two hibernate_infos to determine if they are the same (eg,
935  * we should be performing a hibernate resume on this machine.
936  * Not all fields are checked - just enough to verify that the machine
937  * has the same memory configuration and kernel as the one that
938  * wrote the signature previously.
939  */
940 int
941 hibernate_compare_signature(union hibernate_info *mine,
942     union hibernate_info *disk)
943 {
944 	u_int i;
945 
946 	if (mine->nranges != disk->nranges)
947 		return (1);
948 
949 	if (strcmp(mine->kernel_version, disk->kernel_version) != 0)
950 		return (1);
951 
952 	for (i = 0; i < mine->nranges; i++) {
953 		if ((mine->ranges[i].base != disk->ranges[i].base) ||
954 		    (mine->ranges[i].end != disk->ranges[i].end) )
955 			return (1);
956 	}
957 
958 	return (0);
959 }
960 
961 /*
962  * Reads read_size bytes from the hibernate device specified in
963  * hib_info at offset blkctr. Output is placed into the vaddr specified
964  * at dest.
965  *
966  * Separate offsets and pages are used to handle misaligned reads (reads
967  * that span a page boundary).
968  *
969  * blkctr specifies a relative offset (relative to the start of swap),
970  * not an absolute disk offset
971  *
972  */
973 int
974 hibernate_read_block(union hibernate_info *hib_info, daddr_t blkctr,
975     size_t read_size, vaddr_t dest)
976 {
977 	struct buf *bp;
978 	struct bdevsw *bdsw;
979 	int error;
980 
981 	bp = geteblk(read_size);
982 	bdsw = &bdevsw[major(hib_info->device)];
983 
984 	error = (*bdsw->d_open)(hib_info->device, FREAD, S_IFCHR, curproc);
985 	if (error) {
986 		printf("hibernate_read_block open failed\n");
987 		return (1);
988 	}
989 
990 	bp->b_bcount = read_size;
991 	bp->b_blkno = blkctr;
992 	CLR(bp->b_flags, B_READ | B_WRITE | B_DONE);
993 	SET(bp->b_flags, B_BUSY | B_READ | B_RAW);
994 	bp->b_dev = hib_info->device;
995 	bp->b_cylinder = 0;
996 	(*bdsw->d_strategy)(bp);
997 
998 	error = biowait(bp);
999 	if (error) {
1000 		printf("hibernate_read_block biowait failed %d\n", error);
1001 		error = (*bdsw->d_close)(hib_info->device, 0, S_IFCHR,
1002 		    curproc);
1003 		if (error)
1004 			printf("hibernate_read_block error close failed\n");
1005 		return (1);
1006 	}
1007 
1008 	error = (*bdsw->d_close)(hib_info->device, FREAD, S_IFCHR, curproc);
1009 	if (error) {
1010 		printf("hibernate_read_block close failed\n");
1011 		return (1);
1012 	}
1013 
1014 	bcopy(bp->b_data, (caddr_t)dest, read_size);
1015 
1016 	bp->b_flags |= B_INVAL;
1017 	brelse(bp);
1018 
1019 	return (0);
1020 }
1021 
1022 /*
1023  * Reads the signature block from swap, checks against the current machine's
1024  * information. If the information matches, perform a resume by reading the
1025  * saved image into the pig area, and unpacking.
1026  */
1027 void
1028 hibernate_resume(void)
1029 {
1030 	union hibernate_info hiber_info;
1031 	int s;
1032 
1033 	/* Get current running machine's hibernate info */
1034 	bzero(&hiber_info, sizeof(hiber_info));
1035 	if (get_hibernate_info(&hiber_info, 0))
1036 		return;
1037 
1038 	/* Read hibernate info from disk */
1039 	s = splbio();
1040 
1041 	/* XXX use regular kernel read routine here */
1042 	if (hiber_info.io_func(hiber_info.device, hiber_info.sig_offset,
1043 	    (vaddr_t)&disk_hiber_info, hiber_info.secsize, HIB_R,
1044 	    hiber_info.io_page))
1045 		panic("error in hibernate read");
1046 
1047 	/*
1048 	 * If on-disk and in-memory hibernate signatures match,
1049 	 * this means we should do a resume from hibernate.
1050 	 */
1051 	if (hibernate_compare_signature(&hiber_info, &disk_hiber_info))
1052 		return;
1053 
1054 	/* Read the image from disk into the image (pig) area */
1055 	if (hibernate_read_image(&disk_hiber_info))
1056 		goto fail;
1057 
1058 	/* Point of no return ... */
1059 
1060 	disable_intr();
1061 	cold = 1;
1062 
1063 	/* Switch stacks */
1064 	hibernate_switch_stack_machdep();
1065 
1066 	/*
1067 	 * Image is now in high memory (pig area), copy to correct location
1068 	 * in memory. We'll eventually end up copying on top of ourself, but
1069 	 * we are assured the kernel code here is the same between the
1070 	 * hibernated and resuming kernel, and we are running on our own
1071 	 * stack, so the overwrite is ok.
1072 	 */
1073 	hibernate_unpack_image(&disk_hiber_info);
1074 
1075 	/*
1076 	 * Resume the loaded kernel by jumping to the MD resume vector.
1077 	 * We won't be returning from this call.
1078 	 */
1079 	hibernate_resume_machdep();
1080 
1081 fail:
1082 	printf("Unable to resume hibernated image\n");
1083 }
1084 
1085 /*
1086  * Unpack image from pig area to original location by looping through the
1087  * list of output chunks in the order they should be restored (fchunks).
1088  * This ordering is used to avoid having inflate overwrite a chunk in the
1089  * middle of processing that chunk. This will, of course, happen during the
1090  * final output chunk, where we copy the chunk to the piglet area first,
1091  * before inflating.
1092  */
1093 void
1094 hibernate_unpack_image(union hibernate_info *hiber_info)
1095 {
1096 	struct hibernate_disk_chunk *chunks;
1097 	union hibernate_info local_hiber_info;
1098 	paddr_t image_cur = global_pig_start;
1099 	int *fchunks, i;
1100 	char *pva = (char *)hiber_info->piglet_va;
1101 
1102 	/* Mask off based on arch-specific piglet page size */
1103 	pva = (char *)((paddr_t)pva & (PIGLET_PAGE_MASK));
1104 	fchunks = (int *)(pva + (6 * PAGE_SIZE));
1105 
1106 	chunks = (struct hibernate_disk_chunk *)(pva +  HIBERNATE_CHUNK_SIZE);
1107 
1108 	/* Can't use hiber_info that's passed in after here */
1109 	bcopy(hiber_info, &local_hiber_info, sizeof(union hibernate_info));
1110 
1111 	hibernate_activate_resume_pt_machdep();
1112 
1113 	for (i = 0; i < local_hiber_info.chunk_ctr; i++) {
1114 		/* Reset zlib for inflate */
1115 		if (hibernate_zlib_reset(&local_hiber_info, 0) != Z_OK)
1116 			panic("hibernate failed to reset zlib for inflate");
1117 
1118 		/*
1119 		 * If there is a conflict, copy the chunk to the piglet area
1120 		 * before unpacking it to its original location.
1121 		 */
1122 		if ((chunks[fchunks[i]].flags & HIBERNATE_CHUNK_CONFLICT) == 0)
1123 			hibernate_inflate(&local_hiber_info,
1124 			    chunks[fchunks[i]].base, image_cur,
1125 			    chunks[fchunks[i]].compressed_size);
1126 		else {
1127 			bcopy((caddr_t)image_cur,
1128 			    pva + (HIBERNATE_CHUNK_SIZE * 2),
1129 			    chunks[fchunks[i]].compressed_size);
1130 			hibernate_inflate(&local_hiber_info,
1131 			    chunks[fchunks[i]].base,
1132 			    (vaddr_t)(pva + (HIBERNATE_CHUNK_SIZE * 2)),
1133 			    chunks[fchunks[i]].compressed_size);
1134 		}
1135 		image_cur += chunks[fchunks[i]].compressed_size;
1136 	}
1137 }
1138 
1139 /*
1140  * Write a compressed version of this machine's memory to disk, at the
1141  * precalculated swap offset:
1142  *
1143  * end of swap - signature block size - chunk table size - memory size
1144  *
1145  * The function begins by looping through each phys mem range, cutting each
1146  * one into 4MB chunks. These chunks are then compressed individually
1147  * and written out to disk, in phys mem order. Some chunks might compress
1148  * more than others, and for this reason, each chunk's size is recorded
1149  * in the chunk table, which is written to disk after the image has
1150  * properly been compressed and written (in hibernate_write_chunktable).
1151  *
1152  * When this function is called, the machine is nearly suspended - most
1153  * devices are quiesced/suspended, interrupts are off, and cold has
1154  * been set. This means that there can be no side effects once the
1155  * write has started, and the write function itself can also have no
1156  * side effects.
1157  *
1158  * This function uses the piglet area during this process as follows:
1159  *
1160  * offset from piglet base	use
1161  * -----------------------	--------------------
1162  * 0				i/o allocation area
1163  * PAGE_SIZE			i/o write area
1164  * 2*PAGE_SIZE			temp/scratch page
1165  * 3*PAGE_SIZE			temp/scratch page
1166  * 4*PAGE_SIZE			hiballoc arena
1167  * 5*PAGE_SIZE to 85*PAGE_SIZE	zlib deflate area
1168  * ...
1169  * HIBERNATE_CHUNK_SIZE		chunk table temporary area
1170  *
1171  * Some transient piglet content is saved as part of deflate,
1172  * but it is irrelevant during resume as it will be repurposed
1173  * at that time for other things.
1174  */
1175 int
1176 hibernate_write_chunks(union hibernate_info *hiber_info)
1177 {
1178 	paddr_t range_base, range_end, inaddr, temp_inaddr;
1179 	size_t nblocks, out_remaining, used, offset = 0;
1180 	struct hibernate_disk_chunk *chunks;
1181 	vaddr_t hibernate_io_page = hiber_info->piglet_va + PAGE_SIZE;
1182 	daddr_t blkctr = hiber_info->image_offset;
1183 	int i;
1184 	psize_t rle;
1185 
1186 	hiber_info->chunk_ctr = 0;
1187 
1188 	/*
1189 	 * Allocate VA for the temp and copy page.
1190 	 * These will becomee part of the suspended kernel and will
1191 	 * be freed in hibernate_free, upon resume.
1192 	 */
1193 	hibernate_temp_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1194 	    &kp_none, &kd_nowait);
1195 	if (!hibernate_temp_page)
1196 		return (1);
1197 
1198 	hibernate_copy_page = (vaddr_t)km_alloc(PAGE_SIZE, &kv_any,
1199 	    &kp_none, &kd_nowait);
1200 	if (!hibernate_copy_page)
1201 		return (1);
1202 
1203 	pmap_kenter_pa(hibernate_copy_page,
1204 	    (hiber_info->piglet_pa + 3*PAGE_SIZE), VM_PROT_ALL);
1205 
1206 	/* XXX - not needed on all archs */
1207 	pmap_activate(curproc);
1208 
1209 	chunks = (struct hibernate_disk_chunk *)(hiber_info->piglet_va +
1210 	    HIBERNATE_CHUNK_SIZE);
1211 
1212 	/* Calculate the chunk regions */
1213 	for (i = 0; i < hiber_info->nranges; i++) {
1214 		range_base = hiber_info->ranges[i].base;
1215 		range_end = hiber_info->ranges[i].end;
1216 
1217 		inaddr = range_base;
1218 
1219 		while (inaddr < range_end) {
1220 			chunks[hiber_info->chunk_ctr].base = inaddr;
1221 			if (inaddr + HIBERNATE_CHUNK_SIZE < range_end)
1222 				chunks[hiber_info->chunk_ctr].end = inaddr +
1223 				    HIBERNATE_CHUNK_SIZE;
1224 			else
1225 				chunks[hiber_info->chunk_ctr].end = range_end;
1226 
1227 			inaddr += HIBERNATE_CHUNK_SIZE;
1228 			hiber_info->chunk_ctr ++;
1229 		}
1230 	}
1231 
1232 	/* Compress and write the chunks in the chunktable */
1233 	for (i = 0; i < hiber_info->chunk_ctr; i++) {
1234 		range_base = chunks[i].base;
1235 		range_end = chunks[i].end;
1236 
1237 		chunks[i].offset = blkctr;
1238 
1239 		/* Reset zlib for deflate */
1240 		if (hibernate_zlib_reset(hiber_info, 1) != Z_OK)
1241 			return (1);
1242 
1243 		inaddr = range_base;
1244 
1245 		/*
1246 		 * For each range, loop through its phys mem region
1247 		 * and write out the chunks (the last chunk might be
1248 		 * smaller than the chunk size).
1249 		 */
1250 		while (inaddr < range_end) {
1251 			out_remaining = PAGE_SIZE;
1252 			while (out_remaining > 0 && inaddr < range_end) {
1253 				pmap_kenter_pa(hibernate_temp_page,
1254 				    inaddr & PMAP_PA_MASK, VM_PROT_ALL);
1255 
1256 				/* XXX - not needed on all archs */
1257 				pmap_activate(curproc);
1258 
1259 				bcopy((caddr_t)hibernate_temp_page,
1260 				    (caddr_t)hibernate_copy_page, PAGE_SIZE);
1261 
1262 				/*
1263 				 * Adjust for regions that are not evenly
1264 				 * divisible by PAGE_SIZE
1265 				 */
1266 				temp_inaddr = (inaddr & PAGE_MASK) +
1267 				    hibernate_copy_page;
1268 
1269 				rle = uvm_page_rle(inaddr);
1270 				while (rle > 0 && inaddr < range_end) {
1271 					hibernate_state->hib_stream.avail_in =
1272 					    sizeof(psize_t);
1273 					hibernate_state->hib_stream.avail_out =
1274 					    out_remaining;
1275 					hibernate_state->hib_stream.next_in =
1276 					    (char *)&rle;
1277 					hibernate_state->hib_stream.next_out =
1278 					    (caddr_t)hibernate_io_page +
1279 					    (PAGE_SIZE - out_remaining);
1280 
1281 					if (deflate(&hibernate_state->hib_stream,
1282 					    Z_PARTIAL_FLUSH) != Z_OK)
1283 						return (1);
1284 
1285 					out_remaining =
1286 					    hibernate_state->hib_stream.avail_out;
1287 					inaddr += (rle * PAGE_SIZE);
1288 					if (inaddr > range_end)
1289 						inaddr = range_end;
1290 					else
1291 						rle = uvm_page_rle(inaddr);
1292 				}
1293 
1294 				if (out_remaining == 0) {
1295 					/* Filled up the page */
1296 					nblocks = PAGE_SIZE / hiber_info->secsize;
1297 
1298 					if (hiber_info->io_func(hiber_info->device,
1299 					    blkctr, (vaddr_t)hibernate_io_page,
1300 					    PAGE_SIZE, HIB_W, hiber_info->io_page))
1301 						return (1);
1302 
1303 					blkctr += nblocks;
1304 					out_remaining = PAGE_SIZE;
1305 				}
1306 
1307 				/* Write '0' RLE code */
1308 				if (inaddr < range_end) {
1309 					hibernate_state->hib_stream.avail_in =
1310 					    sizeof(psize_t);
1311 					hibernate_state->hib_stream.avail_out =
1312 					    out_remaining;
1313 					hibernate_state->hib_stream.next_in =
1314 					    (char *)&rle;
1315 					hibernate_state->hib_stream.next_out =
1316 				    	    (caddr_t)hibernate_io_page +
1317 					    (PAGE_SIZE - out_remaining);
1318 
1319 					if (deflate(&hibernate_state->hib_stream,
1320 					    Z_PARTIAL_FLUSH) != Z_OK)
1321 						return (1);
1322 
1323 					out_remaining =
1324 					    hibernate_state->hib_stream.avail_out;
1325 				}
1326 
1327 				if (out_remaining == 0) {
1328 					/* Filled up the page */
1329 					nblocks = PAGE_SIZE / hiber_info->secsize;
1330 
1331 					if (hiber_info->io_func(hiber_info->device,
1332 					    blkctr, (vaddr_t)hibernate_io_page,
1333 					    PAGE_SIZE, HIB_W, hiber_info->io_page))
1334 						return (1);
1335 
1336 					blkctr += nblocks;
1337 					out_remaining = PAGE_SIZE;
1338 				}
1339 
1340 				/* Deflate from temp_inaddr to IO page */
1341 				if (inaddr != range_end)
1342 					inaddr += hibernate_deflate(hiber_info,
1343 					    temp_inaddr, &out_remaining);
1344 			}
1345 
1346 			if (out_remaining == 0) {
1347 				/* Filled up the page */
1348 				nblocks = PAGE_SIZE / hiber_info->secsize;
1349 
1350 				if (hiber_info->io_func(hiber_info->device,
1351 				    blkctr, (vaddr_t)hibernate_io_page,
1352 				    PAGE_SIZE, HIB_W, hiber_info->io_page))
1353 					return (1);
1354 
1355 				blkctr += nblocks;
1356 			}
1357 		}
1358 
1359 		if (inaddr != range_end)
1360 			return (1);
1361 
1362 		/*
1363 		 * End of range. Round up to next secsize bytes
1364 		 * after finishing compress
1365 		 */
1366 		if (out_remaining == 0)
1367 			out_remaining = PAGE_SIZE;
1368 
1369 		/* Finish compress */
1370 		hibernate_state->hib_stream.avail_in = 0;
1371 		hibernate_state->hib_stream.avail_out = out_remaining;
1372 		hibernate_state->hib_stream.next_in = (caddr_t)inaddr;
1373 		hibernate_state->hib_stream.next_out =
1374 		    (caddr_t)hibernate_io_page + (PAGE_SIZE - out_remaining);
1375 
1376 		if (deflate(&hibernate_state->hib_stream, Z_FINISH) !=
1377 		    Z_STREAM_END)
1378 			return (1);
1379 
1380 		out_remaining = hibernate_state->hib_stream.avail_out;
1381 
1382 		used = PAGE_SIZE - out_remaining;
1383 		nblocks = used / hiber_info->secsize;
1384 
1385 		/* Round up to next block if needed */
1386 		if (used % hiber_info->secsize != 0)
1387 			nblocks ++;
1388 
1389 		/* Write final block(s) for this chunk */
1390 		if (hiber_info->io_func(hiber_info->device, blkctr,
1391 		    (vaddr_t)hibernate_io_page, nblocks*hiber_info->secsize,
1392 		    HIB_W, hiber_info->io_page))
1393 			return (1);
1394 
1395 		blkctr += nblocks;
1396 
1397 		offset = blkctr;
1398 		chunks[i].compressed_size = (offset - chunks[i].offset) *
1399 		    hiber_info->secsize;
1400 	}
1401 
1402 	return (0);
1403 }
1404 
1405 /*
1406  * Reset the zlib stream state and allocate a new hiballoc area for either
1407  * inflate or deflate. This function is called once for each hibernate chunk.
1408  * Calling hiballoc_init multiple times is acceptable since the memory it is
1409  * provided is unmanaged memory (stolen). We use the memory provided to us
1410  * by the piglet allocated via the supplied hiber_info.
1411  */
1412 int
1413 hibernate_zlib_reset(union hibernate_info *hiber_info, int deflate)
1414 {
1415 	vaddr_t hibernate_zlib_start;
1416 	size_t hibernate_zlib_size;
1417 	char *pva = (char *)hiber_info->piglet_va;
1418 
1419 	hibernate_state = (struct hibernate_zlib_state *)
1420 	    (pva + (7 * PAGE_SIZE));
1421 
1422 	hibernate_zlib_start = (vaddr_t)(pva + (8 * PAGE_SIZE));
1423 	hibernate_zlib_size = 80 * PAGE_SIZE;
1424 
1425 	bzero((caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1426 	bzero((caddr_t)hibernate_state, PAGE_SIZE);
1427 
1428 	/* Set up stream structure */
1429 	hibernate_state->hib_stream.zalloc = (alloc_func)hibernate_zlib_alloc;
1430 	hibernate_state->hib_stream.zfree = (free_func)hibernate_zlib_free;
1431 
1432 	/* Initialize the hiballoc arena for zlib allocs/frees */
1433 	hiballoc_init(&hibernate_state->hiballoc_arena,
1434 	    (caddr_t)hibernate_zlib_start, hibernate_zlib_size);
1435 
1436 	if (deflate) {
1437 		return deflateInit(&hibernate_state->hib_stream,
1438 		    Z_BEST_SPEED);
1439 	} else
1440 		return inflateInit(&hibernate_state->hib_stream);
1441 }
1442 
1443 /*
1444  * Reads the hibernated memory image from disk, whose location and
1445  * size are recorded in hiber_info. Begin by reading the persisted
1446  * chunk table, which records the original chunk placement location
1447  * and compressed size for each. Next, allocate a pig region of
1448  * sufficient size to hold the compressed image. Next, read the
1449  * chunks into the pig area (calling hibernate_read_chunks to do this),
1450  * and finally, if all of the above succeeds, clear the hibernate signature.
1451  * The function will then return to hibernate_resume, which will proceed
1452  * to unpack the pig image to the correct place in memory.
1453  */
1454 int
1455 hibernate_read_image(union hibernate_info *hiber_info)
1456 {
1457 	size_t compressed_size, disk_size, chunktable_size, pig_sz;
1458 	paddr_t image_start, image_end, pig_start, pig_end;
1459 	struct hibernate_disk_chunk *chunks;
1460 	daddr_t blkctr;
1461 	vaddr_t chunktable = (vaddr_t)NULL;
1462 	paddr_t piglet_chunktable = hiber_info->piglet_pa +
1463 	    HIBERNATE_CHUNK_SIZE;
1464 	int i;
1465 
1466 	/* Calculate total chunk table size in disk blocks */
1467 	chunktable_size = HIBERNATE_CHUNK_TABLE_SIZE / hiber_info->secsize;
1468 
1469 	blkctr = hiber_info->sig_offset - chunktable_size -
1470 			hiber_info->swap_offset;
1471 
1472 	chunktable = (vaddr_t)km_alloc(HIBERNATE_CHUNK_TABLE_SIZE, &kv_any,
1473 	    &kp_none, &kd_nowait);
1474 
1475 	if (!chunktable)
1476 		return (1);
1477 
1478 	/* Read the chunktable from disk into the piglet chunktable */
1479 	for (i = 0; i < HIBERNATE_CHUNK_TABLE_SIZE;
1480 	    i += PAGE_SIZE, blkctr += PAGE_SIZE/hiber_info->secsize) {
1481 		pmap_kenter_pa(chunktable + i, piglet_chunktable + i, VM_PROT_ALL);
1482 		hibernate_read_block(hiber_info, blkctr, PAGE_SIZE,
1483 		    chunktable + i);
1484 	}
1485 
1486 	blkctr = hiber_info->image_offset;
1487 	compressed_size = 0;
1488 	pmap_kenter_pa(chunktable, piglet_chunktable, VM_PROT_ALL);
1489 	chunks = (struct hibernate_disk_chunk *)chunktable;
1490 
1491 	for (i = 0; i < hiber_info->chunk_ctr; i++)
1492 		compressed_size += chunks[i].compressed_size;
1493 
1494 	disk_size = compressed_size;
1495 
1496 	/* Allocate the pig area */
1497 	pig_sz = compressed_size + HIBERNATE_CHUNK_SIZE;
1498 	if (uvm_pmr_alloc_pig(&pig_start, pig_sz) == ENOMEM)
1499 		return (1);
1500 
1501 	pig_end = pig_start + pig_sz;
1502 
1503 	/* Calculate image extents. Pig image must end on a chunk boundary. */
1504 	image_end = pig_end & ~(HIBERNATE_CHUNK_SIZE - 1);
1505 	image_start = pig_start;
1506 
1507 	image_start = image_end - disk_size;
1508 
1509 	hibernate_read_chunks(hiber_info, image_start, image_end, disk_size,
1510 	    chunks);
1511 
1512 	/* Prepare the resume time pmap/page table */
1513 	hibernate_populate_resume_pt(hiber_info, image_start, image_end);
1514 
1515 	/* Read complete, clear the signature and return */
1516 	return hibernate_clear_signature();
1517 }
1518 
1519 /*
1520  * Read the hibernated memory chunks from disk (chunk information at this
1521  * point is stored in the piglet) into the pig area specified by
1522  * [pig_start .. pig_end]. Order the chunks so that the final chunk is the
1523  * only chunk with overlap possibilities.
1524  *
1525  * This function uses the piglet area during this process as follows:
1526  *
1527  * offset from piglet base	use
1528  * -----------------------	--------------------
1529  * 0				i/o allocation area
1530  * PAGE_SIZE			i/o write area
1531  * 2*PAGE_SIZE			temp/scratch page
1532  * 3*PAGE_SIZE			temp/scratch page
1533  * 4*PAGE_SIZE to 6*PAGE_SIZE	chunk ordering area
1534  * 7*PAGE_SIZE			hiballoc arena
1535  * 8*PAGE_SIZE to 88*PAGE_SIZE	zlib deflate area
1536  * ...
1537  * HIBERNATE_CHUNK_SIZE		chunk table temporary area
1538  */
1539 int
1540 hibernate_read_chunks(union hibernate_info *hib_info, paddr_t pig_start,
1541     paddr_t pig_end, size_t image_compr_size,
1542     struct hibernate_disk_chunk *chunks)
1543 {
1544 	paddr_t img_index, img_cur, r1s, r1e, r2s, r2e;
1545 	paddr_t copy_start, copy_end, piglet_cur;
1546 	paddr_t piglet_base = hib_info->piglet_pa;
1547 	paddr_t piglet_end = piglet_base + HIBERNATE_CHUNK_SIZE;
1548 	daddr_t blkctr;
1549 	size_t processed, compressed_size, read_size;
1550 	int i, j, overlap, found, nchunks;
1551 	int nochunks = 0, nfchunks = 0, npchunks = 0;
1552 	int *ochunks, *pchunks, *fchunks;
1553 	vaddr_t tempva = (vaddr_t)NULL, hibernate_fchunk_area = (vaddr_t)NULL;
1554 
1555 	global_pig_start = pig_start;
1556 
1557 	/* XXX - dont need this on all archs */
1558 	pmap_activate(curproc);
1559 
1560 	/*
1561 	 * These mappings go into the resuming kernel's page table, and are
1562 	 * used only during image read. They dissappear from existence
1563 	 * when the suspended kernel is unpacked on top of us.
1564 	 */
1565 	tempva = (vaddr_t)km_alloc(2*PAGE_SIZE, &kv_any, &kp_none, &kd_nowait);
1566 	if (!tempva)
1567 		return (1);
1568 	hibernate_fchunk_area = (vaddr_t)km_alloc(3*PAGE_SIZE, &kv_any,
1569 	    &kp_none, &kd_nowait);
1570 	if (!hibernate_fchunk_area)
1571 		return (1);
1572 
1573 	/* Temporary output chunk ordering */
1574 	ochunks = (int *)hibernate_fchunk_area;
1575 
1576 	/* Piglet chunk ordering */
1577 	pchunks = (int *)(hibernate_fchunk_area + PAGE_SIZE);
1578 
1579 	/* Final chunk ordering */
1580 	fchunks = (int *)(hibernate_fchunk_area + (2*PAGE_SIZE));
1581 
1582 	/* Map the chunk ordering region */
1583 	pmap_kenter_pa(hibernate_fchunk_area,
1584 	    piglet_base + (4*PAGE_SIZE), VM_PROT_ALL);
1585 	pmap_kenter_pa((vaddr_t)pchunks, piglet_base + (5*PAGE_SIZE),
1586 	    VM_PROT_ALL);
1587 	pmap_kenter_pa((vaddr_t)fchunks, piglet_base + (6*PAGE_SIZE),
1588 	    VM_PROT_ALL);
1589 
1590 	nchunks = hib_info->chunk_ctr;
1591 
1592 	/* Initially start all chunks as unplaced */
1593 	for (i = 0; i < nchunks; i++)
1594 		chunks[i].flags = 0;
1595 
1596 	/*
1597 	 * Search the list for chunks that are outside the pig area. These
1598 	 * can be placed first in the final output list.
1599 	 */
1600 	for (i = 0; i < nchunks; i++) {
1601 		if (chunks[i].end <= pig_start || chunks[i].base >= pig_end) {
1602 			ochunks[nochunks] = (u_int8_t)i;
1603 			fchunks[nfchunks] = (u_int8_t)i;
1604 			nochunks++;
1605 			nfchunks++;
1606 			chunks[i].flags |= HIBERNATE_CHUNK_USED;
1607 		}
1608 	}
1609 
1610 	/*
1611 	 * Walk the ordering, place the chunks in ascending memory order.
1612 	 * Conflicts might arise, these are handled next.
1613 	 */
1614 	do {
1615 		img_index = -1;
1616 		found = 0;
1617 		j = -1;
1618 		for (i = 0; i < nchunks; i++)
1619 			if (chunks[i].base < img_index &&
1620 			    chunks[i].flags == 0 ) {
1621 				j = i;
1622 				img_index = chunks[i].base;
1623 			}
1624 
1625 		if (j != -1) {
1626 			found = 1;
1627 			ochunks[nochunks] = (short)j;
1628 			nochunks++;
1629 			chunks[j].flags |= HIBERNATE_CHUNK_PLACED;
1630 		}
1631 	} while (found);
1632 
1633 	img_index = pig_start;
1634 
1635 	/*
1636 	 * Identify chunk output conflicts (chunks whose pig load area
1637 	 * corresponds to their original memory placement location)
1638 	 */
1639 	for (i = 0; i < nochunks ; i++) {
1640 		overlap = 0;
1641 		r1s = img_index;
1642 		r1e = img_index + chunks[ochunks[i]].compressed_size;
1643 		r2s = chunks[ochunks[i]].base;
1644 		r2e = chunks[ochunks[i]].end;
1645 
1646 		overlap = hibernate_check_overlap(r1s, r1e, r2s, r2e);
1647 		if (overlap)
1648 			chunks[ochunks[i]].flags |= HIBERNATE_CHUNK_CONFLICT;
1649 		img_index += chunks[ochunks[i]].compressed_size;
1650 	}
1651 
1652 	/*
1653 	 * Prepare the final output chunk list. Calculate an output
1654 	 * inflate strategy for overlapping chunks if needed.
1655 	 */
1656 	img_index = pig_start;
1657 	for (i = 0; i < nochunks ; i++) {
1658 		/*
1659 		 * If a conflict is detected, consume enough compressed
1660 		 * output chunks to fill the piglet
1661 		 */
1662 		if (chunks[ochunks[i]].flags & HIBERNATE_CHUNK_CONFLICT) {
1663 			copy_start = piglet_base;
1664 			copy_end = piglet_end;
1665 			piglet_cur = piglet_base;
1666 			npchunks = 0;
1667 			j = i;
1668 			while (copy_start < copy_end && j < nochunks) {
1669 				piglet_cur += chunks[ochunks[j]].compressed_size;
1670 				pchunks[npchunks] = ochunks[j];
1671 				npchunks++;
1672 				copy_start += chunks[ochunks[j]].compressed_size;
1673 				img_index += chunks[ochunks[j]].compressed_size;
1674 				i++;
1675 				j++;
1676 			}
1677 
1678 			piglet_cur = piglet_base;
1679 			for (j = 0; j < npchunks; j++) {
1680 				piglet_cur += chunks[pchunks[j]].compressed_size;
1681 				fchunks[nfchunks] = pchunks[j];
1682 				chunks[pchunks[j]].flags |= HIBERNATE_CHUNK_USED;
1683 				nfchunks++;
1684 			}
1685 		} else {
1686 			/*
1687 			 * No conflict, chunk can be added without copying
1688 			 */
1689 			if ((chunks[ochunks[i]].flags &
1690 			    HIBERNATE_CHUNK_USED) == 0) {
1691 				fchunks[nfchunks] = ochunks[i];
1692 				chunks[ochunks[i]].flags |= HIBERNATE_CHUNK_USED;
1693 				nfchunks++;
1694 			}
1695 			img_index += chunks[ochunks[i]].compressed_size;
1696 		}
1697 	}
1698 
1699 	img_index = pig_start;
1700 	for (i = 0; i < nfchunks; i++) {
1701 		piglet_cur = piglet_base;
1702 		img_index += chunks[fchunks[i]].compressed_size;
1703 	}
1704 
1705 	img_cur = pig_start;
1706 
1707 	for (i = 0; i < nfchunks; i++) {
1708 		blkctr = chunks[fchunks[i]].offset - hib_info->swap_offset;
1709 		processed = 0;
1710 		compressed_size = chunks[fchunks[i]].compressed_size;
1711 
1712 		while (processed < compressed_size) {
1713 			pmap_kenter_pa(tempva, img_cur, VM_PROT_ALL);
1714 			pmap_kenter_pa(tempva + PAGE_SIZE, img_cur+PAGE_SIZE,
1715 			    VM_PROT_ALL);
1716 
1717 			/* XXX - not needed on all archs */
1718 			pmap_activate(curproc);
1719 			if (compressed_size - processed >= PAGE_SIZE)
1720 				read_size = PAGE_SIZE;
1721 			else
1722 				read_size = compressed_size - processed;
1723 
1724 			hibernate_read_block(hib_info, blkctr, read_size,
1725 			    tempva + (img_cur & PAGE_MASK));
1726 
1727 			blkctr += (read_size / hib_info->secsize);
1728 
1729 			hibernate_flush();
1730 			pmap_kremove(tempva, PAGE_SIZE);
1731 			pmap_kremove(tempva + PAGE_SIZE, PAGE_SIZE);
1732 			processed += read_size;
1733 			img_cur += read_size;
1734 		}
1735 	}
1736 
1737 	return (0);
1738 }
1739 
1740 /*
1741  * Hibernating a machine comprises the following operations:
1742  *  1. Calculating this machine's hibernate_info information
1743  *  2. Allocating a piglet and saving the piglet's physaddr
1744  *  3. Calculating the memory chunks
1745  *  4. Writing the compressed chunks to disk
1746  *  5. Writing the chunk table
1747  *  6. Writing the signature block (hibernate_info)
1748  *
1749  * On most architectures, the function calling hibernate_suspend would
1750  * then power off the machine using some MD-specific implementation.
1751  */
1752 int
1753 hibernate_suspend(void)
1754 {
1755 	union hibernate_info hib_info;
1756 
1757 	/*
1758 	 * Calculate memory ranges, swap offsets, etc.
1759 	 * This also allocates a piglet whose physaddr is stored in
1760 	 * hib_info->piglet_pa and vaddr stored in hib_info->piglet_va
1761 	 */
1762 	if (get_hibernate_info(&hib_info, 1))
1763 		return (1);
1764 
1765 	global_piglet_va = hib_info.piglet_va;
1766 
1767 	/* XXX - Won't need to zero everything with RLE */
1768 	uvm_pmr_zero_everything();
1769 
1770 	if (hibernate_write_chunks(&hib_info))
1771 		return (1);
1772 
1773 	if (hibernate_write_chunktable(&hib_info))
1774 		return (1);
1775 
1776 	if (hibernate_write_signature(&hib_info))
1777 		return (1);
1778 
1779 	delay(500000);
1780 	return (0);
1781 }
1782 
1783 /*
1784  * Free items allocated during hibernate
1785  */
1786 void
1787 hibernate_free(void)
1788 {
1789 	uvm_pmr_free_piglet(global_piglet_va, 3*HIBERNATE_CHUNK_SIZE);
1790 
1791 	pmap_kremove(hibernate_copy_page, PAGE_SIZE);
1792 	pmap_kremove(hibernate_temp_page, PAGE_SIZE);
1793 	pmap_update(pmap_kernel());
1794 
1795 	km_free((void *)hibernate_copy_page, PAGE_SIZE, &kv_any, &kp_none);
1796 	km_free((void *)hibernate_temp_page, PAGE_SIZE, &kv_any, &kp_none);
1797 }
1798