xref: /openbsd-src/sys/dev/softraid.c (revision aa997e528a848ca5596493c2a801bdd6fb26ae61)
1 /* $OpenBSD: softraid.c,v 1.391 2018/02/08 06:02:09 deraadt Exp $ */
2 /*
3  * Copyright (c) 2007, 2008, 2009 Marco Peereboom <marco@peereboom.us>
4  * Copyright (c) 2008 Chris Kuethe <ckuethe@openbsd.org>
5  * Copyright (c) 2009 Joel Sing <jsing@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include "bio.h"
21 
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/buf.h>
25 #include <sys/device.h>
26 #include <sys/ioctl.h>
27 #include <sys/malloc.h>
28 #include <sys/pool.h>
29 #include <sys/kernel.h>
30 #include <sys/disk.h>
31 #include <sys/rwlock.h>
32 #include <sys/queue.h>
33 #include <sys/fcntl.h>
34 #include <sys/disklabel.h>
35 #include <sys/vnode.h>
36 #include <sys/lock.h>
37 #include <sys/mount.h>
38 #include <sys/sensors.h>
39 #include <sys/stat.h>
40 #include <sys/conf.h>
41 #include <sys/uio.h>
42 #include <sys/task.h>
43 #include <sys/kthread.h>
44 #include <sys/dkio.h>
45 #include <sys/stdint.h>
46 
47 #include <scsi/scsi_all.h>
48 #include <scsi/scsiconf.h>
49 #include <scsi/scsi_disk.h>
50 
51 #include <dev/softraidvar.h>
52 
53 #ifdef HIBERNATE
54 #include <lib/libsa/aes_xts.h>
55 #include <sys/hibernate.h>
56 #include <scsi/sdvar.h>
57 #endif /* HIBERNATE */
58 
59 /* #define SR_FANCY_STATS */
60 
61 #ifdef SR_DEBUG
62 #define SR_FANCY_STATS
63 uint32_t	sr_debug = 0
64 		    /* | SR_D_CMD */
65 		    /* | SR_D_MISC */
66 		    /* | SR_D_INTR */
67 		    /* | SR_D_IOCTL */
68 		    /* | SR_D_CCB */
69 		    /* | SR_D_WU */
70 		    /* | SR_D_META */
71 		    /* | SR_D_DIS */
72 		    /* | SR_D_STATE */
73 		    /* | SR_D_REBUILD */
74 		;
75 #endif
76 
77 struct sr_softc	*softraid0;
78 struct sr_uuid	sr_bootuuid;
79 u_int8_t	sr_bootkey[SR_CRYPTO_MAXKEYBYTES];
80 
81 int		sr_match(struct device *, void *, void *);
82 void		sr_attach(struct device *, struct device *, void *);
83 int		sr_detach(struct device *, int);
84 void		sr_map_root(void);
85 
86 struct cfattach softraid_ca = {
87 	sizeof(struct sr_softc), sr_match, sr_attach, sr_detach,
88 };
89 
90 struct cfdriver softraid_cd = {
91 	NULL, "softraid", DV_DULL
92 };
93 
94 /* scsi & discipline */
95 void			sr_scsi_cmd(struct scsi_xfer *);
96 void			sr_minphys(struct buf *, struct scsi_link *);
97 int			sr_scsi_probe(struct scsi_link *);
98 void			sr_copy_internal_data(struct scsi_xfer *,
99 			    void *, size_t);
100 int			sr_scsi_ioctl(struct scsi_link *, u_long,
101 			    caddr_t, int);
102 int			sr_bio_ioctl(struct device *, u_long, caddr_t);
103 int			sr_bio_handler(struct sr_softc *,
104 			    struct sr_discipline *, u_long, struct bio *);
105 int			sr_ioctl_inq(struct sr_softc *, struct bioc_inq *);
106 int			sr_ioctl_vol(struct sr_softc *, struct bioc_vol *);
107 int			sr_ioctl_disk(struct sr_softc *, struct bioc_disk *);
108 int			sr_ioctl_setstate(struct sr_softc *,
109 			    struct bioc_setstate *);
110 int			sr_ioctl_createraid(struct sr_softc *,
111 			    struct bioc_createraid *, int, void *);
112 int			sr_ioctl_deleteraid(struct sr_softc *,
113 			    struct sr_discipline *, struct bioc_deleteraid *);
114 int			sr_ioctl_discipline(struct sr_softc *,
115 			    struct sr_discipline *, struct bioc_discipline *);
116 int			sr_ioctl_installboot(struct sr_softc *,
117 			    struct sr_discipline *, struct bioc_installboot *);
118 void			sr_chunks_unwind(struct sr_softc *,
119 			    struct sr_chunk_head *);
120 void			sr_discipline_free(struct sr_discipline *);
121 void			sr_discipline_shutdown(struct sr_discipline *, int, int);
122 int			sr_discipline_init(struct sr_discipline *, int);
123 int			sr_alloc_resources(struct sr_discipline *);
124 void			sr_free_resources(struct sr_discipline *);
125 void			sr_set_chunk_state(struct sr_discipline *, int, int);
126 void			sr_set_vol_state(struct sr_discipline *);
127 
128 /* utility functions */
129 void			sr_shutdown(int);
130 void			sr_uuid_generate(struct sr_uuid *);
131 char			*sr_uuid_format(struct sr_uuid *);
132 void			sr_uuid_print(struct sr_uuid *, int);
133 void			sr_checksum_print(u_int8_t *);
134 int			sr_boot_assembly(struct sr_softc *);
135 int			sr_already_assembled(struct sr_discipline *);
136 int			sr_hotspare(struct sr_softc *, dev_t);
137 void			sr_hotspare_rebuild(struct sr_discipline *);
138 int			sr_rebuild_init(struct sr_discipline *, dev_t, int);
139 void			sr_rebuild_start(void *);
140 void			sr_rebuild_thread(void *);
141 void			sr_rebuild(struct sr_discipline *);
142 void			sr_roam_chunks(struct sr_discipline *);
143 int			sr_chunk_in_use(struct sr_softc *, dev_t);
144 int			sr_rw(struct sr_softc *, dev_t, char *, size_t,
145 			    daddr_t, long);
146 void			sr_wu_done_callback(void *);
147 
148 /* don't include these on RAMDISK */
149 #ifndef SMALL_KERNEL
150 void			sr_sensors_refresh(void *);
151 int			sr_sensors_create(struct sr_discipline *);
152 void			sr_sensors_delete(struct sr_discipline *);
153 #endif
154 
155 /* metadata */
156 int			sr_meta_probe(struct sr_discipline *, dev_t *, int);
157 int			sr_meta_attach(struct sr_discipline *, int, int);
158 int			sr_meta_rw(struct sr_discipline *, dev_t, void *, long);
159 int			sr_meta_clear(struct sr_discipline *);
160 void			sr_meta_init(struct sr_discipline *, int, int);
161 void			sr_meta_init_complete(struct sr_discipline *);
162 void			sr_meta_opt_handler(struct sr_discipline *,
163 			    struct sr_meta_opt_hdr *);
164 
165 /* hotplug magic */
166 void			sr_disk_attach(struct disk *, int);
167 
168 struct sr_hotplug_list {
169 	void			(*sh_hotplug)(struct sr_discipline *,
170 				    struct disk *, int);
171 	struct sr_discipline	*sh_sd;
172 
173 	SLIST_ENTRY(sr_hotplug_list) shl_link;
174 };
175 SLIST_HEAD(sr_hotplug_list_head, sr_hotplug_list);
176 
177 struct			sr_hotplug_list_head	sr_hotplug_callbacks;
178 extern void		(*softraid_disk_attach)(struct disk *, int);
179 
180 /* scsi glue */
181 struct scsi_adapter sr_switch = {
182 	sr_scsi_cmd, sr_minphys, sr_scsi_probe, NULL, sr_scsi_ioctl
183 };
184 
185 /* native metadata format */
186 int			sr_meta_native_bootprobe(struct sr_softc *, dev_t,
187 			    struct sr_boot_chunk_head *);
188 #define SR_META_NOTCLAIMED	(0)
189 #define SR_META_CLAIMED		(1)
190 int			sr_meta_native_probe(struct sr_softc *,
191 			   struct sr_chunk *);
192 int			sr_meta_native_attach(struct sr_discipline *, int);
193 int			sr_meta_native_write(struct sr_discipline *, dev_t,
194 			    struct sr_metadata *,void *);
195 
196 #ifdef SR_DEBUG
197 void			sr_meta_print(struct sr_metadata *);
198 #else
199 #define			sr_meta_print(m)
200 #endif
201 
202 /* the metadata driver should remain stateless */
203 struct sr_meta_driver {
204 	daddr_t			smd_offset;	/* metadata location */
205 	u_int32_t		smd_size;	/* size of metadata */
206 
207 	int			(*smd_probe)(struct sr_softc *,
208 				   struct sr_chunk *);
209 	int			(*smd_attach)(struct sr_discipline *, int);
210 	int			(*smd_detach)(struct sr_discipline *);
211 	int			(*smd_read)(struct sr_discipline *, dev_t,
212 				    struct sr_metadata *, void *);
213 	int			(*smd_write)(struct sr_discipline *, dev_t,
214 				    struct sr_metadata *, void *);
215 	int			(*smd_validate)(struct sr_discipline *,
216 				    struct sr_metadata *, void *);
217 } smd[] = {
218 	{ SR_META_OFFSET, SR_META_SIZE * DEV_BSIZE,
219 	  sr_meta_native_probe, sr_meta_native_attach, NULL,
220 	  sr_meta_native_read, sr_meta_native_write, NULL },
221 	{ 0, 0, NULL, NULL, NULL, NULL }
222 };
223 
224 int
225 sr_meta_attach(struct sr_discipline *sd, int chunk_no, int force)
226 {
227 	struct sr_softc		*sc = sd->sd_sc;
228 	struct sr_chunk_head	*cl;
229 	struct sr_chunk		*ch_entry, *chunk1, *chunk2;
230 	int			rv = 1, i = 0;
231 
232 	DNPRINTF(SR_D_META, "%s: sr_meta_attach(%d)\n", DEVNAME(sc), chunk_no);
233 
234 	/* in memory copy of metadata */
235 	sd->sd_meta = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF,
236 	    M_ZERO | M_NOWAIT);
237 	if (!sd->sd_meta) {
238 		sr_error(sc, "could not allocate memory for metadata");
239 		goto bad;
240 	}
241 
242 	if (sd->sd_meta_type != SR_META_F_NATIVE) {
243 		/* in memory copy of foreign metadata */
244 		sd->sd_meta_foreign = malloc(smd[sd->sd_meta_type].smd_size,
245 		    M_DEVBUF, M_ZERO | M_NOWAIT);
246 		if (!sd->sd_meta_foreign) {
247 			/* unwind frees sd_meta */
248 			sr_error(sc, "could not allocate memory for foreign "
249 			    "metadata");
250 			goto bad;
251 		}
252 	}
253 
254 	/* we have a valid list now create an array index */
255 	cl = &sd->sd_vol.sv_chunk_list;
256 	sd->sd_vol.sv_chunks = mallocarray(chunk_no, sizeof(struct sr_chunk *),
257 	    M_DEVBUF, M_WAITOK | M_ZERO);
258 
259 	/* fill out chunk array */
260 	i = 0;
261 	SLIST_FOREACH(ch_entry, cl, src_link)
262 		sd->sd_vol.sv_chunks[i++] = ch_entry;
263 
264 	/* attach metadata */
265 	if (smd[sd->sd_meta_type].smd_attach(sd, force))
266 		goto bad;
267 
268 	/* Force chunks into correct order now that metadata is attached. */
269 	SLIST_INIT(cl);
270 	for (i = 0; i < chunk_no; i++) {
271 		ch_entry = sd->sd_vol.sv_chunks[i];
272 		chunk2 = NULL;
273 		SLIST_FOREACH(chunk1, cl, src_link) {
274 			if (chunk1->src_meta.scmi.scm_chunk_id >
275 			    ch_entry->src_meta.scmi.scm_chunk_id)
276 				break;
277 			chunk2 = chunk1;
278 		}
279 		if (chunk2 == NULL)
280 			SLIST_INSERT_HEAD(cl, ch_entry, src_link);
281 		else
282 			SLIST_INSERT_AFTER(chunk2, ch_entry, src_link);
283 	}
284 	i = 0;
285 	SLIST_FOREACH(ch_entry, cl, src_link)
286 		sd->sd_vol.sv_chunks[i++] = ch_entry;
287 
288 	rv = 0;
289 bad:
290 	return (rv);
291 }
292 
293 int
294 sr_meta_probe(struct sr_discipline *sd, dev_t *dt, int no_chunk)
295 {
296 	struct sr_softc		*sc = sd->sd_sc;
297 	struct vnode		*vn;
298 	struct sr_chunk		*ch_entry, *ch_prev = NULL;
299 	struct sr_chunk_head	*cl;
300 	char			devname[32];
301 	int			i, d, type, found, prevf, error;
302 	dev_t			dev;
303 
304 	DNPRINTF(SR_D_META, "%s: sr_meta_probe(%d)\n", DEVNAME(sc), no_chunk);
305 
306 	if (no_chunk == 0)
307 		goto unwind;
308 
309 	cl = &sd->sd_vol.sv_chunk_list;
310 
311 	for (d = 0, prevf = SR_META_F_INVALID; d < no_chunk; d++) {
312 		ch_entry = malloc(sizeof(struct sr_chunk), M_DEVBUF,
313 		    M_WAITOK | M_ZERO);
314 		/* keep disks in user supplied order */
315 		if (ch_prev)
316 			SLIST_INSERT_AFTER(ch_prev, ch_entry, src_link);
317 		else
318 			SLIST_INSERT_HEAD(cl, ch_entry, src_link);
319 		ch_prev = ch_entry;
320 		dev = dt[d];
321 		ch_entry->src_dev_mm = dev;
322 
323 		if (dev == NODEV) {
324 			ch_entry->src_meta.scm_status = BIOC_SDOFFLINE;
325 			continue;
326 		} else {
327 			sr_meta_getdevname(sc, dev, devname, sizeof(devname));
328 			if (bdevvp(dev, &vn)) {
329 				sr_error(sc, "sr_meta_probe: cannot allocate "
330 				    "vnode");
331 				goto unwind;
332 			}
333 
334 			/*
335 			 * XXX leaving dev open for now; move this to attach
336 			 * and figure out the open/close dance for unwind.
337 			 */
338 			error = VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc);
339 			if (error) {
340 				DNPRINTF(SR_D_META,"%s: sr_meta_probe can't "
341 				    "open %s\n", DEVNAME(sc), devname);
342 				vput(vn);
343 				goto unwind;
344 			}
345 
346 			strlcpy(ch_entry->src_devname, devname,
347 			    sizeof(ch_entry->src_devname));
348 			ch_entry->src_vn = vn;
349 		}
350 
351 		/* determine if this is a device we understand */
352 		for (i = 0, found = SR_META_F_INVALID; smd[i].smd_probe; i++) {
353 			type = smd[i].smd_probe(sc, ch_entry);
354 			if (type == SR_META_F_INVALID)
355 				continue;
356 			else {
357 				found = type;
358 				break;
359 			}
360 		}
361 
362 		if (found == SR_META_F_INVALID)
363 			goto unwind;
364 		if (prevf == SR_META_F_INVALID)
365 			prevf = found;
366 		if (prevf != found) {
367 			DNPRINTF(SR_D_META, "%s: prevf != found\n",
368 			    DEVNAME(sc));
369 			goto unwind;
370 		}
371 	}
372 
373 	return (prevf);
374 unwind:
375 	return (SR_META_F_INVALID);
376 }
377 
378 void
379 sr_meta_getdevname(struct sr_softc *sc, dev_t dev, char *buf, int size)
380 {
381 	int			maj, unit, part;
382 	char			*name;
383 
384 	DNPRINTF(SR_D_META, "%s: sr_meta_getdevname(%p, %d)\n",
385 	    DEVNAME(sc), buf, size);
386 
387 	if (!buf)
388 		return;
389 
390 	maj = major(dev);
391 	part = DISKPART(dev);
392 	unit = DISKUNIT(dev);
393 
394 	name = findblkname(maj);
395 	if (name == NULL)
396 		return;
397 
398 	snprintf(buf, size, "%s%d%c", name, unit, part + 'a');
399 }
400 
401 int
402 sr_rw(struct sr_softc *sc, dev_t dev, char *buf, size_t size, daddr_t blkno,
403     long flags)
404 {
405 	struct vnode		*vp;
406 	struct buf		b;
407 	size_t			bufsize, dma_bufsize;
408 	int			rv = 1;
409 	char			*dma_buf;
410 
411 	DNPRINTF(SR_D_MISC, "%s: sr_rw(0x%x, %p, %zu, %lld 0x%lx)\n",
412 	    DEVNAME(sc), dev, buf, size, (long long)blkno, flags);
413 
414 	dma_bufsize = (size > MAXPHYS) ? MAXPHYS : size;
415 	dma_buf = dma_alloc(dma_bufsize, PR_WAITOK);
416 
417 	if (bdevvp(dev, &vp)) {
418 		printf("%s: sr_rw: failed to allocate vnode\n", DEVNAME(sc));
419 		goto done;
420 	}
421 
422 	while (size > 0) {
423 		DNPRINTF(SR_D_MISC, "%s: dma_buf %p, size %zu, blkno %lld)\n",
424 		    DEVNAME(sc), dma_buf, size, (long long)blkno);
425 
426 		bufsize = (size > MAXPHYS) ? MAXPHYS : size;
427 		if (flags == B_WRITE)
428 			memcpy(dma_buf, buf, bufsize);
429 
430 		bzero(&b, sizeof(b));
431 		b.b_flags = flags | B_PHYS;
432 		b.b_proc = curproc;
433 		b.b_dev = dev;
434 		b.b_iodone = NULL;
435 		b.b_error = 0;
436 		b.b_blkno = blkno;
437 		b.b_data = dma_buf;
438 		b.b_bcount = bufsize;
439 		b.b_bufsize = bufsize;
440 		b.b_resid = bufsize;
441 		b.b_vp = vp;
442 
443 		if ((b.b_flags & B_READ) == 0)
444 			vp->v_numoutput++;
445 
446 		LIST_INIT(&b.b_dep);
447 		VOP_STRATEGY(&b);
448 		biowait(&b);
449 
450 		if (b.b_flags & B_ERROR) {
451 			printf("%s: I/O error %d on dev 0x%x at block %llu\n",
452 			    DEVNAME(sc), b.b_error, dev, b.b_blkno);
453 			goto done;
454 		}
455 
456 		if (flags == B_READ)
457 			memcpy(buf, dma_buf, bufsize);
458 
459 		size -= bufsize;
460 		buf += bufsize;
461 		blkno += howmany(bufsize, DEV_BSIZE);
462 	}
463 
464 	rv = 0;
465 
466 done:
467 	if (vp)
468 		vput(vp);
469 
470 	dma_free(dma_buf, dma_bufsize);
471 
472 	return (rv);
473 }
474 
475 int
476 sr_meta_rw(struct sr_discipline *sd, dev_t dev, void *md, long flags)
477 {
478 	int			rv = 1;
479 
480 	DNPRINTF(SR_D_META, "%s: sr_meta_rw(0x%x, %p, 0x%lx)\n",
481 	    DEVNAME(sd->sd_sc), dev, md, flags);
482 
483 	if (md == NULL) {
484 		printf("%s: sr_meta_rw: invalid metadata pointer\n",
485 		    DEVNAME(sd->sd_sc));
486 		goto done;
487 	}
488 
489 	rv = sr_rw(sd->sd_sc, dev, md, SR_META_SIZE * DEV_BSIZE,
490 	    SR_META_OFFSET, flags);
491 
492 done:
493 	return (rv);
494 }
495 
496 int
497 sr_meta_clear(struct sr_discipline *sd)
498 {
499 	struct sr_softc		*sc = sd->sd_sc;
500 	struct sr_chunk_head	*cl = &sd->sd_vol.sv_chunk_list;
501 	struct sr_chunk		*ch_entry;
502 	void			*m;
503 	int			rv = 1;
504 
505 	DNPRINTF(SR_D_META, "%s: sr_meta_clear\n", DEVNAME(sc));
506 
507 	if (sd->sd_meta_type != SR_META_F_NATIVE) {
508 		sr_error(sc, "cannot clear foreign metadata");
509 		goto done;
510 	}
511 
512 	m = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_WAITOK | M_ZERO);
513 	SLIST_FOREACH(ch_entry, cl, src_link) {
514 		if (sr_meta_native_write(sd, ch_entry->src_dev_mm, m, NULL)) {
515 			/* XXX mark disk offline */
516 			DNPRINTF(SR_D_META, "%s: sr_meta_clear failed to "
517 			    "clear %s\n", DEVNAME(sc), ch_entry->src_devname);
518 			rv++;
519 			continue;
520 		}
521 		bzero(&ch_entry->src_meta, sizeof(ch_entry->src_meta));
522 	}
523 
524 	bzero(sd->sd_meta, SR_META_SIZE * DEV_BSIZE);
525 
526 	free(m, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
527 	rv = 0;
528 done:
529 	return (rv);
530 }
531 
532 void
533 sr_meta_init(struct sr_discipline *sd, int level, int no_chunk)
534 {
535 	struct sr_softc		*sc = sd->sd_sc;
536 	struct sr_metadata	*sm = sd->sd_meta;
537 	struct sr_chunk_head	*cl = &sd->sd_vol.sv_chunk_list;
538 	struct sr_meta_chunk	*scm;
539 	struct sr_chunk		*chunk;
540 	int			cid = 0;
541 	u_int64_t		max_chunk_sz = 0, min_chunk_sz = 0;
542 	u_int32_t		secsize = DEV_BSIZE;
543 
544 	DNPRINTF(SR_D_META, "%s: sr_meta_init\n", DEVNAME(sc));
545 
546 	if (!sm)
547 		return;
548 
549 	/* Initialise volume metadata. */
550 	sm->ssdi.ssd_magic = SR_MAGIC;
551 	sm->ssdi.ssd_version = SR_META_VERSION;
552 	sm->ssdi.ssd_vol_flags = sd->sd_meta_flags;
553 	sm->ssdi.ssd_volid = 0;
554 	sm->ssdi.ssd_chunk_no = no_chunk;
555 	sm->ssdi.ssd_level = level;
556 
557 	sm->ssd_data_blkno = SR_DATA_OFFSET;
558 	sm->ssd_ondisk = 0;
559 
560 	sr_uuid_generate(&sm->ssdi.ssd_uuid);
561 
562 	/* Initialise chunk metadata and get min/max chunk sizes & secsize. */
563 	SLIST_FOREACH(chunk, cl, src_link) {
564 		scm = &chunk->src_meta;
565 		scm->scmi.scm_size = chunk->src_size;
566 		scm->scmi.scm_chunk_id = cid++;
567 		scm->scm_status = BIOC_SDONLINE;
568 		scm->scmi.scm_volid = 0;
569 		strlcpy(scm->scmi.scm_devname, chunk->src_devname,
570 		    sizeof(scm->scmi.scm_devname));
571 		memcpy(&scm->scmi.scm_uuid, &sm->ssdi.ssd_uuid,
572 		    sizeof(scm->scmi.scm_uuid));
573 		sr_checksum(sc, scm, &scm->scm_checksum,
574 		    sizeof(scm->scm_checksum));
575 
576 		if (min_chunk_sz == 0)
577 			min_chunk_sz = scm->scmi.scm_size;
578 		if (chunk->src_secsize > secsize)
579 			secsize = chunk->src_secsize;
580 		min_chunk_sz = MIN(min_chunk_sz, scm->scmi.scm_size);
581 		max_chunk_sz = MAX(max_chunk_sz, scm->scmi.scm_size);
582 	}
583 
584 	sm->ssdi.ssd_secsize = secsize;
585 
586 	/* Equalize chunk sizes. */
587 	SLIST_FOREACH(chunk, cl, src_link)
588 		chunk->src_meta.scmi.scm_coerced_size = min_chunk_sz;
589 
590 	sd->sd_vol.sv_chunk_minsz = min_chunk_sz;
591 	sd->sd_vol.sv_chunk_maxsz = max_chunk_sz;
592 }
593 
594 void
595 sr_meta_init_complete(struct sr_discipline *sd)
596 {
597 #ifdef SR_DEBUG
598 	struct sr_softc		*sc = sd->sd_sc;
599 #endif
600 	struct sr_metadata	*sm = sd->sd_meta;
601 
602 	DNPRINTF(SR_D_META, "%s: sr_meta_complete\n", DEVNAME(sc));
603 
604 	/* Complete initialisation of volume metadata. */
605 	strlcpy(sm->ssdi.ssd_vendor, "OPENBSD", sizeof(sm->ssdi.ssd_vendor));
606 	snprintf(sm->ssdi.ssd_product, sizeof(sm->ssdi.ssd_product),
607 	    "SR %s", sd->sd_name);
608 	snprintf(sm->ssdi.ssd_revision, sizeof(sm->ssdi.ssd_revision),
609 	    "%03d", sm->ssdi.ssd_version);
610 }
611 
612 void
613 sr_meta_opt_handler(struct sr_discipline *sd, struct sr_meta_opt_hdr *om)
614 {
615 	if (om->som_type != SR_OPT_BOOT)
616 		panic("unknown optional metadata type");
617 }
618 
619 void
620 sr_meta_save_callback(void *xsd)
621 {
622 	struct sr_discipline	*sd = xsd;
623 	int			s;
624 
625 	s = splbio();
626 
627 	if (sr_meta_save(sd, SR_META_DIRTY))
628 		printf("%s: save metadata failed\n", DEVNAME(sd->sd_sc));
629 
630 	sd->sd_must_flush = 0;
631 	splx(s);
632 }
633 
634 int
635 sr_meta_save(struct sr_discipline *sd, u_int32_t flags)
636 {
637 	struct sr_softc		*sc = sd->sd_sc;
638 	struct sr_metadata	*sm = sd->sd_meta, *m;
639 	struct sr_meta_driver	*s;
640 	struct sr_chunk		*src;
641 	struct sr_meta_chunk	*cm;
642 	struct sr_workunit	wu;
643 	struct sr_meta_opt_hdr	*omh;
644 	struct sr_meta_opt_item *omi;
645 	int			i;
646 
647 	DNPRINTF(SR_D_META, "%s: sr_meta_save %s\n",
648 	    DEVNAME(sc), sd->sd_meta->ssd_devname);
649 
650 	if (!sm) {
651 		printf("%s: no in memory copy of metadata\n", DEVNAME(sc));
652 		goto bad;
653 	}
654 
655 	/* meta scratchpad */
656 	s = &smd[sd->sd_meta_type];
657 	m = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_ZERO | M_NOWAIT);
658 	if (!m) {
659 		printf("%s: could not allocate metadata scratch area\n",
660 		    DEVNAME(sc));
661 		goto bad;
662 	}
663 
664 	/* from here on out metadata is updated */
665 restart:
666 	sm->ssd_ondisk++;
667 	sm->ssd_meta_flags = flags;
668 	memcpy(m, sm, sizeof(*m));
669 
670 	/* Chunk metadata. */
671 	cm = (struct sr_meta_chunk *)(m + 1);
672 	for (i = 0; i < sm->ssdi.ssd_chunk_no; i++) {
673 		src = sd->sd_vol.sv_chunks[i];
674 		memcpy(cm, &src->src_meta, sizeof(*cm));
675 		cm++;
676 	}
677 
678 	/* Optional metadata. */
679 	omh = (struct sr_meta_opt_hdr *)(cm);
680 	SLIST_FOREACH(omi, &sd->sd_meta_opt, omi_link) {
681 		DNPRINTF(SR_D_META, "%s: saving optional metadata type %u with "
682 		    "length %u\n", DEVNAME(sc), omi->omi_som->som_type,
683 		    omi->omi_som->som_length);
684 		bzero(&omi->omi_som->som_checksum, MD5_DIGEST_LENGTH);
685 		sr_checksum(sc, omi->omi_som, &omi->omi_som->som_checksum,
686 		    omi->omi_som->som_length);
687 		memcpy(omh, omi->omi_som, omi->omi_som->som_length);
688 		omh = (struct sr_meta_opt_hdr *)((u_int8_t *)omh +
689 		    omi->omi_som->som_length);
690 	}
691 
692 	for (i = 0; i < sm->ssdi.ssd_chunk_no; i++) {
693 		src = sd->sd_vol.sv_chunks[i];
694 
695 		/* skip disks that are offline */
696 		if (src->src_meta.scm_status == BIOC_SDOFFLINE)
697 			continue;
698 
699 		/* calculate metadata checksum for correct chunk */
700 		m->ssdi.ssd_chunk_id = i;
701 		sr_checksum(sc, m, &m->ssd_checksum,
702 		    sizeof(struct sr_meta_invariant));
703 
704 #ifdef SR_DEBUG
705 		DNPRINTF(SR_D_META, "%s: sr_meta_save %s: volid: %d "
706 		    "chunkid: %d checksum: ",
707 		    DEVNAME(sc), src->src_meta.scmi.scm_devname,
708 		    m->ssdi.ssd_volid, m->ssdi.ssd_chunk_id);
709 
710 		if (sr_debug & SR_D_META)
711 			sr_checksum_print((u_int8_t *)&m->ssd_checksum);
712 		DNPRINTF(SR_D_META, "\n");
713 		sr_meta_print(m);
714 #endif
715 
716 		/* translate and write to disk */
717 		if (s->smd_write(sd, src->src_dev_mm, m, NULL /* XXX */)) {
718 			printf("%s: could not write metadata to %s\n",
719 			    DEVNAME(sc), src->src_devname);
720 			/* restart the meta write */
721 			src->src_meta.scm_status = BIOC_SDOFFLINE;
722 			/* XXX recalculate volume status */
723 			goto restart;
724 		}
725 	}
726 
727 	/* not all disciplines have sync */
728 	if (sd->sd_scsi_sync) {
729 		bzero(&wu, sizeof(wu));
730 		wu.swu_flags |= SR_WUF_FAKE;
731 		wu.swu_dis = sd;
732 		sd->sd_scsi_sync(&wu);
733 	}
734 	free(m, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
735 	return (0);
736 bad:
737 	return (1);
738 }
739 
740 int
741 sr_meta_read(struct sr_discipline *sd)
742 {
743 	struct sr_softc		*sc = sd->sd_sc;
744 	struct sr_chunk_head	*cl = &sd->sd_vol.sv_chunk_list;
745 	struct sr_metadata	*sm;
746 	struct sr_chunk		*ch_entry;
747 	struct sr_meta_chunk	*cp;
748 	struct sr_meta_driver	*s;
749 	void			*fm = NULL;
750 	int			no_disk = 0, got_meta = 0;
751 
752 	DNPRINTF(SR_D_META, "%s: sr_meta_read\n", DEVNAME(sc));
753 
754 	sm = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_WAITOK | M_ZERO);
755 	s = &smd[sd->sd_meta_type];
756 	if (sd->sd_meta_type != SR_META_F_NATIVE)
757 		fm = malloc(s->smd_size, M_DEVBUF, M_WAITOK | M_ZERO);
758 
759 	cp = (struct sr_meta_chunk *)(sm + 1);
760 	SLIST_FOREACH(ch_entry, cl, src_link) {
761 		/* skip disks that are offline */
762 		if (ch_entry->src_meta.scm_status == BIOC_SDOFFLINE) {
763 			DNPRINTF(SR_D_META,
764 			    "%s: %s chunk marked offline, spoofing status\n",
765 			    DEVNAME(sc), ch_entry->src_devname);
766 			cp++; /* adjust chunk pointer to match failure */
767 			continue;
768 		} else if (s->smd_read(sd, ch_entry->src_dev_mm, sm, fm)) {
769 			/* read and translate */
770 			/* XXX mark chunk offline, elsewhere!! */
771 			ch_entry->src_meta.scm_status = BIOC_SDOFFLINE;
772 			cp++; /* adjust chunk pointer to match failure */
773 			DNPRINTF(SR_D_META, "%s: sr_meta_read failed\n",
774 			    DEVNAME(sc));
775 			continue;
776 		}
777 
778 		if (sm->ssdi.ssd_magic != SR_MAGIC) {
779 			DNPRINTF(SR_D_META, "%s: sr_meta_read !SR_MAGIC\n",
780 			    DEVNAME(sc));
781 			continue;
782 		}
783 
784 		/* validate metadata */
785 		if (sr_meta_validate(sd, ch_entry->src_dev_mm, sm, fm)) {
786 			DNPRINTF(SR_D_META, "%s: invalid metadata\n",
787 			    DEVNAME(sc));
788 			no_disk = -1;
789 			goto done;
790 		}
791 
792 		/* assume first chunk contains metadata */
793 		if (got_meta == 0) {
794 			sr_meta_opt_load(sc, sm, &sd->sd_meta_opt);
795 			memcpy(sd->sd_meta, sm, sizeof(*sd->sd_meta));
796 			got_meta = 1;
797 		}
798 
799 		memcpy(&ch_entry->src_meta, cp, sizeof(ch_entry->src_meta));
800 
801 		no_disk++;
802 		cp++;
803 	}
804 
805 	free(sm, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
806 	free(fm, M_DEVBUF, s->smd_size);
807 
808 done:
809 	DNPRINTF(SR_D_META, "%s: sr_meta_read found %d parts\n", DEVNAME(sc),
810 	    no_disk);
811 	return (no_disk);
812 }
813 
814 void
815 sr_meta_opt_load(struct sr_softc *sc, struct sr_metadata *sm,
816     struct sr_meta_opt_head *som)
817 {
818 	struct sr_meta_opt_hdr	*omh;
819 	struct sr_meta_opt_item *omi;
820 	u_int8_t		checksum[MD5_DIGEST_LENGTH];
821 	int			i;
822 
823 	/* Process optional metadata. */
824 	omh = (struct sr_meta_opt_hdr *)((u_int8_t *)(sm + 1) +
825 	    sizeof(struct sr_meta_chunk) * sm->ssdi.ssd_chunk_no);
826 	for (i = 0; i < sm->ssdi.ssd_opt_no; i++) {
827 
828 		omi = malloc(sizeof(struct sr_meta_opt_item), M_DEVBUF,
829 		    M_WAITOK | M_ZERO);
830 		SLIST_INSERT_HEAD(som, omi, omi_link);
831 
832 		if (omh->som_length == 0) {
833 
834 			/* Load old fixed length optional metadata. */
835 			DNPRINTF(SR_D_META, "%s: old optional metadata of type "
836 			    "%u\n", DEVNAME(sc), omh->som_type);
837 
838 			/* Validate checksum. */
839 			sr_checksum(sc, (void *)omh, &checksum,
840 			    SR_OLD_META_OPT_SIZE - MD5_DIGEST_LENGTH);
841 			if (bcmp(&checksum, (void *)omh + SR_OLD_META_OPT_MD5,
842 			    sizeof(checksum)))
843 				panic("%s: invalid optional metadata "
844 				    "checksum", DEVNAME(sc));
845 
846 			/* Determine correct length. */
847 			switch (omh->som_type) {
848 			case SR_OPT_CRYPTO:
849 				omh->som_length = sizeof(struct sr_meta_crypto);
850 				break;
851 			case SR_OPT_BOOT:
852 				omh->som_length = sizeof(struct sr_meta_boot);
853 				break;
854 			case SR_OPT_KEYDISK:
855 				omh->som_length =
856 				    sizeof(struct sr_meta_keydisk);
857 				break;
858 			default:
859 				panic("unknown old optional metadata "
860 				    "type %u\n", omh->som_type);
861 			}
862 
863 			omi->omi_som = malloc(omh->som_length, M_DEVBUF,
864 			    M_WAITOK | M_ZERO);
865 			memcpy((u_int8_t *)omi->omi_som + sizeof(*omi->omi_som),
866 			    (u_int8_t *)omh + SR_OLD_META_OPT_OFFSET,
867 			    omh->som_length - sizeof(*omi->omi_som));
868 			omi->omi_som->som_type = omh->som_type;
869 			omi->omi_som->som_length = omh->som_length;
870 
871 			omh = (struct sr_meta_opt_hdr *)((void *)omh +
872 			    SR_OLD_META_OPT_SIZE);
873 		} else {
874 
875 			/* Load variable length optional metadata. */
876 			DNPRINTF(SR_D_META, "%s: optional metadata of type %u, "
877 			    "length %u\n", DEVNAME(sc), omh->som_type,
878 			    omh->som_length);
879 			omi->omi_som = malloc(omh->som_length, M_DEVBUF,
880 			    M_WAITOK | M_ZERO);
881 			memcpy(omi->omi_som, omh, omh->som_length);
882 
883 			/* Validate checksum. */
884 			memcpy(&checksum, &omi->omi_som->som_checksum,
885 			    MD5_DIGEST_LENGTH);
886 			bzero(&omi->omi_som->som_checksum, MD5_DIGEST_LENGTH);
887 			sr_checksum(sc, omi->omi_som,
888 			    &omi->omi_som->som_checksum, omh->som_length);
889 			if (bcmp(&checksum, &omi->omi_som->som_checksum,
890 			    sizeof(checksum)))
891 				panic("%s: invalid optional metadata checksum",
892 				    DEVNAME(sc));
893 
894 			omh = (struct sr_meta_opt_hdr *)((void *)omh +
895 			    omh->som_length);
896 		}
897 	}
898 }
899 
900 int
901 sr_meta_validate(struct sr_discipline *sd, dev_t dev, struct sr_metadata *sm,
902     void *fm)
903 {
904 	struct sr_softc		*sc = sd->sd_sc;
905 	struct sr_meta_driver	*s;
906 #ifdef SR_DEBUG
907 	struct sr_meta_chunk	*mc;
908 #endif
909 	u_int8_t		checksum[MD5_DIGEST_LENGTH];
910 	char			devname[32];
911 	int			rv = 1;
912 
913 	DNPRINTF(SR_D_META, "%s: sr_meta_validate(%p)\n", DEVNAME(sc), sm);
914 
915 	sr_meta_getdevname(sc, dev, devname, sizeof(devname));
916 
917 	s = &smd[sd->sd_meta_type];
918 	if (sd->sd_meta_type != SR_META_F_NATIVE)
919 		if (s->smd_validate(sd, sm, fm)) {
920 			sr_error(sc, "invalid foreign metadata");
921 			goto done;
922 		}
923 
924 	/*
925 	 * at this point all foreign metadata has been translated to the native
926 	 * format and will be treated just like the native format
927 	 */
928 
929 	if (sm->ssdi.ssd_magic != SR_MAGIC) {
930 		sr_error(sc, "not valid softraid metadata");
931 		goto done;
932 	}
933 
934 	/* Verify metadata checksum. */
935 	sr_checksum(sc, sm, &checksum, sizeof(struct sr_meta_invariant));
936 	if (bcmp(&checksum, &sm->ssd_checksum, sizeof(checksum))) {
937 		sr_error(sc, "invalid metadata checksum");
938 		goto done;
939 	}
940 
941 	/* Handle changes between versions. */
942 	if (sm->ssdi.ssd_version == 3) {
943 
944 		/*
945 		 * Version 3 - update metadata version and fix up data blkno
946 		 * value since this did not exist in version 3.
947 		 */
948 		if (sm->ssd_data_blkno == 0)
949 			sm->ssd_data_blkno = SR_META_V3_DATA_OFFSET;
950 		sm->ssdi.ssd_secsize = DEV_BSIZE;
951 
952 	} else if (sm->ssdi.ssd_version == 4) {
953 
954 		/*
955 		 * Version 4 - original metadata format did not store
956 		 * data blkno so fix this up if necessary.
957 		 */
958 		if (sm->ssd_data_blkno == 0)
959 			sm->ssd_data_blkno = SR_DATA_OFFSET;
960 		sm->ssdi.ssd_secsize = DEV_BSIZE;
961 
962 	} else if (sm->ssdi.ssd_version == 5) {
963 
964 		/*
965 		 * Version 5 - variable length optional metadata. Migration
966 		 * from earlier fixed length optional metadata is handled
967 		 * in sr_meta_read().
968 		 */
969 		sm->ssdi.ssd_secsize = DEV_BSIZE;
970 
971 	} else if (sm->ssdi.ssd_version == SR_META_VERSION) {
972 
973 		/*
974 		 * Version 6 - store & report a sector size.
975 		 */
976 
977 	} else {
978 
979 		sr_error(sc, "cannot read metadata version %u on %s, "
980 		    "expected version %u or earlier",
981 		    sm->ssdi.ssd_version, devname, SR_META_VERSION);
982 		goto done;
983 
984 	}
985 
986 	/* Update version number and revision string. */
987 	sm->ssdi.ssd_version = SR_META_VERSION;
988 	snprintf(sm->ssdi.ssd_revision, sizeof(sm->ssdi.ssd_revision),
989 	    "%03d", SR_META_VERSION);
990 
991 #ifdef SR_DEBUG
992 	/* warn if disk changed order */
993 	mc = (struct sr_meta_chunk *)(sm + 1);
994 	if (strncmp(mc[sm->ssdi.ssd_chunk_id].scmi.scm_devname, devname,
995 	    sizeof(mc[sm->ssdi.ssd_chunk_id].scmi.scm_devname)))
996 		DNPRINTF(SR_D_META, "%s: roaming device %s -> %s\n",
997 		    DEVNAME(sc), mc[sm->ssdi.ssd_chunk_id].scmi.scm_devname,
998 		    devname);
999 #endif
1000 
1001 	/* we have meta data on disk */
1002 	DNPRINTF(SR_D_META, "%s: sr_meta_validate valid metadata %s\n",
1003 	    DEVNAME(sc), devname);
1004 
1005 	rv = 0;
1006 done:
1007 	return (rv);
1008 }
1009 
1010 int
1011 sr_meta_native_bootprobe(struct sr_softc *sc, dev_t devno,
1012     struct sr_boot_chunk_head *bch)
1013 {
1014 	struct vnode		*vn;
1015 	struct disklabel	label;
1016 	struct sr_metadata	*md = NULL;
1017 	struct sr_discipline	*fake_sd = NULL;
1018 	struct sr_boot_chunk	*bc;
1019 	char			devname[32];
1020 	dev_t			chrdev, rawdev;
1021 	int			error, i;
1022 	int			rv = SR_META_NOTCLAIMED;
1023 
1024 	DNPRINTF(SR_D_META, "%s: sr_meta_native_bootprobe\n", DEVNAME(sc));
1025 
1026 	/*
1027 	 * Use character raw device to avoid SCSI complaints about missing
1028 	 * media on removable media devices.
1029 	 */
1030 	chrdev = blktochr(devno);
1031 	rawdev = MAKEDISKDEV(major(chrdev), DISKUNIT(devno), RAW_PART);
1032 	if (cdevvp(rawdev, &vn)) {
1033 		sr_error(sc, "sr_meta_native_bootprobe: cannot allocate vnode");
1034 		goto done;
1035 	}
1036 
1037 	/* open device */
1038 	error = VOP_OPEN(vn, FREAD, NOCRED, curproc);
1039 	if (error) {
1040 		DNPRINTF(SR_D_META, "%s: sr_meta_native_bootprobe open "
1041 		    "failed\n", DEVNAME(sc));
1042 		vput(vn);
1043 		goto done;
1044 	}
1045 
1046 	/* get disklabel */
1047 	error = VOP_IOCTL(vn, DIOCGDINFO, (caddr_t)&label, FREAD, NOCRED,
1048 	    curproc);
1049 	if (error) {
1050 		DNPRINTF(SR_D_META, "%s: sr_meta_native_bootprobe ioctl "
1051 		    "failed\n", DEVNAME(sc));
1052 		VOP_CLOSE(vn, FREAD, NOCRED, curproc);
1053 		vput(vn);
1054 		goto done;
1055 	}
1056 
1057 	/* we are done, close device */
1058 	error = VOP_CLOSE(vn, FREAD, NOCRED, curproc);
1059 	if (error) {
1060 		DNPRINTF(SR_D_META, "%s: sr_meta_native_bootprobe close "
1061 		    "failed\n", DEVNAME(sc));
1062 		vput(vn);
1063 		goto done;
1064 	}
1065 	vput(vn);
1066 
1067 	md = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_ZERO | M_NOWAIT);
1068 	if (md == NULL) {
1069 		sr_error(sc, "not enough memory for metadata buffer");
1070 		goto done;
1071 	}
1072 
1073 	/* create fake sd to use utility functions */
1074 	fake_sd = malloc(sizeof(struct sr_discipline), M_DEVBUF,
1075 	    M_ZERO | M_NOWAIT);
1076 	if (fake_sd == NULL) {
1077 		sr_error(sc, "not enough memory for fake discipline");
1078 		goto done;
1079 	}
1080 	fake_sd->sd_sc = sc;
1081 	fake_sd->sd_meta_type = SR_META_F_NATIVE;
1082 
1083 	for (i = 0; i < MAXPARTITIONS; i++) {
1084 		if (label.d_partitions[i].p_fstype != FS_RAID)
1085 			continue;
1086 
1087 		/* open partition */
1088 		rawdev = MAKEDISKDEV(major(devno), DISKUNIT(devno), i);
1089 		if (bdevvp(rawdev, &vn)) {
1090 			sr_error(sc, "sr_meta_native_bootprobe: cannot "
1091 			    "allocate vnode for partition");
1092 			goto done;
1093 		}
1094 		error = VOP_OPEN(vn, FREAD, NOCRED, curproc);
1095 		if (error) {
1096 			DNPRINTF(SR_D_META, "%s: sr_meta_native_bootprobe "
1097 			    "open failed, partition %d\n",
1098 			    DEVNAME(sc), i);
1099 			vput(vn);
1100 			continue;
1101 		}
1102 
1103 		if (sr_meta_native_read(fake_sd, rawdev, md, NULL)) {
1104 			sr_error(sc, "native bootprobe could not read native "
1105 			    "metadata");
1106 			VOP_CLOSE(vn, FREAD, NOCRED, curproc);
1107 			vput(vn);
1108 			continue;
1109 		}
1110 
1111 		/* are we a softraid partition? */
1112 		if (md->ssdi.ssd_magic != SR_MAGIC) {
1113 			VOP_CLOSE(vn, FREAD, NOCRED, curproc);
1114 			vput(vn);
1115 			continue;
1116 		}
1117 
1118 		sr_meta_getdevname(sc, rawdev, devname, sizeof(devname));
1119 		if (sr_meta_validate(fake_sd, rawdev, md, NULL) == 0) {
1120 			/* XXX fix M_WAITOK, this is boot time */
1121 			bc = malloc(sizeof(struct sr_boot_chunk),
1122 			    M_DEVBUF, M_WAITOK | M_ZERO);
1123 			bc->sbc_metadata = malloc(sizeof(struct sr_metadata),
1124 			    M_DEVBUF, M_WAITOK | M_ZERO);
1125 			memcpy(bc->sbc_metadata, md, sizeof(struct sr_metadata));
1126 			bc->sbc_mm = rawdev;
1127 			SLIST_INSERT_HEAD(bch, bc, sbc_link);
1128 			rv = SR_META_CLAIMED;
1129 		}
1130 
1131 		/* we are done, close partition */
1132 		VOP_CLOSE(vn, FREAD, NOCRED, curproc);
1133 		vput(vn);
1134 	}
1135 
1136 done:
1137 	free(fake_sd, M_DEVBUF, sizeof(struct sr_discipline));
1138 	free(md, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
1139 
1140 	return (rv);
1141 }
1142 
1143 int
1144 sr_boot_assembly(struct sr_softc *sc)
1145 {
1146 	struct sr_boot_volume_head bvh;
1147 	struct sr_boot_chunk_head bch, kdh;
1148 	struct sr_boot_volume	*bv, *bv1, *bv2;
1149 	struct sr_boot_chunk	*bc, *bcnext, *bc1, *bc2;
1150 	struct sr_disk_head	sdklist;
1151 	struct sr_disk		*sdk;
1152 	struct disk		*dk;
1153 	struct bioc_createraid	bcr;
1154 	struct sr_meta_chunk	*hm;
1155 	struct sr_chunk_head	*cl;
1156 	struct sr_chunk		*hotspare, *chunk, *last;
1157 	u_int64_t		*ondisk = NULL;
1158 	dev_t			*devs = NULL;
1159 	void			*data;
1160 	char			devname[32];
1161 	int			rv = 0, i;
1162 
1163 	DNPRINTF(SR_D_META, "%s: sr_boot_assembly\n", DEVNAME(sc));
1164 
1165 	SLIST_INIT(&sdklist);
1166 	SLIST_INIT(&bvh);
1167 	SLIST_INIT(&bch);
1168 	SLIST_INIT(&kdh);
1169 
1170 	dk = TAILQ_FIRST(&disklist);
1171 	while (dk != NULL) {
1172 
1173 		/* See if this disk has been checked. */
1174 		SLIST_FOREACH(sdk, &sdklist, sdk_link)
1175 			if (sdk->sdk_devno == dk->dk_devno)
1176 				break;
1177 
1178 		if (sdk != NULL || dk->dk_devno == NODEV) {
1179 			dk = TAILQ_NEXT(dk, dk_link);
1180 			continue;
1181 		}
1182 
1183 		/* Add this disk to the list that we've checked. */
1184 		sdk = malloc(sizeof(struct sr_disk), M_DEVBUF,
1185 		    M_NOWAIT | M_ZERO);
1186 		if (sdk == NULL)
1187 			goto unwind;
1188 		sdk->sdk_devno = dk->dk_devno;
1189 		SLIST_INSERT_HEAD(&sdklist, sdk, sdk_link);
1190 
1191 		/* Only check sd(4) and wd(4) devices. */
1192 		if (strncmp(dk->dk_name, "sd", 2) &&
1193 		    strncmp(dk->dk_name, "wd", 2)) {
1194 			dk = TAILQ_NEXT(dk, dk_link);
1195 			continue;
1196 		}
1197 
1198 		/* native softraid uses partitions */
1199 		rw_enter_write(&sc->sc_lock);
1200 		bio_status_init(&sc->sc_status, &sc->sc_dev);
1201 		sr_meta_native_bootprobe(sc, dk->dk_devno, &bch);
1202 		rw_exit_write(&sc->sc_lock);
1203 
1204 		/* probe non-native disks if native failed. */
1205 
1206 		/* Restart scan since we may have slept. */
1207 		dk = TAILQ_FIRST(&disklist);
1208 	}
1209 
1210 	/*
1211 	 * Create a list of volumes and associate chunks with each volume.
1212 	 */
1213 	for (bc = SLIST_FIRST(&bch); bc != NULL; bc = bcnext) {
1214 
1215 		bcnext = SLIST_NEXT(bc, sbc_link);
1216 		SLIST_REMOVE(&bch, bc, sr_boot_chunk, sbc_link);
1217 		bc->sbc_chunk_id = bc->sbc_metadata->ssdi.ssd_chunk_id;
1218 
1219 		/* Handle key disks separately. */
1220 		if (bc->sbc_metadata->ssdi.ssd_level == SR_KEYDISK_LEVEL) {
1221 			SLIST_INSERT_HEAD(&kdh, bc, sbc_link);
1222 			continue;
1223 		}
1224 
1225 		SLIST_FOREACH(bv, &bvh, sbv_link) {
1226 			if (bcmp(&bc->sbc_metadata->ssdi.ssd_uuid,
1227 			    &bv->sbv_uuid,
1228 			    sizeof(bc->sbc_metadata->ssdi.ssd_uuid)) == 0)
1229 				break;
1230 		}
1231 
1232 		if (bv == NULL) {
1233 			bv = malloc(sizeof(struct sr_boot_volume),
1234 			    M_DEVBUF, M_NOWAIT | M_ZERO);
1235 			if (bv == NULL) {
1236 				printf("%s: failed to allocate boot volume\n",
1237 				    DEVNAME(sc));
1238 				goto unwind;
1239 			}
1240 
1241 			bv->sbv_level = bc->sbc_metadata->ssdi.ssd_level;
1242 			bv->sbv_volid = bc->sbc_metadata->ssdi.ssd_volid;
1243 			bv->sbv_chunk_no = bc->sbc_metadata->ssdi.ssd_chunk_no;
1244 			bv->sbv_flags = bc->sbc_metadata->ssdi.ssd_vol_flags;
1245 			memcpy(&bv->sbv_uuid, &bc->sbc_metadata->ssdi.ssd_uuid,
1246 			    sizeof(bc->sbc_metadata->ssdi.ssd_uuid));
1247 			SLIST_INIT(&bv->sbv_chunks);
1248 
1249 			/* Maintain volume order. */
1250 			bv2 = NULL;
1251 			SLIST_FOREACH(bv1, &bvh, sbv_link) {
1252 				if (bv1->sbv_volid > bv->sbv_volid)
1253 					break;
1254 				bv2 = bv1;
1255 			}
1256 			if (bv2 == NULL) {
1257 				DNPRINTF(SR_D_META, "%s: insert volume %u "
1258 				    "at head\n", DEVNAME(sc), bv->sbv_volid);
1259 				SLIST_INSERT_HEAD(&bvh, bv, sbv_link);
1260 			} else {
1261 				DNPRINTF(SR_D_META, "%s: insert volume %u "
1262 				    "after %u\n", DEVNAME(sc), bv->sbv_volid,
1263 				    bv2->sbv_volid);
1264 				SLIST_INSERT_AFTER(bv2, bv, sbv_link);
1265 			}
1266 		}
1267 
1268 		/* Maintain chunk order. */
1269 		bc2 = NULL;
1270 		SLIST_FOREACH(bc1, &bv->sbv_chunks, sbc_link) {
1271 			if (bc1->sbc_chunk_id > bc->sbc_chunk_id)
1272 				break;
1273 			bc2 = bc1;
1274 		}
1275 		if (bc2 == NULL) {
1276 			DNPRINTF(SR_D_META, "%s: volume %u insert chunk %u "
1277 			    "at head\n", DEVNAME(sc), bv->sbv_volid,
1278 			    bc->sbc_chunk_id);
1279 			SLIST_INSERT_HEAD(&bv->sbv_chunks, bc, sbc_link);
1280 		} else {
1281 			DNPRINTF(SR_D_META, "%s: volume %u insert chunk %u "
1282 			    "after %u\n", DEVNAME(sc), bv->sbv_volid,
1283 			    bc->sbc_chunk_id, bc2->sbc_chunk_id);
1284 			SLIST_INSERT_AFTER(bc2, bc, sbc_link);
1285 		}
1286 
1287 		bv->sbv_chunks_found++;
1288 	}
1289 
1290 	/* Allocate memory for device and ondisk version arrays. */
1291 	devs = mallocarray(BIOC_CRMAXLEN, sizeof(dev_t), M_DEVBUF,
1292 	    M_NOWAIT);
1293 	if (devs == NULL) {
1294 		printf("%s: failed to allocate device array\n", DEVNAME(sc));
1295 		goto unwind;
1296 	}
1297 	ondisk = mallocarray(BIOC_CRMAXLEN, sizeof(u_int64_t), M_DEVBUF,
1298 	    M_NOWAIT);
1299 	if (ondisk == NULL) {
1300 		printf("%s: failed to allocate ondisk array\n", DEVNAME(sc));
1301 		goto unwind;
1302 	}
1303 
1304 	/*
1305 	 * Assemble hotspare "volumes".
1306 	 */
1307 	SLIST_FOREACH(bv, &bvh, sbv_link) {
1308 
1309 		/* Check if this is a hotspare "volume". */
1310 		if (bv->sbv_level != SR_HOTSPARE_LEVEL ||
1311 		    bv->sbv_chunk_no != 1)
1312 			continue;
1313 
1314 #ifdef SR_DEBUG
1315 		DNPRINTF(SR_D_META, "%s: assembling hotspare volume ",
1316 		    DEVNAME(sc));
1317 		if (sr_debug & SR_D_META)
1318 			sr_uuid_print(&bv->sbv_uuid, 0);
1319 		DNPRINTF(SR_D_META, " volid %u with %u chunks\n",
1320 		    bv->sbv_volid, bv->sbv_chunk_no);
1321 #endif
1322 
1323 		/* Create hotspare chunk metadata. */
1324 		hotspare = malloc(sizeof(struct sr_chunk), M_DEVBUF,
1325 		    M_NOWAIT | M_ZERO);
1326 		if (hotspare == NULL) {
1327 			printf("%s: failed to allocate hotspare\n",
1328 			    DEVNAME(sc));
1329 			goto unwind;
1330 		}
1331 
1332 		bc = SLIST_FIRST(&bv->sbv_chunks);
1333 		sr_meta_getdevname(sc, bc->sbc_mm, devname, sizeof(devname));
1334 		hotspare->src_dev_mm = bc->sbc_mm;
1335 		strlcpy(hotspare->src_devname, devname,
1336 		    sizeof(hotspare->src_devname));
1337 		hotspare->src_size = bc->sbc_metadata->ssdi.ssd_size;
1338 
1339 		hm = &hotspare->src_meta;
1340 		hm->scmi.scm_volid = SR_HOTSPARE_VOLID;
1341 		hm->scmi.scm_chunk_id = 0;
1342 		hm->scmi.scm_size = bc->sbc_metadata->ssdi.ssd_size;
1343 		hm->scmi.scm_coerced_size = bc->sbc_metadata->ssdi.ssd_size;
1344 		strlcpy(hm->scmi.scm_devname, devname,
1345 		    sizeof(hm->scmi.scm_devname));
1346 		memcpy(&hm->scmi.scm_uuid, &bc->sbc_metadata->ssdi.ssd_uuid,
1347 		    sizeof(struct sr_uuid));
1348 
1349 		sr_checksum(sc, hm, &hm->scm_checksum,
1350 		    sizeof(struct sr_meta_chunk_invariant));
1351 
1352 		hm->scm_status = BIOC_SDHOTSPARE;
1353 
1354 		/* Add chunk to hotspare list. */
1355 		rw_enter_write(&sc->sc_hs_lock);
1356 		cl = &sc->sc_hotspare_list;
1357 		if (SLIST_EMPTY(cl))
1358 			SLIST_INSERT_HEAD(cl, hotspare, src_link);
1359 		else {
1360 			SLIST_FOREACH(chunk, cl, src_link)
1361 				last = chunk;
1362 			SLIST_INSERT_AFTER(last, hotspare, src_link);
1363 		}
1364 		sc->sc_hotspare_no++;
1365 		rw_exit_write(&sc->sc_hs_lock);
1366 
1367 	}
1368 
1369 	/*
1370 	 * Assemble RAID volumes.
1371 	 */
1372 	SLIST_FOREACH(bv, &bvh, sbv_link) {
1373 
1374 		bzero(&bcr, sizeof(bcr));
1375 		data = NULL;
1376 
1377 		/* Check if this is a hotspare "volume". */
1378 		if (bv->sbv_level == SR_HOTSPARE_LEVEL &&
1379 		    bv->sbv_chunk_no == 1)
1380 			continue;
1381 
1382 		/*
1383 		 * Skip volumes that are marked as no auto assemble, unless
1384 		 * this was the volume which we actually booted from.
1385 		 */
1386 		if (bcmp(&sr_bootuuid, &bv->sbv_uuid, sizeof(sr_bootuuid)) != 0)
1387 			if (bv->sbv_flags & BIOC_SCNOAUTOASSEMBLE)
1388 				continue;
1389 
1390 #ifdef SR_DEBUG
1391 		DNPRINTF(SR_D_META, "%s: assembling volume ", DEVNAME(sc));
1392 		if (sr_debug & SR_D_META)
1393 			sr_uuid_print(&bv->sbv_uuid, 0);
1394 		DNPRINTF(SR_D_META, " volid %u with %u chunks\n",
1395 		    bv->sbv_volid, bv->sbv_chunk_no);
1396 #endif
1397 
1398 		/*
1399 		 * If this is a crypto volume, try to find a matching
1400 		 * key disk...
1401 		 */
1402 		bcr.bc_key_disk = NODEV;
1403 		if (bv->sbv_level == 'C') {
1404 			SLIST_FOREACH(bc, &kdh, sbc_link) {
1405 				if (bcmp(&bc->sbc_metadata->ssdi.ssd_uuid,
1406 				    &bv->sbv_uuid,
1407 				    sizeof(bc->sbc_metadata->ssdi.ssd_uuid))
1408 				    == 0)
1409 					bcr.bc_key_disk = bc->sbc_mm;
1410 			}
1411 		}
1412 
1413 		for (i = 0; i < BIOC_CRMAXLEN; i++) {
1414 			devs[i] = NODEV; /* mark device as illegal */
1415 			ondisk[i] = 0;
1416 		}
1417 
1418 		SLIST_FOREACH(bc, &bv->sbv_chunks, sbc_link) {
1419 			if (devs[bc->sbc_chunk_id] != NODEV) {
1420 				bv->sbv_chunks_found--;
1421 				sr_meta_getdevname(sc, bc->sbc_mm, devname,
1422 				    sizeof(devname));
1423 				printf("%s: found duplicate chunk %u for "
1424 				    "volume %u on device %s\n", DEVNAME(sc),
1425 				    bc->sbc_chunk_id, bv->sbv_volid, devname);
1426 			}
1427 
1428 			if (devs[bc->sbc_chunk_id] == NODEV ||
1429 			    bc->sbc_metadata->ssd_ondisk >
1430 			    ondisk[bc->sbc_chunk_id]) {
1431 				devs[bc->sbc_chunk_id] = bc->sbc_mm;
1432 				ondisk[bc->sbc_chunk_id] =
1433 				    bc->sbc_metadata->ssd_ondisk;
1434 				DNPRINTF(SR_D_META, "%s: using ondisk "
1435 				    "metadata version %llu for chunk %u\n",
1436 				    DEVNAME(sc), ondisk[bc->sbc_chunk_id],
1437 				    bc->sbc_chunk_id);
1438 			}
1439 		}
1440 
1441 		if (bv->sbv_chunk_no != bv->sbv_chunks_found) {
1442 			printf("%s: not all chunks were provided; "
1443 			    "attempting to bring volume %d online\n",
1444 			    DEVNAME(sc), bv->sbv_volid);
1445 		}
1446 
1447 		bcr.bc_level = bv->sbv_level;
1448 		bcr.bc_dev_list_len = bv->sbv_chunk_no * sizeof(dev_t);
1449 		bcr.bc_dev_list = devs;
1450 		bcr.bc_flags = BIOC_SCDEVT |
1451 		    (bv->sbv_flags & BIOC_SCNOAUTOASSEMBLE);
1452 
1453 		if (bv->sbv_level == 'C' &&
1454 		    bcmp(&sr_bootuuid, &bv->sbv_uuid, sizeof(sr_bootuuid)) == 0)
1455 			data = sr_bootkey;
1456 
1457 		rw_enter_write(&sc->sc_lock);
1458 		bio_status_init(&sc->sc_status, &sc->sc_dev);
1459 		sr_ioctl_createraid(sc, &bcr, 0, data);
1460 		rw_exit_write(&sc->sc_lock);
1461 
1462 		rv++;
1463 	}
1464 
1465 	/* done with metadata */
1466 unwind:
1467 	/* Free boot volumes and associated chunks. */
1468 	for (bv1 = SLIST_FIRST(&bvh); bv1 != NULL; bv1 = bv2) {
1469 		bv2 = SLIST_NEXT(bv1, sbv_link);
1470 		for (bc1 = SLIST_FIRST(&bv1->sbv_chunks); bc1 != NULL;
1471 		    bc1 = bc2) {
1472 			bc2 = SLIST_NEXT(bc1, sbc_link);
1473 			free(bc1->sbc_metadata, M_DEVBUF, 0);
1474 			free(bc1, M_DEVBUF, 0);
1475 		}
1476 		free(bv1, M_DEVBUF, 0);
1477 	}
1478 	/* Free keydisks chunks. */
1479 	for (bc1 = SLIST_FIRST(&kdh); bc1 != NULL; bc1 = bc2) {
1480 		bc2 = SLIST_NEXT(bc1, sbc_link);
1481 		free(bc1->sbc_metadata, M_DEVBUF, 0);
1482 		free(bc1, M_DEVBUF, 0);
1483 	}
1484 	/* Free unallocated chunks. */
1485 	for (bc1 = SLIST_FIRST(&bch); bc1 != NULL; bc1 = bc2) {
1486 		bc2 = SLIST_NEXT(bc1, sbc_link);
1487 		free(bc1->sbc_metadata, M_DEVBUF, 0);
1488 		free(bc1, M_DEVBUF, 0);
1489 	}
1490 
1491 	while (!SLIST_EMPTY(&sdklist)) {
1492 		sdk = SLIST_FIRST(&sdklist);
1493 		SLIST_REMOVE_HEAD(&sdklist, sdk_link);
1494 		free(sdk, M_DEVBUF, 0);
1495 	}
1496 
1497 	free(devs, M_DEVBUF, BIOC_CRMAXLEN * sizeof(dev_t));
1498 	free(ondisk, M_DEVBUF, BIOC_CRMAXLEN * sizeof(u_int64_t));
1499 
1500 	return (rv);
1501 }
1502 
1503 void
1504 sr_map_root(void)
1505 {
1506 	struct sr_softc		*sc = softraid0;
1507 	struct sr_discipline	*sd;
1508 	struct sr_meta_opt_item	*omi;
1509 	struct sr_meta_boot	*sbm;
1510 	u_char			duid[8];
1511 	int			i;
1512 
1513 	DNPRINTF(SR_D_MISC, "%s: sr_map_root\n", DEVNAME(sc));
1514 
1515 	if (sc == NULL)
1516 		return;
1517 
1518 	bzero(duid, sizeof(duid));
1519 	if (bcmp(rootduid, duid, sizeof(duid)) == 0) {
1520 		DNPRINTF(SR_D_MISC, "%s: root duid is zero\n", DEVNAME(sc));
1521 		return;
1522 	}
1523 
1524 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
1525 		SLIST_FOREACH(omi, &sd->sd_meta_opt, omi_link) {
1526 			if (omi->omi_som->som_type != SR_OPT_BOOT)
1527 				continue;
1528 			sbm = (struct sr_meta_boot *)omi->omi_som;
1529 			for (i = 0; i < SR_MAX_BOOT_DISKS; i++) {
1530 				if (bcmp(rootduid, sbm->sbm_boot_duid[i],
1531 				    sizeof(rootduid)) == 0) {
1532 					memcpy(rootduid, sbm->sbm_root_duid,
1533 					    sizeof(rootduid));
1534 					DNPRINTF(SR_D_MISC, "%s: root duid "
1535 					    "mapped to %s\n", DEVNAME(sc),
1536 					    duid_format(rootduid));
1537 					return;
1538 				}
1539 			}
1540 		}
1541 	}
1542 }
1543 
1544 int
1545 sr_meta_native_probe(struct sr_softc *sc, struct sr_chunk *ch_entry)
1546 {
1547 	struct disklabel	label;
1548 	char			*devname;
1549 	int			error, part;
1550 	u_int64_t		size;
1551 
1552 	DNPRINTF(SR_D_META, "%s: sr_meta_native_probe(%s)\n",
1553 	   DEVNAME(sc), ch_entry->src_devname);
1554 
1555 	devname = ch_entry->src_devname;
1556 	part = DISKPART(ch_entry->src_dev_mm);
1557 
1558 	/* get disklabel */
1559 	error = VOP_IOCTL(ch_entry->src_vn, DIOCGDINFO, (caddr_t)&label, FREAD,
1560 	    NOCRED, curproc);
1561 	if (error) {
1562 		DNPRINTF(SR_D_META, "%s: %s can't obtain disklabel\n",
1563 		    DEVNAME(sc), devname);
1564 		goto unwind;
1565 	}
1566 	memcpy(ch_entry->src_duid, label.d_uid, sizeof(ch_entry->src_duid));
1567 
1568 	/* make sure the partition is of the right type */
1569 	if (label.d_partitions[part].p_fstype != FS_RAID) {
1570 		DNPRINTF(SR_D_META,
1571 		    "%s: %s partition not of type RAID (%d)\n", DEVNAME(sc),
1572 		    devname,
1573 		    label.d_partitions[part].p_fstype);
1574 		goto unwind;
1575 	}
1576 
1577 	size = DL_SECTOBLK(&label, DL_GETPSIZE(&label.d_partitions[part]));
1578 	if (size <= SR_DATA_OFFSET) {
1579 		DNPRINTF(SR_D_META, "%s: %s partition too small\n", DEVNAME(sc),
1580 		    devname);
1581 		goto unwind;
1582 	}
1583 	size -= SR_DATA_OFFSET;
1584 	if (size > INT64_MAX) {
1585 		DNPRINTF(SR_D_META, "%s: %s partition too large\n", DEVNAME(sc),
1586 		    devname);
1587 		goto unwind;
1588 	}
1589 	ch_entry->src_size = size;
1590 	ch_entry->src_secsize = label.d_secsize;
1591 
1592 	DNPRINTF(SR_D_META, "%s: probe found %s size %lld\n", DEVNAME(sc),
1593 	    devname, (long long)size);
1594 
1595 	return (SR_META_F_NATIVE);
1596 unwind:
1597 	DNPRINTF(SR_D_META, "%s: invalid device: %s\n", DEVNAME(sc),
1598 	    devname ? devname : "nodev");
1599 	return (SR_META_F_INVALID);
1600 }
1601 
1602 int
1603 sr_meta_native_attach(struct sr_discipline *sd, int force)
1604 {
1605 	struct sr_softc		*sc = sd->sd_sc;
1606 	struct sr_chunk_head	*cl = &sd->sd_vol.sv_chunk_list;
1607 	struct sr_metadata	*md = NULL;
1608 	struct sr_chunk		*ch_entry, *ch_next;
1609 	struct sr_uuid		uuid;
1610 	u_int64_t		version = 0;
1611 	int			sr, not_sr, rv = 1, d, expected = -1, old_meta = 0;
1612 
1613 	DNPRINTF(SR_D_META, "%s: sr_meta_native_attach\n", DEVNAME(sc));
1614 
1615 	md = malloc(SR_META_SIZE * DEV_BSIZE, M_DEVBUF, M_ZERO | M_NOWAIT);
1616 	if (md == NULL) {
1617 		sr_error(sc, "not enough memory for metadata buffer");
1618 		goto bad;
1619 	}
1620 
1621 	bzero(&uuid, sizeof uuid);
1622 
1623 	sr = not_sr = d = 0;
1624 	SLIST_FOREACH(ch_entry, cl, src_link) {
1625 		if (ch_entry->src_dev_mm == NODEV)
1626 			continue;
1627 
1628 		if (sr_meta_native_read(sd, ch_entry->src_dev_mm, md, NULL)) {
1629 			sr_error(sc, "could not read native metadata");
1630 			goto bad;
1631 		}
1632 
1633 		if (md->ssdi.ssd_magic == SR_MAGIC) {
1634 			sr++;
1635 			ch_entry->src_meta.scmi.scm_chunk_id =
1636 			    md->ssdi.ssd_chunk_id;
1637 			if (d == 0) {
1638 				memcpy(&uuid, &md->ssdi.ssd_uuid, sizeof uuid);
1639 				expected = md->ssdi.ssd_chunk_no;
1640 				version = md->ssd_ondisk;
1641 				d++;
1642 				continue;
1643 			} else if (bcmp(&md->ssdi.ssd_uuid, &uuid,
1644 			    sizeof uuid)) {
1645 				sr_error(sc, "not part of the same volume");
1646 				goto bad;
1647 			}
1648 			if (md->ssd_ondisk != version) {
1649 				old_meta++;
1650 				version = MAX(md->ssd_ondisk, version);
1651 			}
1652 		} else
1653 			not_sr++;
1654 	}
1655 
1656 	if (sr && not_sr && !force) {
1657 		sr_error(sc, "not all chunks are of the native metadata "
1658 		    "format");
1659 		goto bad;
1660 	}
1661 
1662 	/* mixed metadata versions; mark bad disks offline */
1663 	if (old_meta) {
1664 		d = 0;
1665 		for (ch_entry = SLIST_FIRST(cl); ch_entry != NULL;
1666 		    ch_entry = ch_next, d++) {
1667 			ch_next = SLIST_NEXT(ch_entry, src_link);
1668 
1669 			/* XXX do we want to read this again? */
1670 			if (ch_entry->src_dev_mm == NODEV)
1671 				panic("src_dev_mm == NODEV");
1672 			if (sr_meta_native_read(sd, ch_entry->src_dev_mm, md,
1673 			    NULL))
1674 				sr_warn(sc, "could not read native metadata");
1675 			if (md->ssd_ondisk != version)
1676 				sd->sd_vol.sv_chunks[d]->src_meta.scm_status =
1677 				    BIOC_SDOFFLINE;
1678 		}
1679 	}
1680 
1681 	if (expected != sr && !force && expected != -1) {
1682 		DNPRINTF(SR_D_META, "%s: not all chunks were provided, trying "
1683 		    "anyway\n", DEVNAME(sc));
1684 	}
1685 
1686 	rv = 0;
1687 bad:
1688 	free(md, M_DEVBUF, SR_META_SIZE * DEV_BSIZE);
1689 	return (rv);
1690 }
1691 
1692 int
1693 sr_meta_native_read(struct sr_discipline *sd, dev_t dev,
1694     struct sr_metadata *md, void *fm)
1695 {
1696 #ifdef SR_DEBUG
1697 	struct sr_softc		*sc = sd->sd_sc;
1698 #endif
1699 	DNPRINTF(SR_D_META, "%s: sr_meta_native_read(0x%x, %p)\n",
1700 	    DEVNAME(sc), dev, md);
1701 
1702 	return (sr_meta_rw(sd, dev, md, B_READ));
1703 }
1704 
1705 int
1706 sr_meta_native_write(struct sr_discipline *sd, dev_t dev,
1707     struct sr_metadata *md, void *fm)
1708 {
1709 #ifdef SR_DEBUG
1710 	struct sr_softc		*sc = sd->sd_sc;
1711 #endif
1712 	DNPRINTF(SR_D_META, "%s: sr_meta_native_write(0x%x, %p)\n",
1713 	    DEVNAME(sc), dev, md);
1714 
1715 	return (sr_meta_rw(sd, dev, md, B_WRITE));
1716 }
1717 
1718 void
1719 sr_hotplug_register(struct sr_discipline *sd, void *func)
1720 {
1721 	struct sr_hotplug_list	*mhe;
1722 
1723 	DNPRINTF(SR_D_MISC, "%s: sr_hotplug_register: %p\n",
1724 	    DEVNAME(sd->sd_sc), func);
1725 
1726 	/* make sure we aren't on the list yet */
1727 	SLIST_FOREACH(mhe, &sr_hotplug_callbacks, shl_link)
1728 		if (mhe->sh_hotplug == func)
1729 			return;
1730 
1731 	mhe = malloc(sizeof(struct sr_hotplug_list), M_DEVBUF,
1732 	    M_WAITOK | M_ZERO);
1733 	mhe->sh_hotplug = func;
1734 	mhe->sh_sd = sd;
1735 	SLIST_INSERT_HEAD(&sr_hotplug_callbacks, mhe, shl_link);
1736 }
1737 
1738 void
1739 sr_hotplug_unregister(struct sr_discipline *sd, void *func)
1740 {
1741 	struct sr_hotplug_list	*mhe;
1742 
1743 	DNPRINTF(SR_D_MISC, "%s: sr_hotplug_unregister: %s %p\n",
1744 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, func);
1745 
1746 	/* make sure we are on the list yet */
1747 	SLIST_FOREACH(mhe, &sr_hotplug_callbacks, shl_link) {
1748 		if (mhe->sh_hotplug == func)
1749 			break;
1750 	}
1751 	if (mhe != NULL) {
1752 		SLIST_REMOVE(&sr_hotplug_callbacks, mhe,
1753 		    sr_hotplug_list, shl_link);
1754 		free(mhe, M_DEVBUF, 0);
1755 	}
1756 }
1757 
1758 void
1759 sr_disk_attach(struct disk *diskp, int action)
1760 {
1761 	struct sr_hotplug_list	*mhe;
1762 
1763 	SLIST_FOREACH(mhe, &sr_hotplug_callbacks, shl_link)
1764 		if (mhe->sh_sd->sd_ready)
1765 			mhe->sh_hotplug(mhe->sh_sd, diskp, action);
1766 }
1767 
1768 int
1769 sr_match(struct device *parent, void *match, void *aux)
1770 {
1771 	return (1);
1772 }
1773 
1774 void
1775 sr_attach(struct device *parent, struct device *self, void *aux)
1776 {
1777 	struct sr_softc		*sc = (void *)self;
1778 	struct scsibus_attach_args saa;
1779 
1780 	DNPRINTF(SR_D_MISC, "\n%s: sr_attach", DEVNAME(sc));
1781 
1782 	if (softraid0 == NULL)
1783 		softraid0 = sc;
1784 
1785 	rw_init(&sc->sc_lock, "sr_lock");
1786 	rw_init(&sc->sc_hs_lock, "sr_hs_lock");
1787 
1788 	SLIST_INIT(&sr_hotplug_callbacks);
1789 	TAILQ_INIT(&sc->sc_dis_list);
1790 	SLIST_INIT(&sc->sc_hotspare_list);
1791 
1792 #if NBIO > 0
1793 	if (bio_register(&sc->sc_dev, sr_bio_ioctl) != 0)
1794 		printf("%s: controller registration failed", DEVNAME(sc));
1795 #endif /* NBIO > 0 */
1796 
1797 #ifndef SMALL_KERNEL
1798 	strlcpy(sc->sc_sensordev.xname, DEVNAME(sc),
1799 	    sizeof(sc->sc_sensordev.xname));
1800 	sensordev_install(&sc->sc_sensordev);
1801 #endif /* SMALL_KERNEL */
1802 
1803 	printf("\n");
1804 
1805 	sc->sc_link.adapter_softc = sc;
1806 	sc->sc_link.adapter = &sr_switch;
1807 	sc->sc_link.adapter_target = SR_MAX_LD;
1808 	sc->sc_link.adapter_buswidth = SR_MAX_LD;
1809 	sc->sc_link.luns = 1;
1810 
1811 	bzero(&saa, sizeof(saa));
1812 	saa.saa_sc_link = &sc->sc_link;
1813 
1814 	sc->sc_scsibus = (struct scsibus_softc *)config_found(&sc->sc_dev,
1815 	    &saa, scsiprint);
1816 
1817 	softraid_disk_attach = sr_disk_attach;
1818 
1819 	sr_boot_assembly(sc);
1820 
1821 	explicit_bzero(sr_bootkey, sizeof(sr_bootkey));
1822 }
1823 
1824 int
1825 sr_detach(struct device *self, int flags)
1826 {
1827 	struct sr_softc		*sc = (void *)self;
1828 	int			rv;
1829 
1830 	DNPRINTF(SR_D_MISC, "%s: sr_detach\n", DEVNAME(sc));
1831 
1832 	softraid_disk_attach = NULL;
1833 
1834 	sr_shutdown(0);
1835 
1836 #ifndef SMALL_KERNEL
1837 	if (sc->sc_sensor_task != NULL)
1838 		sensor_task_unregister(sc->sc_sensor_task);
1839 	sensordev_deinstall(&sc->sc_sensordev);
1840 #endif /* SMALL_KERNEL */
1841 
1842 	if (sc->sc_scsibus != NULL) {
1843 		rv = config_detach((struct device *)sc->sc_scsibus, flags);
1844 		if (rv != 0)
1845 			return (rv);
1846 		sc->sc_scsibus = NULL;
1847 	}
1848 
1849 	return (0);
1850 }
1851 
1852 void
1853 sr_info(struct sr_softc *sc, const char *fmt, ...)
1854 {
1855 	va_list			ap;
1856 
1857 	rw_assert_wrlock(&sc->sc_lock);
1858 
1859 	va_start(ap, fmt);
1860 	bio_status(&sc->sc_status, 0, BIO_MSG_INFO, fmt, &ap);
1861 	va_end(ap);
1862 }
1863 
1864 void
1865 sr_warn(struct sr_softc *sc, const char *fmt, ...)
1866 {
1867 	va_list			ap;
1868 
1869 	rw_assert_wrlock(&sc->sc_lock);
1870 
1871 	va_start(ap, fmt);
1872 	bio_status(&sc->sc_status, 1, BIO_MSG_WARN, fmt, &ap);
1873 	va_end(ap);
1874 }
1875 
1876 void
1877 sr_error(struct sr_softc *sc, const char *fmt, ...)
1878 {
1879 	va_list			ap;
1880 
1881 	rw_assert_wrlock(&sc->sc_lock);
1882 
1883 	va_start(ap, fmt);
1884 	bio_status(&sc->sc_status, 1, BIO_MSG_ERROR, fmt, &ap);
1885 	va_end(ap);
1886 }
1887 
1888 void
1889 sr_minphys(struct buf *bp, struct scsi_link *sl)
1890 {
1891 	DNPRINTF(SR_D_MISC, "sr_minphys: %ld\n", bp->b_bcount);
1892 
1893 	/* XXX currently using SR_MAXFER = MAXPHYS */
1894 	if (bp->b_bcount > SR_MAXFER)
1895 		bp->b_bcount = SR_MAXFER;
1896 	minphys(bp);
1897 }
1898 
1899 void
1900 sr_copy_internal_data(struct scsi_xfer *xs, void *v, size_t size)
1901 {
1902 	size_t			copy_cnt;
1903 
1904 	DNPRINTF(SR_D_MISC, "sr_copy_internal_data xs: %p size: %zu\n",
1905 	    xs, size);
1906 
1907 	if (xs->datalen) {
1908 		copy_cnt = MIN(size, xs->datalen);
1909 		memcpy(xs->data, v, copy_cnt);
1910 	}
1911 }
1912 
1913 int
1914 sr_ccb_alloc(struct sr_discipline *sd)
1915 {
1916 	struct sr_ccb		*ccb;
1917 	int			i;
1918 
1919 	if (!sd)
1920 		return (1);
1921 
1922 	DNPRINTF(SR_D_CCB, "%s: sr_ccb_alloc\n", DEVNAME(sd->sd_sc));
1923 
1924 	if (sd->sd_ccb)
1925 		return (1);
1926 
1927 	sd->sd_ccb = mallocarray(sd->sd_max_wu,
1928 	    sd->sd_max_ccb_per_wu * sizeof(struct sr_ccb),
1929 	    M_DEVBUF, M_WAITOK | M_ZERO);
1930 	TAILQ_INIT(&sd->sd_ccb_freeq);
1931 	for (i = 0; i < sd->sd_max_wu * sd->sd_max_ccb_per_wu; i++) {
1932 		ccb = &sd->sd_ccb[i];
1933 		ccb->ccb_dis = sd;
1934 		sr_ccb_put(ccb);
1935 	}
1936 
1937 	DNPRINTF(SR_D_CCB, "%s: sr_ccb_alloc ccb: %d\n",
1938 	    DEVNAME(sd->sd_sc), sd->sd_max_wu * sd->sd_max_ccb_per_wu);
1939 
1940 	return (0);
1941 }
1942 
1943 void
1944 sr_ccb_free(struct sr_discipline *sd)
1945 {
1946 	struct sr_ccb		*ccb;
1947 
1948 	if (!sd)
1949 		return;
1950 
1951 	DNPRINTF(SR_D_CCB, "%s: sr_ccb_free %p\n", DEVNAME(sd->sd_sc), sd);
1952 
1953 	while ((ccb = TAILQ_FIRST(&sd->sd_ccb_freeq)) != NULL)
1954 		TAILQ_REMOVE(&sd->sd_ccb_freeq, ccb, ccb_link);
1955 
1956 	free(sd->sd_ccb, M_DEVBUF, 0);
1957 }
1958 
1959 struct sr_ccb *
1960 sr_ccb_get(struct sr_discipline *sd)
1961 {
1962 	struct sr_ccb		*ccb;
1963 	int			s;
1964 
1965 	s = splbio();
1966 
1967 	ccb = TAILQ_FIRST(&sd->sd_ccb_freeq);
1968 	if (ccb) {
1969 		TAILQ_REMOVE(&sd->sd_ccb_freeq, ccb, ccb_link);
1970 		ccb->ccb_state = SR_CCB_INPROGRESS;
1971 	}
1972 
1973 	splx(s);
1974 
1975 	DNPRINTF(SR_D_CCB, "%s: sr_ccb_get: %p\n", DEVNAME(sd->sd_sc),
1976 	    ccb);
1977 
1978 	return (ccb);
1979 }
1980 
1981 void
1982 sr_ccb_put(struct sr_ccb *ccb)
1983 {
1984 	struct sr_discipline	*sd = ccb->ccb_dis;
1985 	int			s;
1986 
1987 	DNPRINTF(SR_D_CCB, "%s: sr_ccb_put: %p\n", DEVNAME(sd->sd_sc),
1988 	    ccb);
1989 
1990 	s = splbio();
1991 
1992 	ccb->ccb_wu = NULL;
1993 	ccb->ccb_state = SR_CCB_FREE;
1994 	ccb->ccb_target = -1;
1995 	ccb->ccb_opaque = NULL;
1996 
1997 	TAILQ_INSERT_TAIL(&sd->sd_ccb_freeq, ccb, ccb_link);
1998 
1999 	splx(s);
2000 }
2001 
2002 struct sr_ccb *
2003 sr_ccb_rw(struct sr_discipline *sd, int chunk, daddr_t blkno,
2004     long len, u_int8_t *data, int xsflags, int ccbflags)
2005 {
2006 	struct sr_chunk		*sc = sd->sd_vol.sv_chunks[chunk];
2007 	struct sr_ccb		*ccb = NULL;
2008 
2009 	ccb = sr_ccb_get(sd);
2010 	if (ccb == NULL)
2011 		goto out;
2012 
2013 	ccb->ccb_flags = ccbflags;
2014 	ccb->ccb_target = chunk;
2015 
2016 	ccb->ccb_buf.b_flags = B_PHYS | B_CALL;
2017 	if (ISSET(xsflags, SCSI_DATA_IN))
2018 		ccb->ccb_buf.b_flags |= B_READ;
2019 	else
2020 		ccb->ccb_buf.b_flags |= B_WRITE;
2021 
2022 	ccb->ccb_buf.b_blkno = blkno + sd->sd_meta->ssd_data_blkno;
2023 	ccb->ccb_buf.b_bcount = len;
2024 	ccb->ccb_buf.b_bufsize = len;
2025 	ccb->ccb_buf.b_resid = len;
2026 	ccb->ccb_buf.b_data = data;
2027 	ccb->ccb_buf.b_error = 0;
2028 	ccb->ccb_buf.b_iodone = sd->sd_scsi_intr;
2029 	ccb->ccb_buf.b_proc = curproc;
2030 	ccb->ccb_buf.b_dev = sc->src_dev_mm;
2031 	ccb->ccb_buf.b_vp = sc->src_vn;
2032 	ccb->ccb_buf.b_bq = NULL;
2033 
2034 	if (!ISSET(ccb->ccb_buf.b_flags, B_READ))
2035 		ccb->ccb_buf.b_vp->v_numoutput++;
2036 
2037 	LIST_INIT(&ccb->ccb_buf.b_dep);
2038 
2039 	DNPRINTF(SR_D_DIS, "%s: %s %s ccb "
2040 	    "b_bcount %ld b_blkno %lld b_flags 0x%0lx b_data %p\n",
2041 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, sd->sd_name,
2042 	    ccb->ccb_buf.b_bcount, (long long)ccb->ccb_buf.b_blkno,
2043 	    ccb->ccb_buf.b_flags, ccb->ccb_buf.b_data);
2044 
2045 out:
2046 	return ccb;
2047 }
2048 
2049 void
2050 sr_ccb_done(struct sr_ccb *ccb)
2051 {
2052 	struct sr_workunit	*wu = ccb->ccb_wu;
2053 	struct sr_discipline	*sd = wu->swu_dis;
2054 	struct sr_softc		*sc = sd->sd_sc;
2055 
2056 	DNPRINTF(SR_D_INTR, "%s: %s %s ccb done b_bcount %ld b_resid %zu"
2057 	    " b_flags 0x%0lx block %lld target %d\n",
2058 	    DEVNAME(sc), sd->sd_meta->ssd_devname, sd->sd_name,
2059 	    ccb->ccb_buf.b_bcount, ccb->ccb_buf.b_resid, ccb->ccb_buf.b_flags,
2060 	    (long long)ccb->ccb_buf.b_blkno, ccb->ccb_target);
2061 
2062 	splassert(IPL_BIO);
2063 
2064 	if (ccb->ccb_target == -1)
2065 		panic("%s: invalid target on wu: %p", DEVNAME(sc), wu);
2066 
2067 	if (ccb->ccb_buf.b_flags & B_ERROR) {
2068 		DNPRINTF(SR_D_INTR, "%s: i/o error on block %lld target %d\n",
2069 		    DEVNAME(sc), (long long)ccb->ccb_buf.b_blkno,
2070 		    ccb->ccb_target);
2071 		if (ISSET(sd->sd_capabilities, SR_CAP_REDUNDANT))
2072 			sd->sd_set_chunk_state(sd, ccb->ccb_target,
2073 			    BIOC_SDOFFLINE);
2074 		else
2075 			printf("%s: %s: i/o error %d @ %s block %lld\n",
2076 			    DEVNAME(sc), sd->sd_meta->ssd_devname,
2077 			    ccb->ccb_buf.b_error, sd->sd_name,
2078 			    (long long)ccb->ccb_buf.b_blkno);
2079 		ccb->ccb_state = SR_CCB_FAILED;
2080 		wu->swu_ios_failed++;
2081 	} else {
2082 		ccb->ccb_state = SR_CCB_OK;
2083 		wu->swu_ios_succeeded++;
2084 	}
2085 
2086 	wu->swu_ios_complete++;
2087 }
2088 
2089 int
2090 sr_wu_alloc(struct sr_discipline *sd, int wu_size)
2091 {
2092 	struct sr_workunit	*wu;
2093 	int			i, no_wu;
2094 
2095 	DNPRINTF(SR_D_WU, "%s: sr_wu_alloc %p %d\n", DEVNAME(sd->sd_sc),
2096 	    sd, sd->sd_max_wu);
2097 
2098 	no_wu = sd->sd_max_wu;
2099 	sd->sd_wu_pending = no_wu;
2100 
2101 	mtx_init(&sd->sd_wu_mtx, IPL_BIO);
2102 	TAILQ_INIT(&sd->sd_wu);
2103 	TAILQ_INIT(&sd->sd_wu_freeq);
2104 	TAILQ_INIT(&sd->sd_wu_pendq);
2105 	TAILQ_INIT(&sd->sd_wu_defq);
2106 
2107 	for (i = 0; i < no_wu; i++) {
2108 		wu = malloc(wu_size, M_DEVBUF, M_WAITOK | M_ZERO);
2109 		TAILQ_INSERT_TAIL(&sd->sd_wu, wu, swu_next);
2110 		TAILQ_INIT(&wu->swu_ccb);
2111 		wu->swu_dis = sd;
2112 		task_set(&wu->swu_task, sr_wu_done_callback, wu);
2113 		sr_wu_put(sd, wu);
2114 	}
2115 
2116 	return (0);
2117 }
2118 
2119 void
2120 sr_wu_free(struct sr_discipline *sd)
2121 {
2122 	struct sr_workunit	*wu;
2123 
2124 	DNPRINTF(SR_D_WU, "%s: sr_wu_free %p\n", DEVNAME(sd->sd_sc), sd);
2125 
2126 	while ((wu = TAILQ_FIRST(&sd->sd_wu_freeq)) != NULL)
2127 		TAILQ_REMOVE(&sd->sd_wu_freeq, wu, swu_link);
2128 	while ((wu = TAILQ_FIRST(&sd->sd_wu_pendq)) != NULL)
2129 		TAILQ_REMOVE(&sd->sd_wu_pendq, wu, swu_link);
2130 	while ((wu = TAILQ_FIRST(&sd->sd_wu_defq)) != NULL)
2131 		TAILQ_REMOVE(&sd->sd_wu_defq, wu, swu_link);
2132 
2133 	while ((wu = TAILQ_FIRST(&sd->sd_wu)) != NULL) {
2134 		TAILQ_REMOVE(&sd->sd_wu, wu, swu_next);
2135 		free(wu, M_DEVBUF, 0);
2136 	}
2137 }
2138 
2139 void *
2140 sr_wu_get(void *xsd)
2141 {
2142 	struct sr_discipline	*sd = (struct sr_discipline *)xsd;
2143 	struct sr_workunit	*wu;
2144 
2145 	mtx_enter(&sd->sd_wu_mtx);
2146 	wu = TAILQ_FIRST(&sd->sd_wu_freeq);
2147 	if (wu) {
2148 		TAILQ_REMOVE(&sd->sd_wu_freeq, wu, swu_link);
2149 		sd->sd_wu_pending++;
2150 	}
2151 	mtx_leave(&sd->sd_wu_mtx);
2152 
2153 	DNPRINTF(SR_D_WU, "%s: sr_wu_get: %p\n", DEVNAME(sd->sd_sc), wu);
2154 
2155 	return (wu);
2156 }
2157 
2158 void
2159 sr_wu_put(void *xsd, void *xwu)
2160 {
2161 	struct sr_discipline	*sd = (struct sr_discipline *)xsd;
2162 	struct sr_workunit	*wu = (struct sr_workunit *)xwu;
2163 
2164 	DNPRINTF(SR_D_WU, "%s: sr_wu_put: %p\n", DEVNAME(sd->sd_sc), wu);
2165 
2166 	sr_wu_release_ccbs(wu);
2167 	sr_wu_init(sd, wu);
2168 
2169 	mtx_enter(&sd->sd_wu_mtx);
2170 	TAILQ_INSERT_TAIL(&sd->sd_wu_freeq, wu, swu_link);
2171 	sd->sd_wu_pending--;
2172 	mtx_leave(&sd->sd_wu_mtx);
2173 }
2174 
2175 void
2176 sr_wu_init(struct sr_discipline *sd, struct sr_workunit *wu)
2177 {
2178 	int			s;
2179 
2180 	s = splbio();
2181 	if (wu->swu_cb_active == 1)
2182 		panic("%s: sr_wu_init got active wu", DEVNAME(sd->sd_sc));
2183 	splx(s);
2184 
2185 	wu->swu_xs = NULL;
2186 	wu->swu_state = SR_WU_FREE;
2187 	wu->swu_flags = 0;
2188 	wu->swu_blk_start = 0;
2189 	wu->swu_blk_end = 0;
2190 	wu->swu_collider = NULL;
2191 }
2192 
2193 void
2194 sr_wu_enqueue_ccb(struct sr_workunit *wu, struct sr_ccb *ccb)
2195 {
2196 	struct sr_discipline	*sd = wu->swu_dis;
2197 	int			s;
2198 
2199 	s = splbio();
2200 	if (wu->swu_cb_active == 1)
2201 		panic("%s: sr_wu_enqueue_ccb got active wu",
2202 		    DEVNAME(sd->sd_sc));
2203 	ccb->ccb_wu = wu;
2204 	wu->swu_io_count++;
2205 	TAILQ_INSERT_TAIL(&wu->swu_ccb, ccb, ccb_link);
2206 	splx(s);
2207 }
2208 
2209 void
2210 sr_wu_release_ccbs(struct sr_workunit *wu)
2211 {
2212 	struct sr_ccb		*ccb;
2213 
2214 	/* Return all ccbs that are associated with this workunit. */
2215 	while ((ccb = TAILQ_FIRST(&wu->swu_ccb)) != NULL) {
2216 		TAILQ_REMOVE(&wu->swu_ccb, ccb, ccb_link);
2217 		sr_ccb_put(ccb);
2218 	}
2219 
2220 	wu->swu_io_count = 0;
2221 	wu->swu_ios_complete = 0;
2222 	wu->swu_ios_failed = 0;
2223 	wu->swu_ios_succeeded = 0;
2224 }
2225 
2226 void
2227 sr_wu_done(struct sr_workunit *wu)
2228 {
2229 	struct sr_discipline	*sd = wu->swu_dis;
2230 
2231 	DNPRINTF(SR_D_INTR, "%s: sr_wu_done count %d completed %d failed %d\n",
2232 	    DEVNAME(sd->sd_sc), wu->swu_io_count, wu->swu_ios_complete,
2233 	    wu->swu_ios_failed);
2234 
2235 	if (wu->swu_ios_complete < wu->swu_io_count)
2236 		return;
2237 
2238 	task_add(sd->sd_taskq, &wu->swu_task);
2239 }
2240 
2241 void
2242 sr_wu_done_callback(void *xwu)
2243 {
2244 	struct sr_workunit	*wu = xwu;
2245 	struct sr_discipline	*sd = wu->swu_dis;
2246 	struct scsi_xfer	*xs = wu->swu_xs;
2247 	struct sr_workunit	*wup;
2248 	int			s;
2249 
2250 	/*
2251 	 * The SR_WUF_DISCIPLINE or SR_WUF_REBUILD flag must be set if
2252 	 * the work unit is not associated with a scsi_xfer.
2253 	 */
2254 	KASSERT(xs != NULL ||
2255 	    (wu->swu_flags & (SR_WUF_DISCIPLINE|SR_WUF_REBUILD)));
2256 
2257 	s = splbio();
2258 
2259 	if (xs != NULL) {
2260 		if (wu->swu_ios_failed)
2261 			xs->error = XS_DRIVER_STUFFUP;
2262 		else
2263 			xs->error = XS_NOERROR;
2264 	}
2265 
2266 	if (sd->sd_scsi_wu_done) {
2267 		if (sd->sd_scsi_wu_done(wu) == SR_WU_RESTART)
2268 			goto done;
2269 	}
2270 
2271 	/* Remove work unit from pending queue. */
2272 	TAILQ_FOREACH(wup, &sd->sd_wu_pendq, swu_link)
2273 		if (wup == wu)
2274 			break;
2275 	if (wup == NULL)
2276 		panic("%s: wu %p not on pending queue",
2277 		    DEVNAME(sd->sd_sc), wu);
2278 	TAILQ_REMOVE(&sd->sd_wu_pendq, wu, swu_link);
2279 
2280 	if (wu->swu_collider) {
2281 		if (wu->swu_ios_failed)
2282 			sr_raid_recreate_wu(wu->swu_collider);
2283 
2284 		/* XXX Should the collider be failed if this xs failed? */
2285 		sr_raid_startwu(wu->swu_collider);
2286 	}
2287 
2288 	/*
2289 	 * If a discipline provides its own sd_scsi_done function, then it
2290 	 * is responsible for calling sr_scsi_done() once I/O is complete.
2291 	 */
2292 	if (wu->swu_flags & SR_WUF_REBUILD)
2293 		wu->swu_flags |= SR_WUF_REBUILDIOCOMP;
2294 	if (wu->swu_flags & SR_WUF_WAKEUP)
2295 		wakeup(wu);
2296 	if (sd->sd_scsi_done)
2297 		sd->sd_scsi_done(wu);
2298 	else if (wu->swu_flags & SR_WUF_DISCIPLINE)
2299 		sr_scsi_wu_put(sd, wu);
2300 	else if (!(wu->swu_flags & SR_WUF_REBUILD))
2301 		sr_scsi_done(sd, xs);
2302 
2303 done:
2304 	splx(s);
2305 }
2306 
2307 struct sr_workunit *
2308 sr_scsi_wu_get(struct sr_discipline *sd, int flags)
2309 {
2310 	return scsi_io_get(&sd->sd_iopool, flags);
2311 }
2312 
2313 void
2314 sr_scsi_wu_put(struct sr_discipline *sd, struct sr_workunit *wu)
2315 {
2316 	scsi_io_put(&sd->sd_iopool, wu);
2317 
2318 	if (sd->sd_sync && sd->sd_wu_pending == 0)
2319 		wakeup(sd);
2320 }
2321 
2322 void
2323 sr_scsi_done(struct sr_discipline *sd, struct scsi_xfer *xs)
2324 {
2325 	DNPRINTF(SR_D_DIS, "%s: sr_scsi_done: xs %p\n", DEVNAME(sd->sd_sc), xs);
2326 
2327 	if (xs->error == XS_NOERROR)
2328 		xs->resid = 0;
2329 
2330 	scsi_done(xs);
2331 
2332 	if (sd->sd_sync && sd->sd_wu_pending == 0)
2333 		wakeup(sd);
2334 }
2335 
2336 void
2337 sr_scsi_cmd(struct scsi_xfer *xs)
2338 {
2339 	struct scsi_link	*link = xs->sc_link;
2340 	struct sr_softc		*sc = link->adapter_softc;
2341 	struct sr_workunit	*wu = xs->io;
2342 	struct sr_discipline	*sd;
2343 
2344 	DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd target %d xs %p flags %#x\n",
2345 	    DEVNAME(sc), link->target, xs, xs->flags);
2346 
2347 	sd = sc->sc_targets[link->target];
2348 	if (sd == NULL)
2349 		panic("%s: sr_scsi_cmd NULL discipline", DEVNAME(sc));
2350 
2351 	if (sd->sd_deleted) {
2352 		printf("%s: %s device is being deleted, failing io\n",
2353 		    DEVNAME(sc), sd->sd_meta->ssd_devname);
2354 		goto stuffup;
2355 	}
2356 
2357 	/* scsi layer *can* re-send wu without calling sr_wu_put(). */
2358 	sr_wu_release_ccbs(wu);
2359 	sr_wu_init(sd, wu);
2360 	wu->swu_state = SR_WU_INPROGRESS;
2361 	wu->swu_xs = xs;
2362 
2363 	switch (xs->cmd->opcode) {
2364 	case READ_COMMAND:
2365 	case READ_BIG:
2366 	case READ_16:
2367 	case WRITE_COMMAND:
2368 	case WRITE_BIG:
2369 	case WRITE_16:
2370 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd: READ/WRITE %02x\n",
2371 		    DEVNAME(sc), xs->cmd->opcode);
2372 		if (sd->sd_scsi_rw(wu))
2373 			goto stuffup;
2374 		break;
2375 
2376 	case SYNCHRONIZE_CACHE:
2377 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd: SYNCHRONIZE_CACHE\n",
2378 		    DEVNAME(sc));
2379 		if (sd->sd_scsi_sync(wu))
2380 			goto stuffup;
2381 		goto complete;
2382 
2383 	case TEST_UNIT_READY:
2384 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd: TEST_UNIT_READY\n",
2385 		    DEVNAME(sc));
2386 		if (sd->sd_scsi_tur(wu))
2387 			goto stuffup;
2388 		goto complete;
2389 
2390 	case START_STOP:
2391 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd: START_STOP\n",
2392 		    DEVNAME(sc));
2393 		if (sd->sd_scsi_start_stop(wu))
2394 			goto stuffup;
2395 		goto complete;
2396 
2397 	case INQUIRY:
2398 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd: INQUIRY\n",
2399 		    DEVNAME(sc));
2400 		if (sd->sd_scsi_inquiry(wu))
2401 			goto stuffup;
2402 		goto complete;
2403 
2404 	case READ_CAPACITY:
2405 	case READ_CAPACITY_16:
2406 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd READ CAPACITY 0x%02x\n",
2407 		    DEVNAME(sc), xs->cmd->opcode);
2408 		if (sd->sd_scsi_read_cap(wu))
2409 			goto stuffup;
2410 		goto complete;
2411 
2412 	case REQUEST_SENSE:
2413 		DNPRINTF(SR_D_CMD, "%s: sr_scsi_cmd REQUEST SENSE\n",
2414 		    DEVNAME(sc));
2415 		if (sd->sd_scsi_req_sense(wu))
2416 			goto stuffup;
2417 		goto complete;
2418 
2419 	default:
2420 		DNPRINTF(SR_D_CMD, "%s: unsupported scsi command %x\n",
2421 		    DEVNAME(sc), xs->cmd->opcode);
2422 		/* XXX might need to add generic function to handle others */
2423 		goto stuffup;
2424 	}
2425 
2426 	return;
2427 stuffup:
2428 	if (sd->sd_scsi_sense.error_code) {
2429 		xs->error = XS_SENSE;
2430 		memcpy(&xs->sense, &sd->sd_scsi_sense, sizeof(xs->sense));
2431 		bzero(&sd->sd_scsi_sense, sizeof(sd->sd_scsi_sense));
2432 	} else {
2433 		xs->error = XS_DRIVER_STUFFUP;
2434 	}
2435 complete:
2436 	sr_scsi_done(sd, xs);
2437 }
2438 
2439 int
2440 sr_scsi_probe(struct scsi_link *link)
2441 {
2442 	struct sr_softc		*sc = link->adapter_softc;
2443 	struct sr_discipline	*sd;
2444 
2445 	KASSERT(link->target < SR_MAX_LD && link->lun == 0);
2446 
2447 	sd = sc->sc_targets[link->target];
2448 	if (sd == NULL)
2449 		return (ENODEV);
2450 
2451 	link->pool = &sd->sd_iopool;
2452 	if (sd->sd_openings)
2453 		link->openings = sd->sd_openings(sd);
2454 	else
2455 		link->openings = sd->sd_max_wu;
2456 
2457 	return (0);
2458 }
2459 
2460 int
2461 sr_scsi_ioctl(struct scsi_link *link, u_long cmd, caddr_t addr, int flag)
2462 {
2463 	struct sr_softc		*sc = link->adapter_softc;
2464 	struct sr_discipline	*sd;
2465 
2466 	sd = sc->sc_targets[link->target];
2467 	if (sd == NULL)
2468 		return (ENODEV);
2469 
2470 	DNPRINTF(SR_D_IOCTL, "%s: %s sr_scsi_ioctl cmd: %#lx\n",
2471 	    DEVNAME(sc), sd->sd_meta->ssd_devname, cmd);
2472 
2473 	/* Pass bio ioctls through to the bio handler. */
2474 	if (IOCGROUP(cmd) == 'B')
2475 		return (sr_bio_handler(sc, sd, cmd, (struct bio *)addr));
2476 
2477 	switch (cmd) {
2478 	case DIOCGCACHE:
2479 	case DIOCSCACHE:
2480 		return (EOPNOTSUPP);
2481 	default:
2482 		return (ENOTTY);
2483 	}
2484 }
2485 
2486 int
2487 sr_bio_ioctl(struct device *dev, u_long cmd, caddr_t addr)
2488 {
2489 	struct sr_softc *sc = (struct sr_softc *) dev;
2490 	DNPRINTF(SR_D_IOCTL, "%s: sr_bio_ioctl\n", DEVNAME(sc));
2491 
2492 	return sr_bio_handler(sc, NULL, cmd, (struct bio *)addr);
2493 }
2494 
2495 int
2496 sr_bio_handler(struct sr_softc *sc, struct sr_discipline *sd, u_long cmd,
2497     struct bio *bio)
2498 {
2499 	int			rv = 0;
2500 
2501 	DNPRINTF(SR_D_IOCTL, "%s: sr_bio_handler ", DEVNAME(sc));
2502 
2503 	rw_enter_write(&sc->sc_lock);
2504 
2505 	bio_status_init(&sc->sc_status, &sc->sc_dev);
2506 
2507 	switch (cmd) {
2508 	case BIOCINQ:
2509 		DNPRINTF(SR_D_IOCTL, "inq\n");
2510 		rv = sr_ioctl_inq(sc, (struct bioc_inq *)bio);
2511 		break;
2512 
2513 	case BIOCVOL:
2514 		DNPRINTF(SR_D_IOCTL, "vol\n");
2515 		rv = sr_ioctl_vol(sc, (struct bioc_vol *)bio);
2516 		break;
2517 
2518 	case BIOCDISK:
2519 		DNPRINTF(SR_D_IOCTL, "disk\n");
2520 		rv = sr_ioctl_disk(sc, (struct bioc_disk *)bio);
2521 		break;
2522 
2523 	case BIOCALARM:
2524 		DNPRINTF(SR_D_IOCTL, "alarm\n");
2525 		/*rv = sr_ioctl_alarm(sc, (struct bioc_alarm *)bio); */
2526 		break;
2527 
2528 	case BIOCBLINK:
2529 		DNPRINTF(SR_D_IOCTL, "blink\n");
2530 		/*rv = sr_ioctl_blink(sc, (struct bioc_blink *)bio); */
2531 		break;
2532 
2533 	case BIOCSETSTATE:
2534 		DNPRINTF(SR_D_IOCTL, "setstate\n");
2535 		rv = sr_ioctl_setstate(sc, (struct bioc_setstate *)bio);
2536 		break;
2537 
2538 	case BIOCCREATERAID:
2539 		DNPRINTF(SR_D_IOCTL, "createraid\n");
2540 		rv = sr_ioctl_createraid(sc, (struct bioc_createraid *)bio,
2541 		    1, NULL);
2542 		break;
2543 
2544 	case BIOCDELETERAID:
2545 		DNPRINTF(SR_D_IOCTL, "deleteraid\n");
2546 		rv = sr_ioctl_deleteraid(sc, sd, (struct bioc_deleteraid *)bio);
2547 		break;
2548 
2549 	case BIOCDISCIPLINE:
2550 		DNPRINTF(SR_D_IOCTL, "discipline\n");
2551 		rv = sr_ioctl_discipline(sc, sd, (struct bioc_discipline *)bio);
2552 		break;
2553 
2554 	case BIOCINSTALLBOOT:
2555 		DNPRINTF(SR_D_IOCTL, "installboot\n");
2556 		rv = sr_ioctl_installboot(sc, sd,
2557 		    (struct bioc_installboot *)bio);
2558 		break;
2559 
2560 	default:
2561 		DNPRINTF(SR_D_IOCTL, "invalid ioctl\n");
2562 		rv = ENOTTY;
2563 	}
2564 
2565 	sc->sc_status.bs_status = (rv ? BIO_STATUS_ERROR : BIO_STATUS_SUCCESS);
2566 
2567 	if (sc->sc_status.bs_msg_count > 0)
2568 		rv = 0;
2569 
2570 	memcpy(&bio->bio_status, &sc->sc_status, sizeof(struct bio_status));
2571 
2572 	rw_exit_write(&sc->sc_lock);
2573 
2574 	return (rv);
2575 }
2576 
2577 int
2578 sr_ioctl_inq(struct sr_softc *sc, struct bioc_inq *bi)
2579 {
2580 	struct sr_discipline	*sd;
2581 	int			vol = 0, disk = 0;
2582 
2583 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
2584 		vol++;
2585 		disk += sd->sd_meta->ssdi.ssd_chunk_no;
2586 	}
2587 
2588 	strlcpy(bi->bi_dev, sc->sc_dev.dv_xname, sizeof(bi->bi_dev));
2589 	bi->bi_novol = vol + sc->sc_hotspare_no;
2590 	bi->bi_nodisk = disk + sc->sc_hotspare_no;
2591 
2592 	return (0);
2593 }
2594 
2595 int
2596 sr_ioctl_vol(struct sr_softc *sc, struct bioc_vol *bv)
2597 {
2598 	int			vol = -1, rv = EINVAL;
2599 	struct sr_discipline	*sd;
2600 	struct sr_chunk		*hotspare;
2601 
2602 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
2603 		vol++;
2604 		if (vol != bv->bv_volid)
2605 			continue;
2606 
2607 		bv->bv_status = sd->sd_vol_status;
2608 		bv->bv_size = sd->sd_meta->ssdi.ssd_size << DEV_BSHIFT;
2609 		bv->bv_level = sd->sd_meta->ssdi.ssd_level;
2610 		bv->bv_nodisk = sd->sd_meta->ssdi.ssd_chunk_no;
2611 
2612 #ifdef CRYPTO
2613 		if (sd->sd_meta->ssdi.ssd_level == 'C' &&
2614 		    sd->mds.mdd_crypto.key_disk != NULL)
2615 			bv->bv_nodisk++;
2616 #endif
2617 		if (bv->bv_status == BIOC_SVREBUILD)
2618 			bv->bv_percent = sr_rebuild_percent(sd);
2619 
2620 		strlcpy(bv->bv_dev, sd->sd_meta->ssd_devname,
2621 		    sizeof(bv->bv_dev));
2622 		strlcpy(bv->bv_vendor, sd->sd_meta->ssdi.ssd_vendor,
2623 		    sizeof(bv->bv_vendor));
2624 		rv = 0;
2625 		goto done;
2626 	}
2627 
2628 	/* Check hotspares list. */
2629 	SLIST_FOREACH(hotspare, &sc->sc_hotspare_list, src_link) {
2630 		vol++;
2631 		if (vol != bv->bv_volid)
2632 			continue;
2633 
2634 		bv->bv_status = BIOC_SVONLINE;
2635 		bv->bv_size = hotspare->src_meta.scmi.scm_size << DEV_BSHIFT;
2636 		bv->bv_level = -1;	/* Hotspare. */
2637 		bv->bv_nodisk = 1;
2638 		strlcpy(bv->bv_dev, hotspare->src_meta.scmi.scm_devname,
2639 		    sizeof(bv->bv_dev));
2640 		strlcpy(bv->bv_vendor, hotspare->src_meta.scmi.scm_devname,
2641 		    sizeof(bv->bv_vendor));
2642 		rv = 0;
2643 		goto done;
2644 	}
2645 
2646 done:
2647 	return (rv);
2648 }
2649 
2650 int
2651 sr_ioctl_disk(struct sr_softc *sc, struct bioc_disk *bd)
2652 {
2653 	struct sr_discipline	*sd;
2654 	struct sr_chunk		*src, *hotspare;
2655 	int			vol = -1, rv = EINVAL;
2656 
2657 	if (bd->bd_diskid < 0)
2658 		goto done;
2659 
2660 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
2661 		vol++;
2662 		if (vol != bd->bd_volid)
2663 			continue;
2664 
2665 		if (bd->bd_diskid < sd->sd_meta->ssdi.ssd_chunk_no)
2666 			src = sd->sd_vol.sv_chunks[bd->bd_diskid];
2667 #ifdef CRYPTO
2668 		else if (bd->bd_diskid == sd->sd_meta->ssdi.ssd_chunk_no &&
2669 		    sd->sd_meta->ssdi.ssd_level == 'C' &&
2670 		    sd->mds.mdd_crypto.key_disk != NULL)
2671 			src = sd->mds.mdd_crypto.key_disk;
2672 #endif
2673 		else
2674 			break;
2675 
2676 		bd->bd_status = src->src_meta.scm_status;
2677 		bd->bd_size = src->src_meta.scmi.scm_size << DEV_BSHIFT;
2678 		bd->bd_channel = vol;
2679 		bd->bd_target = bd->bd_diskid;
2680 		strlcpy(bd->bd_vendor, src->src_meta.scmi.scm_devname,
2681 		    sizeof(bd->bd_vendor));
2682 		rv = 0;
2683 		goto done;
2684 	}
2685 
2686 	/* Check hotspares list. */
2687 	SLIST_FOREACH(hotspare, &sc->sc_hotspare_list, src_link) {
2688 		vol++;
2689 		if (vol != bd->bd_volid)
2690 			continue;
2691 
2692 		if (bd->bd_diskid != 0)
2693 			break;
2694 
2695 		bd->bd_status = hotspare->src_meta.scm_status;
2696 		bd->bd_size = hotspare->src_meta.scmi.scm_size << DEV_BSHIFT;
2697 		bd->bd_channel = vol;
2698 		bd->bd_target = bd->bd_diskid;
2699 		strlcpy(bd->bd_vendor, hotspare->src_meta.scmi.scm_devname,
2700 		    sizeof(bd->bd_vendor));
2701 		rv = 0;
2702 		goto done;
2703 	}
2704 
2705 done:
2706 	return (rv);
2707 }
2708 
2709 int
2710 sr_ioctl_setstate(struct sr_softc *sc, struct bioc_setstate *bs)
2711 {
2712 	int			rv = EINVAL;
2713 	int			vol = -1, found, c;
2714 	struct sr_discipline	*sd;
2715 	struct sr_chunk		*ch_entry;
2716 	struct sr_chunk_head	*cl;
2717 
2718 	if (bs->bs_other_id_type == BIOC_SSOTHER_UNUSED)
2719 		goto done;
2720 
2721 	if (bs->bs_status == BIOC_SSHOTSPARE) {
2722 		rv = sr_hotspare(sc, (dev_t)bs->bs_other_id);
2723 		goto done;
2724 	}
2725 
2726 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
2727 		vol++;
2728 		if (vol == bs->bs_volid)
2729 			break;
2730 	}
2731 	if (sd == NULL)
2732 		goto done;
2733 
2734 	switch (bs->bs_status) {
2735 	case BIOC_SSOFFLINE:
2736 		/* Take chunk offline */
2737 		found = c = 0;
2738 		cl = &sd->sd_vol.sv_chunk_list;
2739 		SLIST_FOREACH(ch_entry, cl, src_link) {
2740 			if (ch_entry->src_dev_mm == bs->bs_other_id) {
2741 				found = 1;
2742 				break;
2743 			}
2744 			c++;
2745 		}
2746 		if (found == 0) {
2747 			sr_error(sc, "chunk not part of array");
2748 			goto done;
2749 		}
2750 
2751 		/* XXX: check current state first */
2752 		sd->sd_set_chunk_state(sd, c, BIOC_SDOFFLINE);
2753 
2754 		if (sr_meta_save(sd, SR_META_DIRTY)) {
2755 			sr_error(sc, "could not save metadata for %s",
2756 			    sd->sd_meta->ssd_devname);
2757 			goto done;
2758 		}
2759 		rv = 0;
2760 		break;
2761 
2762 	case BIOC_SDSCRUB:
2763 		break;
2764 
2765 	case BIOC_SSREBUILD:
2766 		rv = sr_rebuild_init(sd, (dev_t)bs->bs_other_id, 0);
2767 		break;
2768 
2769 	default:
2770 		sr_error(sc, "unsupported state request %d", bs->bs_status);
2771 	}
2772 
2773 done:
2774 	return (rv);
2775 }
2776 
2777 int
2778 sr_chunk_in_use(struct sr_softc *sc, dev_t dev)
2779 {
2780 	struct sr_discipline	*sd;
2781 	struct sr_chunk		*chunk;
2782 	int			i;
2783 
2784 	DNPRINTF(SR_D_MISC, "%s: sr_chunk_in_use(%d)\n", DEVNAME(sc), dev);
2785 
2786 	if (dev == NODEV)
2787 		return BIOC_SDINVALID;
2788 
2789 	/* See if chunk is already in use. */
2790 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
2791 		for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
2792 			chunk = sd->sd_vol.sv_chunks[i];
2793 			if (chunk->src_dev_mm == dev)
2794 				return chunk->src_meta.scm_status;
2795 		}
2796 	}
2797 
2798 	/* Check hotspares list. */
2799 	SLIST_FOREACH(chunk, &sc->sc_hotspare_list, src_link)
2800 		if (chunk->src_dev_mm == dev)
2801 			return chunk->src_meta.scm_status;
2802 
2803 	return BIOC_SDINVALID;
2804 }
2805 
2806 int
2807 sr_hotspare(struct sr_softc *sc, dev_t dev)
2808 {
2809 	struct sr_discipline	*sd = NULL;
2810 	struct sr_metadata	*sm = NULL;
2811 	struct sr_meta_chunk    *hm;
2812 	struct sr_chunk_head	*cl;
2813 	struct sr_chunk		*chunk, *last, *hotspare = NULL;
2814 	struct sr_uuid		uuid;
2815 	struct disklabel	label;
2816 	struct vnode		*vn;
2817 	u_int64_t		size;
2818 	char			devname[32];
2819 	int			rv = EINVAL;
2820 	int			c, part, open = 0;
2821 
2822 	/*
2823 	 * Add device to global hotspares list.
2824 	 */
2825 
2826 	sr_meta_getdevname(sc, dev, devname, sizeof(devname));
2827 
2828 	/* Make sure chunk is not already in use. */
2829 	c = sr_chunk_in_use(sc, dev);
2830 	if (c != BIOC_SDINVALID && c != BIOC_SDOFFLINE) {
2831 		if (c == BIOC_SDHOTSPARE)
2832 			sr_error(sc, "%s is already a hotspare", devname);
2833 		else
2834 			sr_error(sc, "%s is already in use", devname);
2835 		goto done;
2836 	}
2837 
2838 	/* XXX - See if there is an existing degraded volume... */
2839 
2840 	/* Open device. */
2841 	if (bdevvp(dev, &vn)) {
2842 		sr_error(sc, "sr_hotspare: cannot allocate vnode");
2843 		goto done;
2844 	}
2845 	if (VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc)) {
2846 		DNPRINTF(SR_D_META,"%s: sr_hotspare cannot open %s\n",
2847 		    DEVNAME(sc), devname);
2848 		vput(vn);
2849 		goto fail;
2850 	}
2851 	open = 1; /* close dev on error */
2852 
2853 	/* Get partition details. */
2854 	part = DISKPART(dev);
2855 	if (VOP_IOCTL(vn, DIOCGDINFO, (caddr_t)&label, FREAD,
2856 	    NOCRED, curproc)) {
2857 		DNPRINTF(SR_D_META, "%s: sr_hotspare ioctl failed\n",
2858 		    DEVNAME(sc));
2859 		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
2860 		vput(vn);
2861 		goto fail;
2862 	}
2863 	if (label.d_partitions[part].p_fstype != FS_RAID) {
2864 		sr_error(sc, "%s partition not of type RAID (%d)",
2865 		    devname, label.d_partitions[part].p_fstype);
2866 		goto fail;
2867 	}
2868 
2869 	/* Calculate partition size. */
2870 	size = DL_SECTOBLK(&label, DL_GETPSIZE(&label.d_partitions[part]));
2871 	if (size <= SR_DATA_OFFSET) {
2872 		DNPRINTF(SR_D_META, "%s: %s partition too small\n", DEVNAME(sc),
2873 		    devname);
2874 		goto fail;
2875 	}
2876 	size -= SR_DATA_OFFSET;
2877 	if (size > INT64_MAX) {
2878 		DNPRINTF(SR_D_META, "%s: %s partition too large\n", DEVNAME(sc),
2879 		    devname);
2880 		goto fail;
2881 	}
2882 
2883 	/*
2884 	 * Create and populate chunk metadata.
2885 	 */
2886 
2887 	sr_uuid_generate(&uuid);
2888 	hotspare = malloc(sizeof(struct sr_chunk), M_DEVBUF, M_WAITOK | M_ZERO);
2889 
2890 	hotspare->src_dev_mm = dev;
2891 	hotspare->src_vn = vn;
2892 	strlcpy(hotspare->src_devname, devname, sizeof(hm->scmi.scm_devname));
2893 	hotspare->src_size = size;
2894 
2895 	hm = &hotspare->src_meta;
2896 	hm->scmi.scm_volid = SR_HOTSPARE_VOLID;
2897 	hm->scmi.scm_chunk_id = 0;
2898 	hm->scmi.scm_size = size;
2899 	hm->scmi.scm_coerced_size = size;
2900 	strlcpy(hm->scmi.scm_devname, devname, sizeof(hm->scmi.scm_devname));
2901 	memcpy(&hm->scmi.scm_uuid, &uuid, sizeof(struct sr_uuid));
2902 
2903 	sr_checksum(sc, hm, &hm->scm_checksum,
2904 	    sizeof(struct sr_meta_chunk_invariant));
2905 
2906 	hm->scm_status = BIOC_SDHOTSPARE;
2907 
2908 	/*
2909 	 * Create and populate our own discipline and metadata.
2910 	 */
2911 
2912 	sm = malloc(sizeof(struct sr_metadata), M_DEVBUF, M_WAITOK | M_ZERO);
2913 	sm->ssdi.ssd_magic = SR_MAGIC;
2914 	sm->ssdi.ssd_version = SR_META_VERSION;
2915 	sm->ssd_ondisk = 0;
2916 	sm->ssdi.ssd_vol_flags = 0;
2917 	memcpy(&sm->ssdi.ssd_uuid, &uuid, sizeof(struct sr_uuid));
2918 	sm->ssdi.ssd_chunk_no = 1;
2919 	sm->ssdi.ssd_volid = SR_HOTSPARE_VOLID;
2920 	sm->ssdi.ssd_level = SR_HOTSPARE_LEVEL;
2921 	sm->ssdi.ssd_size = size;
2922 	sm->ssdi.ssd_secsize = label.d_secsize;
2923 	strlcpy(sm->ssdi.ssd_vendor, "OPENBSD", sizeof(sm->ssdi.ssd_vendor));
2924 	snprintf(sm->ssdi.ssd_product, sizeof(sm->ssdi.ssd_product),
2925 	    "SR %s", "HOTSPARE");
2926 	snprintf(sm->ssdi.ssd_revision, sizeof(sm->ssdi.ssd_revision),
2927 	    "%03d", SR_META_VERSION);
2928 
2929 	sd = malloc(sizeof(struct sr_discipline), M_DEVBUF, M_WAITOK | M_ZERO);
2930 	sd->sd_sc = sc;
2931 	sd->sd_meta = sm;
2932 	sd->sd_meta_type = SR_META_F_NATIVE;
2933 	sd->sd_vol_status = BIOC_SVONLINE;
2934 	strlcpy(sd->sd_name, "HOTSPARE", sizeof(sd->sd_name));
2935 	SLIST_INIT(&sd->sd_meta_opt);
2936 
2937 	/* Add chunk to volume. */
2938 	sd->sd_vol.sv_chunks = malloc(sizeof(struct sr_chunk *), M_DEVBUF,
2939 	    M_WAITOK | M_ZERO);
2940 	sd->sd_vol.sv_chunks[0] = hotspare;
2941 	SLIST_INIT(&sd->sd_vol.sv_chunk_list);
2942 	SLIST_INSERT_HEAD(&sd->sd_vol.sv_chunk_list, hotspare, src_link);
2943 
2944 	/* Save metadata. */
2945 	if (sr_meta_save(sd, SR_META_DIRTY)) {
2946 		sr_error(sc, "could not save metadata to %s", devname);
2947 		goto fail;
2948 	}
2949 
2950 	/*
2951 	 * Add chunk to hotspare list.
2952 	 */
2953 	rw_enter_write(&sc->sc_hs_lock);
2954 	cl = &sc->sc_hotspare_list;
2955 	if (SLIST_EMPTY(cl))
2956 		SLIST_INSERT_HEAD(cl, hotspare, src_link);
2957 	else {
2958 		SLIST_FOREACH(chunk, cl, src_link)
2959 			last = chunk;
2960 		SLIST_INSERT_AFTER(last, hotspare, src_link);
2961 	}
2962 	sc->sc_hotspare_no++;
2963 	rw_exit_write(&sc->sc_hs_lock);
2964 
2965 	rv = 0;
2966 	goto done;
2967 
2968 fail:
2969 	free(hotspare, M_DEVBUF, 0);
2970 
2971 done:
2972 	if (sd)
2973 		free(sd->sd_vol.sv_chunks, M_DEVBUF, 0);
2974 	free(sd, M_DEVBUF, 0);
2975 	free(sm, M_DEVBUF, 0);
2976 	if (open) {
2977 		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
2978 		vput(vn);
2979 	}
2980 
2981 	return (rv);
2982 }
2983 
2984 void
2985 sr_hotspare_rebuild_callback(void *xsd)
2986 {
2987 	struct sr_discipline *sd = xsd;
2988 	sr_hotspare_rebuild(sd);
2989 }
2990 
2991 void
2992 sr_hotspare_rebuild(struct sr_discipline *sd)
2993 {
2994 	struct sr_softc		*sc = sd->sd_sc;
2995 	struct sr_chunk_head	*cl;
2996 	struct sr_chunk		*hotspare, *chunk = NULL;
2997 	struct sr_workunit	*wu;
2998 	struct sr_ccb		*ccb;
2999 	int			i, s, cid, busy;
3000 
3001 	/*
3002 	 * Attempt to locate a hotspare and initiate rebuild.
3003 	 */
3004 
3005 	/* Find first offline chunk. */
3006 	for (cid = 0; cid < sd->sd_meta->ssdi.ssd_chunk_no; cid++) {
3007 		if (sd->sd_vol.sv_chunks[cid]->src_meta.scm_status ==
3008 		    BIOC_SDOFFLINE) {
3009 			chunk = sd->sd_vol.sv_chunks[cid];
3010 			break;
3011 		}
3012 	}
3013 	if (chunk == NULL) {
3014 		printf("%s: no offline chunk found on %s!\n",
3015 		    DEVNAME(sc), sd->sd_meta->ssd_devname);
3016 		return;
3017 	}
3018 
3019 	/* See if we have a suitable hotspare... */
3020 	rw_enter_write(&sc->sc_hs_lock);
3021 	cl = &sc->sc_hotspare_list;
3022 	SLIST_FOREACH(hotspare, cl, src_link)
3023 		if (hotspare->src_size >= chunk->src_size &&
3024 		    hotspare->src_secsize <= sd->sd_meta->ssdi.ssd_secsize)
3025 			break;
3026 
3027 	if (hotspare != NULL) {
3028 
3029 		printf("%s: %s volume degraded, will attempt to "
3030 		    "rebuild on hotspare %s\n", DEVNAME(sc),
3031 		    sd->sd_meta->ssd_devname, hotspare->src_devname);
3032 
3033 		/*
3034 		 * Ensure that all pending I/O completes on the failed chunk
3035 		 * before trying to initiate a rebuild.
3036 		 */
3037 		i = 0;
3038 		do {
3039 			busy = 0;
3040 
3041 			s = splbio();
3042 			TAILQ_FOREACH(wu, &sd->sd_wu_pendq, swu_link) {
3043 				TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link) {
3044 					if (ccb->ccb_target == cid)
3045 						busy = 1;
3046 				}
3047 			}
3048 			TAILQ_FOREACH(wu, &sd->sd_wu_defq, swu_link) {
3049 				TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link) {
3050 					if (ccb->ccb_target == cid)
3051 						busy = 1;
3052 				}
3053 			}
3054 			splx(s);
3055 
3056 			if (busy) {
3057 				tsleep(sd, PRIBIO, "sr_hotspare", hz);
3058 				i++;
3059 			}
3060 
3061 		} while (busy && i < 120);
3062 
3063 		DNPRINTF(SR_D_META, "%s: waited %i seconds for I/O to "
3064 		    "complete on failed chunk %s\n", DEVNAME(sc),
3065 		    i, chunk->src_devname);
3066 
3067 		if (busy) {
3068 			printf("%s: pending I/O failed to complete on "
3069 			    "failed chunk %s, hotspare rebuild aborted...\n",
3070 			    DEVNAME(sc), chunk->src_devname);
3071 			goto done;
3072 		}
3073 
3074 		s = splbio();
3075 		rw_enter_write(&sc->sc_lock);
3076 		bio_status_init(&sc->sc_status, &sc->sc_dev);
3077 		if (sr_rebuild_init(sd, hotspare->src_dev_mm, 1) == 0) {
3078 
3079 			/* Remove hotspare from available list. */
3080 			sc->sc_hotspare_no--;
3081 			SLIST_REMOVE(cl, hotspare, sr_chunk, src_link);
3082 			free(hotspare, M_DEVBUF, 0);
3083 
3084 		}
3085 		rw_exit_write(&sc->sc_lock);
3086 		splx(s);
3087 	}
3088 done:
3089 	rw_exit_write(&sc->sc_hs_lock);
3090 }
3091 
3092 int
3093 sr_rebuild_init(struct sr_discipline *sd, dev_t dev, int hotspare)
3094 {
3095 	struct sr_softc		*sc = sd->sd_sc;
3096 	struct sr_chunk		*chunk = NULL;
3097 	struct sr_meta_chunk	*meta;
3098 	struct disklabel	label;
3099 	struct vnode		*vn;
3100 	u_int64_t		size;
3101 	int64_t			csize;
3102 	char			devname[32];
3103 	int			rv = EINVAL, open = 0;
3104 	int			cid, i, part, status;
3105 
3106 	/*
3107 	 * Attempt to initiate a rebuild onto the specified device.
3108 	 */
3109 
3110 	if (!(sd->sd_capabilities & SR_CAP_REBUILD)) {
3111 		sr_error(sc, "discipline does not support rebuild");
3112 		goto done;
3113 	}
3114 
3115 	/* make sure volume is in the right state */
3116 	if (sd->sd_vol_status == BIOC_SVREBUILD) {
3117 		sr_error(sc, "rebuild already in progress");
3118 		goto done;
3119 	}
3120 	if (sd->sd_vol_status != BIOC_SVDEGRADED) {
3121 		sr_error(sc, "volume not degraded");
3122 		goto done;
3123 	}
3124 
3125 	/* Find first offline chunk. */
3126 	for (cid = 0; cid < sd->sd_meta->ssdi.ssd_chunk_no; cid++) {
3127 		if (sd->sd_vol.sv_chunks[cid]->src_meta.scm_status ==
3128 		    BIOC_SDOFFLINE) {
3129 			chunk = sd->sd_vol.sv_chunks[cid];
3130 			break;
3131 		}
3132 	}
3133 	if (chunk == NULL) {
3134 		sr_error(sc, "no offline chunks available to rebuild");
3135 		goto done;
3136 	}
3137 
3138 	/* Get coerced size from another online chunk. */
3139 	csize = 0;
3140 	for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
3141 		if (sd->sd_vol.sv_chunks[i]->src_meta.scm_status ==
3142 		    BIOC_SDONLINE) {
3143 			meta = &sd->sd_vol.sv_chunks[i]->src_meta;
3144 			csize = meta->scmi.scm_coerced_size;
3145 			break;
3146 		}
3147 	}
3148 	if (csize == 0) {
3149 		sr_error(sc, "no online chunks available for rebuild");
3150 		goto done;
3151 	}
3152 
3153 	sr_meta_getdevname(sc, dev, devname, sizeof(devname));
3154 	if (bdevvp(dev, &vn)) {
3155 		printf("%s: sr_rebuild_init: can't allocate vnode\n",
3156 		    DEVNAME(sc));
3157 		goto done;
3158 	}
3159 	if (VOP_OPEN(vn, FREAD | FWRITE, NOCRED, curproc)) {
3160 		DNPRINTF(SR_D_META,"%s: sr_ioctl_setstate can't "
3161 		    "open %s\n", DEVNAME(sc), devname);
3162 		vput(vn);
3163 		goto done;
3164 	}
3165 	open = 1; /* close dev on error */
3166 
3167 	/* Get disklabel and check partition. */
3168 	part = DISKPART(dev);
3169 	if (VOP_IOCTL(vn, DIOCGDINFO, (caddr_t)&label, FREAD,
3170 	    NOCRED, curproc)) {
3171 		DNPRINTF(SR_D_META, "%s: sr_ioctl_setstate ioctl failed\n",
3172 		    DEVNAME(sc));
3173 		goto done;
3174 	}
3175 	if (label.d_partitions[part].p_fstype != FS_RAID) {
3176 		sr_error(sc, "%s partition not of type RAID (%d)",
3177 		    devname, label.d_partitions[part].p_fstype);
3178 		goto done;
3179 	}
3180 
3181 	/* Is the partition large enough? */
3182 	size = DL_SECTOBLK(&label, DL_GETPSIZE(&label.d_partitions[part]));
3183 	if (size <= sd->sd_meta->ssd_data_blkno) {
3184 		sr_error(sc, "%s: %s partition too small", DEVNAME(sc),
3185 		    devname);
3186 		goto done;
3187 	}
3188 	size -= sd->sd_meta->ssd_data_blkno;
3189 	if (size > INT64_MAX) {
3190 		sr_error(sc, "%s: %s partition too large", DEVNAME(sc),
3191 		    devname);
3192 		goto done;
3193 	}
3194 	if (size < csize) {
3195 		sr_error(sc, "%s partition too small, at least %lld bytes "
3196 		    "required", devname, (long long)(csize << DEV_BSHIFT));
3197 		goto done;
3198 	} else if (size > csize)
3199 		sr_warn(sc, "%s partition too large, wasting %lld bytes",
3200 		    devname, (long long)((size - csize) << DEV_BSHIFT));
3201 	if (label.d_secsize > sd->sd_meta->ssdi.ssd_secsize) {
3202 		sr_error(sc, "%s sector size too large, <= %u bytes "
3203 		    "required", devname, sd->sd_meta->ssdi.ssd_secsize);
3204 		goto done;
3205 	}
3206 
3207 	/* Ensure that this chunk is not already in use. */
3208 	status = sr_chunk_in_use(sc, dev);
3209 	if (status != BIOC_SDINVALID && status != BIOC_SDOFFLINE &&
3210 	    !(hotspare && status == BIOC_SDHOTSPARE)) {
3211 		sr_error(sc, "%s is already in use", devname);
3212 		goto done;
3213 	}
3214 
3215 	/* Reset rebuild counter since we rebuilding onto a new chunk. */
3216 	sd->sd_meta->ssd_rebuild = 0;
3217 
3218 	open = 0; /* leave dev open from here on out */
3219 
3220 	/* Fix up chunk. */
3221 	memcpy(chunk->src_duid, label.d_uid, sizeof(chunk->src_duid));
3222 	chunk->src_dev_mm = dev;
3223 	chunk->src_vn = vn;
3224 
3225 	/* Reconstruct metadata. */
3226 	meta = &chunk->src_meta;
3227 	meta->scmi.scm_volid = sd->sd_meta->ssdi.ssd_volid;
3228 	meta->scmi.scm_chunk_id = cid;
3229 	strlcpy(meta->scmi.scm_devname, devname,
3230 	    sizeof(meta->scmi.scm_devname));
3231 	meta->scmi.scm_size = size;
3232 	meta->scmi.scm_coerced_size = csize;
3233 	memcpy(&meta->scmi.scm_uuid, &sd->sd_meta->ssdi.ssd_uuid,
3234 	    sizeof(meta->scmi.scm_uuid));
3235 	sr_checksum(sc, meta, &meta->scm_checksum,
3236 	    sizeof(struct sr_meta_chunk_invariant));
3237 
3238 	sd->sd_set_chunk_state(sd, cid, BIOC_SDREBUILD);
3239 
3240 	if (sr_meta_save(sd, SR_META_DIRTY)) {
3241 		sr_error(sc, "could not save metadata to %s", devname);
3242 		open = 1;
3243 		goto done;
3244 	}
3245 
3246 	sr_warn(sc, "rebuild of %s started on %s",
3247 	    sd->sd_meta->ssd_devname, devname);
3248 
3249 	sd->sd_reb_abort = 0;
3250 	kthread_create_deferred(sr_rebuild_start, sd);
3251 
3252 	rv = 0;
3253 done:
3254 	if (open) {
3255 		VOP_CLOSE(vn, FREAD | FWRITE, NOCRED, curproc);
3256 		vput(vn);
3257 	}
3258 
3259 	return (rv);
3260 }
3261 
3262 int
3263 sr_rebuild_percent(struct sr_discipline *sd)
3264 {
3265 	daddr_t			rb, sz;
3266 
3267 	sz = sd->sd_meta->ssdi.ssd_size;
3268 	rb = sd->sd_meta->ssd_rebuild;
3269 
3270 	if (rb > 0)
3271 		return (100 - ((sz * 100 - rb * 100) / sz) - 1);
3272 
3273 	return (0);
3274 }
3275 
3276 void
3277 sr_roam_chunks(struct sr_discipline *sd)
3278 {
3279 	struct sr_softc		*sc = sd->sd_sc;
3280 	struct sr_chunk		*chunk;
3281 	struct sr_meta_chunk	*meta;
3282 	int			roamed = 0;
3283 
3284 	/* Have any chunks roamed? */
3285 	SLIST_FOREACH(chunk, &sd->sd_vol.sv_chunk_list, src_link) {
3286 		meta = &chunk->src_meta;
3287 		if (strncmp(meta->scmi.scm_devname, chunk->src_devname,
3288 		    sizeof(meta->scmi.scm_devname))) {
3289 
3290 			printf("%s: roaming device %s -> %s\n", DEVNAME(sc),
3291 			    meta->scmi.scm_devname, chunk->src_devname);
3292 
3293 			strlcpy(meta->scmi.scm_devname, chunk->src_devname,
3294 			    sizeof(meta->scmi.scm_devname));
3295 
3296 			roamed++;
3297 		}
3298 	}
3299 
3300 	if (roamed)
3301 		sr_meta_save(sd, SR_META_DIRTY);
3302 }
3303 
3304 int
3305 sr_ioctl_createraid(struct sr_softc *sc, struct bioc_createraid *bc,
3306     int user, void *data)
3307 {
3308 	struct sr_meta_opt_item *omi;
3309 	struct sr_chunk_head	*cl;
3310 	struct sr_discipline	*sd = NULL;
3311 	struct sr_chunk		*ch_entry;
3312 	struct scsi_link	*link;
3313 	struct device		*dev;
3314 	char			*uuid, devname[32];
3315 	dev_t			*dt = NULL;
3316 	int			i, no_chunk, rv = EINVAL, target, vol;
3317 	int			no_meta;
3318 
3319 	DNPRINTF(SR_D_IOCTL, "%s: sr_ioctl_createraid(%d)\n",
3320 	    DEVNAME(sc), user);
3321 
3322 	/* user input */
3323 	if (bc->bc_dev_list_len > BIOC_CRMAXLEN)
3324 		goto unwind;
3325 
3326 	dt = malloc(bc->bc_dev_list_len, M_DEVBUF, M_WAITOK | M_ZERO);
3327 	if (user) {
3328 		if (copyin(bc->bc_dev_list, dt, bc->bc_dev_list_len) != 0)
3329 			goto unwind;
3330 	} else
3331 		memcpy(dt, bc->bc_dev_list, bc->bc_dev_list_len);
3332 
3333 	/* Initialise discipline. */
3334 	sd = malloc(sizeof(struct sr_discipline), M_DEVBUF, M_WAITOK | M_ZERO);
3335 	sd->sd_sc = sc;
3336 	SLIST_INIT(&sd->sd_meta_opt);
3337 	sd->sd_taskq = taskq_create("srdis", 1, IPL_BIO, 0);
3338 	if (sd->sd_taskq == NULL) {
3339 		sr_error(sc, "could not create discipline taskq");
3340 		goto unwind;
3341 	}
3342 	if (sr_discipline_init(sd, bc->bc_level)) {
3343 		sr_error(sc, "could not initialize discipline");
3344 		goto unwind;
3345 	}
3346 
3347 	no_chunk = bc->bc_dev_list_len / sizeof(dev_t);
3348 	cl = &sd->sd_vol.sv_chunk_list;
3349 	SLIST_INIT(cl);
3350 
3351 	/* Ensure that chunks are not already in use. */
3352 	for (i = 0; i < no_chunk; i++) {
3353 		if (sr_chunk_in_use(sc, dt[i]) != BIOC_SDINVALID) {
3354 			sr_meta_getdevname(sc, dt[i], devname, sizeof(devname));
3355 			sr_error(sc, "chunk %s already in use", devname);
3356 			goto unwind;
3357 		}
3358 	}
3359 
3360 	sd->sd_meta_type = sr_meta_probe(sd, dt, no_chunk);
3361 	if (sd->sd_meta_type == SR_META_F_INVALID) {
3362 		sr_error(sc, "invalid metadata format");
3363 		goto unwind;
3364 	}
3365 
3366 	if (sr_meta_attach(sd, no_chunk, bc->bc_flags & BIOC_SCFORCE))
3367 		goto unwind;
3368 
3369 	/* force the raid volume by clearing metadata region */
3370 	if (bc->bc_flags & BIOC_SCFORCE) {
3371 		/* make sure disk isn't up and running */
3372 		if (sr_meta_read(sd))
3373 			if (sr_already_assembled(sd)) {
3374 				uuid = sr_uuid_format(
3375 				    &sd->sd_meta->ssdi.ssd_uuid);
3376 				sr_error(sc, "disk %s is currently in use; "
3377 				    "cannot force create", uuid);
3378 				free(uuid, M_DEVBUF, 0);
3379 				goto unwind;
3380 			}
3381 
3382 		if (sr_meta_clear(sd)) {
3383 			sr_error(sc, "failed to clear metadata");
3384 			goto unwind;
3385 		}
3386 	}
3387 
3388 	no_meta = sr_meta_read(sd);
3389 	if (no_meta == -1) {
3390 
3391 		/* Corrupt metadata on one or more chunks. */
3392 		sr_error(sc, "one of the chunks has corrupt metadata; "
3393 		    "aborting assembly");
3394 		goto unwind;
3395 
3396 	} else if (no_meta == 0) {
3397 
3398 		/* Initialise volume and chunk metadata. */
3399 		sr_meta_init(sd, bc->bc_level, no_chunk);
3400 		sd->sd_vol_status = BIOC_SVONLINE;
3401 		sd->sd_meta_flags = bc->bc_flags & BIOC_SCNOAUTOASSEMBLE;
3402 		if (sd->sd_create) {
3403 			if ((i = sd->sd_create(sd, bc, no_chunk,
3404 			    sd->sd_vol.sv_chunk_minsz))) {
3405 				rv = i;
3406 				goto unwind;
3407 			}
3408 		}
3409 		sr_meta_init_complete(sd);
3410 
3411 		DNPRINTF(SR_D_IOCTL,
3412 		    "%s: sr_ioctl_createraid: vol_size: %lld\n",
3413 		    DEVNAME(sc), sd->sd_meta->ssdi.ssd_size);
3414 
3415 		/* Warn if we've wasted chunk space due to coercing. */
3416 		if ((sd->sd_capabilities & SR_CAP_NON_COERCED) == 0 &&
3417 		    sd->sd_vol.sv_chunk_minsz != sd->sd_vol.sv_chunk_maxsz)
3418 			sr_warn(sc, "chunk sizes are not equal; up to %llu "
3419 			    "blocks wasted per chunk",
3420 			    sd->sd_vol.sv_chunk_maxsz -
3421 			    sd->sd_vol.sv_chunk_minsz);
3422 
3423 	} else {
3424 
3425 		/* Ensure we are assembling the correct # of chunks. */
3426 		if (sd->sd_meta->ssdi.ssd_chunk_no != no_chunk) {
3427 			sr_error(sc, "volume chunk count does not match metadata "
3428 			    "chunk count");
3429 			goto unwind;
3430 		}
3431 
3432 		/* Ensure metadata level matches requested assembly level. */
3433 		if (sd->sd_meta->ssdi.ssd_level != bc->bc_level) {
3434 			sr_error(sc, "volume level does not match metadata "
3435 			    "level");
3436 			goto unwind;
3437 		}
3438 
3439 		if (sr_already_assembled(sd)) {
3440 			uuid = sr_uuid_format(&sd->sd_meta->ssdi.ssd_uuid);
3441 			sr_error(sc, "disk %s already assembled", uuid);
3442 			free(uuid, M_DEVBUF, 0);
3443 			goto unwind;
3444 		}
3445 
3446 		if (user == 0 && sd->sd_meta_flags & BIOC_SCNOAUTOASSEMBLE) {
3447 			DNPRINTF(SR_D_META, "%s: disk not auto assembled from "
3448 			    "metadata\n", DEVNAME(sc));
3449 			goto unwind;
3450 		}
3451 
3452 		if (no_meta != no_chunk)
3453 			sr_warn(sc, "trying to bring up %s degraded",
3454 			    sd->sd_meta->ssd_devname);
3455 
3456 		if (sd->sd_meta->ssd_meta_flags & SR_META_DIRTY)
3457 			sr_warn(sc, "%s was not shutdown properly",
3458 			    sd->sd_meta->ssd_devname);
3459 
3460 		SLIST_FOREACH(omi, &sd->sd_meta_opt, omi_link)
3461 			if (sd->sd_meta_opt_handler == NULL ||
3462 			    sd->sd_meta_opt_handler(sd, omi->omi_som) != 0)
3463 				sr_meta_opt_handler(sd, omi->omi_som);
3464 
3465 		if (sd->sd_assemble) {
3466 			if ((i = sd->sd_assemble(sd, bc, no_chunk, data))) {
3467 				rv = i;
3468 				goto unwind;
3469 			}
3470 		}
3471 
3472 		DNPRINTF(SR_D_META, "%s: disk assembled from metadata\n",
3473 		    DEVNAME(sc));
3474 
3475 	}
3476 
3477 	/* Metadata MUST be fully populated by this point. */
3478 	TAILQ_INSERT_TAIL(&sc->sc_dis_list, sd, sd_link);
3479 
3480 	/* Allocate all resources. */
3481 	if ((rv = sd->sd_alloc_resources(sd)))
3482 		goto unwind;
3483 
3484 	/* Adjust flags if necessary. */
3485 	if ((sd->sd_capabilities & SR_CAP_AUTO_ASSEMBLE) &&
3486 	    (bc->bc_flags & BIOC_SCNOAUTOASSEMBLE) !=
3487 	    (sd->sd_meta->ssdi.ssd_vol_flags & BIOC_SCNOAUTOASSEMBLE)) {
3488 		sd->sd_meta->ssdi.ssd_vol_flags &= ~BIOC_SCNOAUTOASSEMBLE;
3489 		sd->sd_meta->ssdi.ssd_vol_flags |=
3490 		    bc->bc_flags & BIOC_SCNOAUTOASSEMBLE;
3491 	}
3492 
3493 	if (sd->sd_capabilities & SR_CAP_SYSTEM_DISK) {
3494 		/* Initialise volume state. */
3495 		sd->sd_set_vol_state(sd);
3496 		if (sd->sd_vol_status == BIOC_SVOFFLINE) {
3497 			sr_error(sc, "%s is offline, will not be brought "
3498 			    "online", sd->sd_meta->ssd_devname);
3499 			goto unwind;
3500 		}
3501 
3502 		/* Setup SCSI iopool. */
3503 		scsi_iopool_init(&sd->sd_iopool, sd, sr_wu_get, sr_wu_put);
3504 
3505 		/*
3506 		 * All checks passed - return ENXIO if volume cannot be created.
3507 		 */
3508 		rv = ENXIO;
3509 
3510 		/*
3511 		 * Find a free target.
3512 		 *
3513 		 * XXX: We reserve sd_target == 0 to indicate the
3514 		 * discipline is not linked into sc->sc_targets, so begin
3515 		 * the search with target = 1.
3516 		 */
3517 		for (target = 1; target < SR_MAX_LD; target++)
3518 			if (sc->sc_targets[target] == NULL)
3519 				break;
3520 		if (target == SR_MAX_LD) {
3521 			sr_error(sc, "no free target for %s",
3522 			    sd->sd_meta->ssd_devname);
3523 			goto unwind;
3524 		}
3525 
3526 		/* Clear sense data. */
3527 		bzero(&sd->sd_scsi_sense, sizeof(sd->sd_scsi_sense));
3528 
3529 		/* Attach discipline and get midlayer to probe it. */
3530 		sd->sd_target = target;
3531 		sc->sc_targets[target] = sd;
3532 		if (scsi_probe_lun(sc->sc_scsibus, target, 0) != 0) {
3533 			sr_error(sc, "scsi_probe_lun failed");
3534 			sc->sc_targets[target] = NULL;
3535 			sd->sd_target = 0;
3536 			goto unwind;
3537 		}
3538 
3539 		link = scsi_get_link(sc->sc_scsibus, target, 0);
3540 		if (link == NULL)
3541 			goto unwind;
3542 
3543 		dev = link->device_softc;
3544 		DNPRINTF(SR_D_IOCTL, "%s: sr device added: %s at target %d\n",
3545 		    DEVNAME(sc), dev->dv_xname, sd->sd_target);
3546 
3547 		/* XXX - Count volumes, not targets. */
3548 		for (i = 0, vol = -1; i <= sd->sd_target; i++)
3549 			if (sc->sc_targets[i])
3550 				vol++;
3551 
3552 		rv = 0;
3553 
3554 		if (sd->sd_meta->ssd_devname[0] != '\0' &&
3555 		    strncmp(sd->sd_meta->ssd_devname, dev->dv_xname,
3556 		    sizeof(dev->dv_xname)))
3557 			sr_warn(sc, "volume %s is roaming, it used to be %s, "
3558 			    "updating metadata", dev->dv_xname,
3559 			    sd->sd_meta->ssd_devname);
3560 
3561 		/* Populate remaining volume metadata. */
3562 		sd->sd_meta->ssdi.ssd_volid = vol;
3563 		strlcpy(sd->sd_meta->ssd_devname, dev->dv_xname,
3564 		    sizeof(sd->sd_meta->ssd_devname));
3565 
3566 		sr_info(sc, "%s volume attached as %s",
3567 		    sd->sd_name, sd->sd_meta->ssd_devname);
3568 
3569 		/* Update device name on any roaming chunks. */
3570 		sr_roam_chunks(sd);
3571 
3572 #ifndef SMALL_KERNEL
3573 		if (sr_sensors_create(sd))
3574 			sr_warn(sc, "unable to create sensor for %s",
3575 			    dev->dv_xname);
3576 #endif /* SMALL_KERNEL */
3577 	} else {
3578 		/* This volume does not attach as a system disk. */
3579 		ch_entry = SLIST_FIRST(cl); /* XXX */
3580 		strlcpy(sd->sd_meta->ssd_devname, ch_entry->src_devname,
3581 		    sizeof(sd->sd_meta->ssd_devname));
3582 
3583 		if (sd->sd_start_discipline(sd))
3584 			goto unwind;
3585 	}
3586 
3587 	/* Save current metadata to disk. */
3588 	rv = sr_meta_save(sd, SR_META_DIRTY);
3589 
3590 	if (sd->sd_vol_status == BIOC_SVREBUILD)
3591 		kthread_create_deferred(sr_rebuild_start, sd);
3592 
3593 	sd->sd_ready = 1;
3594 
3595 	free(dt, M_DEVBUF, bc->bc_dev_list_len);
3596 
3597 	return (rv);
3598 
3599 unwind:
3600 	free(dt, M_DEVBUF, bc->bc_dev_list_len);
3601 
3602 	sr_discipline_shutdown(sd, 0, 0);
3603 
3604 	if (rv == EAGAIN)
3605 		rv = 0;
3606 
3607 	return (rv);
3608 }
3609 
3610 int
3611 sr_ioctl_deleteraid(struct sr_softc *sc, struct sr_discipline *sd,
3612     struct bioc_deleteraid *bd)
3613 {
3614 	int			rv = 1;
3615 
3616 	DNPRINTF(SR_D_IOCTL, "%s: sr_ioctl_deleteraid %s\n",
3617 	    DEVNAME(sc), bd->bd_dev);
3618 
3619 	if (sd == NULL) {
3620 		TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
3621 			if (!strncmp(sd->sd_meta->ssd_devname, bd->bd_dev,
3622 			    sizeof(sd->sd_meta->ssd_devname)))
3623 				break;
3624 		}
3625 		if (sd == NULL) {
3626 			sr_error(sc, "volume %s not found", bd->bd_dev);
3627 			goto bad;
3628 		}
3629 	}
3630 
3631 	sd->sd_deleted = 1;
3632 	sd->sd_meta->ssdi.ssd_vol_flags = BIOC_SCNOAUTOASSEMBLE;
3633 	sr_discipline_shutdown(sd, 1, 0);
3634 
3635 	rv = 0;
3636 bad:
3637 	return (rv);
3638 }
3639 
3640 int
3641 sr_ioctl_discipline(struct sr_softc *sc, struct sr_discipline *sd,
3642     struct bioc_discipline *bd)
3643 {
3644 	int			rv = 1;
3645 
3646 	/* Dispatch a discipline specific ioctl. */
3647 
3648 	DNPRINTF(SR_D_IOCTL, "%s: sr_ioctl_discipline %s\n", DEVNAME(sc),
3649 	    bd->bd_dev);
3650 
3651 	if (sd == NULL) {
3652 		TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
3653 			if (!strncmp(sd->sd_meta->ssd_devname, bd->bd_dev,
3654 			    sizeof(sd->sd_meta->ssd_devname)))
3655 				break;
3656 		}
3657 		if (sd == NULL) {
3658 			sr_error(sc, "volume %s not found", bd->bd_dev);
3659 			goto bad;
3660 		}
3661 	}
3662 
3663 	if (sd->sd_ioctl_handler)
3664 		rv = sd->sd_ioctl_handler(sd, bd);
3665 
3666 bad:
3667 	return (rv);
3668 }
3669 
3670 int
3671 sr_ioctl_installboot(struct sr_softc *sc, struct sr_discipline *sd,
3672     struct bioc_installboot *bb)
3673 {
3674 	void			*bootblk = NULL, *bootldr = NULL;
3675 	struct sr_chunk		*chunk;
3676 	struct sr_meta_opt_item *omi;
3677 	struct sr_meta_boot	*sbm;
3678 	struct disk		*dk;
3679 	u_int32_t		bbs, bls, secsize;
3680 	u_char			duid[8];
3681 	int			rv = EINVAL;
3682 	int			i;
3683 
3684 	DNPRINTF(SR_D_IOCTL, "%s: sr_ioctl_installboot %s\n", DEVNAME(sc),
3685 	    bb->bb_dev);
3686 
3687 	if (sd == NULL) {
3688 		TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
3689 			if (!strncmp(sd->sd_meta->ssd_devname, bb->bb_dev,
3690 			    sizeof(sd->sd_meta->ssd_devname)))
3691 				break;
3692 		}
3693 		if (sd == NULL) {
3694 			sr_error(sc, "volume %s not found", bb->bb_dev);
3695 			goto done;
3696 		}
3697 	}
3698 
3699 	bzero(duid, sizeof(duid));
3700 	TAILQ_FOREACH(dk, &disklist,  dk_link)
3701 		if (!strncmp(dk->dk_name, bb->bb_dev, sizeof(bb->bb_dev)))
3702 			break;
3703 	if (dk == NULL || dk->dk_label == NULL ||
3704 	    (dk->dk_flags & DKF_LABELVALID) == 0 ||
3705 	    bcmp(dk->dk_label->d_uid, &duid, sizeof(duid)) == 0) {
3706 		sr_error(sc, "failed to get DUID for softraid volume");
3707 		goto done;
3708 	}
3709 	memcpy(duid, dk->dk_label->d_uid, sizeof(duid));
3710 
3711 	/* Ensure that boot storage area is large enough. */
3712 	if (sd->sd_meta->ssd_data_blkno < (SR_BOOT_OFFSET + SR_BOOT_SIZE)) {
3713 		sr_error(sc, "insufficient boot storage");
3714 		goto done;
3715 	}
3716 
3717 	if (bb->bb_bootblk_size > SR_BOOT_BLOCKS_SIZE * DEV_BSIZE)
3718 		goto done;
3719 
3720 	if (bb->bb_bootldr_size > SR_BOOT_LOADER_SIZE * DEV_BSIZE)
3721 		goto done;
3722 
3723 	secsize = sd->sd_meta->ssdi.ssd_secsize;
3724 
3725 	/* Copy in boot block. */
3726 	bbs = howmany(bb->bb_bootblk_size, secsize) * secsize;
3727 	bootblk = malloc(bbs, M_DEVBUF, M_WAITOK | M_ZERO);
3728 	if (copyin(bb->bb_bootblk, bootblk, bb->bb_bootblk_size) != 0)
3729 		goto done;
3730 
3731 	/* Copy in boot loader. */
3732 	bls = howmany(bb->bb_bootldr_size, secsize) * secsize;
3733 	bootldr = malloc(bls, M_DEVBUF, M_WAITOK | M_ZERO);
3734 	if (copyin(bb->bb_bootldr, bootldr, bb->bb_bootldr_size) != 0)
3735 		goto done;
3736 
3737 	/* Create or update optional meta for bootable volumes. */
3738 	SLIST_FOREACH(omi, &sd->sd_meta_opt, omi_link)
3739 		if (omi->omi_som->som_type == SR_OPT_BOOT)
3740 			break;
3741 	if (omi == NULL) {
3742 		omi = malloc(sizeof(struct sr_meta_opt_item), M_DEVBUF,
3743 		    M_WAITOK | M_ZERO);
3744 		omi->omi_som = malloc(sizeof(struct sr_meta_crypto), M_DEVBUF,
3745 		    M_WAITOK | M_ZERO);
3746 		omi->omi_som->som_type = SR_OPT_BOOT;
3747 		omi->omi_som->som_length = sizeof(struct sr_meta_boot);
3748 		SLIST_INSERT_HEAD(&sd->sd_meta_opt, omi, omi_link);
3749 		sd->sd_meta->ssdi.ssd_opt_no++;
3750 	}
3751 	sbm = (struct sr_meta_boot *)omi->omi_som;
3752 
3753 	memcpy(sbm->sbm_root_duid, duid, sizeof(sbm->sbm_root_duid));
3754 	bzero(&sbm->sbm_boot_duid, sizeof(sbm->sbm_boot_duid));
3755 	sbm->sbm_bootblk_size = bbs;
3756 	sbm->sbm_bootldr_size = bls;
3757 
3758 	DNPRINTF(SR_D_IOCTL, "sr_ioctl_installboot: root duid is %s\n",
3759 	    duid_format(sbm->sbm_root_duid));
3760 
3761 	/* Save boot block and boot loader to each chunk. */
3762 	for (i = 0; i < sd->sd_meta->ssdi.ssd_chunk_no; i++) {
3763 
3764 		chunk = sd->sd_vol.sv_chunks[i];
3765 		if (chunk->src_meta.scm_status != BIOC_SDONLINE &&
3766 		    chunk->src_meta.scm_status != BIOC_SDREBUILD)
3767 			continue;
3768 
3769 		if (i < SR_MAX_BOOT_DISKS)
3770 			memcpy(&sbm->sbm_boot_duid[i], chunk->src_duid,
3771 			    sizeof(sbm->sbm_boot_duid[i]));
3772 
3773 		/* Save boot blocks. */
3774 		DNPRINTF(SR_D_IOCTL,
3775 		    "sr_ioctl_installboot: saving boot block to %s "
3776 		    "(%u bytes)\n", chunk->src_devname, bbs);
3777 
3778 		if (sr_rw(sc, chunk->src_dev_mm, bootblk, bbs,
3779 		    SR_BOOT_BLOCKS_OFFSET, B_WRITE)) {
3780 			sr_error(sc, "failed to write boot block", DEVNAME(sc));
3781 			goto done;
3782 		}
3783 
3784 		/* Save boot loader.*/
3785 		DNPRINTF(SR_D_IOCTL,
3786 		    "sr_ioctl_installboot: saving boot loader to %s "
3787 		    "(%u bytes)\n", chunk->src_devname, bls);
3788 
3789 		if (sr_rw(sc, chunk->src_dev_mm, bootldr, bls,
3790 		    SR_BOOT_LOADER_OFFSET, B_WRITE)) {
3791 			sr_error(sc, "failed to write boot loader");
3792 			goto done;
3793 		}
3794 	}
3795 
3796 	/* XXX - Install boot block on disk - MD code. */
3797 
3798 	/* Mark volume as bootable and save metadata. */
3799 	sd->sd_meta->ssdi.ssd_vol_flags |= BIOC_SCBOOTABLE;
3800 	if (sr_meta_save(sd, SR_META_DIRTY)) {
3801 		sr_error(sc, "could not save metadata to %s", DEVNAME(sc));
3802 		goto done;
3803 	}
3804 
3805 	rv = 0;
3806 
3807 done:
3808 	free(bootblk, M_DEVBUF, 0);
3809 	free(bootldr, M_DEVBUF, 0);
3810 
3811 	return (rv);
3812 }
3813 
3814 void
3815 sr_chunks_unwind(struct sr_softc *sc, struct sr_chunk_head *cl)
3816 {
3817 	struct sr_chunk		*ch_entry, *ch_next;
3818 
3819 	DNPRINTF(SR_D_IOCTL, "%s: sr_chunks_unwind\n", DEVNAME(sc));
3820 
3821 	if (!cl)
3822 		return;
3823 
3824 	for (ch_entry = SLIST_FIRST(cl); ch_entry != NULL; ch_entry = ch_next) {
3825 		ch_next = SLIST_NEXT(ch_entry, src_link);
3826 
3827 		DNPRINTF(SR_D_IOCTL, "%s: sr_chunks_unwind closing: %s\n",
3828 		    DEVNAME(sc), ch_entry->src_devname);
3829 		if (ch_entry->src_vn) {
3830 			/*
3831 			 * XXX - explicitly lock the vnode until we can resolve
3832 			 * the problem introduced by vnode aliasing... specfs
3833 			 * has no locking, whereas ufs/ffs does!
3834 			 */
3835 			vn_lock(ch_entry->src_vn, LK_EXCLUSIVE |
3836 			    LK_RETRY, curproc);
3837 			VOP_CLOSE(ch_entry->src_vn, FREAD | FWRITE, NOCRED,
3838 			    curproc);
3839 			vput(ch_entry->src_vn);
3840 		}
3841 		free(ch_entry, M_DEVBUF, 0);
3842 	}
3843 	SLIST_INIT(cl);
3844 }
3845 
3846 void
3847 sr_discipline_free(struct sr_discipline *sd)
3848 {
3849 	struct sr_softc		*sc;
3850 	struct sr_discipline	*sdtmp1;
3851 	struct sr_meta_opt_head *som;
3852 	struct sr_meta_opt_item	*omi, *omi_next;
3853 
3854 	if (!sd)
3855 		return;
3856 
3857 	sc = sd->sd_sc;
3858 
3859 	DNPRINTF(SR_D_DIS, "%s: sr_discipline_free %s\n",
3860 	    DEVNAME(sc),
3861 	    sd->sd_meta ? sd->sd_meta->ssd_devname : "nodev");
3862 	if (sd->sd_free_resources)
3863 		sd->sd_free_resources(sd);
3864 	free(sd->sd_vol.sv_chunks, M_DEVBUF, 0);
3865 	free(sd->sd_meta, M_DEVBUF, 0);
3866 	free(sd->sd_meta_foreign, M_DEVBUF, 0);
3867 
3868 	som = &sd->sd_meta_opt;
3869 	for (omi = SLIST_FIRST(som); omi != NULL; omi = omi_next) {
3870 		omi_next = SLIST_NEXT(omi, omi_link);
3871 		free(omi->omi_som, M_DEVBUF, 0);
3872 		free(omi, M_DEVBUF, 0);
3873 	}
3874 
3875 	if (sd->sd_target != 0) {
3876 		KASSERT(sc->sc_targets[sd->sd_target] == sd);
3877 		sc->sc_targets[sd->sd_target] = NULL;
3878 	}
3879 
3880 	TAILQ_FOREACH(sdtmp1, &sc->sc_dis_list, sd_link) {
3881 		if (sdtmp1 == sd)
3882 			break;
3883 	}
3884 	if (sdtmp1 != NULL)
3885 		TAILQ_REMOVE(&sc->sc_dis_list, sd, sd_link);
3886 
3887 	explicit_bzero(sd, sizeof *sd);
3888 	free(sd, M_DEVBUF, 0);
3889 }
3890 
3891 void
3892 sr_discipline_shutdown(struct sr_discipline *sd, int meta_save, int dying)
3893 {
3894 	struct sr_softc		*sc;
3895 	int			s;
3896 
3897 	if (!sd)
3898 		return;
3899 	sc = sd->sd_sc;
3900 
3901 	DNPRINTF(SR_D_DIS, "%s: sr_discipline_shutdown %s\n", DEVNAME(sc),
3902 	    sd->sd_meta ? sd->sd_meta->ssd_devname : "nodev");
3903 
3904 	/* If rebuilding, abort rebuild and drain I/O. */
3905 	if (sd->sd_reb_active) {
3906 		sd->sd_reb_abort = 1;
3907 		while (sd->sd_reb_active)
3908 			tsleep(sd, PWAIT, "sr_shutdown", 1);
3909 	}
3910 
3911 	if (meta_save)
3912 		sr_meta_save(sd, 0);
3913 
3914 	s = splbio();
3915 
3916 	sd->sd_ready = 0;
3917 
3918 	/* make sure there isn't a sync pending and yield */
3919 	wakeup(sd);
3920 	while (sd->sd_sync || sd->sd_must_flush)
3921 		if (tsleep(&sd->sd_sync, MAXPRI, "sr_down", 60 * hz) ==
3922 		    EWOULDBLOCK)
3923 			break;
3924 
3925 	if (dying == -1) {
3926 		sd->sd_ready = 1;
3927 		splx(s);
3928 		return;
3929 	}
3930 
3931 #ifndef SMALL_KERNEL
3932 	sr_sensors_delete(sd);
3933 #endif /* SMALL_KERNEL */
3934 
3935 	if (sd->sd_target != 0)
3936 		scsi_detach_lun(sc->sc_scsibus, sd->sd_target, 0,
3937 		    dying ? 0 : DETACH_FORCE);
3938 
3939 	sr_chunks_unwind(sc, &sd->sd_vol.sv_chunk_list);
3940 
3941 	if (sd->sd_taskq)
3942 		taskq_destroy(sd->sd_taskq);
3943 
3944 	sr_discipline_free(sd);
3945 
3946 	splx(s);
3947 }
3948 
3949 int
3950 sr_discipline_init(struct sr_discipline *sd, int level)
3951 {
3952 	int			rv = 1;
3953 
3954 	/* Initialise discipline function pointers with defaults. */
3955 	sd->sd_alloc_resources = sr_alloc_resources;
3956 	sd->sd_assemble = NULL;
3957 	sd->sd_create = NULL;
3958 	sd->sd_free_resources = sr_free_resources;
3959 	sd->sd_ioctl_handler = NULL;
3960 	sd->sd_openings = NULL;
3961 	sd->sd_meta_opt_handler = NULL;
3962 	sd->sd_rebuild = sr_rebuild;
3963 	sd->sd_scsi_inquiry = sr_raid_inquiry;
3964 	sd->sd_scsi_read_cap = sr_raid_read_cap;
3965 	sd->sd_scsi_tur = sr_raid_tur;
3966 	sd->sd_scsi_req_sense = sr_raid_request_sense;
3967 	sd->sd_scsi_start_stop = sr_raid_start_stop;
3968 	sd->sd_scsi_sync = sr_raid_sync;
3969 	sd->sd_scsi_rw = NULL;
3970 	sd->sd_scsi_intr = sr_raid_intr;
3971 	sd->sd_scsi_wu_done = NULL;
3972 	sd->sd_scsi_done = NULL;
3973 	sd->sd_set_chunk_state = sr_set_chunk_state;
3974 	sd->sd_set_vol_state = sr_set_vol_state;
3975 	sd->sd_start_discipline = NULL;
3976 
3977 	task_set(&sd->sd_meta_save_task, sr_meta_save_callback, sd);
3978 	task_set(&sd->sd_hotspare_rebuild_task, sr_hotspare_rebuild_callback,
3979 	    sd);
3980 
3981 	switch (level) {
3982 	case 0:
3983 		sr_raid0_discipline_init(sd);
3984 		break;
3985 	case 1:
3986 		sr_raid1_discipline_init(sd);
3987 		break;
3988 	case 5:
3989 		sr_raid5_discipline_init(sd);
3990 		break;
3991 	case 6:
3992 		sr_raid6_discipline_init(sd);
3993 		break;
3994 #ifdef CRYPTO
3995 	case 'C':
3996 		sr_crypto_discipline_init(sd);
3997 		break;
3998 #endif
3999 	case 'c':
4000 		sr_concat_discipline_init(sd);
4001 		break;
4002 	default:
4003 		goto bad;
4004 	}
4005 
4006 	rv = 0;
4007 bad:
4008 	return (rv);
4009 }
4010 
4011 int
4012 sr_raid_inquiry(struct sr_workunit *wu)
4013 {
4014 	struct sr_discipline	*sd = wu->swu_dis;
4015 	struct scsi_xfer	*xs = wu->swu_xs;
4016 	struct scsi_inquiry	*cdb = (struct scsi_inquiry *)xs->cmd;
4017 	struct scsi_inquiry_data inq;
4018 
4019 	DNPRINTF(SR_D_DIS, "%s: sr_raid_inquiry\n", DEVNAME(sd->sd_sc));
4020 
4021 	if (xs->cmdlen != sizeof(*cdb))
4022 		return (EINVAL);
4023 
4024 	if (ISSET(cdb->flags, SI_EVPD))
4025 		return (EOPNOTSUPP);
4026 
4027 	bzero(&inq, sizeof(inq));
4028 	inq.device = T_DIRECT;
4029 	inq.dev_qual2 = 0;
4030 	inq.version = 2;
4031 	inq.response_format = 2;
4032 	inq.additional_length = 32;
4033 	inq.flags |= SID_CmdQue;
4034 	strlcpy(inq.vendor, sd->sd_meta->ssdi.ssd_vendor,
4035 	    sizeof(inq.vendor));
4036 	strlcpy(inq.product, sd->sd_meta->ssdi.ssd_product,
4037 	    sizeof(inq.product));
4038 	strlcpy(inq.revision, sd->sd_meta->ssdi.ssd_revision,
4039 	    sizeof(inq.revision));
4040 	sr_copy_internal_data(xs, &inq, sizeof(inq));
4041 
4042 	return (0);
4043 }
4044 
4045 int
4046 sr_raid_read_cap(struct sr_workunit *wu)
4047 {
4048 	struct sr_discipline	*sd = wu->swu_dis;
4049 	struct scsi_xfer	*xs = wu->swu_xs;
4050 	struct scsi_read_cap_data rcd;
4051 	struct scsi_read_cap_data_16 rcd16;
4052 	u_int64_t		addr;
4053 	int			rv = 1;
4054 	u_int32_t		secsize;
4055 
4056 	DNPRINTF(SR_D_DIS, "%s: sr_raid_read_cap\n", DEVNAME(sd->sd_sc));
4057 
4058 	secsize = sd->sd_meta->ssdi.ssd_secsize;
4059 
4060 	addr = ((sd->sd_meta->ssdi.ssd_size * DEV_BSIZE) / secsize) - 1;
4061 	if (xs->cmd->opcode == READ_CAPACITY) {
4062 		bzero(&rcd, sizeof(rcd));
4063 		if (addr > 0xffffffffllu)
4064 			_lto4b(0xffffffff, rcd.addr);
4065 		else
4066 			_lto4b(addr, rcd.addr);
4067 		_lto4b(secsize, rcd.length);
4068 		sr_copy_internal_data(xs, &rcd, sizeof(rcd));
4069 		rv = 0;
4070 	} else if (xs->cmd->opcode == READ_CAPACITY_16) {
4071 		bzero(&rcd16, sizeof(rcd16));
4072 		_lto8b(addr, rcd16.addr);
4073 		_lto4b(secsize, rcd16.length);
4074 		sr_copy_internal_data(xs, &rcd16, sizeof(rcd16));
4075 		rv = 0;
4076 	}
4077 
4078 	return (rv);
4079 }
4080 
4081 int
4082 sr_raid_tur(struct sr_workunit *wu)
4083 {
4084 	struct sr_discipline	*sd = wu->swu_dis;
4085 
4086 	DNPRINTF(SR_D_DIS, "%s: sr_raid_tur\n", DEVNAME(sd->sd_sc));
4087 
4088 	if (sd->sd_vol_status == BIOC_SVOFFLINE) {
4089 		sd->sd_scsi_sense.error_code = SSD_ERRCODE_CURRENT;
4090 		sd->sd_scsi_sense.flags = SKEY_NOT_READY;
4091 		sd->sd_scsi_sense.add_sense_code = 0x04;
4092 		sd->sd_scsi_sense.add_sense_code_qual = 0x11;
4093 		sd->sd_scsi_sense.extra_len = 4;
4094 		return (1);
4095 	} else if (sd->sd_vol_status == BIOC_SVINVALID) {
4096 		sd->sd_scsi_sense.error_code = SSD_ERRCODE_CURRENT;
4097 		sd->sd_scsi_sense.flags = SKEY_HARDWARE_ERROR;
4098 		sd->sd_scsi_sense.add_sense_code = 0x05;
4099 		sd->sd_scsi_sense.add_sense_code_qual = 0x00;
4100 		sd->sd_scsi_sense.extra_len = 4;
4101 		return (1);
4102 	}
4103 
4104 	return (0);
4105 }
4106 
4107 int
4108 sr_raid_request_sense(struct sr_workunit *wu)
4109 {
4110 	struct sr_discipline	*sd = wu->swu_dis;
4111 	struct scsi_xfer	*xs = wu->swu_xs;
4112 
4113 	DNPRINTF(SR_D_DIS, "%s: sr_raid_request_sense\n",
4114 	    DEVNAME(sd->sd_sc));
4115 
4116 	/* use latest sense data */
4117 	memcpy(&xs->sense, &sd->sd_scsi_sense, sizeof(xs->sense));
4118 
4119 	/* clear sense data */
4120 	bzero(&sd->sd_scsi_sense, sizeof(sd->sd_scsi_sense));
4121 
4122 	return (0);
4123 }
4124 
4125 int
4126 sr_raid_start_stop(struct sr_workunit *wu)
4127 {
4128 	struct scsi_xfer	*xs = wu->swu_xs;
4129 	struct scsi_start_stop	*ss = (struct scsi_start_stop *)xs->cmd;
4130 
4131 	DNPRINTF(SR_D_DIS, "%s: sr_raid_start_stop\n",
4132 	    DEVNAME(wu->swu_dis->sd_sc));
4133 
4134 	if (!ss)
4135 		return (1);
4136 
4137 	/*
4138 	 * do nothing!
4139 	 * a softraid discipline should always reflect correct status
4140 	 */
4141 	return (0);
4142 }
4143 
4144 int
4145 sr_raid_sync(struct sr_workunit *wu)
4146 {
4147 	struct sr_discipline	*sd = wu->swu_dis;
4148 	int			s, rv = 0, ios;
4149 
4150 	DNPRINTF(SR_D_DIS, "%s: sr_raid_sync\n", DEVNAME(sd->sd_sc));
4151 
4152 	/* when doing a fake sync don't count the wu */
4153 	ios = (wu->swu_flags & SR_WUF_FAKE) ? 0 : 1;
4154 
4155 	s = splbio();
4156 	sd->sd_sync = 1;
4157 	while (sd->sd_wu_pending > ios) {
4158 		if (tsleep(sd, PRIBIO, "sr_sync", 15 * hz) == EWOULDBLOCK) {
4159 			DNPRINTF(SR_D_DIS, "%s: sr_raid_sync timeout\n",
4160 			    DEVNAME(sd->sd_sc));
4161 			rv = 1;
4162 			break;
4163 		}
4164 	}
4165 	sd->sd_sync = 0;
4166 	splx(s);
4167 
4168 	wakeup(&sd->sd_sync);
4169 
4170 	return (rv);
4171 }
4172 
4173 void
4174 sr_raid_intr(struct buf *bp)
4175 {
4176 	struct sr_ccb		*ccb = (struct sr_ccb *)bp;
4177 	struct sr_workunit	*wu = ccb->ccb_wu;
4178 #ifdef SR_DEBUG
4179 	struct sr_discipline	*sd = wu->swu_dis;
4180 	struct scsi_xfer	*xs = wu->swu_xs;
4181 #endif
4182 	int			s;
4183 
4184 	DNPRINTF(SR_D_INTR, "%s: %s %s intr bp %p xs %p\n",
4185 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, sd->sd_name, bp, xs);
4186 
4187 	s = splbio();
4188 	sr_ccb_done(ccb);
4189 	sr_wu_done(wu);
4190 	splx(s);
4191 }
4192 
4193 void
4194 sr_schedule_wu(struct sr_workunit *wu)
4195 {
4196 	struct sr_discipline	*sd = wu->swu_dis;
4197 	struct sr_workunit	*wup;
4198 	int			s;
4199 
4200 	DNPRINTF(SR_D_WU, "sr_schedule_wu: schedule wu %p state %i "
4201 	    "flags 0x%x\n", wu, wu->swu_state, wu->swu_flags);
4202 
4203 	KASSERT(wu->swu_io_count > 0);
4204 
4205 	s = splbio();
4206 
4207 	/* Construct the work unit, do not schedule it. */
4208 	if (wu->swu_state == SR_WU_CONSTRUCT)
4209 		goto queued;
4210 
4211 	/* Deferred work unit being reconstructed, do not start. */
4212 	if (wu->swu_state == SR_WU_REQUEUE)
4213 		goto queued;
4214 
4215 	/* Current work unit failed, restart. */
4216 	if (wu->swu_state == SR_WU_RESTART)
4217 		goto start;
4218 
4219 	if (wu->swu_state != SR_WU_INPROGRESS)
4220 		panic("sr_schedule_wu: work unit not in progress (state %i)\n",
4221 		    wu->swu_state);
4222 
4223 	/* Walk queue backwards and fill in collider if we have one. */
4224 	TAILQ_FOREACH_REVERSE(wup, &sd->sd_wu_pendq, sr_wu_list, swu_link) {
4225 		if (wu->swu_blk_end < wup->swu_blk_start ||
4226 		    wup->swu_blk_end < wu->swu_blk_start)
4227 			continue;
4228 
4229 		/* Defer work unit due to LBA collision. */
4230 		DNPRINTF(SR_D_WU, "sr_schedule_wu: deferring work unit %p\n",
4231 		    wu);
4232 		wu->swu_state = SR_WU_DEFERRED;
4233 		while (wup->swu_collider)
4234 			wup = wup->swu_collider;
4235 		wup->swu_collider = wu;
4236 		TAILQ_INSERT_TAIL(&sd->sd_wu_defq, wu, swu_link);
4237 		sd->sd_wu_collisions++;
4238 		goto queued;
4239 	}
4240 
4241 start:
4242 	sr_raid_startwu(wu);
4243 
4244 queued:
4245 	splx(s);
4246 }
4247 
4248 void
4249 sr_raid_startwu(struct sr_workunit *wu)
4250 {
4251 	struct sr_discipline	*sd = wu->swu_dis;
4252 	struct sr_ccb		*ccb;
4253 
4254 	DNPRINTF(SR_D_WU, "sr_raid_startwu: start wu %p\n", wu);
4255 
4256 	splassert(IPL_BIO);
4257 
4258 	if (wu->swu_state == SR_WU_DEFERRED) {
4259 		TAILQ_REMOVE(&sd->sd_wu_defq, wu, swu_link);
4260 		wu->swu_state = SR_WU_INPROGRESS;
4261 	}
4262 
4263 	if (wu->swu_state != SR_WU_RESTART)
4264 		TAILQ_INSERT_TAIL(&sd->sd_wu_pendq, wu, swu_link);
4265 
4266 	/* Start all of the individual I/Os. */
4267 	if (wu->swu_cb_active == 1)
4268 		panic("%s: sr_startwu_callback", DEVNAME(sd->sd_sc));
4269 	wu->swu_cb_active = 1;
4270 
4271 	TAILQ_FOREACH(ccb, &wu->swu_ccb, ccb_link)
4272 		VOP_STRATEGY(&ccb->ccb_buf);
4273 
4274 	wu->swu_cb_active = 0;
4275 }
4276 
4277 void
4278 sr_raid_recreate_wu(struct sr_workunit *wu)
4279 {
4280 	struct sr_discipline	*sd = wu->swu_dis;
4281 	struct sr_workunit	*wup = wu;
4282 
4283 	/*
4284 	 * Recreate a work unit by releasing the associated CCBs and reissuing
4285 	 * the SCSI I/O request. This process is then repeated for all of the
4286 	 * colliding work units.
4287 	 */
4288 	do {
4289 		sr_wu_release_ccbs(wup);
4290 
4291 		wup->swu_state = SR_WU_REQUEUE;
4292 		if (sd->sd_scsi_rw(wup))
4293 			panic("could not requeue I/O");
4294 
4295 		wup = wup->swu_collider;
4296 	} while (wup);
4297 }
4298 
4299 int
4300 sr_alloc_resources(struct sr_discipline *sd)
4301 {
4302 	if (sr_wu_alloc(sd, sizeof(struct sr_workunit))) {
4303 		sr_error(sd->sd_sc, "unable to allocate work units");
4304 		return (ENOMEM);
4305 	}
4306 	if (sr_ccb_alloc(sd)) {
4307 		sr_error(sd->sd_sc, "unable to allocate ccbs");
4308 		return (ENOMEM);
4309 	}
4310 
4311 	return (0);
4312 }
4313 
4314 void
4315 sr_free_resources(struct sr_discipline *sd)
4316 {
4317 	sr_wu_free(sd);
4318 	sr_ccb_free(sd);
4319 }
4320 
4321 void
4322 sr_set_chunk_state(struct sr_discipline *sd, int c, int new_state)
4323 {
4324 	int			old_state, s;
4325 
4326 	DNPRINTF(SR_D_STATE, "%s: %s: %s: sr_set_chunk_state %d -> %d\n",
4327 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
4328 	    sd->sd_vol.sv_chunks[c]->src_meta.scmi.scm_devname, c, new_state);
4329 
4330 	/* ok to go to splbio since this only happens in error path */
4331 	s = splbio();
4332 	old_state = sd->sd_vol.sv_chunks[c]->src_meta.scm_status;
4333 
4334 	/* multiple IOs to the same chunk that fail will come through here */
4335 	if (old_state == new_state)
4336 		goto done;
4337 
4338 	switch (old_state) {
4339 	case BIOC_SDONLINE:
4340 		if (new_state == BIOC_SDOFFLINE)
4341 			break;
4342 		else
4343 			goto die;
4344 		break;
4345 
4346 	case BIOC_SDOFFLINE:
4347 		goto die;
4348 
4349 	default:
4350 die:
4351 		splx(s); /* XXX */
4352 		panic("%s: %s: %s: invalid chunk state transition "
4353 		    "%d -> %d", DEVNAME(sd->sd_sc),
4354 		    sd->sd_meta->ssd_devname,
4355 		    sd->sd_vol.sv_chunks[c]->src_meta.scmi.scm_devname,
4356 		    old_state, new_state);
4357 		/* NOTREACHED */
4358 	}
4359 
4360 	sd->sd_vol.sv_chunks[c]->src_meta.scm_status = new_state;
4361 	sd->sd_set_vol_state(sd);
4362 
4363 	sd->sd_must_flush = 1;
4364 	task_add(systq, &sd->sd_meta_save_task);
4365 done:
4366 	splx(s);
4367 }
4368 
4369 void
4370 sr_set_vol_state(struct sr_discipline *sd)
4371 {
4372 	int			states[SR_MAX_STATES];
4373 	int			new_state, i, nd;
4374 	int			old_state = sd->sd_vol_status;
4375 	u_int32_t		s;
4376 
4377 	DNPRINTF(SR_D_STATE, "%s: %s: sr_set_vol_state\n",
4378 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname);
4379 
4380 	nd = sd->sd_meta->ssdi.ssd_chunk_no;
4381 
4382 	for (i = 0; i < SR_MAX_STATES; i++)
4383 		states[i] = 0;
4384 
4385 	for (i = 0; i < nd; i++) {
4386 		s = sd->sd_vol.sv_chunks[i]->src_meta.scm_status;
4387 		if (s >= SR_MAX_STATES)
4388 			panic("%s: %s: %s: invalid chunk state",
4389 			    DEVNAME(sd->sd_sc),
4390 			    sd->sd_meta->ssd_devname,
4391 			    sd->sd_vol.sv_chunks[i]->src_meta.scmi.scm_devname);
4392 		states[s]++;
4393 	}
4394 
4395 	if (states[BIOC_SDONLINE] == nd)
4396 		new_state = BIOC_SVONLINE;
4397 	else
4398 		new_state = BIOC_SVOFFLINE;
4399 
4400 	DNPRINTF(SR_D_STATE, "%s: %s: sr_set_vol_state %d -> %d\n",
4401 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname,
4402 	    old_state, new_state);
4403 
4404 	switch (old_state) {
4405 	case BIOC_SVONLINE:
4406 		if (new_state == BIOC_SVOFFLINE || new_state == BIOC_SVONLINE)
4407 			break;
4408 		else
4409 			goto die;
4410 		break;
4411 
4412 	case BIOC_SVOFFLINE:
4413 		/* XXX this might be a little too much */
4414 		goto die;
4415 
4416 	default:
4417 die:
4418 		panic("%s: %s: invalid volume state transition "
4419 		    "%d -> %d", DEVNAME(sd->sd_sc),
4420 		    sd->sd_meta->ssd_devname,
4421 		    old_state, new_state);
4422 		/* NOTREACHED */
4423 	}
4424 
4425 	sd->sd_vol_status = new_state;
4426 }
4427 
4428 void *
4429 sr_block_get(struct sr_discipline *sd, long length)
4430 {
4431 	return dma_alloc(length, PR_NOWAIT | PR_ZERO);
4432 }
4433 
4434 void
4435 sr_block_put(struct sr_discipline *sd, void *ptr, int length)
4436 {
4437 	dma_free(ptr, length);
4438 }
4439 
4440 void
4441 sr_checksum_print(u_int8_t *md5)
4442 {
4443 	int			i;
4444 
4445 	for (i = 0; i < MD5_DIGEST_LENGTH; i++)
4446 		printf("%02x", md5[i]);
4447 }
4448 
4449 void
4450 sr_checksum(struct sr_softc *sc, void *src, void *md5, u_int32_t len)
4451 {
4452 	MD5_CTX			ctx;
4453 
4454 	DNPRINTF(SR_D_MISC, "%s: sr_checksum(%p %p %d)\n", DEVNAME(sc), src,
4455 	    md5, len);
4456 
4457 	MD5Init(&ctx);
4458 	MD5Update(&ctx, src, len);
4459 	MD5Final(md5, &ctx);
4460 }
4461 
4462 void
4463 sr_uuid_generate(struct sr_uuid *uuid)
4464 {
4465 	arc4random_buf(uuid->sui_id, sizeof(uuid->sui_id));
4466 	/* UUID version 4: random */
4467 	uuid->sui_id[6] &= 0x0f;
4468 	uuid->sui_id[6] |= 0x40;
4469 	/* RFC4122 variant */
4470 	uuid->sui_id[8] &= 0x3f;
4471 	uuid->sui_id[8] |= 0x80;
4472 }
4473 
4474 char *
4475 sr_uuid_format(struct sr_uuid *uuid)
4476 {
4477 	char *uuidstr;
4478 
4479 	uuidstr = malloc(37, M_DEVBUF, M_WAITOK);
4480 
4481 	snprintf(uuidstr, 37,
4482 	    "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-"
4483 	    "%02x%02x%02x%02x%02x%02x",
4484 	    uuid->sui_id[0], uuid->sui_id[1],
4485 	    uuid->sui_id[2], uuid->sui_id[3],
4486 	    uuid->sui_id[4], uuid->sui_id[5],
4487 	    uuid->sui_id[6], uuid->sui_id[7],
4488 	    uuid->sui_id[8], uuid->sui_id[9],
4489 	    uuid->sui_id[10], uuid->sui_id[11],
4490 	    uuid->sui_id[12], uuid->sui_id[13],
4491 	    uuid->sui_id[14], uuid->sui_id[15]);
4492 
4493 	return uuidstr;
4494 }
4495 
4496 void
4497 sr_uuid_print(struct sr_uuid *uuid, int cr)
4498 {
4499 	char *uuidstr;
4500 
4501 	uuidstr = sr_uuid_format(uuid);
4502 	printf("%s%s", uuidstr, (cr ? "\n" : ""));
4503 	free(uuidstr, M_DEVBUF, 37);
4504 }
4505 
4506 int
4507 sr_already_assembled(struct sr_discipline *sd)
4508 {
4509 	struct sr_softc		*sc = sd->sd_sc;
4510 	struct sr_discipline	*sdtmp;
4511 
4512 	TAILQ_FOREACH(sdtmp, &sc->sc_dis_list, sd_link) {
4513 		if (!bcmp(&sd->sd_meta->ssdi.ssd_uuid,
4514 		    &sdtmp->sd_meta->ssdi.ssd_uuid,
4515 		    sizeof(sd->sd_meta->ssdi.ssd_uuid)))
4516 			return (1);
4517 	}
4518 
4519 	return (0);
4520 }
4521 
4522 int32_t
4523 sr_validate_stripsize(u_int32_t b)
4524 {
4525 	int			s = 0;
4526 
4527 	if (b % DEV_BSIZE)
4528 		return (-1);
4529 
4530 	while ((b & 1) == 0) {
4531 		b >>= 1;
4532 		s++;
4533 	}
4534 
4535 	/* only multiple of twos */
4536 	b >>= 1;
4537 	if (b)
4538 		return(-1);
4539 
4540 	return (s);
4541 }
4542 
4543 void
4544 sr_quiesce(void)
4545 {
4546 	struct sr_softc		*sc = softraid0;
4547 	struct sr_discipline	*sd, *nsd;
4548 
4549 	/* Shutdown disciplines in reverse attach order. */
4550 	TAILQ_FOREACH_REVERSE_SAFE(sd, &sc->sc_dis_list,
4551 	    sr_discipline_list, sd_link, nsd)
4552 		sr_discipline_shutdown(sd, 1, -1);
4553 }
4554 
4555 void
4556 sr_shutdown(int dying)
4557 {
4558 	struct sr_softc		*sc = softraid0;
4559 	struct sr_discipline	*sd;
4560 
4561 	DNPRINTF(SR_D_MISC, "%s: sr_shutdown\n", DEVNAME(sc));
4562 
4563 	/*
4564 	 * Since softraid is not under mainbus, we have to explicitly
4565 	 * notify its children that the power is going down, so they
4566 	 * can execute their shutdown hooks.
4567 	 */
4568 	config_suspend((struct device *)sc, DVACT_POWERDOWN);
4569 
4570 	/* Shutdown disciplines in reverse attach order. */
4571 	while ((sd = TAILQ_LAST(&sc->sc_dis_list, sr_discipline_list)) != NULL)
4572 		sr_discipline_shutdown(sd, 1, dying);
4573 }
4574 
4575 int
4576 sr_validate_io(struct sr_workunit *wu, daddr_t *blkno, char *func)
4577 {
4578 	struct sr_discipline	*sd = wu->swu_dis;
4579 	struct scsi_xfer	*xs = wu->swu_xs;
4580 	int			rv = 1;
4581 
4582 	DNPRINTF(SR_D_DIS, "%s: %s 0x%02x\n", DEVNAME(sd->sd_sc), func,
4583 	    xs->cmd->opcode);
4584 
4585 	if (sd->sd_meta->ssd_data_blkno == 0)
4586 		panic("invalid data blkno");
4587 
4588 	if (sd->sd_vol_status == BIOC_SVOFFLINE) {
4589 		DNPRINTF(SR_D_DIS, "%s: %s device offline\n",
4590 		    DEVNAME(sd->sd_sc), func);
4591 		goto bad;
4592 	}
4593 
4594 	if (xs->datalen == 0) {
4595 		printf("%s: %s: illegal block count for %s\n",
4596 		    DEVNAME(sd->sd_sc), func, sd->sd_meta->ssd_devname);
4597 		goto bad;
4598 	}
4599 
4600 	if (xs->cmdlen == 10)
4601 		*blkno = _4btol(((struct scsi_rw_big *)xs->cmd)->addr);
4602 	else if (xs->cmdlen == 16)
4603 		*blkno = _8btol(((struct scsi_rw_16 *)xs->cmd)->addr);
4604 	else if (xs->cmdlen == 6)
4605 		*blkno = _3btol(((struct scsi_rw *)xs->cmd)->addr);
4606 	else {
4607 		printf("%s: %s: illegal cmdlen for %s\n",
4608 		    DEVNAME(sd->sd_sc), func, sd->sd_meta->ssd_devname);
4609 		goto bad;
4610 	}
4611 
4612 	*blkno *= (sd->sd_meta->ssdi.ssd_secsize / DEV_BSIZE);
4613 
4614 	wu->swu_blk_start = *blkno;
4615 	wu->swu_blk_end = *blkno + (xs->datalen >> DEV_BSHIFT) - 1;
4616 
4617 	if (wu->swu_blk_end > sd->sd_meta->ssdi.ssd_size) {
4618 		DNPRINTF(SR_D_DIS, "%s: %s out of bounds start: %lld "
4619 		    "end: %lld length: %d\n",
4620 		    DEVNAME(sd->sd_sc), func, (long long)wu->swu_blk_start,
4621 		    (long long)wu->swu_blk_end, xs->datalen);
4622 
4623 		sd->sd_scsi_sense.error_code = SSD_ERRCODE_CURRENT |
4624 		    SSD_ERRCODE_VALID;
4625 		sd->sd_scsi_sense.flags = SKEY_ILLEGAL_REQUEST;
4626 		sd->sd_scsi_sense.add_sense_code = 0x21;
4627 		sd->sd_scsi_sense.add_sense_code_qual = 0x00;
4628 		sd->sd_scsi_sense.extra_len = 4;
4629 		goto bad;
4630 	}
4631 
4632 	rv = 0;
4633 bad:
4634 	return (rv);
4635 }
4636 
4637 void
4638 sr_rebuild_start(void *arg)
4639 {
4640 	struct sr_discipline	*sd = arg;
4641 	struct sr_softc		*sc = sd->sd_sc;
4642 
4643 	DNPRINTF(SR_D_REBUILD, "%s: %s starting rebuild thread\n",
4644 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname);
4645 
4646 	if (kthread_create(sr_rebuild_thread, sd, &sd->sd_background_proc,
4647 	    DEVNAME(sc)) != 0)
4648 		printf("%s: unable to start background operation\n",
4649 		    DEVNAME(sc));
4650 }
4651 
4652 void
4653 sr_rebuild_thread(void *arg)
4654 {
4655 	struct sr_discipline	*sd = arg;
4656 
4657 	DNPRINTF(SR_D_REBUILD, "%s: %s rebuild thread started\n",
4658 	    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname);
4659 
4660 	sd->sd_reb_active = 1;
4661 	sd->sd_rebuild(sd);
4662 	sd->sd_reb_active = 0;
4663 
4664 	kthread_exit(0);
4665 }
4666 
4667 void
4668 sr_rebuild(struct sr_discipline *sd)
4669 {
4670 	struct sr_softc		*sc = sd->sd_sc;
4671 	u_int64_t		sz, whole_blk, partial_blk, blk, restart;
4672 	daddr_t			lba;
4673 	struct sr_workunit	*wu_r, *wu_w;
4674 	struct scsi_xfer	xs_r, xs_w;
4675 	struct scsi_rw_16	*cr, *cw;
4676 	int			c, s, slept, percent = 0, old_percent = -1;
4677 	u_int8_t		*buf;
4678 
4679 	whole_blk = sd->sd_meta->ssdi.ssd_size / SR_REBUILD_IO_SIZE;
4680 	partial_blk = sd->sd_meta->ssdi.ssd_size % SR_REBUILD_IO_SIZE;
4681 
4682 	restart = sd->sd_meta->ssd_rebuild / SR_REBUILD_IO_SIZE;
4683 	if (restart > whole_blk) {
4684 		printf("%s: bogus rebuild restart offset, starting from 0\n",
4685 		    DEVNAME(sc));
4686 		restart = 0;
4687 	}
4688 	if (restart) {
4689 		/*
4690 		 * XXX there is a hole here; there is a posibility that we
4691 		 * had a restart however the chunk that was supposed to
4692 		 * be rebuilt is no longer valid; we can reach this situation
4693 		 * when a rebuild is in progress and the box crashes and
4694 		 * on reboot the rebuild chunk is different (like zero'd or
4695 		 * replaced).  We need to check the uuid of the chunk that is
4696 		 * being rebuilt to assert this.
4697 		 */
4698 		percent = sr_rebuild_percent(sd);
4699 		printf("%s: resuming rebuild on %s at %d%%\n",
4700 		    DEVNAME(sc), sd->sd_meta->ssd_devname, percent);
4701 	}
4702 
4703 	/* currently this is 64k therefore we can use dma_alloc */
4704 	buf = dma_alloc(SR_REBUILD_IO_SIZE << DEV_BSHIFT, PR_WAITOK);
4705 	for (blk = restart; blk <= whole_blk; blk++) {
4706 		lba = blk * SR_REBUILD_IO_SIZE;
4707 		sz = SR_REBUILD_IO_SIZE;
4708 		if (blk == whole_blk) {
4709 			if (partial_blk == 0)
4710 				break;
4711 			sz = partial_blk;
4712 		}
4713 
4714 		/* get some wu */
4715 		wu_r = sr_scsi_wu_get(sd, 0);
4716 		wu_w = sr_scsi_wu_get(sd, 0);
4717 
4718 		DNPRINTF(SR_D_REBUILD, "%s: %s rebuild wu_r %p, wu_w %p\n",
4719 		    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, wu_r, wu_w);
4720 
4721 		/* setup read io */
4722 		bzero(&xs_r, sizeof xs_r);
4723 		xs_r.error = XS_NOERROR;
4724 		xs_r.flags = SCSI_DATA_IN;
4725 		xs_r.datalen = sz << DEV_BSHIFT;
4726 		xs_r.data = buf;
4727 		xs_r.cmdlen = sizeof(*cr);
4728 		xs_r.cmd = &xs_r.cmdstore;
4729 		cr = (struct scsi_rw_16 *)xs_r.cmd;
4730 		cr->opcode = READ_16;
4731 		_lto4b(sz, cr->length);
4732 		_lto8b(lba, cr->addr);
4733 		wu_r->swu_state = SR_WU_CONSTRUCT;
4734 		wu_r->swu_flags |= SR_WUF_REBUILD;
4735 		wu_r->swu_xs = &xs_r;
4736 		if (sd->sd_scsi_rw(wu_r)) {
4737 			printf("%s: could not create read io\n",
4738 			    DEVNAME(sc));
4739 			goto fail;
4740 		}
4741 
4742 		/* setup write io */
4743 		bzero(&xs_w, sizeof xs_w);
4744 		xs_w.error = XS_NOERROR;
4745 		xs_w.flags = SCSI_DATA_OUT;
4746 		xs_w.datalen = sz << DEV_BSHIFT;
4747 		xs_w.data = buf;
4748 		xs_w.cmdlen = sizeof(*cw);
4749 		xs_w.cmd = &xs_w.cmdstore;
4750 		cw = (struct scsi_rw_16 *)xs_w.cmd;
4751 		cw->opcode = WRITE_16;
4752 		_lto4b(sz, cw->length);
4753 		_lto8b(lba, cw->addr);
4754 		wu_w->swu_state = SR_WU_CONSTRUCT;
4755 		wu_w->swu_flags |= SR_WUF_REBUILD | SR_WUF_WAKEUP;
4756 		wu_w->swu_xs = &xs_w;
4757 		if (sd->sd_scsi_rw(wu_w)) {
4758 			printf("%s: could not create write io\n",
4759 			    DEVNAME(sc));
4760 			goto fail;
4761 		}
4762 
4763 		/*
4764 		 * collide with the read io so that we get automatically
4765 		 * started when the read is done
4766 		 */
4767 		wu_w->swu_state = SR_WU_DEFERRED;
4768 		wu_r->swu_collider = wu_w;
4769 		s = splbio();
4770 		TAILQ_INSERT_TAIL(&sd->sd_wu_defq, wu_w, swu_link);
4771 		splx(s);
4772 
4773 		DNPRINTF(SR_D_REBUILD, "%s: %s rebuild scheduling wu_r %p\n",
4774 		    DEVNAME(sd->sd_sc), sd->sd_meta->ssd_devname, wu_r);
4775 
4776 		wu_r->swu_state = SR_WU_INPROGRESS;
4777 		sr_schedule_wu(wu_r);
4778 
4779 		/* wait for write completion */
4780 		slept = 0;
4781 		while ((wu_w->swu_flags & SR_WUF_REBUILDIOCOMP) == 0) {
4782 			tsleep(wu_w, PRIBIO, "sr_rebuild", 0);
4783 			slept = 1;
4784 		}
4785 		/* yield if we didn't sleep */
4786 		if (slept == 0)
4787 			tsleep(sc, PWAIT, "sr_yield", 1);
4788 
4789 		sr_scsi_wu_put(sd, wu_r);
4790 		sr_scsi_wu_put(sd, wu_w);
4791 
4792 		sd->sd_meta->ssd_rebuild = lba;
4793 
4794 		/* XXX - this should be based on size, not percentage. */
4795 		/* save metadata every percent */
4796 		percent = sr_rebuild_percent(sd);
4797 		if (percent != old_percent && blk != whole_blk) {
4798 			if (sr_meta_save(sd, SR_META_DIRTY))
4799 				printf("%s: could not save metadata to %s\n",
4800 				    DEVNAME(sc), sd->sd_meta->ssd_devname);
4801 			old_percent = percent;
4802 		}
4803 
4804 		if (sd->sd_reb_abort)
4805 			goto abort;
4806 	}
4807 
4808 	/* all done */
4809 	sd->sd_meta->ssd_rebuild = 0;
4810 	for (c = 0; c < sd->sd_meta->ssdi.ssd_chunk_no; c++) {
4811 		if (sd->sd_vol.sv_chunks[c]->src_meta.scm_status ==
4812 		    BIOC_SDREBUILD) {
4813 			sd->sd_set_chunk_state(sd, c, BIOC_SDONLINE);
4814 			break;
4815 		}
4816 	}
4817 
4818 abort:
4819 	if (sr_meta_save(sd, SR_META_DIRTY))
4820 		printf("%s: could not save metadata to %s\n",
4821 		    DEVNAME(sc), sd->sd_meta->ssd_devname);
4822 fail:
4823 	dma_free(buf, SR_REBUILD_IO_SIZE << DEV_BSHIFT);
4824 }
4825 
4826 #ifndef SMALL_KERNEL
4827 int
4828 sr_sensors_create(struct sr_discipline *sd)
4829 {
4830 	struct sr_softc		*sc = sd->sd_sc;
4831 	int			rv = 1;
4832 
4833 	DNPRINTF(SR_D_STATE, "%s: %s: sr_sensors_create\n",
4834 	    DEVNAME(sc), sd->sd_meta->ssd_devname);
4835 
4836 	sd->sd_vol.sv_sensor.type = SENSOR_DRIVE;
4837 	sd->sd_vol.sv_sensor.status = SENSOR_S_UNKNOWN;
4838 	strlcpy(sd->sd_vol.sv_sensor.desc, sd->sd_meta->ssd_devname,
4839 	    sizeof(sd->sd_vol.sv_sensor.desc));
4840 
4841 	sensor_attach(&sc->sc_sensordev, &sd->sd_vol.sv_sensor);
4842 	sd->sd_vol.sv_sensor_attached = 1;
4843 
4844 	if (sc->sc_sensor_task == NULL) {
4845 		sc->sc_sensor_task = sensor_task_register(sc,
4846 		    sr_sensors_refresh, 10);
4847 		if (sc->sc_sensor_task == NULL)
4848 			goto bad;
4849 	}
4850 
4851 	rv = 0;
4852 bad:
4853 	return (rv);
4854 }
4855 
4856 void
4857 sr_sensors_delete(struct sr_discipline *sd)
4858 {
4859 	DNPRINTF(SR_D_STATE, "%s: sr_sensors_delete\n", DEVNAME(sd->sd_sc));
4860 
4861 	if (sd->sd_vol.sv_sensor_attached)
4862 		sensor_detach(&sd->sd_sc->sc_sensordev, &sd->sd_vol.sv_sensor);
4863 }
4864 
4865 void
4866 sr_sensors_refresh(void *arg)
4867 {
4868 	struct sr_softc		*sc = arg;
4869 	struct sr_volume	*sv;
4870 	struct sr_discipline	*sd;
4871 
4872 	DNPRINTF(SR_D_STATE, "%s: sr_sensors_refresh\n", DEVNAME(sc));
4873 
4874 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
4875 		sv = &sd->sd_vol;
4876 
4877 		switch(sd->sd_vol_status) {
4878 		case BIOC_SVOFFLINE:
4879 			sv->sv_sensor.value = SENSOR_DRIVE_FAIL;
4880 			sv->sv_sensor.status = SENSOR_S_CRIT;
4881 			break;
4882 
4883 		case BIOC_SVDEGRADED:
4884 			sv->sv_sensor.value = SENSOR_DRIVE_PFAIL;
4885 			sv->sv_sensor.status = SENSOR_S_WARN;
4886 			break;
4887 
4888 		case BIOC_SVREBUILD:
4889 			sv->sv_sensor.value = SENSOR_DRIVE_REBUILD;
4890 			sv->sv_sensor.status = SENSOR_S_WARN;
4891 			break;
4892 
4893 		case BIOC_SVSCRUB:
4894 		case BIOC_SVONLINE:
4895 			sv->sv_sensor.value = SENSOR_DRIVE_ONLINE;
4896 			sv->sv_sensor.status = SENSOR_S_OK;
4897 			break;
4898 
4899 		default:
4900 			sv->sv_sensor.value = 0; /* unknown */
4901 			sv->sv_sensor.status = SENSOR_S_UNKNOWN;
4902 		}
4903 	}
4904 }
4905 #endif /* SMALL_KERNEL */
4906 
4907 #ifdef SR_FANCY_STATS
4908 void				sr_print_stats(void);
4909 
4910 void
4911 sr_print_stats(void)
4912 {
4913 	struct sr_softc		*sc = softraid0;
4914 	struct sr_discipline	*sd;
4915 
4916 	if (sc == NULL) {
4917 		printf("no softraid softc found\n");
4918 		return;
4919 	}
4920 
4921 	TAILQ_FOREACH(sd, &sc->sc_dis_list, sd_link) {
4922 		printf("%s: ios pending %d, collisions %llu\n",
4923 		    sd->sd_meta->ssd_devname,
4924 		    sd->sd_wu_pending,
4925 		    sd->sd_wu_collisions);
4926 	}
4927 }
4928 #endif /* SR_FANCY_STATS */
4929 
4930 #ifdef SR_DEBUG
4931 void
4932 sr_meta_print(struct sr_metadata *m)
4933 {
4934 	int			i;
4935 	struct sr_meta_chunk	*mc;
4936 	struct sr_meta_opt_hdr	*omh;
4937 
4938 	if (!(sr_debug & SR_D_META))
4939 		return;
4940 
4941 	printf("\tssd_magic 0x%llx\n", m->ssdi.ssd_magic);
4942 	printf("\tssd_version %d\n", m->ssdi.ssd_version);
4943 	printf("\tssd_vol_flags 0x%x\n", m->ssdi.ssd_vol_flags);
4944 	printf("\tssd_uuid ");
4945 	sr_uuid_print(&m->ssdi.ssd_uuid, 1);
4946 	printf("\tssd_chunk_no %d\n", m->ssdi.ssd_chunk_no);
4947 	printf("\tssd_chunk_id %d\n", m->ssdi.ssd_chunk_id);
4948 	printf("\tssd_opt_no %d\n", m->ssdi.ssd_opt_no);
4949 	printf("\tssd_volid %d\n", m->ssdi.ssd_volid);
4950 	printf("\tssd_level %d\n", m->ssdi.ssd_level);
4951 	printf("\tssd_size %lld\n", m->ssdi.ssd_size);
4952 	printf("\tssd_devname %s\n", m->ssd_devname);
4953 	printf("\tssd_vendor %s\n", m->ssdi.ssd_vendor);
4954 	printf("\tssd_product %s\n", m->ssdi.ssd_product);
4955 	printf("\tssd_revision %s\n", m->ssdi.ssd_revision);
4956 	printf("\tssd_strip_size %d\n", m->ssdi.ssd_strip_size);
4957 	printf("\tssd_checksum ");
4958 	sr_checksum_print(m->ssd_checksum);
4959 	printf("\n");
4960 	printf("\tssd_meta_flags 0x%x\n", m->ssd_meta_flags);
4961 	printf("\tssd_ondisk %llu\n", m->ssd_ondisk);
4962 
4963 	mc = (struct sr_meta_chunk *)(m + 1);
4964 	for (i = 0; i < m->ssdi.ssd_chunk_no; i++, mc++) {
4965 		printf("\t\tscm_volid %d\n", mc->scmi.scm_volid);
4966 		printf("\t\tscm_chunk_id %d\n", mc->scmi.scm_chunk_id);
4967 		printf("\t\tscm_devname %s\n", mc->scmi.scm_devname);
4968 		printf("\t\tscm_size %lld\n", mc->scmi.scm_size);
4969 		printf("\t\tscm_coerced_size %lld\n",mc->scmi.scm_coerced_size);
4970 		printf("\t\tscm_uuid ");
4971 		sr_uuid_print(&mc->scmi.scm_uuid, 1);
4972 		printf("\t\tscm_checksum ");
4973 		sr_checksum_print(mc->scm_checksum);
4974 		printf("\n");
4975 		printf("\t\tscm_status %d\n", mc->scm_status);
4976 	}
4977 
4978 	omh = (struct sr_meta_opt_hdr *)((u_int8_t *)(m + 1) +
4979 	    sizeof(struct sr_meta_chunk) * m->ssdi.ssd_chunk_no);
4980 	for (i = 0; i < m->ssdi.ssd_opt_no; i++) {
4981 		printf("\t\t\tsom_type %d\n", omh->som_type);
4982 		printf("\t\t\tsom_checksum ");
4983 		sr_checksum_print(omh->som_checksum);
4984 		printf("\n");
4985 		omh = (struct sr_meta_opt_hdr *)((void *)omh +
4986 		    omh->som_length);
4987 	}
4988 }
4989 
4990 void
4991 sr_dump_block(void *blk, int len)
4992 {
4993 	uint8_t			*b = blk;
4994 	int			i, j, c;
4995 
4996 	for (i = 0; i < len; i += 16) {
4997 		for (j = 0; j < 16; j++)
4998 			printf("%.2x ", b[i + j]);
4999 		printf("  ");
5000 		for (j = 0; j < 16; j++) {
5001 			c = b[i + j];
5002 			if (c < ' ' || c > 'z' || i + j > len)
5003 				c = '.';
5004 			printf("%c", c);
5005 		}
5006 		printf("\n");
5007 	}
5008 }
5009 
5010 void
5011 sr_dump_mem(u_int8_t *p, int len)
5012 {
5013 	int			i;
5014 
5015 	for (i = 0; i < len; i++)
5016 		printf("%02x ", *p++);
5017 	printf("\n");
5018 }
5019 
5020 #endif /* SR_DEBUG */
5021 
5022 #ifdef HIBERNATE
5023 /*
5024  * Side-effect free (no malloc, printf, pool, splx) softraid crypto writer.
5025  *
5026  * This function must perform the following:
5027  * 1. Determine the underlying device's own side-effect free I/O function
5028  *    (eg, ahci_hibernate_io, wd_hibernate_io, etc).
5029  * 2. Store enough information in the provided page argument for subsequent
5030  *    I/O calls (such as the crypto discipline structure for the keys, the
5031  *    offset of the softraid partition on the underlying disk, as well as
5032  *    the offset of the swap partition within the crypto volume.
5033  * 3. Encrypt the incoming data using the sr_discipline keys, then pass
5034  *    the request to the underlying device's own I/O function.
5035  */
5036 int
5037 sr_hibernate_io(dev_t dev, daddr_t blkno, vaddr_t addr, size_t size, int op, void *page)
5038 {
5039 	/* Struct for stashing data obtained on HIB_INIT.
5040 	 * XXX
5041 	 * We share the page with the underlying device's own
5042 	 * side-effect free I/O function, so we pad our data to
5043 	 * the end of the page. Presently this does not overlap
5044 	 * with either of the two other side-effect free i/o
5045 	 * functions (ahci/wd).
5046 	 */
5047 	struct {
5048 		char pad[3072];
5049 		struct sr_discipline *srd;
5050 		hibio_fn subfn;		/* underlying device i/o fn */
5051 		dev_t subdev;		/* underlying device dev_t */
5052 		daddr_t sr_swapoff;	/* ofs of swap part in sr volume */
5053 		char buf[DEV_BSIZE];	/* encryption performed into this buf */
5054 	} *my = page;
5055 	extern struct cfdriver sd_cd;
5056 	char errstr[128], *dl_ret;
5057 	struct sr_chunk *schunk;
5058 	struct sd_softc *sd;
5059 	struct aes_xts_ctx ctx;
5060 	struct sr_softc *sc;
5061 	struct device *dv;
5062 	daddr_t key_blkno;
5063 	uint32_t sub_raidoff;  /* ofs of sr part in underlying dev */
5064 	struct disklabel dl;
5065 	struct partition *pp;
5066 	size_t i, j;
5067 	u_char iv[8];
5068 
5069 	/*
5070 	 * In HIB_INIT, we are passed the swap partition size and offset
5071 	 * in 'size' and 'blkno' respectively. These are relative to the
5072 	 * start of the softraid partition, and we need to save these
5073 	 * for later translation to the underlying device's layout.
5074 	 */
5075 	if (op == HIB_INIT) {
5076 		dv = disk_lookup(&sd_cd, DISKUNIT(dev));
5077 		sd = (struct sd_softc *)dv;
5078 		sc = (struct sr_softc *)dv->dv_parent->dv_parent;
5079 
5080 		/*
5081 		 * Look up the sr discipline. This is used to determine
5082 		 * if we are SR crypto and what the underlying device is.
5083 		 */
5084 		my->srd = sc->sc_targets[sd->sc_link->target];
5085 		DNPRINTF(SR_D_MISC, "sr_hibernate_io: discipline is %s\n",
5086 			my->srd->sd_name);
5087 		if (strncmp(my->srd->sd_name, "CRYPTO",
5088 		    sizeof(my->srd->sd_name)))
5089 			return (ENOTSUP);
5090 
5091 		/* Find the underlying device */
5092 		schunk = my->srd->sd_vol.sv_chunks[0];
5093 		my->subdev = schunk->src_dev_mm;
5094 
5095 		/*
5096 		 * Find the appropriate underlying device side effect free
5097 		 * I/O function, based on the type of device it is.
5098 		 */
5099 		my->subfn = get_hibernate_io_function(my->subdev);
5100 		if (!my->subfn)
5101 			return (ENODEV);
5102 
5103 		/*
5104 		 * Find blkno where this raid partition starts on
5105 		 * the underlying disk.
5106 		 */
5107 		dl_ret = disk_readlabel(&dl, my->subdev, errstr,
5108 		    sizeof(errstr));
5109 		if (dl_ret) {
5110 			printf("Hibernate error reading disklabel: %s\n", dl_ret);
5111 			return (ENOTSUP);
5112 		}
5113 
5114 		pp = &dl.d_partitions[DISKPART(my->subdev)];
5115 		if (pp->p_fstype != FS_RAID || DL_GETPSIZE(pp) == 0)
5116 			return (ENOTSUP);
5117 
5118 		/* Find the blkno of the SR part in the underlying device */
5119 		sub_raidoff = my->srd->sd_meta->ssd_data_blkno +
5120 		    DL_SECTOBLK(&dl, DL_GETPOFFSET(pp));
5121 		DNPRINTF(SR_D_MISC,"sr_hibernate_io: blk trans ofs: %d blks\n",
5122 		    sub_raidoff);
5123 
5124 		/* Save the blkno of the swap partition in the SR disk */
5125 		my->sr_swapoff = blkno;
5126 
5127 		/* Initialize the sub-device */
5128 		return my->subfn(my->subdev, sub_raidoff + blkno,
5129 		    addr, size, op, page);
5130 	}
5131 
5132 	/* Hibernate only uses (and we only support) writes */
5133 	if (op != HIB_W)
5134 		return (ENOTSUP);
5135 
5136 	/*
5137 	 * Blocks act as the IV for the encryption. These block numbers
5138 	 * are relative to the start of the sr partition, but the 'blkno'
5139 	 * passed above is relative to the start of the swap partition
5140 	 * inside the sr partition, so bias appropriately.
5141 	 */
5142 	key_blkno = my->sr_swapoff + blkno;
5143 
5144 	/* Process each disk block one at a time. */
5145 	for (i = 0; i < size; i += DEV_BSIZE) {
5146 		int res;
5147 
5148 		bzero(&ctx, sizeof(ctx));
5149 
5150 		/*
5151 		 * Set encryption key (from the sr discipline stashed
5152 		 * during HIB_INIT. This code is based on the softraid
5153 		 * bootblock code.
5154 		 */
5155 		aes_xts_setkey(&ctx, my->srd->mds.mdd_crypto.scr_key[0], 64);
5156 		/* We encrypt DEV_BSIZE bytes at a time in my->buf */
5157 		memcpy(my->buf, ((char *)addr) + i, DEV_BSIZE);
5158 
5159 		/* Block number is the IV */
5160 		memcpy(&iv, &key_blkno, sizeof(key_blkno));
5161 		aes_xts_reinit(&ctx, iv);
5162 
5163 		/* Encrypt DEV_BSIZE bytes, AES_XTS_BLOCKSIZE bytes at a time */
5164 		for (j = 0; j < DEV_BSIZE; j += AES_XTS_BLOCKSIZE)
5165 			aes_xts_encrypt(&ctx, my->buf + j);
5166 
5167 		/*
5168 		 * Write one block out from my->buf to the underlying device
5169 		 * using its own side-effect free I/O function.
5170 		 */
5171 		res = my->subfn(my->subdev, blkno + (i / DEV_BSIZE),
5172 		    (vaddr_t)(my->buf), DEV_BSIZE, op, page);
5173 		if (res != 0)
5174 			return (res);
5175 		key_blkno++;
5176 	}
5177 	return (0);
5178 }
5179 #endif /* HIBERNATE */
5180