1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
135     struct g_raid_subdisk *sd);
137     int no, off_t off, off_t len, u_int mask);
146 	strip_size = vol->v_strip_size;  in V2P()
152 	*disk = (nstrip * N) % vol->v_disks_count;  in V2P()
154 	*offset = ((nstrip * N) / vol->v_disks_count) * strip_size;  in V2P()
164 	strip_size = vol->v_strip_size;  in P2V()
168 	nstrip = (offset / strip_size) * vol->v_disks_count + disk;  in P2V()
181 	if (tr->tro_volume->v_raid_level != G_RAID_VOLUME_RL_RAID1E ||  in g_raid_tr_taste_raid1e()
182 	    tr->tro_volume->v_raid_level_qualifier != G_RAID_VOLUME_RLQ_R1EA)  in g_raid_tr_taste_raid1e()
184 	trs->trso_starting = 1;  in g_raid_tr_taste_raid1e()
192 	struct g_raid_subdisk *sd, *bestsd, *worstsd;  in g_raid_tr_update_state_raid1e_even()  local
195 	sc = vol->v_softc;  in g_raid_tr_update_state_raid1e_even()
197 	for (i = 0; i < vol->v_disks_count / N; i++) {  in g_raid_tr_update_state_raid1e_even()
198 		bestsd = &vol->v_subdisks[i * N];  in g_raid_tr_update_state_raid1e_even()
200 			sd = &vol->v_subdisks[i * N + j];  in g_raid_tr_update_state_raid1e_even()
201 			if (sd->sd_state > bestsd->sd_state)  in g_raid_tr_update_state_raid1e_even()
202 				bestsd = sd;  in g_raid_tr_update_state_raid1e_even()
203 			else if (sd->sd_state == bestsd->sd_state &&  in g_raid_tr_update_state_raid1e_even()
204 			    (sd->sd_state == G_RAID_SUBDISK_S_REBUILD ||  in g_raid_tr_update_state_raid1e_even()
205 			     sd->sd_state == G_RAID_SUBDISK_S_RESYNC) &&  in g_raid_tr_update_state_raid1e_even()
206 			    sd->sd_rebuild_pos > bestsd->sd_rebuild_pos)  in g_raid_tr_update_state_raid1e_even()
207 				bestsd = sd;  in g_raid_tr_update_state_raid1e_even()
209 		if (bestsd->sd_state >= G_RAID_SUBDISK_S_UNINITIALIZED &&  in g_raid_tr_update_state_raid1e_even()
210 		    bestsd->sd_state != G_RAID_SUBDISK_S_ACTIVE) {  in g_raid_tr_update_state_raid1e_even()
214 			    vol->v_name, bestsd->sd_pos,  in g_raid_tr_update_state_raid1e_even()
215 			    g_raid_subdisk_state2str(bestsd->sd_state));  in g_raid_tr_update_state_raid1e_even()
219 			    vol, bestsd, bestsd->sd_disk);  in g_raid_tr_update_state_raid1e_even()
221 		worstsd = &vol->v_subdisks[i * N];  in g_raid_tr_update_state_raid1e_even()
223 			sd = &vol->v_subdisks[i * N + j];  in g_raid_tr_update_state_raid1e_even()
224 			if (sd->sd_state < worstsd->sd_state)  in g_raid_tr_update_state_raid1e_even()
225 				worstsd = sd;  in g_raid_tr_update_state_raid1e_even()
227 		if (worstsd->sd_state == G_RAID_SUBDISK_S_ACTIVE)  in g_raid_tr_update_state_raid1e_even()
229 		else if (worstsd->sd_state >= G_RAID_SUBDISK_S_STALE)  in g_raid_tr_update_state_raid1e_even()
231 		else if (bestsd->sd_state == G_RAID_SUBDISK_S_ACTIVE)  in g_raid_tr_update_state_raid1e_even()
245 	struct g_raid_subdisk *sd, *bestsd, *worstsd;  in g_raid_tr_update_state_raid1e_odd()  local
248 	sc = vol->v_softc;  in g_raid_tr_update_state_raid1e_odd()
250 	    vol->v_disks_count)  in g_raid_tr_update_state_raid1e_odd()
252 	for (i = 0; i < vol->v_disks_count; i++) {  in g_raid_tr_update_state_raid1e_odd()
253 		sd = &vol->v_subdisks[i];  in g_raid_tr_update_state_raid1e_odd()
254 		if (sd->sd_state == G_RAID_SUBDISK_S_UNINITIALIZED) {  in g_raid_tr_update_state_raid1e_odd()
258 			    vol->v_name, sd->sd_pos,  in g_raid_tr_update_state_raid1e_odd()
259 			    g_raid_subdisk_state2str(sd->sd_state));  in g_raid_tr_update_state_raid1e_odd()
260 			g_raid_change_subdisk_state(sd,  in g_raid_tr_update_state_raid1e_odd()
262 			g_raid_write_metadata(sc, vol, sd, sd->sd_disk);  in g_raid_tr_update_state_raid1e_odd()
266 	for (i = 0; i < vol->v_disks_count; i++) {  in g_raid_tr_update_state_raid1e_odd()
267 		bestsd = &vol->v_subdisks[i];  in g_raid_tr_update_state_raid1e_odd()
268 		worstsd = &vol->v_subdisks[i];  in g_raid_tr_update_state_raid1e_odd()
270 			sd = &vol->v_subdisks[(i + j) % vol->v_disks_count];  in g_raid_tr_update_state_raid1e_odd()
271 			if (sd->sd_state > bestsd->sd_state)  in g_raid_tr_update_state_raid1e_odd()
272 				bestsd = sd;  in g_raid_tr_update_state_raid1e_odd()
273 			else if (sd->sd_state == bestsd->sd_state &&  in g_raid_tr_update_state_raid1e_odd()
274 			    (sd->sd_state == G_RAID_SUBDISK_S_REBUILD ||  in g_raid_tr_update_state_raid1e_odd()
275 			     sd->sd_state == G_RAID_SUBDISK_S_RESYNC) &&  in g_raid_tr_update_state_raid1e_odd()
276 			    sd->sd_rebuild_pos > bestsd->sd_rebuild_pos)  in g_raid_tr_update_state_raid1e_odd()
277 				bestsd = sd;  in g_raid_tr_update_state_raid1e_odd()
278 			if (sd->sd_state < worstsd->sd_state)  in g_raid_tr_update_state_raid1e_odd()
279 				worstsd = sd;  in g_raid_tr_update_state_raid1e_odd()
281 		if (worstsd->sd_state == G_RAID_SUBDISK_S_ACTIVE)  in g_raid_tr_update_state_raid1e_odd()
283 		else if (worstsd->sd_state >= G_RAID_SUBDISK_S_STALE)  in g_raid_tr_update_state_raid1e_odd()
285 		else if (bestsd->sd_state >= G_RAID_SUBDISK_S_STALE)  in g_raid_tr_update_state_raid1e_odd()
297     struct g_raid_subdisk *sd)  in g_raid_tr_update_state_raid1e()  argument
303 	sc = vol->v_softc;  in g_raid_tr_update_state_raid1e()
304 	trs = (struct g_raid_tr_raid1e_object *)vol->v_tr;  in g_raid_tr_update_state_raid1e()
305 	if (trs->trso_stopping &&  in g_raid_tr_update_state_raid1e()
306 	    (trs->trso_flags & TR_RAID1E_F_DOING_SOME) == 0)  in g_raid_tr_update_state_raid1e()
308 	else if (trs->trso_starting)  in g_raid_tr_update_state_raid1e()
311 		if ((vol->v_disks_count % N) == 0)  in g_raid_tr_update_state_raid1e()
316 	if (s != vol->v_state) {  in g_raid_tr_update_state_raid1e()
321 		if (!trs->trso_starting && !trs->trso_stopping)  in g_raid_tr_update_state_raid1e()
324 	if (!trs->trso_starting && !trs->trso_stopping)  in g_raid_tr_update_state_raid1e()
325 		g_raid_tr_raid1e_maybe_rebuild(vol->v_tr, sd);  in g_raid_tr_update_state_raid1e()
330 g_raid_tr_raid1e_fail_disk(struct g_raid_softc *sc, struct g_raid_subdisk *sd,  in g_raid_tr_raid1e_fail_disk()  argument
335 	vol = sd->sd_volume;  in g_raid_tr_raid1e_fail_disk()
349 	     vol->v_disks_count) &&  in g_raid_tr_raid1e_fail_disk()
350 	    (sd->sd_state >= G_RAID_SUBDISK_S_UNINITIALIZED))  in g_raid_tr_raid1e_fail_disk()
352 	g_raid_fail_disk(sc, sd, disk);  in g_raid_tr_raid1e_fail_disk()
359 	struct g_raid_subdisk *sd;  in g_raid_tr_raid1e_rebuild_done()  local
361 	vol = trs->trso_base.tro_volume;  in g_raid_tr_raid1e_rebuild_done()
362 	sd = trs->trso_failed_sd;  in g_raid_tr_raid1e_rebuild_done()
363 	g_raid_write_metadata(vol->v_softc, vol, sd, sd->sd_disk);  in g_raid_tr_raid1e_rebuild_done()
364 	free(trs->trso_buffer, M_TR_RAID1E);  in g_raid_tr_raid1e_rebuild_done()
365 	trs->trso_buffer = NULL;  in g_raid_tr_raid1e_rebuild_done()
366 	trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;  in g_raid_tr_raid1e_rebuild_done()
367 	trs->trso_type = TR_RAID1E_NONE;  in g_raid_tr_raid1e_rebuild_done()
368 	trs->trso_recover_slabs = 0;  in g_raid_tr_raid1e_rebuild_done()
369 	trs->trso_failed_sd = NULL;  in g_raid_tr_raid1e_rebuild_done()
377 	struct g_raid_subdisk *sd;  in g_raid_tr_raid1e_rebuild_finish()  local
380 	sd = trs->trso_failed_sd;  in g_raid_tr_raid1e_rebuild_finish()
381 	G_RAID_DEBUG1(0, tr->tro_volume->v_softc,  in g_raid_tr_raid1e_rebuild_finish()
382 	    "Subdisk %s:%d-%s rebuild completed.",  in g_raid_tr_raid1e_rebuild_finish()
383 	    sd->sd_volume->v_name, sd->sd_pos,  in g_raid_tr_raid1e_rebuild_finish()
384 	    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]");  in g_raid_tr_raid1e_rebuild_finish()
385 	g_raid_change_subdisk_state(sd, G_RAID_SUBDISK_S_ACTIVE);  in g_raid_tr_raid1e_rebuild_finish()
386 	sd->sd_rebuild_pos = 0;  in g_raid_tr_raid1e_rebuild_finish()
394 	struct g_raid_subdisk *sd;  in g_raid_tr_raid1e_rebuild_abort()  local
397 	vol = tr->tro_volume;  in g_raid_tr_raid1e_rebuild_abort()
399 	sd = trs->trso_failed_sd;  in g_raid_tr_raid1e_rebuild_abort()
400 	if (trs->trso_flags & TR_RAID1E_F_DOING_SOME) {  in g_raid_tr_raid1e_rebuild_abort()
401 		G_RAID_DEBUG1(1, vol->v_softc,  in g_raid_tr_raid1e_rebuild_abort()
402 		    "Subdisk %s:%d-%s rebuild is aborting.",  in g_raid_tr_raid1e_rebuild_abort()
403 		    sd->sd_volume->v_name, sd->sd_pos,  in g_raid_tr_raid1e_rebuild_abort()
404 		    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]");  in g_raid_tr_raid1e_rebuild_abort()
405 		trs->trso_flags |= TR_RAID1E_F_ABORT;  in g_raid_tr_raid1e_rebuild_abort()
407 		G_RAID_DEBUG1(0, vol->v_softc,  in g_raid_tr_raid1e_rebuild_abort()
408 		    "Subdisk %s:%d-%s rebuild aborted.",  in g_raid_tr_raid1e_rebuild_abort()
409 		    sd->sd_volume->v_name, sd->sd_pos,  in g_raid_tr_raid1e_rebuild_abort()
410 		    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]");  in g_raid_tr_raid1e_rebuild_abort()
411 		trs->trso_flags &= ~TR_RAID1E_F_ABORT;  in g_raid_tr_raid1e_rebuild_abort()
412 		if (trs->trso_flags & TR_RAID1E_F_LOCKED) {  in g_raid_tr_raid1e_rebuild_abort()
413 			trs->trso_flags &= ~TR_RAID1E_F_LOCKED;  in g_raid_tr_raid1e_rebuild_abort()
414 			g_raid_unlock_range(tr->tro_volume,  in g_raid_tr_raid1e_rebuild_abort()
415 			    trs->trso_lock_pos, trs->trso_lock_len);  in g_raid_tr_raid1e_rebuild_abort()
427 	struct g_raid_subdisk *sd;  in g_raid_tr_raid1e_rebuild_some()  local
433 	if (trs->trso_flags & TR_RAID1E_F_DOING_SOME)  in g_raid_tr_raid1e_rebuild_some()
435 	vol = tr->tro_volume;  in g_raid_tr_raid1e_rebuild_some()
436 	sc = vol->v_softc;  in g_raid_tr_raid1e_rebuild_some()
437 	sd = trs->trso_failed_sd;  in g_raid_tr_raid1e_rebuild_some()
440 		if (sd->sd_rebuild_pos >= sd->sd_size) {  in g_raid_tr_raid1e_rebuild_some()
445 		P2V(vol, sd->sd_pos, sd->sd_rebuild_pos, &virtual, &copy);  in g_raid_tr_raid1e_rebuild_some()
450 		    sd->sd_size - sd->sd_rebuild_pos);  in g_raid_tr_raid1e_rebuild_some()
451 		if ((vol->v_disks_count % N) != 0)  in g_raid_tr_raid1e_rebuild_some()
452 			len = MIN(len, vol->v_strip_size - start);  in g_raid_tr_raid1e_rebuild_some()
457 			/* There is no any valid disk. */  in g_raid_tr_raid1e_rebuild_some()
465 		G_RAID_DEBUG1(3, sc, "Skipping rebuild for range %ju - %ju",  in g_raid_tr_raid1e_rebuild_some()
466 		    sd->sd_rebuild_pos, sd->sd_rebuild_pos + len);  in g_raid_tr_raid1e_rebuild_some()
467 		sd->sd_rebuild_pos += len;  in g_raid_tr_raid1e_rebuild_some()
470 	bp = &trs->trso_bio;  in g_raid_tr_raid1e_rebuild_some()
472 	bp->bio_offset = offset + start +  in g_raid_tr_raid1e_rebuild_some()
473 	    ((disk + best >= vol->v_disks_count) ? vol->v_strip_size : 0);  in g_raid_tr_raid1e_rebuild_some()
474 	bp->bio_length = len;  in g_raid_tr_raid1e_rebuild_some()
475 	bp->bio_data = trs->trso_buffer;  in g_raid_tr_raid1e_rebuild_some()
476 	bp->bio_cmd = BIO_READ;  in g_raid_tr_raid1e_rebuild_some()
477 	bp->bio_cflags = G_RAID_BIO_FLAG_SYNC;  in g_raid_tr_raid1e_rebuild_some()
478 	bp->bio_caller1 = &vol->v_subdisks[(disk + best) % vol->v_disks_count];  in g_raid_tr_raid1e_rebuild_some()
484 	if (start + len > vol->v_strip_size) {  in g_raid_tr_raid1e_rebuild_some()
485 		P2V(vol, sd->sd_pos, sd->sd_rebuild_pos + len, &vend, &copy);  in g_raid_tr_raid1e_rebuild_some()
486 		len = vend - virtual;  in g_raid_tr_raid1e_rebuild_some()
488 	trs->trso_flags |= TR_RAID1E_F_DOING_SOME;  in g_raid_tr_raid1e_rebuild_some()
489 	trs->trso_flags |= TR_RAID1E_F_LOCKED;  in g_raid_tr_raid1e_rebuild_some()
490 	trs->trso_lock_pos = virtual;  in g_raid_tr_raid1e_rebuild_some()
491 	trs->trso_lock_len = len;  in g_raid_tr_raid1e_rebuild_some()
493 	g_raid_lock_range(sd->sd_volume, virtual, len, NULL, bp);  in g_raid_tr_raid1e_rebuild_some()
501 	struct g_raid_subdisk *sd;  in g_raid_tr_raid1e_rebuild_start()  local
503 	vol = tr->tro_volume;  in g_raid_tr_raid1e_rebuild_start()
505 	if (trs->trso_failed_sd) {  in g_raid_tr_raid1e_rebuild_start()
506 		G_RAID_DEBUG1(1, vol->v_softc,  in g_raid_tr_raid1e_rebuild_start()
508 		    (intmax_t)trs->trso_failed_sd->sd_rebuild_pos);  in g_raid_tr_raid1e_rebuild_start()
511 	sd = g_raid_get_subdisk(vol, G_RAID_SUBDISK_S_RESYNC);  in g_raid_tr_raid1e_rebuild_start()
512 	if (sd == NULL)  in g_raid_tr_raid1e_rebuild_start()
513 		sd = g_raid_get_subdisk(vol, G_RAID_SUBDISK_S_REBUILD);  in g_raid_tr_raid1e_rebuild_start()
514 	if (sd == NULL) {  in g_raid_tr_raid1e_rebuild_start()
515 		sd = g_raid_get_subdisk(vol, G_RAID_SUBDISK_S_STALE);  in g_raid_tr_raid1e_rebuild_start()
516 		if (sd != NULL) {  in g_raid_tr_raid1e_rebuild_start()
517 			sd->sd_rebuild_pos = 0;  in g_raid_tr_raid1e_rebuild_start()
518 			g_raid_change_subdisk_state(sd,  in g_raid_tr_raid1e_rebuild_start()
520 			g_raid_write_metadata(vol->v_softc, vol, sd, NULL);  in g_raid_tr_raid1e_rebuild_start()
522 			sd = g_raid_get_subdisk(vol,  in g_raid_tr_raid1e_rebuild_start()
524 			if (sd == NULL)  in g_raid_tr_raid1e_rebuild_start()
525 				sd = g_raid_get_subdisk(vol,  in g_raid_tr_raid1e_rebuild_start()
527 			if (sd != NULL) {  in g_raid_tr_raid1e_rebuild_start()
528 				sd->sd_rebuild_pos = 0;  in g_raid_tr_raid1e_rebuild_start()
529 				g_raid_change_subdisk_state(sd,  in g_raid_tr_raid1e_rebuild_start()
531 				g_raid_write_metadata(vol->v_softc,  in g_raid_tr_raid1e_rebuild_start()
532 				    vol, sd, NULL);  in g_raid_tr_raid1e_rebuild_start()
536 	if (sd == NULL) {  in g_raid_tr_raid1e_rebuild_start()
537 		G_RAID_DEBUG1(1, vol->v_softc,  in g_raid_tr_raid1e_rebuild_start()
538 		    "No failed disk to rebuild.  night night.");  in g_raid_tr_raid1e_rebuild_start()
541 	trs->trso_failed_sd = sd;  in g_raid_tr_raid1e_rebuild_start()
542 	G_RAID_DEBUG1(0, vol->v_softc,  in g_raid_tr_raid1e_rebuild_start()
543 	    "Subdisk %s:%d-%s rebuild start at %jd.",  in g_raid_tr_raid1e_rebuild_start()
544 	    sd->sd_volume->v_name, sd->sd_pos,  in g_raid_tr_raid1e_rebuild_start()
545 	    sd->sd_disk ? g_raid_get_diskname(sd->sd_disk) : "[none]",  in g_raid_tr_raid1e_rebuild_start()
546 	    trs->trso_failed_sd->sd_rebuild_pos);  in g_raid_tr_raid1e_rebuild_start()
547 	trs->trso_type = TR_RAID1E_REBUILD;  in g_raid_tr_raid1e_rebuild_start()
548 	trs->trso_buffer = malloc(g_raid1e_rebuild_slab, M_TR_RAID1E, M_WAITOK);  in g_raid_tr_raid1e_rebuild_start()
549 	trs->trso_meta_update = g_raid1e_rebuild_meta_update;  in g_raid_tr_raid1e_rebuild_start()
555     struct g_raid_subdisk *sd)  in g_raid_tr_raid1e_maybe_rebuild()  argument
561 	vol = tr->tro_volume;  in g_raid_tr_raid1e_maybe_rebuild()
563 	if (trs->trso_stopping)  in g_raid_tr_raid1e_maybe_rebuild()
567 	switch(trs->trso_type) {  in g_raid_tr_raid1e_maybe_rebuild()
569 		if (vol->v_state < G_RAID_VOLUME_S_DEGRADED)  in g_raid_tr_raid1e_maybe_rebuild()
581 		if (vol->v_state < G_RAID_VOLUME_S_DEGRADED || nr == 0 ||  in g_raid_tr_raid1e_maybe_rebuild()
582 		    trs->trso_failed_sd == sd)  in g_raid_tr_raid1e_maybe_rebuild()
592     struct g_raid_subdisk *sd, u_int event)  in g_raid_tr_event_raid1e()  argument
595 	g_raid_tr_update_state_raid1e(tr->tro_volume, sd);  in g_raid_tr_event_raid1e()
606 	vol = tr->tro_volume;  in g_raid_tr_start_raid1e()
607 	trs->trso_starting = 0;  in g_raid_tr_start_raid1e()
619 	vol = tr->tro_volume;  in g_raid_tr_stop_raid1e()
620 	trs->trso_starting = 0;  in g_raid_tr_stop_raid1e()
621 	trs->trso_stopping = 1;  in g_raid_tr_stop_raid1e()
630 #define ABS(x)		(((x) >= 0) ? (x) : (-(x)))
633     int no, off_t off, off_t len, u_int mask)  in g_raid_tr_raid1e_select_read_disk()  argument
635 	struct g_raid_subdisk *sd;  in g_raid_tr_raid1e_select_read_disk()  local
639 	best = -1;  in g_raid_tr_raid1e_select_read_disk()
642 		sd = &vol->v_subdisks[(no + i) % vol->v_disks_count];  in g_raid_tr_raid1e_select_read_disk()
644 		if (no + i >= vol->v_disks_count)  in g_raid_tr_raid1e_select_read_disk()
645 			offset += vol->v_strip_size;  in g_raid_tr_raid1e_select_read_disk()
647 		prio = G_RAID_SUBDISK_LOAD(sd);  in g_raid_tr_raid1e_select_read_disk()
648 		if ((mask & (1 << sd->sd_pos)) != 0)  in g_raid_tr_raid1e_select_read_disk()
650 		switch (sd->sd_state) {  in g_raid_tr_raid1e_select_read_disk()
654 			if (offset + off < sd->sd_rebuild_pos)  in g_raid_tr_raid1e_select_read_disk()
661 			if (offset + off < sd->sd_rebuild_pos)  in g_raid_tr_raid1e_select_read_disk()
667 		prio += min(sd->sd_recovery, 255) << 16;  in g_raid_tr_raid1e_select_read_disk()
668 		/* If disk head is precisely in position - highly prefer it. */  in g_raid_tr_raid1e_select_read_disk()
669 		if (G_RAID_SUBDISK_POS(sd) == offset)  in g_raid_tr_raid1e_select_read_disk()
670 			prio -= 2 * G_RAID_SUBDISK_LOAD_SCALE;  in g_raid_tr_raid1e_select_read_disk()
672 		/* If disk head is close to position - prefer it. */  in g_raid_tr_raid1e_select_read_disk()
673 		if (ABS(G_RAID_SUBDISK_POS(sd) - offset) <  in g_raid_tr_raid1e_select_read_disk()
675 			prio -= 1 * G_RAID_SUBDISK_LOAD_SCALE;  in g_raid_tr_raid1e_select_read_disk()
688 	struct g_raid_subdisk *sd;  in g_raid_tr_iostart_raid1e_read()  local
693 	u_int no, strip_size;  in g_raid_tr_iostart_raid1e_read()  local
696 	vol = tr->tro_volume;  in g_raid_tr_iostart_raid1e_read()
697 	if ((bp->bio_flags & BIO_UNMAPPED) != 0)  in g_raid_tr_iostart_raid1e_read()
700 		addr = bp->bio_data;  in g_raid_tr_iostart_raid1e_read()
701 	strip_size = vol->v_strip_size;  in g_raid_tr_iostart_raid1e_read()
702 	V2P(vol, bp->bio_offset, &no, &offset, &start);  in g_raid_tr_iostart_raid1e_read()
703 	remain = bp->bio_length;  in g_raid_tr_iostart_raid1e_read()
706 		length = MIN(strip_size - start, remain);  in g_raid_tr_iostart_raid1e_read()
708 		    no, offset, length, 0);  in g_raid_tr_iostart_raid1e_read()
709 		KASSERT(best >= 0, ("No readable disk in volume %s!",  in g_raid_tr_iostart_raid1e_read()
710 		    vol->v_name));  in g_raid_tr_iostart_raid1e_read()
711 		no += best;  in g_raid_tr_iostart_raid1e_read()
712 		if (no >= vol->v_disks_count) {  in g_raid_tr_iostart_raid1e_read()
713 			no -= vol->v_disks_count;  in g_raid_tr_iostart_raid1e_read()
719 		cbp->bio_offset = offset + start;  in g_raid_tr_iostart_raid1e_read()
720 		cbp->bio_length = length;  in g_raid_tr_iostart_raid1e_read()
721 		if ((bp->bio_flags & BIO_UNMAPPED) != 0) {  in g_raid_tr_iostart_raid1e_read()
722 			cbp->bio_ma_offset += (uintptr_t)addr;  in g_raid_tr_iostart_raid1e_read()
723 			cbp->bio_ma += cbp->bio_ma_offset / PAGE_SIZE;  in g_raid_tr_iostart_raid1e_read()
724 			cbp->bio_ma_offset %= PAGE_SIZE;  in g_raid_tr_iostart_raid1e_read()
725 			cbp->bio_ma_n = round_page(cbp->bio_ma_offset +  in g_raid_tr_iostart_raid1e_read()
726 			    cbp->bio_length) / PAGE_SIZE;  in g_raid_tr_iostart_raid1e_read()
728 			cbp->bio_data = addr;  in g_raid_tr_iostart_raid1e_read()
729 		cbp->bio_caller1 = &vol->v_subdisks[no];  in g_raid_tr_iostart_raid1e_read()
731 		no += N - best;  in g_raid_tr_iostart_raid1e_read()
732 		if (no >= vol->v_disks_count) {  in g_raid_tr_iostart_raid1e_read()
733 			no -= vol->v_disks_count;  in g_raid_tr_iostart_raid1e_read()
736 		remain -= length;  in g_raid_tr_iostart_raid1e_read()
741 		sd = cbp->bio_caller1;  in g_raid_tr_iostart_raid1e_read()
742 		cbp->bio_caller1 = NULL;  in g_raid_tr_iostart_raid1e_read()
743 		g_raid_subdisk_iostart(sd, cbp);  in g_raid_tr_iostart_raid1e_read()
749 	if (bp->bio_error == 0)  in g_raid_tr_iostart_raid1e_read()
750 		bp->bio_error = ENOMEM;  in g_raid_tr_iostart_raid1e_read()
751 	g_raid_iodone(bp, bp->bio_error);  in g_raid_tr_iostart_raid1e_read()
758 	struct g_raid_subdisk *sd;  in g_raid_tr_iostart_raid1e_write()  local
763 	u_int no, strip_size;  in g_raid_tr_iostart_raid1e_write()  local
766 	vol = tr->tro_volume;  in g_raid_tr_iostart_raid1e_write()
767 	if ((bp->bio_flags & BIO_UNMAPPED) != 0)  in g_raid_tr_iostart_raid1e_write()
770 		addr = bp->bio_data;  in g_raid_tr_iostart_raid1e_write()
771 	strip_size = vol->v_strip_size;  in g_raid_tr_iostart_raid1e_write()
772 	V2P(vol, bp->bio_offset, &no, &offset, &start);  in g_raid_tr_iostart_raid1e_write()
773 	remain = bp->bio_length;  in g_raid_tr_iostart_raid1e_write()
776 		length = MIN(strip_size - start, remain);  in g_raid_tr_iostart_raid1e_write()
778 			sd = &vol->v_subdisks[no];  in g_raid_tr_iostart_raid1e_write()
779 			switch (sd->sd_state) {  in g_raid_tr_iostart_raid1e_write()
785 				if (offset + start >= sd->sd_rebuild_pos)  in g_raid_tr_iostart_raid1e_write()
794 			cbp->bio_offset = offset + start;  in g_raid_tr_iostart_raid1e_write()
795 			cbp->bio_length = length;  in g_raid_tr_iostart_raid1e_write()
796 			if ((bp->bio_flags & BIO_UNMAPPED) != 0 &&  in g_raid_tr_iostart_raid1e_write()
797 			    bp->bio_cmd != BIO_DELETE) {  in g_raid_tr_iostart_raid1e_write()
798 				cbp->bio_ma_offset += (uintptr_t)addr;  in g_raid_tr_iostart_raid1e_write()
799 				cbp->bio_ma += cbp->bio_ma_offset / PAGE_SIZE;  in g_raid_tr_iostart_raid1e_write()
800 				cbp->bio_ma_offset %= PAGE_SIZE;  in g_raid_tr_iostart_raid1e_write()
801 				cbp->bio_ma_n = round_page(cbp->bio_ma_offset +  in g_raid_tr_iostart_raid1e_write()
802 				    cbp->bio_length) / PAGE_SIZE;  in g_raid_tr_iostart_raid1e_write()
804 				cbp->bio_data = addr;  in g_raid_tr_iostart_raid1e_write()
805 			cbp->bio_caller1 = sd;  in g_raid_tr_iostart_raid1e_write()
808 			if (++no >= vol->v_disks_count) {  in g_raid_tr_iostart_raid1e_write()
809 				no = 0;  in g_raid_tr_iostart_raid1e_write()
813 		remain -= length;  in g_raid_tr_iostart_raid1e_write()
814 		if (bp->bio_cmd != BIO_DELETE)  in g_raid_tr_iostart_raid1e_write()
819 		sd = cbp->bio_caller1;  in g_raid_tr_iostart_raid1e_write()
820 		cbp->bio_caller1 = NULL;  in g_raid_tr_iostart_raid1e_write()
821 		g_raid_subdisk_iostart(sd, cbp);  in g_raid_tr_iostart_raid1e_write()
827 	if (bp->bio_error == 0)  in g_raid_tr_iostart_raid1e_write()
828 		bp->bio_error = ENOMEM;  in g_raid_tr_iostart_raid1e_write()
829 	g_raid_iodone(bp, bp->bio_error);  in g_raid_tr_iostart_raid1e_write()
838 	vol = tr->tro_volume;  in g_raid_tr_iostart_raid1e()
840 	if (vol->v_state != G_RAID_VOLUME_S_OPTIMAL &&  in g_raid_tr_iostart_raid1e()
841 	    vol->v_state != G_RAID_VOLUME_S_SUBOPTIMAL &&  in g_raid_tr_iostart_raid1e()
842 	    vol->v_state != G_RAID_VOLUME_S_DEGRADED) {  in g_raid_tr_iostart_raid1e()
852 	if (trs->trso_failed_sd != NULL &&  in g_raid_tr_iostart_raid1e()
853 	    !(bp->bio_cflags & G_RAID_BIO_FLAG_SPECIAL)) {  in g_raid_tr_iostart_raid1e()
855 		trs->trso_recover_slabs = 0;  in g_raid_tr_iostart_raid1e()
856 		if (--trs->trso_fair_io <= 0) {  in g_raid_tr_iostart_raid1e()
857 			trs->trso_fair_io = g_raid1e_rebuild_fair_io;  in g_raid_tr_iostart_raid1e()
861 	switch (bp->bio_cmd) {  in g_raid_tr_iostart_raid1e()
875 		    bp->bio_cmd, vol->v_name));  in g_raid_tr_iostart_raid1e()
882     struct g_raid_subdisk *sd, struct bio *bp)  in g_raid_tr_iodone_raid1e()  argument
894 	vol = tr->tro_volume;  in g_raid_tr_iodone_raid1e()
895 	if (bp->bio_cflags & G_RAID_BIO_FLAG_SYNC) {  in g_raid_tr_iodone_raid1e()
896 		if (trs->trso_type == TR_RAID1E_REBUILD) {  in g_raid_tr_iodone_raid1e()
897 			nsd = trs->trso_failed_sd;  in g_raid_tr_iodone_raid1e()
898 			if (bp->bio_cmd == BIO_READ) {  in g_raid_tr_iodone_raid1e()
900 				if (trs->trso_flags & TR_RAID1E_F_ABORT) {  in g_raid_tr_iodone_raid1e()
901 					trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;  in g_raid_tr_iodone_raid1e()
907 				if (bp->bio_error != 0) {  in g_raid_tr_iodone_raid1e()
911 					    bp->bio_error);  in g_raid_tr_iodone_raid1e()
920 				    bp->bio_error);  in g_raid_tr_iodone_raid1e()
921 				bp->bio_cmd = BIO_WRITE;  in g_raid_tr_iodone_raid1e()
922 				bp->bio_cflags = G_RAID_BIO_FLAG_SYNC;  in g_raid_tr_iodone_raid1e()
923 				bp->bio_offset = nsd->sd_rebuild_pos;  in g_raid_tr_iodone_raid1e()
934 				    bp->bio_error);  in g_raid_tr_iodone_raid1e()
935 				if (bp->bio_error != 0 ||  in g_raid_tr_iodone_raid1e()
936 				    trs->trso_flags & TR_RAID1E_F_ABORT) {  in g_raid_tr_iodone_raid1e()
937 					if ((trs->trso_flags &  in g_raid_tr_iodone_raid1e()
939 						g_raid_tr_raid1e_fail_disk(sd->sd_softc,  in g_raid_tr_iodone_raid1e()
940 						    nsd, nsd->sd_disk);  in g_raid_tr_iodone_raid1e()
942 					trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;  in g_raid_tr_iodone_raid1e()
947 				trs->trso_flags &= ~TR_RAID1E_F_LOCKED;  in g_raid_tr_iodone_raid1e()
948 				g_raid_unlock_range(tr->tro_volume,  in g_raid_tr_iodone_raid1e()
949 				    trs->trso_lock_pos, trs->trso_lock_len);  in g_raid_tr_iodone_raid1e()
950 				nsd->sd_rebuild_pos += bp->bio_length;  in g_raid_tr_iodone_raid1e()
951 				if (nsd->sd_rebuild_pos >= nsd->sd_size) {  in g_raid_tr_iodone_raid1e()
957 				if (trs->trso_stopping) {  in g_raid_tr_iodone_raid1e()
958 					trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;  in g_raid_tr_iodone_raid1e()
963 				if (--trs->trso_meta_update <= 0) {  in g_raid_tr_iodone_raid1e()
964 					g_raid_write_metadata(vol->v_softc,  in g_raid_tr_iodone_raid1e()
965 					    vol, nsd, nsd->sd_disk);  in g_raid_tr_iodone_raid1e()
966 					trs->trso_meta_update =  in g_raid_tr_iodone_raid1e()
969 					if ((vol->v_disks_count % N) != 0 &&  in g_raid_tr_iodone_raid1e()
970 					    vol->v_strip_size <  in g_raid_tr_iodone_raid1e()
972 						trs->trso_meta_update *=  in g_raid_tr_iodone_raid1e()
974 						trs->trso_meta_update /=  in g_raid_tr_iodone_raid1e()
975 						    vol->v_strip_size;  in g_raid_tr_iodone_raid1e()
978 				trs->trso_flags &= ~TR_RAID1E_F_DOING_SOME;  in g_raid_tr_iodone_raid1e()
979 				if (--trs->trso_recover_slabs <= 0)  in g_raid_tr_iodone_raid1e()
984 		} else if (trs->trso_type == TR_RAID1E_RESYNC) {  in g_raid_tr_iodone_raid1e()
986 			 * read good sd, read bad sd in parallel.  when both  in g_raid_tr_iodone_raid1e()
994 	pbp = bp->bio_parent;  in g_raid_tr_iodone_raid1e()
995 	pbp->bio_inbed++;  in g_raid_tr_iodone_raid1e()
996 	mask = (intptr_t)bp->bio_caller2;  in g_raid_tr_iodone_raid1e()
997 	if (bp->bio_cmd == BIO_READ && bp->bio_error != 0) {  in g_raid_tr_iodone_raid1e()
1003 		sd->sd_disk->d_read_errs++;  in g_raid_tr_iodone_raid1e()
1006 		    bp->bio_error, sd->sd_disk->d_read_errs);  in g_raid_tr_iodone_raid1e()
1015 		if (sd->sd_disk->d_read_errs > g_raid_read_err_thresh)  in g_raid_tr_iodone_raid1e()
1016 			g_raid_tr_raid1e_fail_disk(sd->sd_softc, sd, sd->sd_disk);  in g_raid_tr_iodone_raid1e()
1021 		P2V(vol, sd->sd_pos, bp->bio_offset, &virtual, &copy);  in g_raid_tr_iodone_raid1e()
1030 			if (disk >= vol->v_disks_count) {  in g_raid_tr_iodone_raid1e()
1031 				disk -= vol->v_disks_count;  in g_raid_tr_iodone_raid1e()
1032 				offset += vol->v_strip_size;  in g_raid_tr_iodone_raid1e()
1034 			cbp->bio_offset = offset + start;  in g_raid_tr_iodone_raid1e()
1035 			cbp->bio_length = bp->bio_length;  in g_raid_tr_iodone_raid1e()
1036 			cbp->bio_data = bp->bio_data;  in g_raid_tr_iodone_raid1e()
1037 			cbp->bio_ma = bp->bio_ma;  in g_raid_tr_iodone_raid1e()
1038 			cbp->bio_ma_offset = bp->bio_ma_offset;  in g_raid_tr_iodone_raid1e()
1039 			cbp->bio_ma_n = bp->bio_ma_n;  in g_raid_tr_iodone_raid1e()
1041 			nsd = &vol->v_subdisks[disk];  in g_raid_tr_iodone_raid1e()
1043 			    nsd->sd_pos);  in g_raid_tr_iodone_raid1e()
1047 				sd->sd_recovery++;  in g_raid_tr_iodone_raid1e()
1048 			cbp->bio_caller2 = (void *)mask;  in g_raid_tr_iodone_raid1e()
1050 				cbp->bio_caller1 = nsd;  in g_raid_tr_iodone_raid1e()
1052 				g_raid_lock_range(sd->sd_volume,  in g_raid_tr_iodone_raid1e()
1053 				    virtual, cbp->bio_length, pbp, cbp);  in g_raid_tr_iodone_raid1e()
1067 	if (bp->bio_cmd == BIO_READ &&  in g_raid_tr_iodone_raid1e()
1068 	    bp->bio_error == 0 &&  in g_raid_tr_iodone_raid1e()
1073 		P2V(vol, sd->sd_pos, bp->bio_offset, &virtual, &copy);  in g_raid_tr_iodone_raid1e()
1081 			if (disk >= vol->v_disks_count) {  in g_raid_tr_iodone_raid1e()
1082 				disk -= vol->v_disks_count;  in g_raid_tr_iodone_raid1e()
1083 				offset += vol->v_strip_size;  in g_raid_tr_iodone_raid1e()
1085 			cbp->bio_offset = offset + start;  in g_raid_tr_iodone_raid1e()
1086 			cbp->bio_cmd = BIO_WRITE;  in g_raid_tr_iodone_raid1e()
1087 			cbp->bio_cflags = G_RAID_BIO_FLAG_REMAP;  in g_raid_tr_iodone_raid1e()
1088 			cbp->bio_caller2 = (void *)mask;  in g_raid_tr_iodone_raid1e()
1092 			g_raid_subdisk_iostart(&vol->v_subdisks[disk], cbp);  in g_raid_tr_iodone_raid1e()
1108 		P2V(vol, sd->sd_pos, bp->bio_offset, &virtual, &copy);  in g_raid_tr_iodone_raid1e()
1113 				vol->v_subdisks[(disk + copy) %  in g_raid_tr_iodone_raid1e()
1114 				    vol->v_disks_count].sd_recovery--;  in g_raid_tr_iodone_raid1e()
1117 		if (bp->bio_cmd == BIO_WRITE && bp->bio_error) {  in g_raid_tr_iodone_raid1e()
1120 			g_raid_tr_raid1e_fail_disk(sd->sd_softc, sd, sd->sd_disk);  in g_raid_tr_iodone_raid1e()
1121 			bp->bio_error = 0;  in g_raid_tr_iodone_raid1e()
1123 		G_RAID_LOGREQ(2, bp, "REMAP done %d.", bp->bio_error);  in g_raid_tr_iodone_raid1e()
1124 		g_raid_unlock_range(sd->sd_volume, virtual, bp->bio_length);  in g_raid_tr_iodone_raid1e()
1126 	if (pbp->bio_cmd != BIO_READ) {  in g_raid_tr_iodone_raid1e()
1127 		if (pbp->bio_inbed == 1 || pbp->bio_error != 0)  in g_raid_tr_iodone_raid1e()
1128 			pbp->bio_error = bp->bio_error;  in g_raid_tr_iodone_raid1e()
1129 		if (pbp->bio_cmd == BIO_WRITE && bp->bio_error != 0) {  in g_raid_tr_iodone_raid1e()
1131 			g_raid_tr_raid1e_fail_disk(sd->sd_softc, sd, sd->sd_disk);  in g_raid_tr_iodone_raid1e()
1133 		error = pbp->bio_error;  in g_raid_tr_iodone_raid1e()
1135 		error = bp->bio_error;  in g_raid_tr_iodone_raid1e()
1137 	if (pbp->bio_children == pbp->bio_inbed) {  in g_raid_tr_iodone_raid1e()
1138 		pbp->bio_completed = pbp->bio_length;  in g_raid_tr_iodone_raid1e()
1148 	struct g_raid_subdisk *sd;  in g_raid_tr_kerneldump_raid1e()  local
1152 	u_int no, strip_size;  in g_raid_tr_kerneldump_raid1e()  local
1155 	vol = tr->tro_volume;  in g_raid_tr_kerneldump_raid1e()
1157 	strip_size = vol->v_strip_size;  in g_raid_tr_kerneldump_raid1e()
1158 	V2P(vol, boffset, &no, &offset, &start);  in g_raid_tr_kerneldump_raid1e()
1162 		length = MIN(strip_size - start, remain);  in g_raid_tr_kerneldump_raid1e()
1164 			sd = &vol->v_subdisks[no];  in g_raid_tr_kerneldump_raid1e()
1165 			switch (sd->sd_state) {  in g_raid_tr_kerneldump_raid1e()
1171 				if (offset + start >= sd->sd_rebuild_pos)  in g_raid_tr_kerneldump_raid1e()
1177 			error = g_raid_subdisk_kerneldump(sd, addr,  in g_raid_tr_kerneldump_raid1e()
1182 			if (++no >= vol->v_disks_count) {  in g_raid_tr_kerneldump_raid1e()
1183 				no = 0;  in g_raid_tr_kerneldump_raid1e()
1187 		remain -= length;  in g_raid_tr_kerneldump_raid1e()
1198 	struct g_raid_subdisk *sd;  in g_raid_tr_locked_raid1e()  local
1201 	sd = (struct g_raid_subdisk *)bp->bio_caller1;  in g_raid_tr_locked_raid1e()
1202 	g_raid_subdisk_iostart(sd, bp);  in g_raid_tr_locked_raid1e()
1213 	vol = tr->tro_volume;  in g_raid_tr_idle_raid1e()
1215 	trs->trso_fair_io = g_raid1e_rebuild_fair_io;  in g_raid_tr_idle_raid1e()
1216 	trs->trso_recover_slabs = g_raid1e_rebuild_cluster_idle;  in g_raid_tr_idle_raid1e()
1218 	if ((vol->v_disks_count % N) != 0 &&  in g_raid_tr_idle_raid1e()
1219 	    vol->v_strip_size < g_raid1e_rebuild_slab) {  in g_raid_tr_idle_raid1e()
1220 		trs->trso_recover_slabs *= g_raid1e_rebuild_slab;  in g_raid_tr_idle_raid1e()
1221 		trs->trso_recover_slabs /= vol->v_strip_size;  in g_raid_tr_idle_raid1e()
1223 	if (trs->trso_type == TR_RAID1E_REBUILD)  in g_raid_tr_idle_raid1e()
1235 	if (trs->trso_buffer != NULL) {  in g_raid_tr_free_raid1e()
1236 		free(trs->trso_buffer, M_TR_RAID1E);  in g_raid_tr_free_raid1e()
1237 		trs->trso_buffer = NULL;  in g_raid_tr_free_raid1e()