xref: /netbsd-src/sys/dev/raidframe/rf_states.c (revision cb861154c176d3dcc8ff846f449e3c16a5f5edb5)
1 /*	$NetBSD: rf_states.c,v 1.46 2011/04/27 07:55:15 mrg Exp $	*/
2 /*
3  * Copyright (c) 1995 Carnegie-Mellon University.
4  * All rights reserved.
5  *
6  * Author: Mark Holland, William V. Courtright II, Robby Findler
7  *
8  * Permission to use, copy, modify and distribute this software and
9  * its documentation is hereby granted, provided that both the copyright
10  * notice and this permission notice appear in all copies of the
11  * software, derivative works or modified versions, and any portions
12  * thereof, and that both notices appear in supporting documentation.
13  *
14  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
15  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
16  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
17  *
18  * Carnegie Mellon requests users of this software to return to
19  *
20  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
21  *  School of Computer Science
22  *  Carnegie Mellon University
23  *  Pittsburgh PA 15213-3890
24  *
25  * any improvements or extensions that they make and grant Carnegie the
26  * rights to redistribute these changes.
27  */
28 
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: rf_states.c,v 1.46 2011/04/27 07:55:15 mrg Exp $");
31 
32 #include <sys/errno.h>
33 
34 #include "rf_archs.h"
35 #include "rf_threadstuff.h"
36 #include "rf_raid.h"
37 #include "rf_dag.h"
38 #include "rf_desc.h"
39 #include "rf_aselect.h"
40 #include "rf_general.h"
41 #include "rf_states.h"
42 #include "rf_dagutils.h"
43 #include "rf_driver.h"
44 #include "rf_engine.h"
45 #include "rf_map.h"
46 #include "rf_etimer.h"
47 #include "rf_kintf.h"
48 #include "rf_paritymap.h"
49 
50 #ifndef RF_DEBUG_STATES
51 #define RF_DEBUG_STATES 0
52 #endif
53 
54 /* prototypes for some of the available states.
55 
56    States must:
57 
58      - not block.
59 
60      - either schedule rf_ContinueRaidAccess as a callback and return
61        RF_TRUE, or complete all of their work and return RF_FALSE.
62 
63      - increment desc->state when they have finished their work.
64 */
65 
66 #if RF_DEBUG_STATES
67 static char *
68 StateName(RF_AccessState_t state)
69 {
70 	switch (state) {
71 		case rf_QuiesceState:return "QuiesceState";
72 	case rf_MapState:
73 		return "MapState";
74 	case rf_LockState:
75 		return "LockState";
76 	case rf_CreateDAGState:
77 		return "CreateDAGState";
78 	case rf_ExecuteDAGState:
79 		return "ExecuteDAGState";
80 	case rf_ProcessDAGState:
81 		return "ProcessDAGState";
82 	case rf_CleanupState:
83 		return "CleanupState";
84 	case rf_LastState:
85 		return "LastState";
86 	case rf_IncrAccessesCountState:
87 		return "IncrAccessesCountState";
88 	case rf_DecrAccessesCountState:
89 		return "DecrAccessesCountState";
90 	default:
91 		return "!!! UnnamedState !!!";
92 	}
93 }
94 #endif
95 
96 void
97 rf_ContinueRaidAccess(RF_RaidAccessDesc_t *desc)
98 {
99 	int     suspended = RF_FALSE;
100 	int     current_state_index = desc->state;
101 	RF_AccessState_t current_state = desc->states[current_state_index];
102 #if RF_DEBUG_STATES
103 	int     unit = desc->raidPtr->raidid;
104 #endif
105 
106 	do {
107 
108 		current_state_index = desc->state;
109 		current_state = desc->states[current_state_index];
110 
111 		switch (current_state) {
112 
113 		case rf_QuiesceState:
114 			suspended = rf_State_Quiesce(desc);
115 			break;
116 		case rf_IncrAccessesCountState:
117 			suspended = rf_State_IncrAccessCount(desc);
118 			break;
119 		case rf_MapState:
120 			suspended = rf_State_Map(desc);
121 			break;
122 		case rf_LockState:
123 			suspended = rf_State_Lock(desc);
124 			break;
125 		case rf_CreateDAGState:
126 			suspended = rf_State_CreateDAG(desc);
127 			break;
128 		case rf_ExecuteDAGState:
129 			suspended = rf_State_ExecuteDAG(desc);
130 			break;
131 		case rf_ProcessDAGState:
132 			suspended = rf_State_ProcessDAG(desc);
133 			break;
134 		case rf_CleanupState:
135 			suspended = rf_State_Cleanup(desc);
136 			break;
137 		case rf_DecrAccessesCountState:
138 			suspended = rf_State_DecrAccessCount(desc);
139 			break;
140 		case rf_LastState:
141 			suspended = rf_State_LastState(desc);
142 			break;
143 		}
144 
145 		/* after this point, we cannot dereference desc since
146 		 * desc may have been freed. desc is only freed in
147 		 * LastState, so if we renter this function or loop
148 		 * back up, desc should be valid. */
149 
150 #if RF_DEBUG_STATES
151 		if (rf_printStatesDebug) {
152 			printf("raid%d: State: %-24s StateIndex: %3i desc: 0x%ld %s\n",
153 			       unit, StateName(current_state),
154 			       current_state_index, (long) desc,
155 			       suspended ? "callback scheduled" : "looping");
156 		}
157 #endif
158 	} while (!suspended && current_state != rf_LastState);
159 
160 	return;
161 }
162 
163 
164 void
165 rf_ContinueDagAccess(RF_DagList_t *dagList)
166 {
167 #if RF_ACC_TRACE > 0
168 	RF_AccTraceEntry_t *tracerec = &(dagList->desc->tracerec);
169 	RF_Etimer_t timer;
170 #endif
171 	RF_RaidAccessDesc_t *desc;
172 	RF_DagHeader_t *dag_h;
173 	int     i;
174 
175 	desc = dagList->desc;
176 
177 #if RF_ACC_TRACE > 0
178 	timer = tracerec->timer;
179 	RF_ETIMER_STOP(timer);
180 	RF_ETIMER_EVAL(timer);
181 	tracerec->specific.user.exec_us = RF_ETIMER_VAL_US(timer);
182 	RF_ETIMER_START(tracerec->timer);
183 #endif
184 
185 	/* skip to dag which just finished */
186 	dag_h = dagList->dags;
187 	for (i = 0; i < dagList->numDagsDone; i++) {
188 		dag_h = dag_h->next;
189 	}
190 
191 	/* check to see if retry is required */
192 	if (dag_h->status == rf_rollBackward) {
193 		/* when a dag fails, mark desc status as bad and allow
194 		 * all other dags in the desc to execute to
195 		 * completion.  then, free all dags and start over */
196 		desc->status = 1;	/* bad status */
197 #if 0
198 		printf("raid%d: DAG failure: %c addr 0x%lx "
199 		       "(%ld) nblk 0x%x (%d) buf 0x%lx state %d\n",
200 		       desc->raidPtr->raidid, desc->type,
201 		       (long) desc->raidAddress,
202 		       (long) desc->raidAddress, (int) desc->numBlocks,
203 		       (int) desc->numBlocks,
204 		       (unsigned long) (desc->bufPtr), desc->state);
205 #endif
206 	}
207 	dagList->numDagsDone++;
208 	rf_ContinueRaidAccess(desc);
209 }
210 
211 int
212 rf_State_LastState(RF_RaidAccessDesc_t *desc)
213 {
214 	void    (*callbackFunc) (RF_CBParam_t) = desc->callbackFunc;
215 	RF_CBParam_t callbackArg;
216 
217 	callbackArg.p = desc->callbackArg;
218 
219 	/*
220 	 * If this is not an async request, wake up the caller
221 	 */
222 	if (desc->async_flag == 0)
223 		wakeup(desc->bp);
224 
225 	/*
226 	 * That's all the IO for this one... unbusy the 'disk'.
227 	 */
228 
229 	rf_disk_unbusy(desc);
230 
231 	/*
232 	 * Wakeup any requests waiting to go.
233 	 */
234 
235 	RF_LOCK_MUTEX(((RF_Raid_t *) desc->raidPtr)->mutex);
236 	((RF_Raid_t *) desc->raidPtr)->openings++;
237 	RF_UNLOCK_MUTEX(((RF_Raid_t *) desc->raidPtr)->mutex);
238 
239 	rf_lock_mutex2(desc->raidPtr->iodone_lock);
240 	rf_signal_cond2(desc->raidPtr->iodone_cv);
241 	rf_unlock_mutex2(desc->raidPtr->iodone_lock);
242 
243 	/*
244 	 * The parity_map hook has to go here, because the iodone
245 	 * callback goes straight into the kintf layer.
246 	 */
247 	if (desc->raidPtr->parity_map != NULL &&
248 	    desc->type == RF_IO_TYPE_WRITE)
249 		rf_paritymap_end(desc->raidPtr->parity_map,
250 		    desc->raidAddress, desc->numBlocks);
251 
252 	/* printf("Calling biodone on 0x%x\n",desc->bp); */
253 	biodone(desc->bp);	/* access came through ioctl */
254 
255 	if (callbackFunc)
256 		callbackFunc(callbackArg);
257 	rf_FreeRaidAccDesc(desc);
258 
259 	return RF_FALSE;
260 }
261 
262 int
263 rf_State_IncrAccessCount(RF_RaidAccessDesc_t *desc)
264 {
265 	RF_Raid_t *raidPtr;
266 
267 	raidPtr = desc->raidPtr;
268 	/* Bummer. We have to do this to be 100% safe w.r.t. the increment
269 	 * below */
270 	RF_LOCK_MUTEX(raidPtr->access_suspend_mutex);
271 	raidPtr->accs_in_flight++;	/* used to detect quiescence */
272 	RF_UNLOCK_MUTEX(raidPtr->access_suspend_mutex);
273 
274 	desc->state++;
275 	return RF_FALSE;
276 }
277 
278 int
279 rf_State_DecrAccessCount(RF_RaidAccessDesc_t *desc)
280 {
281 	RF_Raid_t *raidPtr;
282 
283 	raidPtr = desc->raidPtr;
284 
285 	RF_LOCK_MUTEX(raidPtr->access_suspend_mutex);
286 	raidPtr->accs_in_flight--;
287 	if (raidPtr->accesses_suspended && raidPtr->accs_in_flight == 0) {
288 		rf_SignalQuiescenceLock(raidPtr);
289 	}
290 	RF_UNLOCK_MUTEX(raidPtr->access_suspend_mutex);
291 
292 	desc->state++;
293 	return RF_FALSE;
294 }
295 
296 int
297 rf_State_Quiesce(RF_RaidAccessDesc_t *desc)
298 {
299 #if RF_ACC_TRACE > 0
300 	RF_AccTraceEntry_t *tracerec = &desc->tracerec;
301 	RF_Etimer_t timer;
302 #endif
303 	RF_CallbackDesc_t *cb;
304 	RF_Raid_t *raidPtr;
305 	int     suspended = RF_FALSE;
306 	int need_cb, used_cb;
307 
308 	raidPtr = desc->raidPtr;
309 
310 #if RF_ACC_TRACE > 0
311 	RF_ETIMER_START(timer);
312 	RF_ETIMER_START(desc->timer);
313 #endif
314 
315 	need_cb = 0;
316 	used_cb = 0;
317 	cb = NULL;
318 
319 	RF_LOCK_MUTEX(raidPtr->access_suspend_mutex);
320 	/* Do an initial check to see if we might need a callback structure */
321 	if (raidPtr->accesses_suspended) {
322 		need_cb = 1;
323 	}
324 	RF_UNLOCK_MUTEX(raidPtr->access_suspend_mutex);
325 
326 	if (need_cb) {
327 		/* create a callback if we might need it...
328 		   and we likely do. */
329 		cb = rf_AllocCallbackDesc();
330 	}
331 
332 	RF_LOCK_MUTEX(raidPtr->access_suspend_mutex);
333 	if (raidPtr->accesses_suspended) {
334 		cb->callbackFunc = (void (*) (RF_CBParam_t)) rf_ContinueRaidAccess;
335 		cb->callbackArg.p = (void *) desc;
336 		cb->next = raidPtr->quiesce_wait_list;
337 		raidPtr->quiesce_wait_list = cb;
338 		suspended = RF_TRUE;
339 		used_cb = 1;
340 	}
341 	RF_UNLOCK_MUTEX(raidPtr->access_suspend_mutex);
342 
343 	if ((need_cb == 1) && (used_cb == 0)) {
344 		rf_FreeCallbackDesc(cb);
345 	}
346 
347 #if RF_ACC_TRACE > 0
348 	RF_ETIMER_STOP(timer);
349 	RF_ETIMER_EVAL(timer);
350 	tracerec->specific.user.suspend_ovhd_us += RF_ETIMER_VAL_US(timer);
351 #endif
352 
353 #if RF_DEBUG_QUIESCE
354 	if (suspended && rf_quiesceDebug)
355 		printf("Stalling access due to quiescence lock\n");
356 #endif
357 	desc->state++;
358 	return suspended;
359 }
360 
361 int
362 rf_State_Map(RF_RaidAccessDesc_t *desc)
363 {
364 	RF_Raid_t *raidPtr = desc->raidPtr;
365 #if RF_ACC_TRACE > 0
366 	RF_AccTraceEntry_t *tracerec = &desc->tracerec;
367 	RF_Etimer_t timer;
368 
369 	RF_ETIMER_START(timer);
370 #endif
371 
372 	if (!(desc->asmap = rf_MapAccess(raidPtr, desc->raidAddress, desc->numBlocks,
373 		    desc->bufPtr, RF_DONT_REMAP)))
374 		RF_PANIC();
375 
376 #if RF_ACC_TRACE > 0
377 	RF_ETIMER_STOP(timer);
378 	RF_ETIMER_EVAL(timer);
379 	tracerec->specific.user.map_us = RF_ETIMER_VAL_US(timer);
380 #endif
381 
382 	desc->state++;
383 	return RF_FALSE;
384 }
385 
386 int
387 rf_State_Lock(RF_RaidAccessDesc_t *desc)
388 {
389 #if RF_ACC_TRACE > 0
390 	RF_AccTraceEntry_t *tracerec = &desc->tracerec;
391 	RF_Etimer_t timer;
392 #endif
393 	RF_Raid_t *raidPtr = desc->raidPtr;
394 	RF_AccessStripeMapHeader_t *asmh = desc->asmap;
395 	RF_AccessStripeMap_t *asm_p;
396 	RF_StripeNum_t lastStripeID = -1;
397 	int     suspended = RF_FALSE;
398 
399 #if RF_ACC_TRACE > 0
400 	RF_ETIMER_START(timer);
401 #endif
402 
403 	/* acquire each lock that we don't already hold */
404 	for (asm_p = asmh->stripeMap; asm_p; asm_p = asm_p->next) {
405 		RF_ASSERT(RF_IO_IS_R_OR_W(desc->type));
406 		if (!rf_suppressLocksAndLargeWrites &&
407 		    asm_p->parityInfo &&
408 		    !(desc->flags & RF_DAG_SUPPRESS_LOCKS) &&
409 		    !(asm_p->flags & RF_ASM_FLAGS_LOCK_TRIED)) {
410 			asm_p->flags |= RF_ASM_FLAGS_LOCK_TRIED;
411 				/* locks must be acquired hierarchically */
412 			RF_ASSERT(asm_p->stripeID > lastStripeID);
413 			lastStripeID = asm_p->stripeID;
414 
415 			RF_INIT_LOCK_REQ_DESC(asm_p->lockReqDesc, desc->type,
416 					      (void (*) (struct buf *)) rf_ContinueRaidAccess, desc, asm_p,
417 					      raidPtr->Layout.dataSectorsPerStripe);
418 			if (rf_AcquireStripeLock(raidPtr->lockTable, asm_p->stripeID,
419 						 &asm_p->lockReqDesc)) {
420 				suspended = RF_TRUE;
421 				break;
422 			}
423 		}
424 		if (desc->type == RF_IO_TYPE_WRITE &&
425 		    raidPtr->status == rf_rs_reconstructing) {
426 			if (!(asm_p->flags & RF_ASM_FLAGS_FORCE_TRIED)) {
427 				int     val;
428 
429 				asm_p->flags |= RF_ASM_FLAGS_FORCE_TRIED;
430 				val = rf_ForceOrBlockRecon(raidPtr, asm_p,
431 							   (void (*) (RF_Raid_t *, void *)) rf_ContinueRaidAccess, desc);
432 				if (val == 0) {
433 					asm_p->flags |= RF_ASM_FLAGS_RECON_BLOCKED;
434 				} else {
435 					suspended = RF_TRUE;
436 					break;
437 				}
438 			} else {
439 #if RF_DEBUG_PSS > 0
440 				if (rf_pssDebug) {
441 					printf("raid%d: skipping force/block because already done, psid %ld\n",
442 					       desc->raidPtr->raidid,
443 					       (long) asm_p->stripeID);
444 				}
445 #endif
446 			}
447 		} else {
448 #if RF_DEBUG_PSS > 0
449 			if (rf_pssDebug) {
450 				printf("raid%d: skipping force/block because not write or not under recon, psid %ld\n",
451 				       desc->raidPtr->raidid,
452 				       (long) asm_p->stripeID);
453 			}
454 #endif
455 		}
456 	}
457 #if RF_ACC_TRACE > 0
458 	RF_ETIMER_STOP(timer);
459 	RF_ETIMER_EVAL(timer);
460 	tracerec->specific.user.lock_us += RF_ETIMER_VAL_US(timer);
461 #endif
462 	if (suspended)
463 		return (RF_TRUE);
464 
465 	desc->state++;
466 	return (RF_FALSE);
467 }
468 /*
469  * the following three states create, execute, and post-process dags
470  * the error recovery unit is a single dag.
471  * by default, SelectAlgorithm creates an array of dags, one per parity stripe
472  * in some tricky cases, multiple dags per stripe are created
473  *   - dags within a parity stripe are executed sequentially (arbitrary order)
474  *   - dags for distinct parity stripes are executed concurrently
475  *
476  * repeat until all dags complete successfully -or- dag selection fails
477  *
478  * while !done
479  *   create dag(s) (SelectAlgorithm)
480  *   if dag
481  *     execute dag (DispatchDAG)
482  *     if dag successful
483  *       done (SUCCESS)
484  *     else
485  *       !done (RETRY - start over with new dags)
486  *   else
487  *     done (FAIL)
488  */
489 int
490 rf_State_CreateDAG(RF_RaidAccessDesc_t *desc)
491 {
492 #if RF_ACC_TRACE > 0
493 	RF_AccTraceEntry_t *tracerec = &desc->tracerec;
494 	RF_Etimer_t timer;
495 #endif
496 	RF_DagHeader_t *dag_h;
497 	RF_DagList_t *dagList;
498 	struct buf *bp;
499 	int     i, selectStatus;
500 
501 	/* generate a dag for the access, and fire it off.  When the dag
502 	 * completes, we'll get re-invoked in the next state. */
503 #if RF_ACC_TRACE > 0
504 	RF_ETIMER_START(timer);
505 #endif
506 	/* SelectAlgorithm returns one or more dags */
507 	selectStatus = rf_SelectAlgorithm(desc, desc->flags | RF_DAG_SUPPRESS_LOCKS);
508 #if RF_DEBUG_VALIDATE_DAG
509 	if (rf_printDAGsDebug) {
510 		dagList = desc->dagList;
511 		for (i = 0; i < desc->numStripes; i++) {
512 			rf_PrintDAGList(dagList->dags);
513 			dagList = dagList->next;
514 		}
515 	}
516 #endif /* RF_DEBUG_VALIDATE_DAG */
517 #if RF_ACC_TRACE > 0
518 	RF_ETIMER_STOP(timer);
519 	RF_ETIMER_EVAL(timer);
520 	/* update time to create all dags */
521 	tracerec->specific.user.dag_create_us = RF_ETIMER_VAL_US(timer);
522 #endif
523 
524 	desc->status = 0;	/* good status */
525 
526 	if (selectStatus || (desc->numRetries > RF_RETRY_THRESHOLD)) {
527 		/* failed to create a dag */
528 		/* this happens when there are too many faults or incomplete
529 		 * dag libraries */
530 		if (selectStatus) {
531 			printf("raid%d: failed to create a dag. "
532 			       "Too many component failures.\n",
533 			       desc->raidPtr->raidid);
534 		} else {
535 			printf("raid%d: IO failed after %d retries.\n",
536 			       desc->raidPtr->raidid, RF_RETRY_THRESHOLD);
537 		}
538 
539 		desc->status = 1; /* bad status */
540 		/* skip straight to rf_State_Cleanup() */
541 		desc->state = rf_CleanupState;
542 		bp = (struct buf *)desc->bp;
543 		bp->b_error = EIO;
544 		bp->b_resid = bp->b_bcount;
545 	} else {
546 		/* bind dags to desc */
547 		dagList = desc->dagList;
548 		for (i = 0; i < desc->numStripes; i++) {
549 			dag_h = dagList->dags;
550 			while (dag_h) {
551 				dag_h->bp = (struct buf *) desc->bp;
552 #if RF_ACC_TRACE > 0
553 				dag_h->tracerec = tracerec;
554 #endif
555 				dag_h = dag_h->next;
556 			}
557 			dagList = dagList->next;
558 		}
559 		desc->flags |= RF_DAG_DISPATCH_RETURNED;
560 		desc->state++;	/* next state should be rf_State_ExecuteDAG */
561 	}
562 	return RF_FALSE;
563 }
564 
565 
566 
567 /* the access has an list of dagLists, one dagList per parity stripe.
568  * fire the first dag in each parity stripe (dagList).
569  * dags within a stripe (dagList) must be executed sequentially
570  *  - this preserves atomic parity update
571  * dags for independents parity groups (stripes) are fired concurrently */
572 
573 int
574 rf_State_ExecuteDAG(RF_RaidAccessDesc_t *desc)
575 {
576 	int     i;
577 	RF_DagHeader_t *dag_h;
578 	RF_DagList_t *dagList;
579 
580 	/* next state is always rf_State_ProcessDAG important to do
581 	 * this before firing the first dag (it may finish before we
582 	 * leave this routine) */
583 	desc->state++;
584 
585 	/* sweep dag array, a stripe at a time, firing the first dag
586 	 * in each stripe */
587 	dagList = desc->dagList;
588 	for (i = 0; i < desc->numStripes; i++) {
589 		RF_ASSERT(dagList->numDags > 0);
590 		RF_ASSERT(dagList->numDagsDone == 0);
591 		RF_ASSERT(dagList->numDagsFired == 0);
592 #if RF_ACC_TRACE > 0
593 		RF_ETIMER_START(dagList->tracerec.timer);
594 #endif
595 		/* fire first dag in this stripe */
596 		dag_h = dagList->dags;
597 		RF_ASSERT(dag_h);
598 		dagList->numDagsFired++;
599 		rf_DispatchDAG(dag_h, (void (*) (void *)) rf_ContinueDagAccess, dagList);
600 		dagList = dagList->next;
601 	}
602 
603 	/* the DAG will always call the callback, even if there was no
604 	 * blocking, so we are always suspended in this state */
605 	return RF_TRUE;
606 }
607 
608 
609 
610 /* rf_State_ProcessDAG is entered when a dag completes.
611  * first, check to all dags in the access have completed
612  * if not, fire as many dags as possible */
613 
614 int
615 rf_State_ProcessDAG(RF_RaidAccessDesc_t *desc)
616 {
617 	RF_AccessStripeMapHeader_t *asmh = desc->asmap;
618 	RF_Raid_t *raidPtr = desc->raidPtr;
619 	RF_DagHeader_t *dag_h;
620 	int     i, j, done = RF_TRUE;
621 	RF_DagList_t *dagList, *temp;
622 
623 	/* check to see if this is the last dag */
624 	dagList = desc->dagList;
625 	for (i = 0; i < desc->numStripes; i++) {
626 		if (dagList->numDags != dagList->numDagsDone)
627 			done = RF_FALSE;
628 		dagList = dagList->next;
629 	}
630 
631 	if (done) {
632 		if (desc->status) {
633 			/* a dag failed, retry */
634 			/* free all dags */
635 			dagList = desc->dagList;
636 			for (i = 0; i < desc->numStripes; i++) {
637 				rf_FreeDAG(dagList->dags);
638 				temp = dagList;
639 				dagList = dagList->next;
640 				rf_FreeDAGList(temp);
641 			}
642 			desc->dagList = NULL;
643 
644 			rf_MarkFailuresInASMList(raidPtr, asmh);
645 
646 			/* note the retry so that we'll bail in
647 			   rf_State_CreateDAG() once we've retired
648 			   the IO RF_RETRY_THRESHOLD times */
649 
650 			desc->numRetries++;
651 
652 			/* back up to rf_State_CreateDAG */
653 			desc->state = desc->state - 2;
654 			return RF_FALSE;
655 		} else {
656 			/* move on to rf_State_Cleanup */
657 			desc->state++;
658 		}
659 		return RF_FALSE;
660 	} else {
661 		/* more dags to execute */
662 		/* see if any are ready to be fired.  if so, fire them */
663 		/* don't fire the initial dag in a list, it's fired in
664 		 * rf_State_ExecuteDAG */
665 		dagList = desc->dagList;
666 		for (i = 0; i < desc->numStripes; i++) {
667 			if ((dagList->numDagsDone < dagList->numDags)
668 			    && (dagList->numDagsDone == dagList->numDagsFired)
669 			    && (dagList->numDagsFired > 0)) {
670 #if RF_ACC_TRACE > 0
671 				RF_ETIMER_START(dagList->tracerec.timer);
672 #endif
673 				/* fire next dag in this stripe */
674 				/* first, skip to next dag awaiting execution */
675 				dag_h = dagList->dags;
676 				for (j = 0; j < dagList->numDagsDone; j++)
677 					dag_h = dag_h->next;
678 				dagList->numDagsFired++;
679 				rf_DispatchDAG(dag_h, (void (*) (void *)) rf_ContinueDagAccess,
680 				    dagList);
681 			}
682 			dagList = dagList->next;
683 		}
684 		return RF_TRUE;
685 	}
686 }
687 /* only make it this far if all dags complete successfully */
688 int
689 rf_State_Cleanup(RF_RaidAccessDesc_t *desc)
690 {
691 #if RF_ACC_TRACE > 0
692 	RF_AccTraceEntry_t *tracerec = &desc->tracerec;
693 	RF_Etimer_t timer;
694 #endif
695 	RF_AccessStripeMapHeader_t *asmh = desc->asmap;
696 	RF_Raid_t *raidPtr = desc->raidPtr;
697 	RF_AccessStripeMap_t *asm_p;
698 	RF_DagList_t *dagList;
699 	int i;
700 
701 	desc->state++;
702 
703 #if RF_ACC_TRACE > 0
704 	timer = tracerec->timer;
705 	RF_ETIMER_STOP(timer);
706 	RF_ETIMER_EVAL(timer);
707 	tracerec->specific.user.dag_retry_us = RF_ETIMER_VAL_US(timer);
708 
709 	/* the RAID I/O is complete.  Clean up. */
710 	tracerec->specific.user.dag_retry_us = 0;
711 
712 	RF_ETIMER_START(timer);
713 #endif
714 	/* free all dags */
715 	dagList = desc->dagList;
716 	for (i = 0; i < desc->numStripes; i++) {
717 		rf_FreeDAG(dagList->dags);
718 		dagList = dagList->next;
719 	}
720 #if RF_ACC_TRACE > 0
721 	RF_ETIMER_STOP(timer);
722 	RF_ETIMER_EVAL(timer);
723 	tracerec->specific.user.cleanup_us = RF_ETIMER_VAL_US(timer);
724 
725 	RF_ETIMER_START(timer);
726 #endif
727 	for (asm_p = asmh->stripeMap; asm_p; asm_p = asm_p->next) {
728 		if (!rf_suppressLocksAndLargeWrites &&
729 		    asm_p->parityInfo &&
730 		    !(desc->flags & RF_DAG_SUPPRESS_LOCKS)) {
731 			RF_ASSERT_VALID_LOCKREQ(&asm_p->lockReqDesc);
732 			rf_ReleaseStripeLock(raidPtr->lockTable,
733 					     asm_p->stripeID,
734 					     &asm_p->lockReqDesc);
735 		}
736 		if (asm_p->flags & RF_ASM_FLAGS_RECON_BLOCKED) {
737 			rf_UnblockRecon(raidPtr, asm_p);
738 		}
739 	}
740 #if RF_ACC_TRACE > 0
741 	RF_ETIMER_STOP(timer);
742 	RF_ETIMER_EVAL(timer);
743 	tracerec->specific.user.lock_us += RF_ETIMER_VAL_US(timer);
744 
745 	RF_ETIMER_START(timer);
746 #endif
747 	rf_FreeAccessStripeMap(asmh);
748 #if RF_ACC_TRACE > 0
749 	RF_ETIMER_STOP(timer);
750 	RF_ETIMER_EVAL(timer);
751 	tracerec->specific.user.cleanup_us += RF_ETIMER_VAL_US(timer);
752 
753 	RF_ETIMER_STOP(desc->timer);
754 	RF_ETIMER_EVAL(desc->timer);
755 
756 	timer = desc->tracerec.tot_timer;
757 	RF_ETIMER_STOP(timer);
758 	RF_ETIMER_EVAL(timer);
759 	desc->tracerec.total_us = RF_ETIMER_VAL_US(timer);
760 
761 	rf_LogTraceRec(raidPtr, tracerec);
762 #endif
763 	desc->flags |= RF_DAG_ACCESS_COMPLETE;
764 
765 	return RF_FALSE;
766 }
767