xref: /netbsd-src/sys/dev/raidframe/rf_driver.c (revision 53b02e147d4ed531c0d2a5ca9b3e8026ba3e99b5)
1 /*	$NetBSD: rf_driver.c,v 1.139 2021/07/23 02:35:14 oster Exp $	*/
2 /*-
3  * Copyright (c) 1999 The NetBSD Foundation, Inc.
4  * All rights reserved.
5  *
6  * This code is derived from software contributed to The NetBSD Foundation
7  * by Greg Oster
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
19  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
20  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
22  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
23  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
25  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
26  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
27  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
28  * POSSIBILITY OF SUCH DAMAGE.
29  */
30 
31 /*
32  * Copyright (c) 1995 Carnegie-Mellon University.
33  * All rights reserved.
34  *
35  * Author: Mark Holland, Khalil Amiri, Claudson Bornstein, William V. Courtright II,
36  *         Robby Findler, Daniel Stodolsky, Rachad Youssef, Jim Zelenka
37  *
38  * Permission to use, copy, modify and distribute this software and
39  * its documentation is hereby granted, provided that both the copyright
40  * notice and this permission notice appear in all copies of the
41  * software, derivative works or modified versions, and any portions
42  * thereof, and that both notices appear in supporting documentation.
43  *
44  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
45  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
46  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
47  *
48  * Carnegie Mellon requests users of this software to return to
49  *
50  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
51  *  School of Computer Science
52  *  Carnegie Mellon University
53  *  Pittsburgh PA 15213-3890
54  *
55  * any improvements or extensions that they make and grant Carnegie the
56  * rights to redistribute these changes.
57  */
58 
59 /******************************************************************************
60  *
61  * rf_driver.c -- main setup, teardown, and access routines for the RAID driver
62  *
63  * all routines are prefixed with rf_ (raidframe), to avoid conficts.
64  *
65  ******************************************************************************/
66 
67 
68 #include <sys/cdefs.h>
69 __KERNEL_RCSID(0, "$NetBSD: rf_driver.c,v 1.139 2021/07/23 02:35:14 oster Exp $");
70 
71 #ifdef _KERNEL_OPT
72 #include "opt_raid_diagnostic.h"
73 #endif
74 
75 #include <sys/param.h>
76 #include <sys/systm.h>
77 #include <sys/ioctl.h>
78 #include <sys/fcntl.h>
79 #include <sys/vnode.h>
80 
81 
82 #include "rf_archs.h"
83 #include "rf_threadstuff.h"
84 
85 #include <sys/errno.h>
86 
87 #include "rf_raid.h"
88 #include "rf_dag.h"
89 #include "rf_aselect.h"
90 #include "rf_diskqueue.h"
91 #include "rf_parityscan.h"
92 #include "rf_alloclist.h"
93 #include "rf_dagutils.h"
94 #include "rf_utils.h"
95 #include "rf_etimer.h"
96 #include "rf_acctrace.h"
97 #include "rf_general.h"
98 #include "rf_desc.h"
99 #include "rf_states.h"
100 #include "rf_decluster.h"
101 #include "rf_map.h"
102 #include "rf_revent.h"
103 #include "rf_callback.h"
104 #include "rf_engine.h"
105 #include "rf_mcpair.h"
106 #include "rf_nwayxor.h"
107 #include "rf_copyback.h"
108 #include "rf_driver.h"
109 #include "rf_options.h"
110 #include "rf_shutdown.h"
111 #include "rf_kintf.h"
112 #include "rf_paritymap.h"
113 
114 #include <sys/buf.h>
115 
116 #ifndef RF_ACCESS_DEBUG
117 #define RF_ACCESS_DEBUG 0
118 #endif
119 
120 /* rad == RF_RaidAccessDesc_t */
121 #define RF_MAX_FREE_RAD 128
122 #define RF_MIN_FREE_RAD  32
123 
124 /* main configuration routines */
125 static int raidframe_booted = 0;
126 
127 static void rf_ConfigureDebug(RF_Config_t * cfgPtr);
128 static void set_debug_option(char *name, long val);
129 static void rf_UnconfigureArray(void);
130 static void rf_ShutdownRDFreeList(void *);
131 static int rf_ConfigureRDFreeList(RF_ShutdownList_t **, RF_Raid_t *, RF_Config_t *);
132 
133 rf_declare_mutex2(rf_printf_mutex);	/* debug only:  avoids interleaved
134 					 * printfs by different stripes */
135 
136 #define SIGNAL_QUIESCENT_COND(_raid_) \
137 	rf_broadcast_cond2((_raid_)->access_suspend_cv)
138 #define WAIT_FOR_QUIESCENCE(_raid_) \
139 	rf_wait_cond2((_raid_)->access_suspend_cv, \
140 		      (_raid_)->access_suspend_mutex)
141 
142 static int configureCount = 0;	/* number of active configurations */
143 static int isconfigged = 0;	/* is basic raidframe (non per-array)
144 				 * stuff configured */
145 static rf_declare_mutex2(configureMutex); /* used to lock the configuration
146 					   * stuff */
147 static RF_ShutdownList_t *globalShutdown;	/* non array-specific
148 						 * stuff */
149 
150 static int rf_ConfigureRDFreeList(RF_ShutdownList_t ** listp, RF_Raid_t *raidPtr, RF_Config_t *cfgPtr);
151 static int rf_AllocEmergBuffers(RF_Raid_t *);
152 static void rf_FreeEmergBuffers(RF_Raid_t *);
153 static void rf_destroy_mutex_cond(RF_Raid_t *);
154 static void rf_alloc_mutex_cond(RF_Raid_t *);
155 
156 /* called at system boot time */
157 int
158 rf_BootRaidframe(bool boot)
159 {
160 
161 	if (boot) {
162 		if (raidframe_booted)
163 			return (EBUSY);
164 		raidframe_booted = 1;
165 		rf_init_mutex2(configureMutex, IPL_NONE);
166  		configureCount = 0;
167 		isconfigged = 0;
168 		globalShutdown = NULL;
169 	} else {
170 		rf_destroy_mutex2(configureMutex);
171 		raidframe_booted = 0;
172 	}
173 	return (0);
174 }
175 
176 /*
177  * Called whenever an array is shutdown
178  */
179 static void
180 rf_UnconfigureArray(void)
181 {
182 
183 	rf_lock_mutex2(configureMutex);
184 	if (--configureCount == 0) {	/* if no active configurations, shut
185 					 * everything down */
186 		rf_destroy_mutex2(rf_printf_mutex);
187 		isconfigged = 0;
188 		rf_ShutdownList(&globalShutdown);
189 
190 		/*
191 	         * We must wait until now, because the AllocList module
192 	         * uses the DebugMem module.
193 	         */
194 #if RF_DEBUG_MEM
195 		if (rf_memDebug)
196 			rf_print_unfreed();
197 #endif
198 	}
199 	rf_unlock_mutex2(configureMutex);
200 }
201 
202 /*
203  * Called to shut down an array.
204  */
205 int
206 rf_Shutdown(RF_Raid_t *raidPtr)
207 {
208 
209 	if (!raidPtr->valid) {
210 		RF_ERRORMSG("Attempt to shut down unconfigured RAIDframe driver.  Aborting shutdown\n");
211 		return (EINVAL);
212 	}
213 	/*
214          * wait for outstanding IOs to land
215          * As described in rf_raid.h, we use the rad_freelist lock
216          * to protect the per-array info about outstanding descs
217          * since we need to do freelist locking anyway, and this
218          * cuts down on the amount of serialization we've got going
219          * on.
220          */
221 	rf_lock_mutex2(raidPtr->rad_lock);
222 	if (raidPtr->waitShutdown) {
223 		rf_unlock_mutex2(raidPtr->rad_lock);
224 		return (EBUSY);
225 	}
226 	raidPtr->waitShutdown = 1;
227 	while (raidPtr->nAccOutstanding) {
228 		rf_wait_cond2(raidPtr->outstandingCond, raidPtr->rad_lock);
229 	}
230 
231 	/* Wait for any parity re-writes to stop... */
232 	while (raidPtr->parity_rewrite_in_progress) {
233 		printf("raid%d: Waiting for parity re-write to exit...\n",
234 		       raidPtr->raidid);
235 		rf_wait_cond2(raidPtr->parity_rewrite_cv, raidPtr->rad_lock);
236 	}
237 	rf_unlock_mutex2(raidPtr->rad_lock);
238 
239 	/* Wait for any reconstruction to stop... */
240 	rf_lock_mutex2(raidPtr->mutex);
241 	while (raidPtr->reconInProgress) {
242 		printf("raid%d: Waiting for reconstruction to stop...\n",
243 		       raidPtr->raidid);
244 		rf_wait_cond2(raidPtr->waitForReconCond, raidPtr->mutex);
245 	}
246 	rf_unlock_mutex2(raidPtr->mutex);
247 
248 	raidPtr->valid = 0;
249 
250 	if (raidPtr->parity_map != NULL)
251 		rf_paritymap_detach(raidPtr);
252 
253 	rf_update_component_labels(raidPtr, RF_FINAL_COMPONENT_UPDATE);
254 
255 	rf_UnconfigureVnodes(raidPtr);
256 
257 	rf_FreeEmergBuffers(raidPtr);
258 
259 	rf_ShutdownList(&raidPtr->shutdownList);
260 
261 	rf_destroy_mutex_cond(raidPtr);
262 
263 	rf_UnconfigureArray();
264 
265 	return (0);
266 }
267 
268 
269 #define DO_INIT_CONFIGURE(f) { \
270 	rc = f (&globalShutdown); \
271 	if (rc) { \
272 		RF_ERRORMSG2("RAIDFRAME: failed %s with %d\n", RF_STRING(f), rc); \
273 		rf_ShutdownList(&globalShutdown); \
274 		configureCount--; \
275 		rf_unlock_mutex2(configureMutex); \
276 		rf_destroy_mutex2(rf_printf_mutex); \
277 		return(rc); \
278 	} \
279 }
280 
281 #define DO_RAID_FAIL() { \
282 	rf_UnconfigureVnodes(raidPtr); \
283 	rf_FreeEmergBuffers(raidPtr); \
284 	rf_ShutdownList(&raidPtr->shutdownList); \
285 	rf_UnconfigureArray(); \
286 	rf_destroy_mutex_cond(raidPtr); \
287 }
288 
289 #define DO_RAID_INIT_CONFIGURE(f) { \
290 	rc = f (&raidPtr->shutdownList, raidPtr, cfgPtr); \
291 	if (rc) { \
292 		RF_ERRORMSG2("RAIDFRAME: failed %s with %d\n", RF_STRING(f), rc); \
293 		DO_RAID_FAIL(); \
294 		return(rc); \
295 	} \
296 }
297 
298 int
299 rf_Configure(RF_Raid_t *raidPtr, RF_Config_t *cfgPtr, RF_AutoConfig_t *ac)
300 {
301 	RF_RowCol_t col;
302 	int rc;
303 	bool swapped = false;
304 	bool first = true;
305 
306 	rf_lock_mutex2(configureMutex);
307 	configureCount++;
308 	if (isconfigged == 0) {
309 		rf_init_mutex2(rf_printf_mutex, IPL_VM);
310 
311 		/* initialize globals */
312 		DO_INIT_CONFIGURE(rf_ConfigureAllocList);
313 
314 		/*
315 	         * Yes, this does make debugging general to the whole
316 	         * system instead of being array specific. Bummer, drag.
317 		 */
318 		rf_ConfigureDebug(cfgPtr);
319 		DO_INIT_CONFIGURE(rf_ConfigureDebugMem);
320 #if RF_ACC_TRACE > 0
321 		DO_INIT_CONFIGURE(rf_ConfigureAccessTrace);
322 #endif
323 		DO_INIT_CONFIGURE(rf_ConfigureNWayXor);
324 		DO_INIT_CONFIGURE(rf_ConfigureDAGFuncs);
325 		DO_INIT_CONFIGURE(rf_ConfigureCopyback);
326 		isconfigged = 1;
327 	}
328 	rf_unlock_mutex2(configureMutex);
329 
330 	rf_alloc_mutex_cond(raidPtr);
331 
332 	/* set up the cleanup list.  Do this after ConfigureDebug so that
333 	 * value of memDebug will be set */
334 
335 	rf_MakeAllocList(raidPtr->cleanupList);
336 	if (raidPtr->cleanupList == NULL) {
337 		DO_RAID_FAIL();
338 		return (ENOMEM);
339 	}
340 	rf_ShutdownCreate(&raidPtr->shutdownList,
341 			  (void (*) (void *)) rf_FreeAllocList,
342 			  raidPtr->cleanupList);
343 
344 	raidPtr->numCol = cfgPtr->numCol;
345 	raidPtr->numSpare = cfgPtr->numSpare;
346 
347 	raidPtr->status = rf_rs_optimal;
348 	raidPtr->reconControl = NULL;
349 
350 	DO_RAID_INIT_CONFIGURE(rf_ConfigureMapModule);
351 	DO_RAID_INIT_CONFIGURE(rf_ConfigureReconEvent);
352 	DO_RAID_INIT_CONFIGURE(rf_ConfigureCallback);
353 	DO_RAID_INIT_CONFIGURE(rf_ConfigureRDFreeList);
354 	DO_RAID_INIT_CONFIGURE(rf_ConfigureStripeLockFreeList);
355 	DO_RAID_INIT_CONFIGURE(rf_ConfigureMCPair);
356 	DO_RAID_INIT_CONFIGURE(rf_ConfigureDAGs);
357 	DO_RAID_INIT_CONFIGURE(rf_ConfigureReconstruction);
358 	DO_RAID_INIT_CONFIGURE(rf_ConfigureDiskQueueSystem);
359 	DO_RAID_INIT_CONFIGURE(rf_ConfigurePSStatus);
360 
361 	DO_RAID_INIT_CONFIGURE(rf_ConfigureEngine);
362 	DO_RAID_INIT_CONFIGURE(rf_ConfigureStripeLocks);
363 
364 	raidPtr->nAccOutstanding = 0;
365 	raidPtr->waitShutdown = 0;
366 
367 	if (ac!=NULL) {
368 		/* We have an AutoConfig structure..  Don't do the
369 		   normal disk configuration... call the auto config
370 		   stuff */
371 		rf_AutoConfigureDisks(raidPtr, cfgPtr, ac);
372 	} else {
373 		DO_RAID_INIT_CONFIGURE(rf_ConfigureDisks);
374 		DO_RAID_INIT_CONFIGURE(rf_ConfigureSpareDisks);
375 	}
376 	/* do this after ConfigureDisks & ConfigureSpareDisks to be sure dev
377 	 * no. is set */
378 	DO_RAID_INIT_CONFIGURE(rf_ConfigureDiskQueues);
379 
380 	DO_RAID_INIT_CONFIGURE(rf_ConfigureLayout);
381 
382 
383 
384 
385 	/* Initialize per-RAID PSS bits */
386 	rf_InitPSStatus(raidPtr);
387 
388 #if RF_INCLUDE_CHAINDECLUSTER > 0
389 	for (col = 0; col < raidPtr->numCol; col++) {
390 		/*
391 		 * XXX better distribution
392 		 */
393 		raidPtr->hist_diskreq[col] = 0;
394 	}
395 #endif
396 	raidPtr->numNewFailures = 0;
397 	raidPtr->copyback_in_progress = 0;
398 	raidPtr->parity_rewrite_in_progress = 0;
399 	raidPtr->adding_hot_spare = 0;
400 	raidPtr->recon_in_progress = 0;
401 
402 	raidPtr->maxOutstanding = cfgPtr->maxOutstandingDiskReqs;
403 
404 	/* autoconfigure and root_partition will actually get filled in
405 	   after the config is done */
406 	raidPtr->autoconfigure = 0;
407 	raidPtr->root_partition = 0;
408 	raidPtr->last_unit = raidPtr->raidid;
409 	raidPtr->config_order = 0;
410 
411 	if (rf_keepAccTotals) {
412 		raidPtr->keep_acc_totals = 1;
413 	}
414 
415 	/* Allocate a bunch of buffers to be used in low-memory conditions */
416 	raidPtr->iobuf = NULL;
417 
418 	rc = rf_AllocEmergBuffers(raidPtr);
419 	if (rc) {
420 		printf("raid%d: Unable to allocate emergency buffers.\n",
421 		       raidPtr->raidid);
422 		DO_RAID_FAIL();
423 		return(rc);
424 	}
425 
426 	/* Set up parity map stuff, if applicable. */
427 #ifndef RF_NO_PARITY_MAP
428 	rf_paritymap_attach(raidPtr, cfgPtr->force);
429 #endif
430 
431 	raidPtr->valid = 1;
432 
433 	printf("raid%d: %s\n", raidPtr->raidid,
434 	       raidPtr->Layout.map->configName);
435 	printf("raid%d: Components:", raidPtr->raidid);
436 
437 	for (col = 0; col < raidPtr->numCol; col++) {
438 		RF_ComponentLabel_t *clabel;
439 		bool compswapped;
440 
441 		printf(" %s", raidPtr->Disks[col].devname);
442 		if (RF_DEAD_DISK(raidPtr->Disks[col].status)) {
443 			printf("[**FAILED**]");
444 		}
445 		clabel = raidget_component_label(raidPtr, col);
446 		compswapped = clabel->version ==
447 			      bswap32(RF_COMPONENT_LABEL_VERSION);
448 		if (first)
449 			swapped = compswapped;
450 		else if (swapped != compswapped)
451 			printf("raid%d: Component %d has different endian "
452 			       "than first component.", raidPtr->raidid, col);
453 	}
454 	printf("\n");
455 	printf("raid%d: Total Sectors: %" PRIu64 " (%" PRIu64 " MB)\n",
456 	       raidPtr->raidid,
457 	       raidPtr->totalSectors,
458 	       (raidPtr->totalSectors / 1024 *
459 				(1 << raidPtr->logBytesPerSector) / 1024));
460 	if (swapped)
461 		printf("raid%d: Using swapped-endian component labels.\n",
462 		    raidPtr->raidid);
463 
464 	return (0);
465 }
466 
467 
468 /*
469 
470   Routines to allocate and free the "emergency buffers" for a given
471   RAID set.  These emergency buffers will be used when the kernel runs
472   out of kernel memory.
473 
474  */
475 
476 static int
477 rf_AllocEmergBuffers(RF_Raid_t *raidPtr)
478 {
479 	void *tmpbuf;
480 	RF_VoidPointerListElem_t *vple;
481 	int i;
482 
483 	/* XXX next line needs tuning... */
484 	raidPtr->numEmergencyBuffers = 10 * raidPtr->numCol;
485 #if DEBUG
486 	printf("raid%d: allocating %d buffers of %d bytes.\n",
487 	       raidPtr->raidid,
488 	       raidPtr->numEmergencyBuffers,
489 	       (int)(raidPtr->Layout.sectorsPerStripeUnit <<
490 	       raidPtr->logBytesPerSector));
491 #endif
492 	for (i = 0; i < raidPtr->numEmergencyBuffers; i++) {
493 		tmpbuf = malloc( raidPtr->Layout.sectorsPerStripeUnit <<
494 				 raidPtr->logBytesPerSector,
495 				 M_RAIDFRAME, M_WAITOK);
496 		if (tmpbuf) {
497 			vple = rf_AllocVPListElem(raidPtr);
498 			vple->p= tmpbuf;
499 			vple->next = raidPtr->iobuf;
500 			raidPtr->iobuf = vple;
501 			raidPtr->iobuf_count++;
502 		} else {
503 			printf("raid%d: failed to allocate emergency buffer!\n",
504 			       raidPtr->raidid);
505 			return 1;
506 		}
507 	}
508 
509 	/* XXX next line needs tuning too... */
510 	raidPtr->numEmergencyStripeBuffers = 10;
511         for (i = 0; i < raidPtr->numEmergencyStripeBuffers; i++) {
512                 tmpbuf = malloc( raidPtr->numCol * (raidPtr->Layout.sectorsPerStripeUnit <<
513                                  raidPtr->logBytesPerSector),
514                                  M_RAIDFRAME, M_WAITOK);
515                 if (tmpbuf) {
516                         vple = rf_AllocVPListElem(raidPtr);
517                         vple->p= tmpbuf;
518                         vple->next = raidPtr->stripebuf;
519                         raidPtr->stripebuf = vple;
520                         raidPtr->stripebuf_count++;
521                 } else {
522                         printf("raid%d: failed to allocate emergency stripe buffer!\n",
523                                raidPtr->raidid);
524 			return 1;
525                 }
526         }
527 
528 	return (0);
529 }
530 
531 static void
532 rf_FreeEmergBuffers(RF_Raid_t *raidPtr)
533 {
534 	RF_VoidPointerListElem_t *tmp;
535 
536 	/* Free the emergency IO buffers */
537 	while (raidPtr->iobuf != NULL) {
538 		tmp = raidPtr->iobuf;
539 		raidPtr->iobuf = raidPtr->iobuf->next;
540 		free(tmp->p, M_RAIDFRAME);
541 		rf_FreeVPListElem(raidPtr,tmp);
542 	}
543 
544 	/* Free the emergency stripe buffers */
545 	while (raidPtr->stripebuf != NULL) {
546 		tmp = raidPtr->stripebuf;
547 		raidPtr->stripebuf = raidPtr->stripebuf->next;
548 		free(tmp->p, M_RAIDFRAME);
549 		rf_FreeVPListElem(raidPtr, tmp);
550 	}
551 }
552 
553 
554 static void
555 rf_ShutdownRDFreeList(void *arg)
556 {
557 	RF_Raid_t *raidPtr;
558 
559 	raidPtr = (RF_Raid_t *) arg;
560 
561 	pool_destroy(&raidPtr->pools.rad);
562 }
563 
564 static int
565 rf_ConfigureRDFreeList(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr,
566 		       RF_Config_t *cfgPtr)
567 {
568 
569 	rf_pool_init(raidPtr, raidPtr->poolNames.rad, &raidPtr->pools.rad, sizeof(RF_RaidAccessDesc_t),
570 		     "rad", RF_MIN_FREE_RAD, RF_MAX_FREE_RAD);
571 	rf_ShutdownCreate(listp, rf_ShutdownRDFreeList, raidPtr);
572 	return (0);
573 }
574 
575 RF_RaidAccessDesc_t *
576 rf_AllocRaidAccDesc(RF_Raid_t *raidPtr, RF_IoType_t type,
577 		    RF_RaidAddr_t raidAddress, RF_SectorCount_t numBlocks,
578 		    void *bufPtr, void *bp, RF_RaidAccessFlags_t flags,
579 		    const RF_AccessState_t *states)
580 {
581 	RF_RaidAccessDesc_t *desc;
582 
583 	desc = pool_get(&raidPtr->pools.rad, PR_WAITOK);
584 
585 	rf_lock_mutex2(raidPtr->rad_lock);
586 	if (raidPtr->waitShutdown) {
587 		/*
588 	         * Actually, we're shutting the array down. Free the desc
589 	         * and return NULL.
590 	         */
591 
592 		rf_unlock_mutex2(raidPtr->rad_lock);
593 		pool_put(&raidPtr->pools.rad, desc);
594 		return (NULL);
595 	}
596 	raidPtr->nAccOutstanding++;
597 
598 	rf_unlock_mutex2(raidPtr->rad_lock);
599 
600 	desc->raidPtr = (void *) raidPtr;
601 	desc->type = type;
602 	desc->raidAddress = raidAddress;
603 	desc->numBlocks = numBlocks;
604 	desc->bufPtr = bufPtr;
605 	desc->bp = bp;
606 	desc->flags = flags;
607 	desc->states = states;
608 	desc->state = 0;
609 	desc->dagList = NULL;
610 
611 	desc->status = 0;
612 	desc->numRetries = 0;
613 #if RF_ACC_TRACE > 0
614 	memset(&desc->tracerec, 0, sizeof(desc->tracerec));
615 #endif
616 	desc->callbackFunc = NULL;
617 	desc->callbackArg = NULL;
618 	desc->next = NULL;
619 	desc->iobufs = NULL;
620 	desc->stripebufs = NULL;
621 
622 	return (desc);
623 }
624 
625 void
626 rf_FreeRaidAccDesc(RF_RaidAccessDesc_t *desc)
627 {
628 	RF_Raid_t *raidPtr = desc->raidPtr;
629 	RF_DagList_t *dagList, *temp;
630 	RF_VoidPointerListElem_t *tmp;
631 
632 	RF_ASSERT(desc);
633 
634 	/* Cleanup the dagList(s) */
635 	dagList = desc->dagList;
636 	while(dagList != NULL) {
637 		temp = dagList;
638 		dagList = dagList->next;
639 		rf_FreeDAGList(raidPtr, temp);
640 	}
641 
642 	while (desc->iobufs) {
643 		tmp = desc->iobufs;
644 		desc->iobufs = desc->iobufs->next;
645 		rf_FreeIOBuffer(raidPtr, tmp);
646 	}
647 
648 	while (desc->stripebufs) {
649 		tmp = desc->stripebufs;
650 		desc->stripebufs = desc->stripebufs->next;
651 		rf_FreeStripeBuffer(raidPtr, tmp);
652 	}
653 
654 	pool_put(&raidPtr->pools.rad, desc);
655 	rf_lock_mutex2(raidPtr->rad_lock);
656 	raidPtr->nAccOutstanding--;
657 	if (raidPtr->waitShutdown) {
658 		rf_signal_cond2(raidPtr->outstandingCond);
659 	}
660 	rf_unlock_mutex2(raidPtr->rad_lock);
661 }
662 /*********************************************************************
663  * Main routine for performing an access.
664  * Accesses are retried until a DAG can not be selected.  This occurs
665  * when either the DAG library is incomplete or there are too many
666  * failures in a parity group.
667  *
668  * type should be read or write.  bp_in is a buf pointer.  void *to
669  * facilitate ignoring it outside the kernel
670  ********************************************************************/
671 int
672 rf_DoAccess(RF_Raid_t * raidPtr, RF_IoType_t type, RF_RaidAddr_t raidAddress, RF_SectorCount_t numBlocks,
673 	    void *bufPtr, struct buf *bp, RF_RaidAccessFlags_t flags)
674 {
675 	RF_RaidAccessDesc_t *desc;
676 	void *lbufPtr = bufPtr;
677 
678 	raidAddress += rf_raidSectorOffset;
679 
680 #if RF_ACCESS_DEBUG
681 	if (rf_accessDebug) {
682 
683 		printf("logBytes is: %d %d %d\n", raidPtr->raidid,
684 		    raidPtr->logBytesPerSector,
685 		    (int) rf_RaidAddressToByte(raidPtr, numBlocks));
686 		printf("raid%d: %s raidAddr %d (stripeid %d-%d) numBlocks %d (%d bytes) buf 0x%lx\n", raidPtr->raidid,
687 		    (type == RF_IO_TYPE_READ) ? "READ" : "WRITE", (int) raidAddress,
688 		    (int) rf_RaidAddressToStripeID(&raidPtr->Layout, raidAddress),
689 		    (int) rf_RaidAddressToStripeID(&raidPtr->Layout, raidAddress + numBlocks - 1),
690 		    (int) numBlocks,
691 		    (int) rf_RaidAddressToByte(raidPtr, numBlocks),
692 		    (long) bufPtr);
693 	}
694 #endif
695 
696 	desc = rf_AllocRaidAccDesc(raidPtr, type, raidAddress,
697 	    numBlocks, lbufPtr, bp, flags, raidPtr->Layout.map->states);
698 
699 	if (desc == NULL) {
700 		return (ENOMEM);
701 	}
702 #if RF_ACC_TRACE > 0
703 	RF_ETIMER_START(desc->tracerec.tot_timer);
704 #endif
705 
706 	if (raidPtr->parity_map != NULL &&
707 	    type == RF_IO_TYPE_WRITE)
708 		rf_paritymap_begin(raidPtr->parity_map, raidAddress,
709 		    numBlocks);
710 
711 	rf_ContinueRaidAccess(desc);
712 
713 	return (0);
714 }
715 #if 0
716 /* force the array into reconfigured mode without doing reconstruction */
717 int
718 rf_SetReconfiguredMode(RF_Raid_t *raidPtr, int col)
719 {
720 	if (!(raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) {
721 		printf("Can't set reconfigured mode in dedicated-spare array\n");
722 		RF_PANIC();
723 	}
724 	rf_lock_mutex2(raidPtr->mutex);
725 	raidPtr->numFailures++;
726 	raidPtr->Disks[col].status = rf_ds_dist_spared;
727 	raidPtr->status = rf_rs_reconfigured;
728 	rf_update_component_labels(raidPtr, RF_NORMAL_COMPONENT_UPDATE);
729 	/* install spare table only if declustering + distributed sparing
730 	 * architecture. */
731 	if (raidPtr->Layout.map->flags & RF_BD_DECLUSTERED)
732 		rf_InstallSpareTable(raidPtr, col);
733 	rf_unlock_mutex2(raidPtr->mutex);
734 	return (0);
735 }
736 #endif
737 
738 int
739 rf_FailDisk(RF_Raid_t *raidPtr, int fcol, int initRecon)
740 {
741 
742 	/* need to suspend IO's here -- if there are DAGs in flight
743 	   and we pull the rug out from under ci_vp, Bad Things
744 	   can happen.  */
745 
746 	rf_SuspendNewRequestsAndWait(raidPtr);
747 
748 	rf_lock_mutex2(raidPtr->mutex);
749 	if (raidPtr->Disks[fcol].status != rf_ds_failed) {
750 		/* must be failing something that is valid, or else it's
751 		   already marked as failed (in which case we don't
752 		   want to mark it failed again!) */
753 		raidPtr->numFailures++;
754 		raidPtr->Disks[fcol].status = rf_ds_failed;
755 		raidPtr->status = rf_rs_degraded;
756 	}
757 	rf_unlock_mutex2(raidPtr->mutex);
758 
759 	rf_update_component_labels(raidPtr, RF_NORMAL_COMPONENT_UPDATE);
760 
761 	/* Close the component, so that it's not "locked" if someone
762 	   else want's to use it! */
763 
764 	rf_close_component(raidPtr, raidPtr->raid_cinfo[fcol].ci_vp,
765 			   raidPtr->Disks[fcol].auto_configured);
766 
767 	rf_lock_mutex2(raidPtr->mutex);
768 	raidPtr->raid_cinfo[fcol].ci_vp = NULL;
769 
770 	/* Need to mark the component as not being auto_configured
771 	   (in case it was previously). */
772 
773 	raidPtr->Disks[fcol].auto_configured = 0;
774 	rf_unlock_mutex2(raidPtr->mutex);
775 	/* now we can allow IO to continue -- we'll be suspending it
776 	   again in rf_ReconstructFailedDisk() if we have to.. */
777 
778 	rf_ResumeNewRequests(raidPtr);
779 
780 	if (initRecon)
781 		rf_ReconstructFailedDisk(raidPtr, fcol);
782 	return (0);
783 }
784 /* releases a thread that is waiting for the array to become quiesced.
785  * access_suspend_mutex should be locked upon calling this
786  */
787 void
788 rf_SignalQuiescenceLock(RF_Raid_t *raidPtr)
789 {
790 #if RF_DEBUG_QUIESCE
791 	if (rf_quiesceDebug) {
792 		printf("raid%d: Signalling quiescence lock\n",
793 		       raidPtr->raidid);
794 	}
795 #endif
796 	raidPtr->access_suspend_release = 1;
797 
798 	if (raidPtr->waiting_for_quiescence) {
799 		SIGNAL_QUIESCENT_COND(raidPtr);
800 	}
801 }
802 /* suspends all new requests to the array.  No effect on accesses that are in flight.  */
803 int
804 rf_SuspendNewRequestsAndWait(RF_Raid_t *raidPtr)
805 {
806 #if RF_DEBUG_QUIESCE
807 	if (rf_quiesceDebug)
808 		printf("raid%d: Suspending new reqs\n", raidPtr->raidid);
809 #endif
810 	rf_lock_mutex2(raidPtr->access_suspend_mutex);
811 	raidPtr->accesses_suspended++;
812 	raidPtr->waiting_for_quiescence = (raidPtr->accs_in_flight == 0) ? 0 : 1;
813 
814 	if (raidPtr->waiting_for_quiescence) {
815 		raidPtr->access_suspend_release = 0;
816 		while (!raidPtr->access_suspend_release) {
817 #if RF_DEBUG_QUIESCE
818 			printf("raid%d: Suspending: Waiting for Quiescence\n",
819 			       raidPtr->raidid);
820 #endif
821 			WAIT_FOR_QUIESCENCE(raidPtr);
822 			raidPtr->waiting_for_quiescence = 0;
823 		}
824 	}
825 #if RF_DEBUG_QUIESCE
826 	printf("raid%d: Quiescence reached..\n", raidPtr->raidid);
827 #endif
828 
829 	rf_unlock_mutex2(raidPtr->access_suspend_mutex);
830 	return (raidPtr->waiting_for_quiescence);
831 }
832 /* wake up everyone waiting for quiescence to be released */
833 void
834 rf_ResumeNewRequests(RF_Raid_t *raidPtr)
835 {
836 	RF_CallbackFuncDesc_t *t, *cb;
837 
838 #if RF_DEBUG_QUIESCE
839 	if (rf_quiesceDebug)
840 		printf("raid%d: Resuming new requests\n", raidPtr->raidid);
841 #endif
842 
843 	rf_lock_mutex2(raidPtr->access_suspend_mutex);
844 	raidPtr->accesses_suspended--;
845 	if (raidPtr->accesses_suspended == 0)
846 		cb = raidPtr->quiesce_wait_list;
847 	else
848 		cb = NULL;
849 	raidPtr->quiesce_wait_list = NULL;
850 	rf_unlock_mutex2(raidPtr->access_suspend_mutex);
851 
852 	while (cb) {
853 		t = cb;
854 		cb = cb->next;
855 		(t->callbackFunc) (t->callbackArg);
856 		rf_FreeCallbackFuncDesc(raidPtr, t);
857 	}
858 }
859 /*****************************************************************************************
860  *
861  * debug routines
862  *
863  ****************************************************************************************/
864 
865 static void
866 set_debug_option(char *name, long val)
867 {
868 	RF_DebugName_t *p;
869 
870 	for (p = rf_debugNames; p->name; p++) {
871 		if (!strcmp(p->name, name)) {
872 			*(p->ptr) = val;
873 			printf("[Set debug variable %s to %ld]\n", name, val);
874 			return;
875 		}
876 	}
877 	RF_ERRORMSG1("Unknown debug string \"%s\"\n", name);
878 }
879 
880 
881 /* would like to use sscanf here, but apparently not available in kernel */
882 /*ARGSUSED*/
883 static void
884 rf_ConfigureDebug(RF_Config_t *cfgPtr)
885 {
886 	char   *val_p, *name_p, *white_p;
887 	long    val;
888 	int     i;
889 
890 	rf_ResetDebugOptions();
891 	for (i = 0; i < RF_MAXDBGV && cfgPtr->debugVars[i][0]; i++) {
892 		name_p = rf_find_non_white(&cfgPtr->debugVars[i][0]);
893 		white_p = rf_find_white(name_p);	/* skip to start of 2nd
894 							 * word */
895 		val_p = rf_find_non_white(white_p);
896 		if (*val_p == '0' && *(val_p + 1) == 'x')
897 			val = rf_htoi(val_p + 2);
898 		else
899 			val = rf_atoi(val_p);
900 		*white_p = '\0';
901 		set_debug_option(name_p, val);
902 	}
903 }
904 
905 void
906 rf_print_panic_message(int line, const char *file)
907 {
908 	kern_assert("raidframe error at line %d file %s", line, file);
909 }
910 
911 #ifdef RAID_DIAGNOSTIC
912 void
913 rf_print_assert_panic_message(int line,	const char *file, const char *condition)
914 {
915 	kern_assert("raidframe error at line %d file %s (failed asserting %s)\n",
916 	    line, file, condition);
917 }
918 #endif
919 
920 void
921 rf_print_unable_to_init_mutex(const char *file, int line, int rc)
922 {
923 	RF_ERRORMSG3("Unable to init mutex file %s line %d rc=%d\n",
924 		     file, line, rc);
925 }
926 
927 void
928 rf_print_unable_to_add_shutdown(const char *file, int line, int rc)
929 {
930 	RF_ERRORMSG3("Unable to add to shutdown list file %s line %d rc=%d\n",
931 		     file, line, rc);
932 }
933 
934 static void
935 rf_alloc_mutex_cond(RF_Raid_t *raidPtr)
936 {
937 
938 	rf_init_mutex2(raidPtr->mutex, IPL_VM);
939 
940 	rf_init_cond2(raidPtr->outstandingCond, "rfocond");
941 	rf_init_cond2(raidPtr->parity_rewrite_cv, "rfprwshutdown");
942 	rf_init_mutex2(raidPtr->rad_lock, IPL_VM);
943 
944 	rf_init_mutex2(raidPtr->access_suspend_mutex, IPL_VM);
945 	rf_init_cond2(raidPtr->access_suspend_cv, "rfquiesce");
946 
947 	rf_init_cond2(raidPtr->waitForReconCond, "rfrcnw");
948 
949 	rf_init_cond2(raidPtr->adding_hot_spare_cv, "raidhs");
950 }
951 
952 static void
953 rf_destroy_mutex_cond(RF_Raid_t *raidPtr)
954 {
955 
956 	rf_destroy_cond2(raidPtr->waitForReconCond);
957 	rf_destroy_cond2(raidPtr->adding_hot_spare_cv);
958 
959 	rf_destroy_mutex2(raidPtr->access_suspend_mutex);
960 	rf_destroy_cond2(raidPtr->access_suspend_cv);
961 
962 	rf_destroy_cond2(raidPtr->parity_rewrite_cv);
963 	rf_destroy_cond2(raidPtr->outstandingCond);
964 	rf_destroy_mutex2(raidPtr->rad_lock);
965 
966 	rf_destroy_mutex2(raidPtr->mutex);
967 }
968