1 /* $NetBSD: rf_diskqueue.c,v 1.1 1998/11/13 04:20:29 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /**************************************************************************************** 30 * 31 * rf_diskqueue.c -- higher-level disk queue code 32 * 33 * the routines here are a generic wrapper around the actual queueing 34 * routines. The code here implements thread scheduling, synchronization, 35 * and locking ops (see below) on top of the lower-level queueing code. 36 * 37 * to support atomic RMW, we implement "locking operations". When a locking op 38 * is dispatched to the lower levels of the driver, the queue is locked, and no further 39 * I/Os are dispatched until the queue receives & completes a corresponding "unlocking 40 * operation". This code relies on the higher layers to guarantee that a locking 41 * op will always be eventually followed by an unlocking op. The model is that 42 * the higher layers are structured so locking and unlocking ops occur in pairs, i.e. 43 * an unlocking op cannot be generated until after a locking op reports completion. 44 * There is no good way to check to see that an unlocking op "corresponds" to the 45 * op that currently has the queue locked, so we make no such attempt. Since by 46 * definition there can be only one locking op outstanding on a disk, this should 47 * not be a problem. 48 * 49 * In the kernel, we allow multiple I/Os to be concurrently dispatched to the disk 50 * driver. In order to support locking ops in this environment, when we decide to 51 * do a locking op, we stop dispatching new I/Os and wait until all dispatched I/Os 52 * have completed before dispatching the locking op. 53 * 54 * Unfortunately, the code is different in the 3 different operating states 55 * (user level, kernel, simulator). In the kernel, I/O is non-blocking, and 56 * we have no disk threads to dispatch for us. Therefore, we have to dispatch 57 * new I/Os to the scsi driver at the time of enqueue, and also at the time 58 * of completion. At user level, I/O is blocking, and so only the disk threads 59 * may dispatch I/Os. Thus at user level, all we can do at enqueue time is 60 * enqueue and wake up the disk thread to do the dispatch. 61 * 62 ***************************************************************************************/ 63 64 /* 65 * : 66 * 67 * Log: rf_diskqueue.c,v 68 * Revision 1.50 1996/08/07 21:08:38 jimz 69 * b_proc -> kb_proc 70 * 71 * Revision 1.49 1996/07/05 20:36:14 jimz 72 * make rf_ConfigureDiskQueueSystem return 0 73 * 74 * Revision 1.48 1996/06/18 20:53:11 jimz 75 * fix up disk queueing (remove configure routine, 76 * add shutdown list arg to create routines) 77 * 78 * Revision 1.47 1996/06/14 14:16:36 jimz 79 * fix handling of bogus queue type 80 * 81 * Revision 1.46 1996/06/13 20:41:44 jimz 82 * add scan, cscan, random queueing 83 * 84 * Revision 1.45 1996/06/11 01:27:50 jimz 85 * Fixed bug where diskthread shutdown would crash or hang. This 86 * turned out to be two distinct bugs: 87 * (1) [crash] The thread shutdown code wasn't properly waiting for 88 * all the diskthreads to complete. This caused diskthreads that were 89 * exiting+cleaning up to unlock a destroyed mutex. 90 * (2) [hang] TerminateDiskQueues wasn't locking, and DiskIODequeue 91 * only checked for termination _after_ a wakeup if the queues were 92 * empty. This was a race where the termination wakeup could be lost 93 * by the dequeueing thread, and the system would hang waiting for the 94 * thread to exit, while the thread waited for an I/O or a signal to 95 * check the termination flag. 96 * 97 * Revision 1.44 1996/06/10 11:55:47 jimz 98 * Straightened out some per-array/not-per-array distinctions, fixed 99 * a couple bugs related to confusion. Added shutdown lists. Removed 100 * layout shutdown function (now subsumed by shutdown lists). 101 * 102 * Revision 1.43 1996/06/09 02:36:46 jimz 103 * lots of little crufty cleanup- fixup whitespace 104 * issues, comment #ifdefs, improve typing in some 105 * places (esp size-related) 106 * 107 * Revision 1.42 1996/06/07 22:26:27 jimz 108 * type-ify which_ru (RF_ReconUnitNum_t) 109 * 110 * Revision 1.41 1996/06/07 21:33:04 jimz 111 * begin using consistent types for sector numbers, 112 * stripe numbers, row+col numbers, recon unit numbers 113 * 114 * Revision 1.40 1996/06/06 17:28:04 jimz 115 * track sector number of last I/O dequeued 116 * 117 * Revision 1.39 1996/06/06 01:14:13 jimz 118 * fix crashing bug when tracerec is NULL (ie, from copyback) 119 * initialize req->queue 120 * 121 * Revision 1.38 1996/06/05 19:38:32 jimz 122 * fixed up disk queueing types config 123 * added sstf disk queueing 124 * fixed exit bug on diskthreads (ref-ing bad mem) 125 * 126 * Revision 1.37 1996/06/05 18:06:02 jimz 127 * Major code cleanup. The Great Renaming is now done. 128 * Better modularity. Better typing. Fixed a bunch of 129 * synchronization bugs. Made a lot of global stuff 130 * per-desc or per-array. Removed dead code. 131 * 132 * Revision 1.36 1996/05/30 23:22:16 jimz 133 * bugfixes of serialization, timing problems 134 * more cleanup 135 * 136 * Revision 1.35 1996/05/30 12:59:18 jimz 137 * make etimer happier, more portable 138 * 139 * Revision 1.34 1996/05/30 11:29:41 jimz 140 * Numerous bug fixes. Stripe lock release code disagreed with the taking code 141 * about when stripes should be locked (I made it consistent: no parity, no lock) 142 * There was a lot of extra serialization of I/Os which I've removed- a lot of 143 * it was to calculate values for the cache code, which is no longer with us. 144 * More types, function, macro cleanup. Added code to properly quiesce the array 145 * on shutdown. Made a lot of stuff array-specific which was (bogusly) general 146 * before. Fixed memory allocation, freeing bugs. 147 * 148 * Revision 1.33 1996/05/27 18:56:37 jimz 149 * more code cleanup 150 * better typing 151 * compiles in all 3 environments 152 * 153 * Revision 1.32 1996/05/24 22:17:04 jimz 154 * continue code + namespace cleanup 155 * typed a bunch of flags 156 * 157 * Revision 1.31 1996/05/24 01:59:45 jimz 158 * another checkpoint in code cleanup for release 159 * time to sync kernel tree 160 * 161 * Revision 1.30 1996/05/23 21:46:35 jimz 162 * checkpoint in code cleanup (release prep) 163 * lots of types, function names have been fixed 164 * 165 * Revision 1.29 1996/05/23 00:33:23 jimz 166 * code cleanup: move all debug decls to rf_options.c, all extern 167 * debug decls to rf_options.h, all debug vars preceded by rf_ 168 * 169 * Revision 1.28 1996/05/20 16:14:29 jimz 170 * switch to rf_{mutex,cond}_{init,destroy} 171 * 172 * Revision 1.27 1996/05/18 19:51:34 jimz 173 * major code cleanup- fix syntax, make some types consistent, 174 * add prototypes, clean out dead code, et cetera 175 * 176 * Revision 1.26 1996/05/16 19:21:49 wvcii 177 * fixed typo in init_dqd 178 * 179 * Revision 1.25 1996/05/16 16:02:51 jimz 180 * switch to RF_FREELIST stuff for DiskQueueData 181 * 182 * Revision 1.24 1996/05/10 16:24:14 jimz 183 * new cvscan function names 184 * 185 * Revision 1.23 1996/05/01 16:27:54 jimz 186 * don't use ccmn bp management 187 * 188 * Revision 1.22 1995/12/12 18:10:06 jimz 189 * MIN -> RF_MIN, MAX -> RF_MAX, ASSERT -> RF_ASSERT 190 * fix 80-column brain damage in comments 191 * 192 * Revision 1.21 1995/12/01 15:59:59 root 193 * added copyright info 194 * 195 * Revision 1.20 1995/11/07 16:27:20 wvcii 196 * added Peek() function to diskqueuesw 197 * non-locking accesses are never blocked (assume clients enforce proper 198 * respect for lock acquisition) 199 * 200 * Revision 1.19 1995/10/05 18:56:52 jimz 201 * fix req handling in IOComplete 202 * 203 * Revision 1.18 1995/10/04 20:13:50 wvcii 204 * added asserts to monitor numOutstanding queueLength 205 * 206 * Revision 1.17 1995/10/04 07:43:52 wvcii 207 * queue->numOutstanding now valid for user & sim 208 * added queue->queueLength 209 * user tested & verified, sim untested 210 * 211 * Revision 1.16 1995/09/12 00:21:19 wvcii 212 * added support for tracing disk queue time 213 * 214 */ 215 216 #include "rf_types.h" 217 #include "rf_threadstuff.h" 218 #include "rf_threadid.h" 219 #include "rf_raid.h" 220 #include "rf_diskqueue.h" 221 #include "rf_alloclist.h" 222 #include "rf_acctrace.h" 223 #include "rf_etimer.h" 224 #include "rf_configure.h" 225 #include "rf_general.h" 226 #include "rf_freelist.h" 227 #include "rf_debugprint.h" 228 #include "rf_shutdown.h" 229 #include "rf_cvscan.h" 230 #include "rf_sstf.h" 231 #include "rf_fifo.h" 232 233 #ifdef SIMULATE 234 #include "rf_diskevent.h" 235 #endif /* SIMULATE */ 236 237 #if !defined(__NetBSD__) 238 extern struct buf *ubc_bufget(); 239 #endif 240 241 static int init_dqd(RF_DiskQueueData_t *); 242 static void clean_dqd(RF_DiskQueueData_t *); 243 static void rf_ShutdownDiskQueueSystem(void *); 244 /* From rf_kintf.c */ 245 int rf_DispatchKernelIO(RF_DiskQueue_t *,RF_DiskQueueData_t *); 246 247 248 #define Dprintf1(s,a) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),NULL,NULL,NULL,NULL,NULL,NULL,NULL) 249 #define Dprintf2(s,a,b) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),(void *)((unsigned long)b),NULL,NULL,NULL,NULL,NULL,NULL) 250 #define Dprintf3(s,a,b,c) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),(void *)((unsigned long)b),(void *)((unsigned long)c),NULL,NULL,NULL,NULL,NULL) 251 #define Dprintf4(s,a,b,c,d) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),(void *)((unsigned long)b),(void *)((unsigned long)c),(void *)((unsigned long)d),NULL,NULL,NULL,NULL) 252 #define Dprintf5(s,a,b,c,d,e) if (rf_queueDebug) rf_debug_printf(s,(void *)((unsigned long)a),(void *)((unsigned long)b),(void *)((unsigned long)c),(void *)((unsigned long)d),(void *)((unsigned long)e),NULL,NULL,NULL) 253 254 #if !defined(KERNEL) && !defined(SIMULATE) 255 256 /* queue must be locked before invoking this */ 257 #define SIGNAL_DISK_QUEUE(_q_,_wh_) \ 258 { \ 259 if ( (_q_)->numWaiting > 0) { \ 260 (_q_)->numWaiting--; \ 261 RF_SIGNAL_COND( ((_q_)->cond) ); \ 262 } \ 263 } 264 265 /* queue must be locked before invoking this */ 266 #define WAIT_DISK_QUEUE(_q_,_wh_) \ 267 { \ 268 (_q_)->numWaiting++; \ 269 RF_WAIT_COND( ((_q_)->cond), ((_q_)->mutex) ); \ 270 } 271 272 #else /* !defined(KERNEL) && !defined(SIMULATE) */ 273 274 #define SIGNAL_DISK_QUEUE(_q_,_wh_) 275 #define WAIT_DISK_QUEUE(_q_,_wh_) 276 277 #endif /* !defined(KERNEL) && !defined(SIMULATE) */ 278 279 /***************************************************************************************** 280 * 281 * the disk queue switch defines all the functions used in the different queueing 282 * disciplines 283 * queue ID, init routine, enqueue routine, dequeue routine 284 * 285 ****************************************************************************************/ 286 287 static RF_DiskQueueSW_t diskqueuesw[] = { 288 {"fifo", /* FIFO */ 289 rf_FifoCreate, 290 rf_FifoEnqueue, 291 rf_FifoDequeue, 292 rf_FifoPeek, 293 rf_FifoPromote}, 294 295 {"cvscan", /* cvscan */ 296 rf_CvscanCreate, 297 rf_CvscanEnqueue, 298 rf_CvscanDequeue, 299 rf_CvscanPeek, 300 rf_CvscanPromote }, 301 302 {"sstf", /* shortest seek time first */ 303 rf_SstfCreate, 304 rf_SstfEnqueue, 305 rf_SstfDequeue, 306 rf_SstfPeek, 307 rf_SstfPromote}, 308 309 {"scan", /* SCAN (two-way elevator) */ 310 rf_ScanCreate, 311 rf_SstfEnqueue, 312 rf_ScanDequeue, 313 rf_ScanPeek, 314 rf_SstfPromote}, 315 316 {"cscan", /* CSCAN (one-way elevator) */ 317 rf_CscanCreate, 318 rf_SstfEnqueue, 319 rf_CscanDequeue, 320 rf_CscanPeek, 321 rf_SstfPromote}, 322 323 #if !defined(KERNEL) && RF_INCLUDE_QUEUE_RANDOM > 0 324 /* to make a point to Chris :-> */ 325 {"random", /* random */ 326 rf_FifoCreate, 327 rf_FifoEnqueue, 328 rf_RandomDequeue, 329 rf_RandomPeek, 330 rf_FifoPromote}, 331 #endif /* !KERNEL && RF_INCLUDE_QUEUE_RANDOM > 0 */ 332 }; 333 #define NUM_DISK_QUEUE_TYPES (sizeof(diskqueuesw)/sizeof(RF_DiskQueueSW_t)) 334 335 static RF_FreeList_t *rf_dqd_freelist; 336 337 #define RF_MAX_FREE_DQD 256 338 #define RF_DQD_INC 16 339 #define RF_DQD_INITIAL 64 340 341 #ifdef __NetBSD__ 342 #ifdef _KERNEL 343 #include <sys/buf.h> 344 #endif 345 #endif 346 347 static int init_dqd(dqd) 348 RF_DiskQueueData_t *dqd; 349 { 350 #ifdef KERNEL 351 #ifdef __NetBSD__ 352 /* XXX not sure if the following malloc is appropriate... probably not quite... */ 353 dqd->bp = (struct buf *) malloc( sizeof(struct buf), M_DEVBUF, M_NOWAIT); 354 memset(dqd->bp,0,sizeof(struct buf)); /* if you don't do it, nobody else will.. */ 355 /* XXX */ 356 /* printf("NEED TO IMPLEMENT THIS BETTER!\n"); */ 357 #else 358 dqd->bp = ubc_bufget(); 359 #endif 360 if (dqd->bp == NULL) { 361 return(ENOMEM); 362 } 363 #endif /* KERNEL */ 364 return(0); 365 } 366 367 static void clean_dqd(dqd) 368 RF_DiskQueueData_t *dqd; 369 { 370 #ifdef KERNEL 371 #ifdef __NetBSD__ 372 /* printf("NEED TO IMPLEMENT THIS BETTER(2)!\n"); */ 373 /* XXX ? */ 374 free( dqd->bp, M_DEVBUF ); 375 #else 376 ubc_buffree(dqd->bp); 377 #endif 378 379 #endif /* KERNEL */ 380 } 381 382 /* configures a single disk queue */ 383 static int config_disk_queue( 384 RF_Raid_t *raidPtr, 385 RF_DiskQueue_t *diskqueue, 386 RF_RowCol_t r, /* row & col -- debug only. BZZT not any more... */ 387 RF_RowCol_t c, 388 RF_DiskQueueSW_t *p, 389 RF_SectorCount_t sectPerDisk, 390 dev_t dev, 391 int maxOutstanding, 392 RF_ShutdownList_t **listp, 393 RF_AllocListElem_t *clList) 394 { 395 int rc; 396 397 diskqueue->row = r; 398 diskqueue->col = c; 399 diskqueue->qPtr = p; 400 diskqueue->qHdr = (p->Create)(sectPerDisk, clList, listp); 401 diskqueue->dev = dev; 402 diskqueue->numOutstanding = 0; 403 diskqueue->queueLength = 0; 404 diskqueue->maxOutstanding = maxOutstanding; 405 diskqueue->curPriority = RF_IO_NORMAL_PRIORITY; 406 diskqueue->nextLockingOp = NULL; 407 diskqueue->unlockingOp = NULL; 408 diskqueue->numWaiting=0; 409 diskqueue->flags = 0; 410 diskqueue->raidPtr = raidPtr; 411 #if defined(__NetBSD__) && defined(_KERNEL) 412 diskqueue->rf_cinfo = &raidPtr->raid_cinfo[r][c]; 413 #endif 414 rc = rf_create_managed_mutex(listp, &diskqueue->mutex); 415 if (rc) { 416 RF_ERRORMSG3("Unable to init mutex file %s line %d rc=%d\n", __FILE__, 417 __LINE__, rc); 418 return(rc); 419 } 420 rc = rf_create_managed_cond(listp, &diskqueue->cond); 421 if (rc) { 422 RF_ERRORMSG3("Unable to init cond file %s line %d rc=%d\n", __FILE__, 423 __LINE__, rc); 424 return(rc); 425 } 426 return(0); 427 } 428 429 static void rf_ShutdownDiskQueueSystem(ignored) 430 void *ignored; 431 { 432 RF_FREELIST_DESTROY_CLEAN(rf_dqd_freelist,next,(RF_DiskQueueData_t *),clean_dqd); 433 } 434 435 int rf_ConfigureDiskQueueSystem(listp) 436 RF_ShutdownList_t **listp; 437 { 438 int rc; 439 440 RF_FREELIST_CREATE(rf_dqd_freelist, RF_MAX_FREE_DQD, 441 RF_DQD_INC, sizeof(RF_DiskQueueData_t)); 442 if (rf_dqd_freelist == NULL) 443 return(ENOMEM); 444 rc = rf_ShutdownCreate(listp, rf_ShutdownDiskQueueSystem, NULL); 445 if (rc) { 446 RF_ERRORMSG3("Unable to add to shutdown list file %s line %d rc=%d\n", 447 __FILE__, __LINE__, rc); 448 rf_ShutdownDiskQueueSystem(NULL); 449 return(rc); 450 } 451 RF_FREELIST_PRIME_INIT(rf_dqd_freelist, RF_DQD_INITIAL,next, 452 (RF_DiskQueueData_t *),init_dqd); 453 return(0); 454 } 455 456 #ifndef KERNEL 457 /* this is called prior to shutdown to wakeup everyone waiting on a disk queue 458 * and tell them to exit 459 */ 460 void rf_TerminateDiskQueues(raidPtr) 461 RF_Raid_t *raidPtr; 462 { 463 RF_RowCol_t r, c; 464 465 raidPtr->terminate_disk_queues = 1; 466 for (r=0; r<raidPtr->numRow; r++) { 467 for (c=0; c<raidPtr->numCol + ((r==0) ? raidPtr->numSpare : 0); c++) { 468 RF_LOCK_QUEUE_MUTEX(&raidPtr->Queues[r][c], "TerminateDiskQueues"); 469 RF_BROADCAST_COND(raidPtr->Queues[r][c].cond); 470 RF_UNLOCK_QUEUE_MUTEX(&raidPtr->Queues[r][c], "TerminateDiskQueues"); 471 } 472 } 473 } 474 #endif /* !KERNEL */ 475 476 int rf_ConfigureDiskQueues( 477 RF_ShutdownList_t **listp, 478 RF_Raid_t *raidPtr, 479 RF_Config_t *cfgPtr) 480 { 481 RF_DiskQueue_t **diskQueues, *spareQueues; 482 RF_DiskQueueSW_t *p; 483 RF_RowCol_t r, c; 484 int rc, i; 485 486 raidPtr->maxQueueDepth = cfgPtr->maxOutstandingDiskReqs; 487 488 for(p=NULL,i=0;i<NUM_DISK_QUEUE_TYPES;i++) { 489 if (!strcmp(diskqueuesw[i].queueType, cfgPtr->diskQueueType)) { 490 p = &diskqueuesw[i]; 491 break; 492 } 493 } 494 if (p == NULL) { 495 RF_ERRORMSG2("Unknown queue type \"%s\". Using %s\n",cfgPtr->diskQueueType, diskqueuesw[0].queueType); 496 p = &diskqueuesw[0]; 497 } 498 499 RF_CallocAndAdd(diskQueues, raidPtr->numRow, sizeof(RF_DiskQueue_t *), (RF_DiskQueue_t **), raidPtr->cleanupList); 500 if (diskQueues == NULL) { 501 return(ENOMEM); 502 } 503 raidPtr->Queues = diskQueues; 504 for (r=0; r<raidPtr->numRow; r++) { 505 RF_CallocAndAdd(diskQueues[r], raidPtr->numCol + ((r==0) ? raidPtr->numSpare : 0), sizeof(RF_DiskQueue_t), (RF_DiskQueue_t *), raidPtr->cleanupList); 506 if (diskQueues[r] == NULL) 507 return(ENOMEM); 508 for (c=0; c<raidPtr->numCol; c++) { 509 rc = config_disk_queue(raidPtr, &diskQueues[r][c], r, c, p, 510 raidPtr->sectorsPerDisk, raidPtr->Disks[r][c].dev, 511 cfgPtr->maxOutstandingDiskReqs, listp, raidPtr->cleanupList); 512 if (rc) 513 return(rc); 514 } 515 } 516 517 spareQueues = &raidPtr->Queues[0][raidPtr->numCol]; 518 for (r=0; r<raidPtr->numSpare; r++) { 519 rc = config_disk_queue(raidPtr, &spareQueues[r], 520 0, raidPtr->numCol+r, p, 521 raidPtr->sectorsPerDisk, 522 raidPtr->Disks[0][raidPtr->numCol+r].dev, 523 cfgPtr->maxOutstandingDiskReqs, listp, 524 raidPtr->cleanupList); 525 if (rc) 526 return(rc); 527 } 528 return(0); 529 } 530 531 /* Enqueue a disk I/O 532 * 533 * Unfortunately, we have to do things differently in the different 534 * environments (simulator, user-level, kernel). 535 * At user level, all I/O is blocking, so we have 1 or more threads/disk 536 * and the thread that enqueues is different from the thread that dequeues. 537 * In the kernel, I/O is non-blocking and so we'd like to have multiple 538 * I/Os outstanding on the physical disks when possible. 539 * 540 * when any request arrives at a queue, we have two choices: 541 * dispatch it to the lower levels 542 * queue it up 543 * 544 * kernel rules for when to do what: 545 * locking request: queue empty => dispatch and lock queue, 546 * else queue it 547 * unlocking req : always dispatch it 548 * normal req : queue empty => dispatch it & set priority 549 * queue not full & priority is ok => dispatch it 550 * else queue it 551 * 552 * user-level rules: 553 * always enqueue. In the special case of an unlocking op, enqueue 554 * in a special way that will cause the unlocking op to be the next 555 * thing dequeued. 556 * 557 * simulator rules: 558 * Do the same as at user level, with the sleeps and wakeups suppressed. 559 */ 560 void rf_DiskIOEnqueue(queue, req, pri) 561 RF_DiskQueue_t *queue; 562 RF_DiskQueueData_t *req; 563 int pri; 564 { 565 int tid; 566 567 RF_ETIMER_START(req->qtime); 568 rf_get_threadid(tid); 569 RF_ASSERT(req->type == RF_IO_TYPE_NOP || req->numSector); 570 req->priority = pri; 571 572 if (rf_queueDebug && (req->numSector == 0)) { 573 printf("Warning: Enqueueing zero-sector access\n"); 574 } 575 576 #ifdef KERNEL 577 /* 578 * kernel 579 */ 580 RF_LOCK_QUEUE_MUTEX( queue, "DiskIOEnqueue" ); 581 /* locking request */ 582 if (RF_LOCKING_REQ(req)) { 583 if (RF_QUEUE_EMPTY(queue)) { 584 Dprintf3("Dispatching pri %d locking op to r %d c %d (queue empty)\n",pri,queue->row, queue->col); 585 RF_LOCK_QUEUE(queue); 586 rf_DispatchKernelIO(queue, req); 587 } else { 588 queue->queueLength++; /* increment count of number of requests waiting in this queue */ 589 Dprintf3("Enqueueing pri %d locking op to r %d c %d (queue not empty)\n",pri,queue->row, queue->col); 590 req->queue = (void *)queue; 591 (queue->qPtr->Enqueue)(queue->qHdr, req, pri); 592 } 593 } 594 /* unlocking request */ 595 else if (RF_UNLOCKING_REQ(req)) { /* we'll do the actual unlock when this I/O completes */ 596 Dprintf3("Dispatching pri %d unlocking op to r %d c %d\n",pri,queue->row, queue->col); 597 RF_ASSERT(RF_QUEUE_LOCKED(queue)); 598 rf_DispatchKernelIO(queue, req); 599 } 600 /* normal request */ 601 else if (RF_OK_TO_DISPATCH(queue, req)) { 602 Dprintf3("Dispatching pri %d regular op to r %d c %d (ok to dispatch)\n",pri,queue->row, queue->col); 603 rf_DispatchKernelIO(queue, req); 604 } else { 605 queue->queueLength++; /* increment count of number of requests waiting in this queue */ 606 Dprintf3("Enqueueing pri %d regular op to r %d c %d (not ok to dispatch)\n",pri,queue->row, queue->col); 607 req->queue = (void *)queue; 608 (queue->qPtr->Enqueue)(queue->qHdr, req, pri); 609 } 610 RF_UNLOCK_QUEUE_MUTEX( queue, "DiskIOEnqueue" ); 611 612 #else /* KERNEL */ 613 /* 614 * user-level 615 */ 616 RF_LOCK_QUEUE_MUTEX( queue, "DiskIOEnqueue" ); 617 queue->queueLength++; /* increment count of number of requests waiting in this queue */ 618 /* unlocking request */ 619 if (RF_UNLOCKING_REQ(req)) { 620 Dprintf4("[%d] enqueueing pri %d unlocking op & signalling r %d c %d\n", tid, pri, queue->row, queue->col); 621 RF_ASSERT(RF_QUEUE_LOCKED(queue) && queue->unlockingOp == NULL); 622 queue->unlockingOp = req; 623 } 624 /* locking and normal requests */ 625 else { 626 req->queue = (void *)queue; 627 Dprintf5("[%d] enqueueing pri %d %s op & signalling r %d c %d\n", tid, pri, 628 (RF_LOCKING_REQ(req)) ? "locking" : "regular",queue->row,queue->col); 629 (queue->qPtr->Enqueue)(queue->qHdr, req, pri); 630 } 631 SIGNAL_DISK_QUEUE( queue, "DiskIOEnqueue"); 632 RF_UNLOCK_QUEUE_MUTEX( queue, "DiskIOEnqueue" ); 633 #endif /* KERNEL */ 634 } 635 636 #if !defined(KERNEL) && !defined(SIMULATE) 637 /* user-level only: tell all threads to wake up & recheck the queue */ 638 void rf_BroadcastOnQueue(queue) 639 RF_DiskQueue_t *queue; 640 { 641 int i; 642 643 if (queue->maxOutstanding > 1) for (i=0; i<queue->maxOutstanding; i++) { 644 SIGNAL_DISK_QUEUE(queue, "BroadcastOnQueue" ); 645 } 646 } 647 #endif /* !KERNEL && !SIMULATE */ 648 649 #ifndef KERNEL /* not used in kernel */ 650 651 RF_DiskQueueData_t *rf_DiskIODequeue(queue) 652 RF_DiskQueue_t *queue; 653 { 654 RF_DiskQueueData_t *p, *headItem; 655 int tid; 656 657 rf_get_threadid(tid); 658 RF_LOCK_QUEUE_MUTEX( queue, "DiskIODequeue" ); 659 for (p=NULL; !p; ) { 660 if (queue->unlockingOp) { 661 /* unlocking request */ 662 RF_ASSERT(RF_QUEUE_LOCKED(queue)); 663 p = queue->unlockingOp; 664 queue->unlockingOp = NULL; 665 Dprintf4("[%d] dequeueing pri %d unlocking op r %d c %d\n", tid, p->priority, queue->row,queue->col); 666 } 667 else { 668 headItem = (queue->qPtr->Peek)(queue->qHdr); 669 if (headItem) { 670 if (RF_LOCKING_REQ(headItem)) { 671 /* locking request */ 672 if (!RF_QUEUE_LOCKED(queue)) { 673 /* queue isn't locked, so dequeue the request & lock the queue */ 674 p = (queue->qPtr->Dequeue)( queue->qHdr ); 675 if (p) 676 Dprintf4("[%d] dequeueing pri %d locking op r %d c %d\n", tid, p->priority, queue->row, queue->col); 677 else 678 Dprintf3("[%d] no dequeue -- raw queue empty r %d c %d\n", tid, queue->row, queue->col); 679 } 680 else { 681 /* queue already locked, no dequeue occurs */ 682 Dprintf3("[%d] no dequeue -- queue is locked r %d c %d\n", tid, queue->row, queue->col); 683 p = NULL; 684 } 685 } 686 else { 687 /* normal request, always dequeue and assume caller already has lock (if needed) */ 688 p = (queue->qPtr->Dequeue)( queue->qHdr ); 689 if (p) 690 Dprintf4("[%d] dequeueing pri %d regular op r %d c %d\n", tid, p->priority, queue->row, queue->col); 691 else 692 Dprintf3("[%d] no dequeue -- raw queue empty r %d c %d\n", tid, queue->row, queue->col); 693 } 694 } 695 else { 696 Dprintf3("[%d] no dequeue -- raw queue empty r %d c %d\n", tid, queue->row, queue->col); 697 } 698 } 699 700 if (queue->raidPtr->terminate_disk_queues) { 701 p = NULL; 702 break; 703 } 704 #ifdef SIMULATE 705 break; /* in simulator, return NULL on empty queue instead of blocking */ 706 #else /* SIMULATE */ 707 if (!p) { 708 Dprintf3("[%d] nothing to dequeue: waiting r %d c %d\n", tid, queue->row, queue->col); 709 WAIT_DISK_QUEUE( queue, "DiskIODequeue" ); 710 } 711 #endif /* SIMULATE */ 712 } 713 714 if (p) { 715 queue->queueLength--; /* decrement count of number of requests waiting in this queue */ 716 RF_ASSERT(queue->queueLength >= 0); 717 queue->numOutstanding++; 718 queue->last_deq_sector = p->sectorOffset; 719 /* record the amount of time this request spent in the disk queue */ 720 RF_ETIMER_STOP(p->qtime); 721 RF_ETIMER_EVAL(p->qtime); 722 if (p->tracerec) 723 p->tracerec->diskqueue_us += RF_ETIMER_VAL_US(p->qtime); 724 } 725 726 if (p && RF_LOCKING_REQ(p)) { 727 RF_ASSERT(!RF_QUEUE_LOCKED(queue)); 728 Dprintf3("[%d] locking queue r %d c %d\n",tid,queue->row,queue->col); 729 RF_LOCK_QUEUE(queue); 730 } 731 RF_UNLOCK_QUEUE_MUTEX( queue, "DiskIODequeue" ); 732 733 return(p); 734 } 735 736 #else /* !KERNEL */ 737 738 /* get the next set of I/Os started, kernel version only */ 739 void rf_DiskIOComplete(queue, req, status) 740 RF_DiskQueue_t *queue; 741 RF_DiskQueueData_t *req; 742 int status; 743 { 744 int done=0; 745 746 RF_LOCK_QUEUE_MUTEX( queue, "DiskIOComplete" ); 747 748 /* unlock the queue: 749 (1) after an unlocking req completes 750 (2) after a locking req fails 751 */ 752 if (RF_UNLOCKING_REQ(req) || (RF_LOCKING_REQ(req) && status)) { 753 Dprintf2("DiskIOComplete: unlocking queue at r %d c %d\n", queue->row, queue->col); 754 RF_ASSERT(RF_QUEUE_LOCKED(queue) && (queue->unlockingOp == NULL)); 755 RF_UNLOCK_QUEUE(queue); 756 } 757 758 queue->numOutstanding--; 759 RF_ASSERT(queue->numOutstanding >= 0); 760 761 /* dispatch requests to the disk until we find one that we can't. */ 762 /* no reason to continue once we've filled up the queue */ 763 /* no reason to even start if the queue is locked */ 764 765 while (!done && !RF_QUEUE_FULL(queue) && !RF_QUEUE_LOCKED(queue)) { 766 if (queue->nextLockingOp) { 767 req = queue->nextLockingOp; queue->nextLockingOp = NULL; 768 Dprintf3("DiskIOComplete: a pri %d locking req was pending at r %d c %d\n",req->priority,queue->row, queue->col); 769 } else { 770 req = (queue->qPtr->Dequeue)( queue->qHdr ); 771 Dprintf3("DiskIOComplete: extracting pri %d req from queue at r %d c %d\n",req->priority,queue->row, queue->col); 772 } 773 if (req) { 774 queue->queueLength--; /* decrement count of number of requests waiting in this queue */ 775 RF_ASSERT(queue->queueLength >= 0); 776 } 777 if (!req) done=1; 778 else if (RF_LOCKING_REQ(req)) { 779 if (RF_QUEUE_EMPTY(queue)) { /* dispatch it */ 780 Dprintf3("DiskIOComplete: dispatching pri %d locking req to r %d c %d (queue empty)\n",req->priority,queue->row, queue->col); 781 RF_LOCK_QUEUE(queue); 782 rf_DispatchKernelIO(queue, req); 783 done = 1; 784 } else { /* put it aside to wait for the queue to drain */ 785 Dprintf3("DiskIOComplete: postponing pri %d locking req to r %d c %d\n",req->priority,queue->row, queue->col); 786 RF_ASSERT(queue->nextLockingOp == NULL); 787 queue->nextLockingOp = req; 788 done = 1; 789 } 790 } else if (RF_UNLOCKING_REQ(req)) { /* should not happen: unlocking ops should not get queued */ 791 RF_ASSERT(RF_QUEUE_LOCKED(queue)); /* support it anyway for the future */ 792 Dprintf3("DiskIOComplete: dispatching pri %d unl req to r %d c %d (SHOULD NOT SEE THIS)\n",req->priority,queue->row, queue->col); 793 rf_DispatchKernelIO(queue, req); 794 done = 1; 795 } else if (RF_OK_TO_DISPATCH(queue, req)) { 796 Dprintf3("DiskIOComplete: dispatching pri %d regular req to r %d c %d (ok to dispatch)\n",req->priority,queue->row, queue->col); 797 rf_DispatchKernelIO(queue, req); 798 } else { /* we can't dispatch it, so just re-enqueue it. */ 799 /* potential trouble here if disk queues batch reqs */ 800 Dprintf3("DiskIOComplete: re-enqueueing pri %d regular req to r %d c %d\n",req->priority,queue->row, queue->col); 801 queue->queueLength++; 802 (queue->qPtr->Enqueue)(queue->qHdr, req, req->priority); 803 done = 1; 804 } 805 } 806 807 RF_UNLOCK_QUEUE_MUTEX( queue, "DiskIOComplete" ); 808 } 809 #endif /* !KERNEL */ 810 811 /* promotes accesses tagged with the given parityStripeID from low priority 812 * to normal priority. This promotion is optional, meaning that a queue 813 * need not implement it. If there is no promotion routine associated with 814 * a queue, this routine does nothing and returns -1. 815 */ 816 int rf_DiskIOPromote(queue, parityStripeID, which_ru) 817 RF_DiskQueue_t *queue; 818 RF_StripeNum_t parityStripeID; 819 RF_ReconUnitNum_t which_ru; 820 { 821 int retval; 822 823 if (!queue->qPtr->Promote) 824 return(-1); 825 RF_LOCK_QUEUE_MUTEX( queue, "DiskIOPromote" ); 826 retval = (queue->qPtr->Promote)( queue->qHdr, parityStripeID, which_ru ); 827 RF_UNLOCK_QUEUE_MUTEX( queue, "DiskIOPromote" ); 828 return(retval); 829 } 830 831 RF_DiskQueueData_t *rf_CreateDiskQueueData( 832 RF_IoType_t typ, 833 RF_SectorNum_t ssect, 834 RF_SectorCount_t nsect, 835 caddr_t buf, 836 RF_StripeNum_t parityStripeID, 837 RF_ReconUnitNum_t which_ru, 838 int (*wakeF)(void *,int), 839 void *arg, 840 RF_DiskQueueData_t *next, 841 RF_AccTraceEntry_t *tracerec, 842 void *raidPtr, 843 RF_DiskQueueDataFlags_t flags, 844 void *kb_proc) 845 { 846 RF_DiskQueueData_t *p; 847 848 RF_FREELIST_GET_INIT(rf_dqd_freelist,p,next,(RF_DiskQueueData_t *),init_dqd); 849 850 p->sectorOffset = ssect + rf_protectedSectors; 851 p->numSector = nsect; 852 p->type = typ; 853 p->buf = buf; 854 p->parityStripeID= parityStripeID; 855 p->which_ru = which_ru; 856 p->CompleteFunc = wakeF; 857 p->argument = arg; 858 p->next = next; 859 p->tracerec = tracerec; 860 p->priority = RF_IO_NORMAL_PRIORITY; 861 p->AuxFunc = NULL; 862 p->buf2 = NULL; 863 #ifdef SIMULATE 864 p->owner = rf_GetCurrentOwner(); 865 #endif /* SIMULATE */ 866 p->raidPtr = raidPtr; 867 p->flags = flags; 868 #ifdef KERNEL 869 p->b_proc = kb_proc; 870 #endif /* KERNEL */ 871 return(p); 872 } 873 874 RF_DiskQueueData_t *rf_CreateDiskQueueDataFull( 875 RF_IoType_t typ, 876 RF_SectorNum_t ssect, 877 RF_SectorCount_t nsect, 878 caddr_t buf, 879 RF_StripeNum_t parityStripeID, 880 RF_ReconUnitNum_t which_ru, 881 int (*wakeF)(void *,int), 882 void *arg, 883 RF_DiskQueueData_t *next, 884 RF_AccTraceEntry_t *tracerec, 885 int priority, 886 int (*AuxFunc)(void *,...), 887 caddr_t buf2, 888 void *raidPtr, 889 RF_DiskQueueDataFlags_t flags, 890 void *kb_proc) 891 { 892 RF_DiskQueueData_t *p; 893 894 RF_FREELIST_GET_INIT(rf_dqd_freelist,p,next,(RF_DiskQueueData_t *),init_dqd); 895 896 p->sectorOffset = ssect + rf_protectedSectors; 897 p->numSector = nsect; 898 p->type = typ; 899 p->buf = buf; 900 p->parityStripeID= parityStripeID; 901 p->which_ru = which_ru; 902 p->CompleteFunc = wakeF; 903 p->argument = arg; 904 p->next = next; 905 p->tracerec = tracerec; 906 p->priority = priority; 907 p->AuxFunc = AuxFunc; 908 p->buf2 = buf2; 909 #ifdef SIMULATE 910 p->owner = rf_GetCurrentOwner(); 911 #endif /* SIMULATE */ 912 p->raidPtr = raidPtr; 913 p->flags = flags; 914 #ifdef KERNEL 915 p->b_proc = kb_proc; 916 #endif /* KERNEL */ 917 return(p); 918 } 919 920 void rf_FreeDiskQueueData(p) 921 RF_DiskQueueData_t *p; 922 { 923 RF_FREELIST_FREE_CLEAN(rf_dqd_freelist,p,next,clean_dqd); 924 } 925