1 /* $NetBSD: rf_dagffrd.c,v 1.4 2000/01/07 03:40:58 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland, Daniel Stodolsky, William V. Courtright II 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /* 30 * rf_dagffrd.c 31 * 32 * code for creating fault-free read DAGs 33 * 34 */ 35 36 #include "rf_types.h" 37 #include "rf_raid.h" 38 #include "rf_dag.h" 39 #include "rf_dagutils.h" 40 #include "rf_dagfuncs.h" 41 #include "rf_debugMem.h" 42 #include "rf_memchunk.h" 43 #include "rf_general.h" 44 #include "rf_dagffrd.h" 45 46 /****************************************************************************** 47 * 48 * General comments on DAG creation: 49 * 50 * All DAGs in this file use roll-away error recovery. Each DAG has a single 51 * commit node, usually called "Cmt." If an error occurs before the Cmt node 52 * is reached, the execution engine will halt forward execution and work 53 * backward through the graph, executing the undo functions. Assuming that 54 * each node in the graph prior to the Cmt node are undoable and atomic - or - 55 * does not make changes to permanent state, the graph will fail atomically. 56 * If an error occurs after the Cmt node executes, the engine will roll-forward 57 * through the graph, blindly executing nodes until it reaches the end. 58 * If a graph reaches the end, it is assumed to have completed successfully. 59 * 60 * A graph has only 1 Cmt node. 61 * 62 */ 63 64 65 /****************************************************************************** 66 * 67 * The following wrappers map the standard DAG creation interface to the 68 * DAG creation routines. Additionally, these wrappers enable experimentation 69 * with new DAG structures by providing an extra level of indirection, allowing 70 * the DAG creation routines to be replaced at this single point. 71 */ 72 73 void 74 rf_CreateFaultFreeReadDAG( 75 RF_Raid_t * raidPtr, 76 RF_AccessStripeMap_t * asmap, 77 RF_DagHeader_t * dag_h, 78 void *bp, 79 RF_RaidAccessFlags_t flags, 80 RF_AllocListElem_t * allocList) 81 { 82 rf_CreateNonredundantDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 83 RF_IO_TYPE_READ); 84 } 85 86 87 /****************************************************************************** 88 * 89 * DAG creation code begins here 90 */ 91 92 /****************************************************************************** 93 * 94 * creates a DAG to perform a nonredundant read or write of data within one 95 * stripe. 96 * For reads, this DAG is as follows: 97 * 98 * /---- read ----\ 99 * Header -- Block ---- read ---- Commit -- Terminate 100 * \---- read ----/ 101 * 102 * For writes, this DAG is as follows: 103 * 104 * /---- write ----\ 105 * Header -- Commit ---- write ---- Block -- Terminate 106 * \---- write ----/ 107 * 108 * There is one disk node per stripe unit accessed, and all disk nodes are in 109 * parallel. 110 * 111 * Tricky point here: The first disk node (read or write) is created 112 * normally. Subsequent disk nodes are created by copying the first one, 113 * and modifying a few params. The "succedents" and "antecedents" fields are 114 * _not_ re-created in each node, but rather left pointing to the same array 115 * that was malloc'd when the first node was created. Thus, it's essential 116 * that when this DAG is freed, the succedents and antecedents fields be freed 117 * in ONLY ONE of the read nodes. This does not apply to the "params" field 118 * because it is recreated for each READ node. 119 * 120 * Note that normal-priority accesses do not need to be tagged with their 121 * parity stripe ID, because they will never be promoted. Hence, I've 122 * commented-out the code to do this, and marked it with UNNEEDED. 123 * 124 *****************************************************************************/ 125 126 void 127 rf_CreateNonredundantDAG( 128 RF_Raid_t * raidPtr, 129 RF_AccessStripeMap_t * asmap, 130 RF_DagHeader_t * dag_h, 131 void *bp, 132 RF_RaidAccessFlags_t flags, 133 RF_AllocListElem_t * allocList, 134 RF_IoType_t type) 135 { 136 RF_DagNode_t *nodes, *diskNodes, *blockNode, *commitNode, *termNode; 137 RF_PhysDiskAddr_t *pda = asmap->physInfo; 138 int (*doFunc) (RF_DagNode_t *), (*undoFunc) (RF_DagNode_t *); 139 int i, n, totalNumNodes; 140 char *name; 141 142 n = asmap->numStripeUnitsAccessed; 143 dag_h->creator = "NonredundantDAG"; 144 145 RF_ASSERT(RF_IO_IS_R_OR_W(type)); 146 switch (type) { 147 case RF_IO_TYPE_READ: 148 doFunc = rf_DiskReadFunc; 149 undoFunc = rf_DiskReadUndoFunc; 150 name = "R "; 151 if (rf_dagDebug) 152 printf("[Creating non-redundant read DAG]\n"); 153 break; 154 case RF_IO_TYPE_WRITE: 155 doFunc = rf_DiskWriteFunc; 156 undoFunc = rf_DiskWriteUndoFunc; 157 name = "W "; 158 if (rf_dagDebug) 159 printf("[Creating non-redundant write DAG]\n"); 160 break; 161 default: 162 RF_PANIC(); 163 } 164 165 /* 166 * For reads, the dag can not commit until the block node is reached. 167 * for writes, the dag commits immediately. 168 */ 169 dag_h->numCommitNodes = 1; 170 dag_h->numCommits = 0; 171 dag_h->numSuccedents = 1; 172 173 /* 174 * Node count: 175 * 1 block node 176 * n data reads (or writes) 177 * 1 commit node 178 * 1 terminator node 179 */ 180 RF_ASSERT(n > 0); 181 totalNumNodes = n + 3; 182 RF_CallocAndAdd(nodes, totalNumNodes, sizeof(RF_DagNode_t), 183 (RF_DagNode_t *), allocList); 184 i = 0; 185 diskNodes = &nodes[i]; 186 i += n; 187 blockNode = &nodes[i]; 188 i += 1; 189 commitNode = &nodes[i]; 190 i += 1; 191 termNode = &nodes[i]; 192 i += 1; 193 RF_ASSERT(i == totalNumNodes); 194 195 /* initialize nodes */ 196 switch (type) { 197 case RF_IO_TYPE_READ: 198 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 199 NULL, n, 0, 0, 0, dag_h, "Nil", allocList); 200 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 201 NULL, 1, n, 0, 0, dag_h, "Cmt", allocList); 202 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc, 203 NULL, 0, 1, 0, 0, dag_h, "Trm", allocList); 204 break; 205 case RF_IO_TYPE_WRITE: 206 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 207 NULL, 1, 0, 0, 0, dag_h, "Nil", allocList); 208 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, rf_NullNodeUndoFunc, 209 NULL, n, 1, 0, 0, dag_h, "Cmt", allocList); 210 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, rf_TerminateUndoFunc, 211 NULL, 0, n, 0, 0, dag_h, "Trm", allocList); 212 break; 213 default: 214 RF_PANIC(); 215 } 216 217 for (i = 0; i < n; i++) { 218 RF_ASSERT(pda != NULL); 219 rf_InitNode(&diskNodes[i], rf_wait, RF_FALSE, doFunc, undoFunc, rf_GenericWakeupFunc, 220 1, 1, 4, 0, dag_h, name, allocList); 221 diskNodes[i].params[0].p = pda; 222 diskNodes[i].params[1].p = pda->bufPtr; 223 /* parity stripe id is not necessary */ 224 diskNodes[i].params[2].v = 0; 225 diskNodes[i].params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, 0); 226 pda = pda->next; 227 } 228 229 /* 230 * Connect nodes. 231 */ 232 233 /* connect hdr to block node */ 234 RF_ASSERT(blockNode->numAntecedents == 0); 235 dag_h->succedents[0] = blockNode; 236 237 if (type == RF_IO_TYPE_READ) { 238 /* connecting a nonredundant read DAG */ 239 RF_ASSERT(blockNode->numSuccedents == n); 240 RF_ASSERT(commitNode->numAntecedents == n); 241 for (i = 0; i < n; i++) { 242 /* connect block node to each read node */ 243 RF_ASSERT(diskNodes[i].numAntecedents == 1); 244 blockNode->succedents[i] = &diskNodes[i]; 245 diskNodes[i].antecedents[0] = blockNode; 246 diskNodes[i].antType[0] = rf_control; 247 248 /* connect each read node to the commit node */ 249 RF_ASSERT(diskNodes[i].numSuccedents == 1); 250 diskNodes[i].succedents[0] = commitNode; 251 commitNode->antecedents[i] = &diskNodes[i]; 252 commitNode->antType[i] = rf_control; 253 } 254 /* connect the commit node to the term node */ 255 RF_ASSERT(commitNode->numSuccedents == 1); 256 RF_ASSERT(termNode->numAntecedents == 1); 257 RF_ASSERT(termNode->numSuccedents == 0); 258 commitNode->succedents[0] = termNode; 259 termNode->antecedents[0] = commitNode; 260 termNode->antType[0] = rf_control; 261 } else { 262 /* connecting a nonredundant write DAG */ 263 /* connect the block node to the commit node */ 264 RF_ASSERT(blockNode->numSuccedents == 1); 265 RF_ASSERT(commitNode->numAntecedents == 1); 266 blockNode->succedents[0] = commitNode; 267 commitNode->antecedents[0] = blockNode; 268 commitNode->antType[0] = rf_control; 269 270 RF_ASSERT(commitNode->numSuccedents == n); 271 RF_ASSERT(termNode->numAntecedents == n); 272 RF_ASSERT(termNode->numSuccedents == 0); 273 for (i = 0; i < n; i++) { 274 /* connect the commit node to each write node */ 275 RF_ASSERT(diskNodes[i].numAntecedents == 1); 276 commitNode->succedents[i] = &diskNodes[i]; 277 diskNodes[i].antecedents[0] = commitNode; 278 diskNodes[i].antType[0] = rf_control; 279 280 /* connect each write node to the term node */ 281 RF_ASSERT(diskNodes[i].numSuccedents == 1); 282 diskNodes[i].succedents[0] = termNode; 283 termNode->antecedents[i] = &diskNodes[i]; 284 termNode->antType[i] = rf_control; 285 } 286 } 287 } 288 /****************************************************************************** 289 * Create a fault-free read DAG for RAID level 1 290 * 291 * Hdr -> Nil -> Rmir -> Cmt -> Trm 292 * 293 * The "Rmir" node schedules a read from the disk in the mirror pair with the 294 * shortest disk queue. the proper queue is selected at Rmir execution. this 295 * deferred mapping is unlike other archs in RAIDframe which generally fix 296 * mapping at DAG creation time. 297 * 298 * Parameters: raidPtr - description of the physical array 299 * asmap - logical & physical addresses for this access 300 * bp - buffer ptr (for holding read data) 301 * flags - general flags (e.g. disk locking) 302 * allocList - list of memory allocated in DAG creation 303 *****************************************************************************/ 304 305 static void 306 CreateMirrorReadDAG( 307 RF_Raid_t * raidPtr, 308 RF_AccessStripeMap_t * asmap, 309 RF_DagHeader_t * dag_h, 310 void *bp, 311 RF_RaidAccessFlags_t flags, 312 RF_AllocListElem_t * allocList, 313 int (*readfunc) (RF_DagNode_t * node)) 314 { 315 RF_DagNode_t *readNodes, *nodes, *blockNode, *commitNode, *termNode; 316 RF_PhysDiskAddr_t *data_pda = asmap->physInfo; 317 RF_PhysDiskAddr_t *parity_pda = asmap->parityInfo; 318 int i, n, totalNumNodes; 319 320 n = asmap->numStripeUnitsAccessed; 321 dag_h->creator = "RaidOneReadDAG"; 322 if (rf_dagDebug) { 323 printf("[Creating RAID level 1 read DAG]\n"); 324 } 325 /* 326 * This dag can not commit until the commit node is reached 327 * errors prior to the commit point imply the dag has failed. 328 */ 329 dag_h->numCommitNodes = 1; 330 dag_h->numCommits = 0; 331 dag_h->numSuccedents = 1; 332 333 /* 334 * Node count: 335 * n data reads 336 * 1 block node 337 * 1 commit node 338 * 1 terminator node 339 */ 340 RF_ASSERT(n > 0); 341 totalNumNodes = n + 3; 342 RF_CallocAndAdd(nodes, totalNumNodes, sizeof(RF_DagNode_t), 343 (RF_DagNode_t *), allocList); 344 i = 0; 345 readNodes = &nodes[i]; 346 i += n; 347 blockNode = &nodes[i]; 348 i += 1; 349 commitNode = &nodes[i]; 350 i += 1; 351 termNode = &nodes[i]; 352 i += 1; 353 RF_ASSERT(i == totalNumNodes); 354 355 /* initialize nodes */ 356 rf_InitNode(blockNode, rf_wait, RF_FALSE, rf_NullNodeFunc, 357 rf_NullNodeUndoFunc, NULL, n, 0, 0, 0, dag_h, "Nil", allocList); 358 rf_InitNode(commitNode, rf_wait, RF_TRUE, rf_NullNodeFunc, 359 rf_NullNodeUndoFunc, NULL, 1, n, 0, 0, dag_h, "Cmt", allocList); 360 rf_InitNode(termNode, rf_wait, RF_FALSE, rf_TerminateFunc, 361 rf_TerminateUndoFunc, NULL, 0, 1, 0, 0, dag_h, "Trm", allocList); 362 363 for (i = 0; i < n; i++) { 364 RF_ASSERT(data_pda != NULL); 365 RF_ASSERT(parity_pda != NULL); 366 rf_InitNode(&readNodes[i], rf_wait, RF_FALSE, readfunc, 367 rf_DiskReadMirrorUndoFunc, rf_GenericWakeupFunc, 1, 1, 5, 0, dag_h, 368 "Rmir", allocList); 369 readNodes[i].params[0].p = data_pda; 370 readNodes[i].params[1].p = data_pda->bufPtr; 371 /* parity stripe id is not necessary */ 372 readNodes[i].params[2].p = 0; 373 readNodes[i].params[3].v = RF_CREATE_PARAM3(RF_IO_NORMAL_PRIORITY, 0, 0, 0); 374 readNodes[i].params[4].p = parity_pda; 375 data_pda = data_pda->next; 376 parity_pda = parity_pda->next; 377 } 378 379 /* 380 * Connect nodes 381 */ 382 383 /* connect hdr to block node */ 384 RF_ASSERT(blockNode->numAntecedents == 0); 385 dag_h->succedents[0] = blockNode; 386 387 /* connect block node to read nodes */ 388 RF_ASSERT(blockNode->numSuccedents == n); 389 for (i = 0; i < n; i++) { 390 RF_ASSERT(readNodes[i].numAntecedents == 1); 391 blockNode->succedents[i] = &readNodes[i]; 392 readNodes[i].antecedents[0] = blockNode; 393 readNodes[i].antType[0] = rf_control; 394 } 395 396 /* connect read nodes to commit node */ 397 RF_ASSERT(commitNode->numAntecedents == n); 398 for (i = 0; i < n; i++) { 399 RF_ASSERT(readNodes[i].numSuccedents == 1); 400 readNodes[i].succedents[0] = commitNode; 401 commitNode->antecedents[i] = &readNodes[i]; 402 commitNode->antType[i] = rf_control; 403 } 404 405 /* connect commit node to term node */ 406 RF_ASSERT(commitNode->numSuccedents == 1); 407 RF_ASSERT(termNode->numAntecedents == 1); 408 RF_ASSERT(termNode->numSuccedents == 0); 409 commitNode->succedents[0] = termNode; 410 termNode->antecedents[0] = commitNode; 411 termNode->antType[0] = rf_control; 412 } 413 414 void 415 rf_CreateMirrorIdleReadDAG( 416 RF_Raid_t * raidPtr, 417 RF_AccessStripeMap_t * asmap, 418 RF_DagHeader_t * dag_h, 419 void *bp, 420 RF_RaidAccessFlags_t flags, 421 RF_AllocListElem_t * allocList) 422 { 423 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 424 rf_DiskReadMirrorIdleFunc); 425 } 426 427 void 428 rf_CreateMirrorPartitionReadDAG( 429 RF_Raid_t * raidPtr, 430 RF_AccessStripeMap_t * asmap, 431 RF_DagHeader_t * dag_h, 432 void *bp, 433 RF_RaidAccessFlags_t flags, 434 RF_AllocListElem_t * allocList) 435 { 436 CreateMirrorReadDAG(raidPtr, asmap, dag_h, bp, flags, allocList, 437 rf_DiskReadMirrorPartitionFunc); 438 } 439