1 /* $NetBSD: rf_decluster.c,v 1.7 2001/10/04 15:58:52 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /*---------------------------------------------------------------------- 30 * 31 * rf_decluster.c -- code related to the declustered layout 32 * 33 * Created 10-21-92 (MCH) 34 * 35 * Nov 93: adding support for distributed sparing. This code is a little 36 * complex: the basic layout used is as follows: 37 * let F = (v-1)/GCD(r,v-1). The spare space for each set of 38 * F consecutive fulltables is grouped together and placed after 39 * that set of tables. 40 * +------------------------------+ 41 * | F fulltables | 42 * | Spare Space | 43 * | F fulltables | 44 * | Spare Space | 45 * | ... | 46 * +------------------------------+ 47 * 48 *--------------------------------------------------------------------*/ 49 50 #include <dev/raidframe/raidframevar.h> 51 52 #include "rf_archs.h" 53 #include "rf_raid.h" 54 #include "rf_decluster.h" 55 #include "rf_debugMem.h" 56 #include "rf_utils.h" 57 #include "rf_alloclist.h" 58 #include "rf_general.h" 59 #include "rf_shutdown.h" 60 61 62 extern int rf_copyback_in_progress; /* debug only */ 63 64 /* found in rf_kintf.c */ 65 int rf_GetSpareTableFromDaemon(RF_SparetWait_t * req); 66 67 #if (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) 68 69 /* configuration code */ 70 71 int 72 rf_ConfigureDeclustered( 73 RF_ShutdownList_t ** listp, 74 RF_Raid_t * raidPtr, 75 RF_Config_t * cfgPtr) 76 { 77 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 78 int b, v, k, r, lambda; /* block design params */ 79 int i, j; 80 RF_RowCol_t *first_avail_slot; 81 RF_StripeCount_t complete_FT_count, numCompleteFullTablesPerDisk; 82 RF_DeclusteredConfigInfo_t *info; 83 RF_StripeCount_t PUsPerDisk, spareRegionDepthInPUs, numCompleteSpareRegionsPerDisk, 84 extraPUsPerDisk; 85 RF_StripeCount_t totSparePUsPerDisk; 86 RF_SectorNum_t diskOffsetOfLastFullTableInSUs; 87 RF_SectorCount_t SpareSpaceInSUs; 88 char *cfgBuf = (char *) (cfgPtr->layoutSpecific); 89 RF_StripeNum_t l, SUID; 90 91 SUID = l = 0; 92 numCompleteSpareRegionsPerDisk = 0; 93 94 /* 1. create layout specific structure */ 95 RF_MallocAndAdd(info, sizeof(RF_DeclusteredConfigInfo_t), (RF_DeclusteredConfigInfo_t *), raidPtr->cleanupList); 96 if (info == NULL) 97 return (ENOMEM); 98 layoutPtr->layoutSpecificInfo = (void *) info; 99 info->SpareTable = NULL; 100 101 /* 2. extract parameters from the config structure */ 102 if (layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) { 103 (void) bcopy(cfgBuf, info->sparemap_fname, RF_SPAREMAP_NAME_LEN); 104 } 105 cfgBuf += RF_SPAREMAP_NAME_LEN; 106 107 b = *((int *) cfgBuf); 108 cfgBuf += sizeof(int); 109 v = *((int *) cfgBuf); 110 cfgBuf += sizeof(int); 111 k = *((int *) cfgBuf); 112 cfgBuf += sizeof(int); 113 r = *((int *) cfgBuf); 114 cfgBuf += sizeof(int); 115 lambda = *((int *) cfgBuf); 116 cfgBuf += sizeof(int); 117 raidPtr->noRotate = *((int *) cfgBuf); 118 cfgBuf += sizeof(int); 119 120 /* the sparemaps are generated assuming that parity is rotated, so we 121 * issue a warning if both distributed sparing and no-rotate are on at 122 * the same time */ 123 if ((layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) && raidPtr->noRotate) { 124 RF_ERRORMSG("Warning: distributed sparing specified without parity rotation.\n"); 125 } 126 if (raidPtr->numCol != v) { 127 RF_ERRORMSG2("RAID: config error: table element count (%d) not equal to no. of cols (%d)\n", v, raidPtr->numCol); 128 return (EINVAL); 129 } 130 /* 3. set up the values used in the mapping code */ 131 info->BlocksPerTable = b; 132 info->Lambda = lambda; 133 info->NumParityReps = info->groupSize = k; 134 info->SUsPerTable = b * (k - 1) * layoutPtr->SUsPerPU; /* b blks, k-1 SUs each */ 135 info->SUsPerFullTable = k * info->SUsPerTable; /* rot k times */ 136 info->PUsPerBlock = k - 1; 137 info->SUsPerBlock = info->PUsPerBlock * layoutPtr->SUsPerPU; 138 info->TableDepthInPUs = (b * k) / v; 139 info->FullTableDepthInPUs = info->TableDepthInPUs * k; /* k repetitions */ 140 141 /* used only in distributed sparing case */ 142 info->FullTablesPerSpareRegion = (v - 1) / rf_gcd(r, v - 1); /* (v-1)/gcd fulltables */ 143 info->TablesPerSpareRegion = k * info->FullTablesPerSpareRegion; 144 info->SpareSpaceDepthPerRegionInSUs = (r * info->TablesPerSpareRegion / (v - 1)) * layoutPtr->SUsPerPU; 145 146 /* check to make sure the block design is sufficiently small */ 147 if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) { 148 if (info->FullTableDepthInPUs * layoutPtr->SUsPerPU + info->SpareSpaceDepthPerRegionInSUs > layoutPtr->stripeUnitsPerDisk) { 149 RF_ERRORMSG3("RAID: config error: Full Table depth (%d) + Spare Space (%d) larger than disk size (%d) (BD too big)\n", 150 (int) info->FullTableDepthInPUs, 151 (int) info->SpareSpaceDepthPerRegionInSUs, 152 (int) layoutPtr->stripeUnitsPerDisk); 153 return (EINVAL); 154 } 155 } else { 156 if (info->TableDepthInPUs * layoutPtr->SUsPerPU > layoutPtr->stripeUnitsPerDisk) { 157 RF_ERRORMSG2("RAID: config error: Table depth (%d) larger than disk size (%d) (BD too big)\n", 158 (int) (info->TableDepthInPUs * layoutPtr->SUsPerPU), \ 159 (int) layoutPtr->stripeUnitsPerDisk); 160 return (EINVAL); 161 } 162 } 163 164 165 /* compute the size of each disk, and the number of tables in the last 166 * fulltable (which need not be complete) */ 167 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 168 169 PUsPerDisk = layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU; 170 spareRegionDepthInPUs = (info->TablesPerSpareRegion * info->TableDepthInPUs + 171 (info->TablesPerSpareRegion * info->TableDepthInPUs) / (v - 1)); 172 info->SpareRegionDepthInSUs = spareRegionDepthInPUs * layoutPtr->SUsPerPU; 173 174 numCompleteSpareRegionsPerDisk = PUsPerDisk / spareRegionDepthInPUs; 175 info->NumCompleteSRs = numCompleteSpareRegionsPerDisk; 176 extraPUsPerDisk = PUsPerDisk % spareRegionDepthInPUs; 177 178 /* assume conservatively that we need the full amount of spare 179 * space in one region in order to provide spares for the 180 * partial spare region at the end of the array. We set "i" 181 * to the number of tables in the partial spare region. This 182 * may actually include some fulltables. */ 183 extraPUsPerDisk -= (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU); 184 if (extraPUsPerDisk <= 0) 185 i = 0; 186 else 187 i = extraPUsPerDisk / info->TableDepthInPUs; 188 189 complete_FT_count = raidPtr->numRow * (numCompleteSpareRegionsPerDisk * (info->TablesPerSpareRegion / k) + i / k); 190 info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable; 191 info->ExtraTablesPerDisk = i % k; 192 193 /* note that in the last spare region, the spare space is 194 * complete even though data/parity space is not */ 195 totSparePUsPerDisk = (numCompleteSpareRegionsPerDisk + 1) * (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU); 196 info->TotSparePUsPerDisk = totSparePUsPerDisk; 197 198 layoutPtr->stripeUnitsPerDisk = 199 ((complete_FT_count / raidPtr->numRow) * info->FullTableDepthInPUs + /* data & parity space */ 200 info->ExtraTablesPerDisk * info->TableDepthInPUs + 201 totSparePUsPerDisk /* spare space */ 202 ) * layoutPtr->SUsPerPU; 203 layoutPtr->dataStripeUnitsPerDisk = 204 (complete_FT_count * info->FullTableDepthInPUs + info->ExtraTablesPerDisk * info->TableDepthInPUs) 205 * layoutPtr->SUsPerPU * (k - 1) / k; 206 207 } else { 208 /* non-dist spare case: force each disk to contain an 209 * integral number of tables */ 210 layoutPtr->stripeUnitsPerDisk /= (info->TableDepthInPUs * layoutPtr->SUsPerPU); 211 layoutPtr->stripeUnitsPerDisk *= (info->TableDepthInPUs * layoutPtr->SUsPerPU); 212 213 /* compute the number of tables in the last fulltable, which 214 * need not be complete */ 215 complete_FT_count = 216 ((layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU) / info->FullTableDepthInPUs) * raidPtr->numRow; 217 218 info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable; 219 info->ExtraTablesPerDisk = 220 ((layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU) / info->TableDepthInPUs) % k; 221 } 222 223 raidPtr->sectorsPerDisk = layoutPtr->stripeUnitsPerDisk * layoutPtr->sectorsPerStripeUnit; 224 225 /* find the disk offset of the stripe unit where the last fulltable 226 * starts */ 227 numCompleteFullTablesPerDisk = complete_FT_count / raidPtr->numRow; 228 diskOffsetOfLastFullTableInSUs = numCompleteFullTablesPerDisk * info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 229 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 230 SpareSpaceInSUs = numCompleteSpareRegionsPerDisk * info->SpareSpaceDepthPerRegionInSUs; 231 diskOffsetOfLastFullTableInSUs += SpareSpaceInSUs; 232 info->DiskOffsetOfLastSpareSpaceChunkInSUs = 233 diskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU; 234 } 235 info->DiskOffsetOfLastFullTableInSUs = diskOffsetOfLastFullTableInSUs; 236 info->numCompleteFullTablesPerDisk = numCompleteFullTablesPerDisk; 237 238 /* 4. create and initialize the lookup tables */ 239 info->LayoutTable = rf_make_2d_array(b, k, raidPtr->cleanupList); 240 if (info->LayoutTable == NULL) 241 return (ENOMEM); 242 info->OffsetTable = rf_make_2d_array(b, k, raidPtr->cleanupList); 243 if (info->OffsetTable == NULL) 244 return (ENOMEM); 245 info->BlockTable = rf_make_2d_array(info->TableDepthInPUs * layoutPtr->SUsPerPU, raidPtr->numCol, raidPtr->cleanupList); 246 if (info->BlockTable == NULL) 247 return (ENOMEM); 248 249 first_avail_slot = rf_make_1d_array(v, NULL); 250 if (first_avail_slot == NULL) 251 return (ENOMEM); 252 253 for (i = 0; i < b; i++) 254 for (j = 0; j < k; j++) 255 info->LayoutTable[i][j] = *cfgBuf++; 256 257 /* initialize offset table */ 258 for (i = 0; i < b; i++) 259 for (j = 0; j < k; j++) { 260 info->OffsetTable[i][j] = first_avail_slot[info->LayoutTable[i][j]]; 261 first_avail_slot[info->LayoutTable[i][j]]++; 262 } 263 264 /* initialize block table */ 265 for (SUID = l = 0; l < layoutPtr->SUsPerPU; l++) { 266 for (i = 0; i < b; i++) { 267 for (j = 0; j < k; j++) { 268 info->BlockTable[(info->OffsetTable[i][j] * layoutPtr->SUsPerPU) + l] 269 [info->LayoutTable[i][j]] = SUID; 270 } 271 SUID++; 272 } 273 } 274 275 rf_free_1d_array(first_avail_slot, v); 276 277 /* 5. set up the remaining redundant-but-useful parameters */ 278 279 raidPtr->totalSectors = (k * complete_FT_count + raidPtr->numRow * info->ExtraTablesPerDisk) * 280 info->SUsPerTable * layoutPtr->sectorsPerStripeUnit; 281 layoutPtr->numStripe = (raidPtr->totalSectors / layoutPtr->sectorsPerStripeUnit) / (k - 1); 282 283 /* strange evaluation order below to try and minimize overflow 284 * problems */ 285 286 layoutPtr->dataSectorsPerStripe = (k - 1) * layoutPtr->sectorsPerStripeUnit; 287 layoutPtr->bytesPerStripeUnit = layoutPtr->sectorsPerStripeUnit << raidPtr->logBytesPerSector; 288 layoutPtr->numDataCol = k - 1; 289 layoutPtr->numParityCol = 1; 290 291 return (0); 292 } 293 /* declustering with distributed sparing */ 294 static void rf_ShutdownDeclusteredDS(RF_ThreadArg_t); 295 static void 296 rf_ShutdownDeclusteredDS(arg) 297 RF_ThreadArg_t arg; 298 { 299 RF_DeclusteredConfigInfo_t *info; 300 RF_Raid_t *raidPtr; 301 302 raidPtr = (RF_Raid_t *) arg; 303 info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 304 if (info->SpareTable) 305 rf_FreeSpareTable(raidPtr); 306 } 307 308 int 309 rf_ConfigureDeclusteredDS( 310 RF_ShutdownList_t ** listp, 311 RF_Raid_t * raidPtr, 312 RF_Config_t * cfgPtr) 313 { 314 int rc; 315 316 rc = rf_ConfigureDeclustered(listp, raidPtr, cfgPtr); 317 if (rc) 318 return (rc); 319 rc = rf_ShutdownCreate(listp, rf_ShutdownDeclusteredDS, raidPtr); 320 if (rc) { 321 RF_ERRORMSG1("Got %d adding shutdown event for DeclusteredDS\n", rc); 322 rf_ShutdownDeclusteredDS(raidPtr); 323 return (rc); 324 } 325 return (0); 326 } 327 328 void 329 rf_MapSectorDeclustered(raidPtr, raidSector, row, col, diskSector, remap) 330 RF_Raid_t *raidPtr; 331 RF_RaidAddr_t raidSector; 332 RF_RowCol_t *row; 333 RF_RowCol_t *col; 334 RF_SectorNum_t *diskSector; 335 int remap; 336 { 337 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 338 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 339 RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit; 340 RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset; 341 RF_StripeNum_t BlockID, BlockOffset, RepIndex; 342 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 343 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 344 RF_StripeNum_t base_suid = 0, outSU, SpareRegion = 0, SpareSpace = 0; 345 346 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 347 348 FullTableID = SUID / sus_per_fulltable; /* fulltable ID within array 349 * (across rows) */ 350 if (raidPtr->numRow == 1) 351 *row = 0; /* avoid a mod and a div in the common case */ 352 else { 353 *row = FullTableID % raidPtr->numRow; 354 FullTableID /= raidPtr->numRow; /* convert to fulltable ID on 355 * this disk */ 356 } 357 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 358 SpareRegion = FullTableID / info->FullTablesPerSpareRegion; 359 SpareSpace = SpareRegion * info->SpareSpaceDepthPerRegionInSUs; 360 } 361 FullTableOffset = SUID % sus_per_fulltable; 362 TableID = FullTableOffset / info->SUsPerTable; 363 TableOffset = FullTableOffset - TableID * info->SUsPerTable; 364 BlockID = TableOffset / info->PUsPerBlock; 365 BlockOffset = TableOffset - BlockID * info->PUsPerBlock; 366 BlockID %= info->BlocksPerTable; 367 RepIndex = info->PUsPerBlock - TableID; 368 if (!raidPtr->noRotate) 369 BlockOffset += ((BlockOffset >= RepIndex) ? 1 : 0); 370 *col = info->LayoutTable[BlockID][BlockOffset]; 371 372 /* remap to distributed spare space if indicated */ 373 if (remap) { 374 RF_ASSERT(raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared || 375 (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal)); 376 rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU); 377 } else { 378 379 outSU = base_suid; 380 outSU += FullTableID * fulltable_depth; /* offs to strt of FT */ 381 outSU += SpareSpace; /* skip rsvd spare space */ 382 outSU += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU; /* offs to strt of tble */ 383 outSU += info->OffsetTable[BlockID][BlockOffset] * layoutPtr->SUsPerPU; /* offs to the PU */ 384 } 385 outSU += TableOffset / (info->BlocksPerTable * info->PUsPerBlock); /* offs to the SU within 386 * a PU */ 387 388 /* convert SUs to sectors, and, if not aligned to SU boundary, add in 389 * offset to sector. */ 390 *diskSector = outSU * layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit); 391 392 RF_ASSERT(*col != -1); 393 } 394 395 396 /* prototyping this inexplicably causes the compile of the layout table (rf_layout.c) to fail */ 397 void 398 rf_MapParityDeclustered( 399 RF_Raid_t * raidPtr, 400 RF_RaidAddr_t raidSector, 401 RF_RowCol_t * row, 402 RF_RowCol_t * col, 403 RF_SectorNum_t * diskSector, 404 int remap) 405 { 406 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 407 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 408 RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit; 409 RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset; 410 RF_StripeNum_t BlockID, BlockOffset, RepIndex; 411 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 412 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 413 RF_StripeNum_t base_suid = 0, outSU, SpareRegion = 0, SpareSpace = 0; 414 415 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 416 417 /* compute row & (possibly) spare space exactly as before */ 418 FullTableID = SUID / sus_per_fulltable; 419 if (raidPtr->numRow == 1) 420 *row = 0; /* avoid a mod and a div in the common case */ 421 else { 422 *row = FullTableID % raidPtr->numRow; 423 FullTableID /= raidPtr->numRow; /* convert to fulltable ID on 424 * this disk */ 425 } 426 if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) { 427 SpareRegion = FullTableID / info->FullTablesPerSpareRegion; 428 SpareSpace = SpareRegion * info->SpareSpaceDepthPerRegionInSUs; 429 } 430 /* compute BlockID and RepIndex exactly as before */ 431 FullTableOffset = SUID % sus_per_fulltable; 432 TableID = FullTableOffset / info->SUsPerTable; 433 TableOffset = FullTableOffset - TableID * info->SUsPerTable; 434 /* TableOffset = FullTableOffset % info->SUsPerTable; */ 435 /* BlockID = (TableOffset / info->PUsPerBlock) % 436 * info->BlocksPerTable; */ 437 BlockID = TableOffset / info->PUsPerBlock; 438 /* BlockOffset = TableOffset % info->PUsPerBlock; */ 439 BlockOffset = TableOffset - BlockID * info->PUsPerBlock; 440 BlockID %= info->BlocksPerTable; 441 442 /* the parity block is in the position indicated by RepIndex */ 443 RepIndex = (raidPtr->noRotate) ? info->PUsPerBlock : info->PUsPerBlock - TableID; 444 *col = info->LayoutTable[BlockID][RepIndex]; 445 446 if (remap) { 447 RF_ASSERT(raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared || 448 (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal)); 449 rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU); 450 } else { 451 452 /* compute sector as before, except use RepIndex instead of 453 * BlockOffset */ 454 outSU = base_suid; 455 outSU += FullTableID * fulltable_depth; 456 outSU += SpareSpace; /* skip rsvd spare space */ 457 outSU += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU; 458 outSU += info->OffsetTable[BlockID][RepIndex] * layoutPtr->SUsPerPU; 459 } 460 461 outSU += TableOffset / (info->BlocksPerTable * info->PUsPerBlock); 462 *diskSector = outSU * layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit); 463 464 RF_ASSERT(*col != -1); 465 } 466 /* returns an array of ints identifying the disks that comprise the stripe containing the indicated address. 467 * the caller must _never_ attempt to modify this array. 468 */ 469 void 470 rf_IdentifyStripeDeclustered( 471 RF_Raid_t * raidPtr, 472 RF_RaidAddr_t addr, 473 RF_RowCol_t ** diskids, 474 RF_RowCol_t * outRow) 475 { 476 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 477 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 478 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 479 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 480 RF_StripeNum_t base_suid = 0; 481 RF_StripeNum_t SUID = rf_RaidAddressToStripeUnitID(layoutPtr, addr); 482 RF_StripeNum_t stripeID, FullTableID; 483 int tableOffset; 484 485 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 486 FullTableID = SUID / sus_per_fulltable; /* fulltable ID within array 487 * (across rows) */ 488 *outRow = FullTableID % raidPtr->numRow; 489 stripeID = rf_StripeUnitIDToStripeID(layoutPtr, SUID); /* find stripe offset 490 * into array */ 491 tableOffset = (stripeID % info->BlocksPerTable); /* find offset into 492 * block design table */ 493 *diskids = info->LayoutTable[tableOffset]; 494 } 495 /* This returns the default head-separation limit, which is measured 496 * in "required units for reconstruction". Each time a disk fetches 497 * a unit, it bumps a counter. The head-sep code prohibits any disk 498 * from getting more than headSepLimit counter values ahead of any 499 * other. 500 * 501 * We assume here that the number of floating recon buffers is already 502 * set. There are r stripes to be reconstructed in each table, and so 503 * if we have a total of B buffers, we can have at most B/r tables 504 * under recon at any one time. In each table, lambda units are required 505 * from each disk, so given B buffers, the head sep limit has to be 506 * (lambda*B)/r units. We subtract one to avoid weird boundary cases. 507 * 508 * for example, suppose were given 50 buffers, r=19, and lambda=4 as in 509 * the 20.5 design. There are 19 stripes/table to be reconstructed, so 510 * we can have 50/19 tables concurrently under reconstruction, which means 511 * we can allow the fastest disk to get 50/19 tables ahead of the slower 512 * disk. There are lambda "required units" for each disk, so the fastest 513 * disk can get 4*50/19 = 10 counter values ahead of the slowest. 514 * 515 * If numBufsToAccumulate is not 1, we need to limit the head sep further 516 * because multiple bufs will be required for each stripe under recon. 517 */ 518 RF_HeadSepLimit_t 519 rf_GetDefaultHeadSepLimitDeclustered( 520 RF_Raid_t * raidPtr) 521 { 522 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 523 524 return (info->Lambda * raidPtr->numFloatingReconBufs / info->TableDepthInPUs / rf_numBufsToAccumulate); 525 } 526 /* returns the default number of recon buffers to use. The value 527 * is somewhat arbitrary...it's intended to be large enough to allow 528 * for a reasonably large head-sep limit, but small enough that you 529 * don't use up all your system memory with buffers. 530 */ 531 int 532 rf_GetDefaultNumFloatingReconBuffersDeclustered(RF_Raid_t * raidPtr) 533 { 534 return (100 * rf_numBufsToAccumulate); 535 } 536 /* sectors in the last fulltable of the array need to be handled 537 * specially since this fulltable can be incomplete. this function 538 * changes the values of certain params to handle this. 539 * 540 * the idea here is that MapSector et. al. figure out which disk the 541 * addressed unit lives on by computing the modulos of the unit number 542 * with the number of units per fulltable, table, etc. In the last 543 * fulltable, there are fewer units per fulltable, so we need to adjust 544 * the number of user data units per fulltable to reflect this. 545 * 546 * so, we (1) convert the fulltable size and depth parameters to 547 * the size of the partial fulltable at the end, (2) compute the 548 * disk sector offset where this fulltable starts, and (3) convert 549 * the users stripe unit number from an offset into the array to 550 * an offset into the last fulltable. 551 */ 552 void 553 rf_decluster_adjust_params( 554 RF_RaidLayout_t * layoutPtr, 555 RF_StripeNum_t * SUID, 556 RF_StripeCount_t * sus_per_fulltable, 557 RF_StripeCount_t * fulltable_depth, 558 RF_StripeNum_t * base_suid) 559 { 560 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 561 562 if (*SUID >= info->FullTableLimitSUID) { 563 /* new full table size is size of last full table on disk */ 564 *sus_per_fulltable = info->ExtraTablesPerDisk * info->SUsPerTable; 565 566 /* new full table depth is corresponding depth */ 567 *fulltable_depth = info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU; 568 569 /* set up the new base offset */ 570 *base_suid = info->DiskOffsetOfLastFullTableInSUs; 571 572 /* convert users array address to an offset into the last 573 * fulltable */ 574 *SUID -= info->FullTableLimitSUID; 575 } 576 } 577 /* 578 * map a stripe ID to a parity stripe ID. 579 * See comment above RaidAddressToParityStripeID in layout.c. 580 */ 581 void 582 rf_MapSIDToPSIDDeclustered( 583 RF_RaidLayout_t * layoutPtr, 584 RF_StripeNum_t stripeID, 585 RF_StripeNum_t * psID, 586 RF_ReconUnitNum_t * which_ru) 587 { 588 RF_DeclusteredConfigInfo_t *info; 589 590 info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 591 592 *psID = (stripeID / (layoutPtr->SUsPerPU * info->BlocksPerTable)) 593 * info->BlocksPerTable + (stripeID % info->BlocksPerTable); 594 *which_ru = (stripeID % (info->BlocksPerTable * layoutPtr->SUsPerPU)) 595 / info->BlocksPerTable; 596 RF_ASSERT((*which_ru) < layoutPtr->SUsPerPU / layoutPtr->SUsPerRU); 597 } 598 /* 599 * Called from MapSector and MapParity to retarget an access at the spare unit. 600 * Modifies the "col" and "outSU" parameters only. 601 */ 602 void 603 rf_remap_to_spare_space( 604 RF_RaidLayout_t * layoutPtr, 605 RF_DeclusteredConfigInfo_t * info, 606 RF_RowCol_t row, 607 RF_StripeNum_t FullTableID, 608 RF_StripeNum_t TableID, 609 RF_SectorNum_t BlockID, 610 RF_StripeNum_t base_suid, 611 RF_StripeNum_t SpareRegion, 612 RF_RowCol_t * outCol, 613 RF_StripeNum_t * outSU) 614 { 615 RF_StripeNum_t ftID, spareTableStartSU, TableInSpareRegion, lastSROffset, 616 which_ft; 617 618 /* 619 * note that FullTableID and hence SpareRegion may have gotten 620 * tweaked by rf_decluster_adjust_params. We detect this by 621 * noticing that base_suid is not 0. 622 */ 623 if (base_suid == 0) { 624 ftID = FullTableID; 625 } else { 626 /* 627 * There may be > 1.0 full tables in the last (i.e. partial) 628 * spare region. find out which of these we're in. 629 */ 630 lastSROffset = info->NumCompleteSRs * info->SpareRegionDepthInSUs; 631 which_ft = (info->DiskOffsetOfLastFullTableInSUs - lastSROffset) / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU); 632 633 /* compute the actual full table ID */ 634 ftID = info->DiskOffsetOfLastFullTableInSUs / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU) + which_ft; 635 SpareRegion = info->NumCompleteSRs; 636 } 637 TableInSpareRegion = (ftID * info->NumParityReps + TableID) % info->TablesPerSpareRegion; 638 639 *outCol = info->SpareTable[TableInSpareRegion][BlockID].spareDisk; 640 RF_ASSERT(*outCol != -1); 641 642 spareTableStartSU = (SpareRegion == info->NumCompleteSRs) ? 643 info->DiskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU : 644 (SpareRegion + 1) * info->SpareRegionDepthInSUs - info->SpareSpaceDepthPerRegionInSUs; 645 *outSU = spareTableStartSU + info->SpareTable[TableInSpareRegion][BlockID].spareBlockOffsetInSUs; 646 if (*outSU >= layoutPtr->stripeUnitsPerDisk) { 647 printf("rf_remap_to_spare_space: invalid remapped disk SU offset %ld\n", (long) *outSU); 648 } 649 } 650 651 #endif /* (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) */ 652 653 654 int 655 rf_InstallSpareTable( 656 RF_Raid_t * raidPtr, 657 RF_RowCol_t frow, 658 RF_RowCol_t fcol) 659 { 660 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 661 RF_SparetWait_t *req; 662 int retcode; 663 664 RF_Malloc(req, sizeof(*req), (RF_SparetWait_t *)); 665 req->C = raidPtr->numCol; 666 req->G = raidPtr->Layout.numDataCol + raidPtr->Layout.numParityCol; 667 req->fcol = fcol; 668 req->SUsPerPU = raidPtr->Layout.SUsPerPU; 669 req->TablesPerSpareRegion = info->TablesPerSpareRegion; 670 req->BlocksPerTable = info->BlocksPerTable; 671 req->TableDepthInPUs = info->TableDepthInPUs; 672 req->SpareSpaceDepthPerRegionInSUs = info->SpareSpaceDepthPerRegionInSUs; 673 674 retcode = rf_GetSpareTableFromDaemon(req); 675 RF_ASSERT(!retcode); /* XXX -- fix this to recover gracefully -- 676 * XXX */ 677 return (retcode); 678 } 679 /* 680 * Invoked via ioctl to install a spare table in the kernel. 681 */ 682 int 683 rf_SetSpareTable(raidPtr, data) 684 RF_Raid_t *raidPtr; 685 void *data; 686 { 687 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 688 RF_SpareTableEntry_t **ptrs; 689 int i, retcode; 690 691 /* what we need to copyin is a 2-d array, so first copyin the user 692 * pointers to the rows in the table */ 693 RF_Malloc(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **)); 694 retcode = copyin((caddr_t) data, (caddr_t) ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 695 696 if (retcode) 697 return (retcode); 698 699 /* now allocate kernel space for the row pointers */ 700 RF_Malloc(info->SpareTable, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **)); 701 702 /* now allocate kernel space for each row in the table, and copy it in 703 * from user space */ 704 for (i = 0; i < info->TablesPerSpareRegion; i++) { 705 RF_Malloc(info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t), (RF_SpareTableEntry_t *)); 706 retcode = copyin(ptrs[i], info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t)); 707 if (retcode) { 708 info->SpareTable = NULL; /* blow off the memory 709 * we've allocated */ 710 return (retcode); 711 } 712 } 713 714 /* free up the temporary array we used */ 715 RF_Free(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 716 717 return (0); 718 } 719 720 RF_ReconUnitCount_t 721 rf_GetNumSpareRUsDeclustered(raidPtr) 722 RF_Raid_t *raidPtr; 723 { 724 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 725 726 return (((RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo)->TotSparePUsPerDisk); 727 } 728 729 void 730 rf_FreeSpareTable(raidPtr) 731 RF_Raid_t *raidPtr; 732 { 733 long i; 734 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 735 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 736 RF_SpareTableEntry_t **table = info->SpareTable; 737 738 for (i = 0; i < info->TablesPerSpareRegion; i++) { 739 RF_Free(table[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t)); 740 } 741 RF_Free(table, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 742 info->SpareTable = (RF_SpareTableEntry_t **) NULL; 743 } 744