1 /* $NetBSD: rf_decluster.c,v 1.6 2001/01/26 04:40:03 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /*---------------------------------------------------------------------- 30 * 31 * rf_decluster.c -- code related to the declustered layout 32 * 33 * Created 10-21-92 (MCH) 34 * 35 * Nov 93: adding support for distributed sparing. This code is a little 36 * complex: the basic layout used is as follows: 37 * let F = (v-1)/GCD(r,v-1). The spare space for each set of 38 * F consecutive fulltables is grouped together and placed after 39 * that set of tables. 40 * +------------------------------+ 41 * | F fulltables | 42 * | Spare Space | 43 * | F fulltables | 44 * | Spare Space | 45 * | ... | 46 * +------------------------------+ 47 * 48 *--------------------------------------------------------------------*/ 49 50 #include "rf_archs.h" 51 #include "rf_types.h" 52 #include "rf_raid.h" 53 #include "rf_raidframe.h" 54 #include "rf_configure.h" 55 #include "rf_decluster.h" 56 #include "rf_debugMem.h" 57 #include "rf_utils.h" 58 #include "rf_alloclist.h" 59 #include "rf_general.h" 60 #include "rf_shutdown.h" 61 62 63 extern int rf_copyback_in_progress; /* debug only */ 64 65 /* found in rf_kintf.c */ 66 int rf_GetSpareTableFromDaemon(RF_SparetWait_t * req); 67 68 #if (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) 69 70 /* configuration code */ 71 72 int 73 rf_ConfigureDeclustered( 74 RF_ShutdownList_t ** listp, 75 RF_Raid_t * raidPtr, 76 RF_Config_t * cfgPtr) 77 { 78 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 79 int b, v, k, r, lambda; /* block design params */ 80 int i, j; 81 RF_RowCol_t *first_avail_slot; 82 RF_StripeCount_t complete_FT_count, numCompleteFullTablesPerDisk; 83 RF_DeclusteredConfigInfo_t *info; 84 RF_StripeCount_t PUsPerDisk, spareRegionDepthInPUs, numCompleteSpareRegionsPerDisk, 85 extraPUsPerDisk; 86 RF_StripeCount_t totSparePUsPerDisk; 87 RF_SectorNum_t diskOffsetOfLastFullTableInSUs; 88 RF_SectorCount_t SpareSpaceInSUs; 89 char *cfgBuf = (char *) (cfgPtr->layoutSpecific); 90 RF_StripeNum_t l, SUID; 91 92 SUID = l = 0; 93 numCompleteSpareRegionsPerDisk = 0; 94 95 /* 1. create layout specific structure */ 96 RF_MallocAndAdd(info, sizeof(RF_DeclusteredConfigInfo_t), (RF_DeclusteredConfigInfo_t *), raidPtr->cleanupList); 97 if (info == NULL) 98 return (ENOMEM); 99 layoutPtr->layoutSpecificInfo = (void *) info; 100 info->SpareTable = NULL; 101 102 /* 2. extract parameters from the config structure */ 103 if (layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) { 104 (void) bcopy(cfgBuf, info->sparemap_fname, RF_SPAREMAP_NAME_LEN); 105 } 106 cfgBuf += RF_SPAREMAP_NAME_LEN; 107 108 b = *((int *) cfgBuf); 109 cfgBuf += sizeof(int); 110 v = *((int *) cfgBuf); 111 cfgBuf += sizeof(int); 112 k = *((int *) cfgBuf); 113 cfgBuf += sizeof(int); 114 r = *((int *) cfgBuf); 115 cfgBuf += sizeof(int); 116 lambda = *((int *) cfgBuf); 117 cfgBuf += sizeof(int); 118 raidPtr->noRotate = *((int *) cfgBuf); 119 cfgBuf += sizeof(int); 120 121 /* the sparemaps are generated assuming that parity is rotated, so we 122 * issue a warning if both distributed sparing and no-rotate are on at 123 * the same time */ 124 if ((layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) && raidPtr->noRotate) { 125 RF_ERRORMSG("Warning: distributed sparing specified without parity rotation.\n"); 126 } 127 if (raidPtr->numCol != v) { 128 RF_ERRORMSG2("RAID: config error: table element count (%d) not equal to no. of cols (%d)\n", v, raidPtr->numCol); 129 return (EINVAL); 130 } 131 /* 3. set up the values used in the mapping code */ 132 info->BlocksPerTable = b; 133 info->Lambda = lambda; 134 info->NumParityReps = info->groupSize = k; 135 info->SUsPerTable = b * (k - 1) * layoutPtr->SUsPerPU; /* b blks, k-1 SUs each */ 136 info->SUsPerFullTable = k * info->SUsPerTable; /* rot k times */ 137 info->PUsPerBlock = k - 1; 138 info->SUsPerBlock = info->PUsPerBlock * layoutPtr->SUsPerPU; 139 info->TableDepthInPUs = (b * k) / v; 140 info->FullTableDepthInPUs = info->TableDepthInPUs * k; /* k repetitions */ 141 142 /* used only in distributed sparing case */ 143 info->FullTablesPerSpareRegion = (v - 1) / rf_gcd(r, v - 1); /* (v-1)/gcd fulltables */ 144 info->TablesPerSpareRegion = k * info->FullTablesPerSpareRegion; 145 info->SpareSpaceDepthPerRegionInSUs = (r * info->TablesPerSpareRegion / (v - 1)) * layoutPtr->SUsPerPU; 146 147 /* check to make sure the block design is sufficiently small */ 148 if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) { 149 if (info->FullTableDepthInPUs * layoutPtr->SUsPerPU + info->SpareSpaceDepthPerRegionInSUs > layoutPtr->stripeUnitsPerDisk) { 150 RF_ERRORMSG3("RAID: config error: Full Table depth (%d) + Spare Space (%d) larger than disk size (%d) (BD too big)\n", 151 (int) info->FullTableDepthInPUs, 152 (int) info->SpareSpaceDepthPerRegionInSUs, 153 (int) layoutPtr->stripeUnitsPerDisk); 154 return (EINVAL); 155 } 156 } else { 157 if (info->TableDepthInPUs * layoutPtr->SUsPerPU > layoutPtr->stripeUnitsPerDisk) { 158 RF_ERRORMSG2("RAID: config error: Table depth (%d) larger than disk size (%d) (BD too big)\n", 159 (int) (info->TableDepthInPUs * layoutPtr->SUsPerPU), \ 160 (int) layoutPtr->stripeUnitsPerDisk); 161 return (EINVAL); 162 } 163 } 164 165 166 /* compute the size of each disk, and the number of tables in the last 167 * fulltable (which need not be complete) */ 168 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 169 170 PUsPerDisk = layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU; 171 spareRegionDepthInPUs = (info->TablesPerSpareRegion * info->TableDepthInPUs + 172 (info->TablesPerSpareRegion * info->TableDepthInPUs) / (v - 1)); 173 info->SpareRegionDepthInSUs = spareRegionDepthInPUs * layoutPtr->SUsPerPU; 174 175 numCompleteSpareRegionsPerDisk = PUsPerDisk / spareRegionDepthInPUs; 176 info->NumCompleteSRs = numCompleteSpareRegionsPerDisk; 177 extraPUsPerDisk = PUsPerDisk % spareRegionDepthInPUs; 178 179 /* assume conservatively that we need the full amount of spare 180 * space in one region in order to provide spares for the 181 * partial spare region at the end of the array. We set "i" 182 * to the number of tables in the partial spare region. This 183 * may actually include some fulltables. */ 184 extraPUsPerDisk -= (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU); 185 if (extraPUsPerDisk <= 0) 186 i = 0; 187 else 188 i = extraPUsPerDisk / info->TableDepthInPUs; 189 190 complete_FT_count = raidPtr->numRow * (numCompleteSpareRegionsPerDisk * (info->TablesPerSpareRegion / k) + i / k); 191 info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable; 192 info->ExtraTablesPerDisk = i % k; 193 194 /* note that in the last spare region, the spare space is 195 * complete even though data/parity space is not */ 196 totSparePUsPerDisk = (numCompleteSpareRegionsPerDisk + 1) * (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU); 197 info->TotSparePUsPerDisk = totSparePUsPerDisk; 198 199 layoutPtr->stripeUnitsPerDisk = 200 ((complete_FT_count / raidPtr->numRow) * info->FullTableDepthInPUs + /* data & parity space */ 201 info->ExtraTablesPerDisk * info->TableDepthInPUs + 202 totSparePUsPerDisk /* spare space */ 203 ) * layoutPtr->SUsPerPU; 204 layoutPtr->dataStripeUnitsPerDisk = 205 (complete_FT_count * info->FullTableDepthInPUs + info->ExtraTablesPerDisk * info->TableDepthInPUs) 206 * layoutPtr->SUsPerPU * (k - 1) / k; 207 208 } else { 209 /* non-dist spare case: force each disk to contain an 210 * integral number of tables */ 211 layoutPtr->stripeUnitsPerDisk /= (info->TableDepthInPUs * layoutPtr->SUsPerPU); 212 layoutPtr->stripeUnitsPerDisk *= (info->TableDepthInPUs * layoutPtr->SUsPerPU); 213 214 /* compute the number of tables in the last fulltable, which 215 * need not be complete */ 216 complete_FT_count = 217 ((layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU) / info->FullTableDepthInPUs) * raidPtr->numRow; 218 219 info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable; 220 info->ExtraTablesPerDisk = 221 ((layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU) / info->TableDepthInPUs) % k; 222 } 223 224 raidPtr->sectorsPerDisk = layoutPtr->stripeUnitsPerDisk * layoutPtr->sectorsPerStripeUnit; 225 226 /* find the disk offset of the stripe unit where the last fulltable 227 * starts */ 228 numCompleteFullTablesPerDisk = complete_FT_count / raidPtr->numRow; 229 diskOffsetOfLastFullTableInSUs = numCompleteFullTablesPerDisk * info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 230 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 231 SpareSpaceInSUs = numCompleteSpareRegionsPerDisk * info->SpareSpaceDepthPerRegionInSUs; 232 diskOffsetOfLastFullTableInSUs += SpareSpaceInSUs; 233 info->DiskOffsetOfLastSpareSpaceChunkInSUs = 234 diskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU; 235 } 236 info->DiskOffsetOfLastFullTableInSUs = diskOffsetOfLastFullTableInSUs; 237 info->numCompleteFullTablesPerDisk = numCompleteFullTablesPerDisk; 238 239 /* 4. create and initialize the lookup tables */ 240 info->LayoutTable = rf_make_2d_array(b, k, raidPtr->cleanupList); 241 if (info->LayoutTable == NULL) 242 return (ENOMEM); 243 info->OffsetTable = rf_make_2d_array(b, k, raidPtr->cleanupList); 244 if (info->OffsetTable == NULL) 245 return (ENOMEM); 246 info->BlockTable = rf_make_2d_array(info->TableDepthInPUs * layoutPtr->SUsPerPU, raidPtr->numCol, raidPtr->cleanupList); 247 if (info->BlockTable == NULL) 248 return (ENOMEM); 249 250 first_avail_slot = rf_make_1d_array(v, NULL); 251 if (first_avail_slot == NULL) 252 return (ENOMEM); 253 254 for (i = 0; i < b; i++) 255 for (j = 0; j < k; j++) 256 info->LayoutTable[i][j] = *cfgBuf++; 257 258 /* initialize offset table */ 259 for (i = 0; i < b; i++) 260 for (j = 0; j < k; j++) { 261 info->OffsetTable[i][j] = first_avail_slot[info->LayoutTable[i][j]]; 262 first_avail_slot[info->LayoutTable[i][j]]++; 263 } 264 265 /* initialize block table */ 266 for (SUID = l = 0; l < layoutPtr->SUsPerPU; l++) { 267 for (i = 0; i < b; i++) { 268 for (j = 0; j < k; j++) { 269 info->BlockTable[(info->OffsetTable[i][j] * layoutPtr->SUsPerPU) + l] 270 [info->LayoutTable[i][j]] = SUID; 271 } 272 SUID++; 273 } 274 } 275 276 rf_free_1d_array(first_avail_slot, v); 277 278 /* 5. set up the remaining redundant-but-useful parameters */ 279 280 raidPtr->totalSectors = (k * complete_FT_count + raidPtr->numRow * info->ExtraTablesPerDisk) * 281 info->SUsPerTable * layoutPtr->sectorsPerStripeUnit; 282 layoutPtr->numStripe = (raidPtr->totalSectors / layoutPtr->sectorsPerStripeUnit) / (k - 1); 283 284 /* strange evaluation order below to try and minimize overflow 285 * problems */ 286 287 layoutPtr->dataSectorsPerStripe = (k - 1) * layoutPtr->sectorsPerStripeUnit; 288 layoutPtr->bytesPerStripeUnit = layoutPtr->sectorsPerStripeUnit << raidPtr->logBytesPerSector; 289 layoutPtr->numDataCol = k - 1; 290 layoutPtr->numParityCol = 1; 291 292 return (0); 293 } 294 /* declustering with distributed sparing */ 295 static void rf_ShutdownDeclusteredDS(RF_ThreadArg_t); 296 static void 297 rf_ShutdownDeclusteredDS(arg) 298 RF_ThreadArg_t arg; 299 { 300 RF_DeclusteredConfigInfo_t *info; 301 RF_Raid_t *raidPtr; 302 303 raidPtr = (RF_Raid_t *) arg; 304 info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 305 if (info->SpareTable) 306 rf_FreeSpareTable(raidPtr); 307 } 308 309 int 310 rf_ConfigureDeclusteredDS( 311 RF_ShutdownList_t ** listp, 312 RF_Raid_t * raidPtr, 313 RF_Config_t * cfgPtr) 314 { 315 int rc; 316 317 rc = rf_ConfigureDeclustered(listp, raidPtr, cfgPtr); 318 if (rc) 319 return (rc); 320 rc = rf_ShutdownCreate(listp, rf_ShutdownDeclusteredDS, raidPtr); 321 if (rc) { 322 RF_ERRORMSG1("Got %d adding shutdown event for DeclusteredDS\n", rc); 323 rf_ShutdownDeclusteredDS(raidPtr); 324 return (rc); 325 } 326 return (0); 327 } 328 329 void 330 rf_MapSectorDeclustered(raidPtr, raidSector, row, col, diskSector, remap) 331 RF_Raid_t *raidPtr; 332 RF_RaidAddr_t raidSector; 333 RF_RowCol_t *row; 334 RF_RowCol_t *col; 335 RF_SectorNum_t *diskSector; 336 int remap; 337 { 338 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 339 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 340 RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit; 341 RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset; 342 RF_StripeNum_t BlockID, BlockOffset, RepIndex; 343 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 344 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 345 RF_StripeNum_t base_suid = 0, outSU, SpareRegion = 0, SpareSpace = 0; 346 347 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 348 349 FullTableID = SUID / sus_per_fulltable; /* fulltable ID within array 350 * (across rows) */ 351 if (raidPtr->numRow == 1) 352 *row = 0; /* avoid a mod and a div in the common case */ 353 else { 354 *row = FullTableID % raidPtr->numRow; 355 FullTableID /= raidPtr->numRow; /* convert to fulltable ID on 356 * this disk */ 357 } 358 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 359 SpareRegion = FullTableID / info->FullTablesPerSpareRegion; 360 SpareSpace = SpareRegion * info->SpareSpaceDepthPerRegionInSUs; 361 } 362 FullTableOffset = SUID % sus_per_fulltable; 363 TableID = FullTableOffset / info->SUsPerTable; 364 TableOffset = FullTableOffset - TableID * info->SUsPerTable; 365 BlockID = TableOffset / info->PUsPerBlock; 366 BlockOffset = TableOffset - BlockID * info->PUsPerBlock; 367 BlockID %= info->BlocksPerTable; 368 RepIndex = info->PUsPerBlock - TableID; 369 if (!raidPtr->noRotate) 370 BlockOffset += ((BlockOffset >= RepIndex) ? 1 : 0); 371 *col = info->LayoutTable[BlockID][BlockOffset]; 372 373 /* remap to distributed spare space if indicated */ 374 if (remap) { 375 RF_ASSERT(raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared || 376 (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal)); 377 rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU); 378 } else { 379 380 outSU = base_suid; 381 outSU += FullTableID * fulltable_depth; /* offs to strt of FT */ 382 outSU += SpareSpace; /* skip rsvd spare space */ 383 outSU += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU; /* offs to strt of tble */ 384 outSU += info->OffsetTable[BlockID][BlockOffset] * layoutPtr->SUsPerPU; /* offs to the PU */ 385 } 386 outSU += TableOffset / (info->BlocksPerTable * info->PUsPerBlock); /* offs to the SU within 387 * a PU */ 388 389 /* convert SUs to sectors, and, if not aligned to SU boundary, add in 390 * offset to sector. */ 391 *diskSector = outSU * layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit); 392 393 RF_ASSERT(*col != -1); 394 } 395 396 397 /* prototyping this inexplicably causes the compile of the layout table (rf_layout.c) to fail */ 398 void 399 rf_MapParityDeclustered( 400 RF_Raid_t * raidPtr, 401 RF_RaidAddr_t raidSector, 402 RF_RowCol_t * row, 403 RF_RowCol_t * col, 404 RF_SectorNum_t * diskSector, 405 int remap) 406 { 407 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 408 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 409 RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit; 410 RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset; 411 RF_StripeNum_t BlockID, BlockOffset, RepIndex; 412 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 413 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 414 RF_StripeNum_t base_suid = 0, outSU, SpareRegion = 0, SpareSpace = 0; 415 416 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 417 418 /* compute row & (possibly) spare space exactly as before */ 419 FullTableID = SUID / sus_per_fulltable; 420 if (raidPtr->numRow == 1) 421 *row = 0; /* avoid a mod and a div in the common case */ 422 else { 423 *row = FullTableID % raidPtr->numRow; 424 FullTableID /= raidPtr->numRow; /* convert to fulltable ID on 425 * this disk */ 426 } 427 if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) { 428 SpareRegion = FullTableID / info->FullTablesPerSpareRegion; 429 SpareSpace = SpareRegion * info->SpareSpaceDepthPerRegionInSUs; 430 } 431 /* compute BlockID and RepIndex exactly as before */ 432 FullTableOffset = SUID % sus_per_fulltable; 433 TableID = FullTableOffset / info->SUsPerTable; 434 TableOffset = FullTableOffset - TableID * info->SUsPerTable; 435 /* TableOffset = FullTableOffset % info->SUsPerTable; */ 436 /* BlockID = (TableOffset / info->PUsPerBlock) % 437 * info->BlocksPerTable; */ 438 BlockID = TableOffset / info->PUsPerBlock; 439 /* BlockOffset = TableOffset % info->PUsPerBlock; */ 440 BlockOffset = TableOffset - BlockID * info->PUsPerBlock; 441 BlockID %= info->BlocksPerTable; 442 443 /* the parity block is in the position indicated by RepIndex */ 444 RepIndex = (raidPtr->noRotate) ? info->PUsPerBlock : info->PUsPerBlock - TableID; 445 *col = info->LayoutTable[BlockID][RepIndex]; 446 447 if (remap) { 448 RF_ASSERT(raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared || 449 (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal)); 450 rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU); 451 } else { 452 453 /* compute sector as before, except use RepIndex instead of 454 * BlockOffset */ 455 outSU = base_suid; 456 outSU += FullTableID * fulltable_depth; 457 outSU += SpareSpace; /* skip rsvd spare space */ 458 outSU += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU; 459 outSU += info->OffsetTable[BlockID][RepIndex] * layoutPtr->SUsPerPU; 460 } 461 462 outSU += TableOffset / (info->BlocksPerTable * info->PUsPerBlock); 463 *diskSector = outSU * layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit); 464 465 RF_ASSERT(*col != -1); 466 } 467 /* returns an array of ints identifying the disks that comprise the stripe containing the indicated address. 468 * the caller must _never_ attempt to modify this array. 469 */ 470 void 471 rf_IdentifyStripeDeclustered( 472 RF_Raid_t * raidPtr, 473 RF_RaidAddr_t addr, 474 RF_RowCol_t ** diskids, 475 RF_RowCol_t * outRow) 476 { 477 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 478 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 479 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 480 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 481 RF_StripeNum_t base_suid = 0; 482 RF_StripeNum_t SUID = rf_RaidAddressToStripeUnitID(layoutPtr, addr); 483 RF_StripeNum_t stripeID, FullTableID; 484 int tableOffset; 485 486 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 487 FullTableID = SUID / sus_per_fulltable; /* fulltable ID within array 488 * (across rows) */ 489 *outRow = FullTableID % raidPtr->numRow; 490 stripeID = rf_StripeUnitIDToStripeID(layoutPtr, SUID); /* find stripe offset 491 * into array */ 492 tableOffset = (stripeID % info->BlocksPerTable); /* find offset into 493 * block design table */ 494 *diskids = info->LayoutTable[tableOffset]; 495 } 496 /* This returns the default head-separation limit, which is measured 497 * in "required units for reconstruction". Each time a disk fetches 498 * a unit, it bumps a counter. The head-sep code prohibits any disk 499 * from getting more than headSepLimit counter values ahead of any 500 * other. 501 * 502 * We assume here that the number of floating recon buffers is already 503 * set. There are r stripes to be reconstructed in each table, and so 504 * if we have a total of B buffers, we can have at most B/r tables 505 * under recon at any one time. In each table, lambda units are required 506 * from each disk, so given B buffers, the head sep limit has to be 507 * (lambda*B)/r units. We subtract one to avoid weird boundary cases. 508 * 509 * for example, suppose were given 50 buffers, r=19, and lambda=4 as in 510 * the 20.5 design. There are 19 stripes/table to be reconstructed, so 511 * we can have 50/19 tables concurrently under reconstruction, which means 512 * we can allow the fastest disk to get 50/19 tables ahead of the slower 513 * disk. There are lambda "required units" for each disk, so the fastest 514 * disk can get 4*50/19 = 10 counter values ahead of the slowest. 515 * 516 * If numBufsToAccumulate is not 1, we need to limit the head sep further 517 * because multiple bufs will be required for each stripe under recon. 518 */ 519 RF_HeadSepLimit_t 520 rf_GetDefaultHeadSepLimitDeclustered( 521 RF_Raid_t * raidPtr) 522 { 523 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 524 525 return (info->Lambda * raidPtr->numFloatingReconBufs / info->TableDepthInPUs / rf_numBufsToAccumulate); 526 } 527 /* returns the default number of recon buffers to use. The value 528 * is somewhat arbitrary...it's intended to be large enough to allow 529 * for a reasonably large head-sep limit, but small enough that you 530 * don't use up all your system memory with buffers. 531 */ 532 int 533 rf_GetDefaultNumFloatingReconBuffersDeclustered(RF_Raid_t * raidPtr) 534 { 535 return (100 * rf_numBufsToAccumulate); 536 } 537 /* sectors in the last fulltable of the array need to be handled 538 * specially since this fulltable can be incomplete. this function 539 * changes the values of certain params to handle this. 540 * 541 * the idea here is that MapSector et. al. figure out which disk the 542 * addressed unit lives on by computing the modulos of the unit number 543 * with the number of units per fulltable, table, etc. In the last 544 * fulltable, there are fewer units per fulltable, so we need to adjust 545 * the number of user data units per fulltable to reflect this. 546 * 547 * so, we (1) convert the fulltable size and depth parameters to 548 * the size of the partial fulltable at the end, (2) compute the 549 * disk sector offset where this fulltable starts, and (3) convert 550 * the users stripe unit number from an offset into the array to 551 * an offset into the last fulltable. 552 */ 553 void 554 rf_decluster_adjust_params( 555 RF_RaidLayout_t * layoutPtr, 556 RF_StripeNum_t * SUID, 557 RF_StripeCount_t * sus_per_fulltable, 558 RF_StripeCount_t * fulltable_depth, 559 RF_StripeNum_t * base_suid) 560 { 561 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 562 563 if (*SUID >= info->FullTableLimitSUID) { 564 /* new full table size is size of last full table on disk */ 565 *sus_per_fulltable = info->ExtraTablesPerDisk * info->SUsPerTable; 566 567 /* new full table depth is corresponding depth */ 568 *fulltable_depth = info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU; 569 570 /* set up the new base offset */ 571 *base_suid = info->DiskOffsetOfLastFullTableInSUs; 572 573 /* convert users array address to an offset into the last 574 * fulltable */ 575 *SUID -= info->FullTableLimitSUID; 576 } 577 } 578 /* 579 * map a stripe ID to a parity stripe ID. 580 * See comment above RaidAddressToParityStripeID in layout.c. 581 */ 582 void 583 rf_MapSIDToPSIDDeclustered( 584 RF_RaidLayout_t * layoutPtr, 585 RF_StripeNum_t stripeID, 586 RF_StripeNum_t * psID, 587 RF_ReconUnitNum_t * which_ru) 588 { 589 RF_DeclusteredConfigInfo_t *info; 590 591 info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 592 593 *psID = (stripeID / (layoutPtr->SUsPerPU * info->BlocksPerTable)) 594 * info->BlocksPerTable + (stripeID % info->BlocksPerTable); 595 *which_ru = (stripeID % (info->BlocksPerTable * layoutPtr->SUsPerPU)) 596 / info->BlocksPerTable; 597 RF_ASSERT((*which_ru) < layoutPtr->SUsPerPU / layoutPtr->SUsPerRU); 598 } 599 /* 600 * Called from MapSector and MapParity to retarget an access at the spare unit. 601 * Modifies the "col" and "outSU" parameters only. 602 */ 603 void 604 rf_remap_to_spare_space( 605 RF_RaidLayout_t * layoutPtr, 606 RF_DeclusteredConfigInfo_t * info, 607 RF_RowCol_t row, 608 RF_StripeNum_t FullTableID, 609 RF_StripeNum_t TableID, 610 RF_SectorNum_t BlockID, 611 RF_StripeNum_t base_suid, 612 RF_StripeNum_t SpareRegion, 613 RF_RowCol_t * outCol, 614 RF_StripeNum_t * outSU) 615 { 616 RF_StripeNum_t ftID, spareTableStartSU, TableInSpareRegion, lastSROffset, 617 which_ft; 618 619 /* 620 * note that FullTableID and hence SpareRegion may have gotten 621 * tweaked by rf_decluster_adjust_params. We detect this by 622 * noticing that base_suid is not 0. 623 */ 624 if (base_suid == 0) { 625 ftID = FullTableID; 626 } else { 627 /* 628 * There may be > 1.0 full tables in the last (i.e. partial) 629 * spare region. find out which of these we're in. 630 */ 631 lastSROffset = info->NumCompleteSRs * info->SpareRegionDepthInSUs; 632 which_ft = (info->DiskOffsetOfLastFullTableInSUs - lastSROffset) / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU); 633 634 /* compute the actual full table ID */ 635 ftID = info->DiskOffsetOfLastFullTableInSUs / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU) + which_ft; 636 SpareRegion = info->NumCompleteSRs; 637 } 638 TableInSpareRegion = (ftID * info->NumParityReps + TableID) % info->TablesPerSpareRegion; 639 640 *outCol = info->SpareTable[TableInSpareRegion][BlockID].spareDisk; 641 RF_ASSERT(*outCol != -1); 642 643 spareTableStartSU = (SpareRegion == info->NumCompleteSRs) ? 644 info->DiskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU : 645 (SpareRegion + 1) * info->SpareRegionDepthInSUs - info->SpareSpaceDepthPerRegionInSUs; 646 *outSU = spareTableStartSU + info->SpareTable[TableInSpareRegion][BlockID].spareBlockOffsetInSUs; 647 if (*outSU >= layoutPtr->stripeUnitsPerDisk) { 648 printf("rf_remap_to_spare_space: invalid remapped disk SU offset %ld\n", (long) *outSU); 649 } 650 } 651 652 #endif /* (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) */ 653 654 655 int 656 rf_InstallSpareTable( 657 RF_Raid_t * raidPtr, 658 RF_RowCol_t frow, 659 RF_RowCol_t fcol) 660 { 661 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 662 RF_SparetWait_t *req; 663 int retcode; 664 665 RF_Malloc(req, sizeof(*req), (RF_SparetWait_t *)); 666 req->C = raidPtr->numCol; 667 req->G = raidPtr->Layout.numDataCol + raidPtr->Layout.numParityCol; 668 req->fcol = fcol; 669 req->SUsPerPU = raidPtr->Layout.SUsPerPU; 670 req->TablesPerSpareRegion = info->TablesPerSpareRegion; 671 req->BlocksPerTable = info->BlocksPerTable; 672 req->TableDepthInPUs = info->TableDepthInPUs; 673 req->SpareSpaceDepthPerRegionInSUs = info->SpareSpaceDepthPerRegionInSUs; 674 675 retcode = rf_GetSpareTableFromDaemon(req); 676 RF_ASSERT(!retcode); /* XXX -- fix this to recover gracefully -- 677 * XXX */ 678 return (retcode); 679 } 680 /* 681 * Invoked via ioctl to install a spare table in the kernel. 682 */ 683 int 684 rf_SetSpareTable(raidPtr, data) 685 RF_Raid_t *raidPtr; 686 void *data; 687 { 688 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 689 RF_SpareTableEntry_t **ptrs; 690 int i, retcode; 691 692 /* what we need to copyin is a 2-d array, so first copyin the user 693 * pointers to the rows in the table */ 694 RF_Malloc(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **)); 695 retcode = copyin((caddr_t) data, (caddr_t) ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 696 697 if (retcode) 698 return (retcode); 699 700 /* now allocate kernel space for the row pointers */ 701 RF_Malloc(info->SpareTable, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **)); 702 703 /* now allocate kernel space for each row in the table, and copy it in 704 * from user space */ 705 for (i = 0; i < info->TablesPerSpareRegion; i++) { 706 RF_Malloc(info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t), (RF_SpareTableEntry_t *)); 707 retcode = copyin(ptrs[i], info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t)); 708 if (retcode) { 709 info->SpareTable = NULL; /* blow off the memory 710 * we've allocated */ 711 return (retcode); 712 } 713 } 714 715 /* free up the temporary array we used */ 716 RF_Free(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 717 718 return (0); 719 } 720 721 RF_ReconUnitCount_t 722 rf_GetNumSpareRUsDeclustered(raidPtr) 723 RF_Raid_t *raidPtr; 724 { 725 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 726 727 return (((RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo)->TotSparePUsPerDisk); 728 } 729 730 void 731 rf_FreeSpareTable(raidPtr) 732 RF_Raid_t *raidPtr; 733 { 734 long i; 735 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 736 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 737 RF_SpareTableEntry_t **table = info->SpareTable; 738 739 for (i = 0; i < info->TablesPerSpareRegion; i++) { 740 RF_Free(table[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t)); 741 } 742 RF_Free(table, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 743 info->SpareTable = (RF_SpareTableEntry_t **) NULL; 744 } 745