1 /* $NetBSD: rf_decluster.c,v 1.15 2003/12/30 21:59:03 oster Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /*---------------------------------------------------------------------- 30 * 31 * rf_decluster.c -- code related to the declustered layout 32 * 33 * Created 10-21-92 (MCH) 34 * 35 * Nov 93: adding support for distributed sparing. This code is a little 36 * complex: the basic layout used is as follows: 37 * let F = (v-1)/GCD(r,v-1). The spare space for each set of 38 * F consecutive fulltables is grouped together and placed after 39 * that set of tables. 40 * +------------------------------+ 41 * | F fulltables | 42 * | Spare Space | 43 * | F fulltables | 44 * | Spare Space | 45 * | ... | 46 * +------------------------------+ 47 * 48 *--------------------------------------------------------------------*/ 49 50 #include <sys/cdefs.h> 51 __KERNEL_RCSID(0, "$NetBSD: rf_decluster.c,v 1.15 2003/12/30 21:59:03 oster Exp $"); 52 53 #include <dev/raidframe/raidframevar.h> 54 55 #include "rf_archs.h" 56 #include "rf_raid.h" 57 #include "rf_decluster.h" 58 #include "rf_debugMem.h" 59 #include "rf_utils.h" 60 #include "rf_alloclist.h" 61 #include "rf_general.h" 62 #include "rf_kintf.h" 63 #include "rf_shutdown.h" 64 65 #if (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) 66 67 /* configuration code */ 68 69 int 70 rf_ConfigureDeclustered(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr, 71 RF_Config_t *cfgPtr) 72 { 73 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 74 int b, v, k, r, lambda; /* block design params */ 75 int i, j; 76 RF_RowCol_t *first_avail_slot; 77 RF_StripeCount_t complete_FT_count, numCompleteFullTablesPerDisk; 78 RF_DeclusteredConfigInfo_t *info; 79 RF_StripeCount_t PUsPerDisk, spareRegionDepthInPUs, numCompleteSpareRegionsPerDisk, 80 extraPUsPerDisk; 81 RF_StripeCount_t totSparePUsPerDisk; 82 RF_SectorNum_t diskOffsetOfLastFullTableInSUs; 83 RF_SectorCount_t SpareSpaceInSUs; 84 char *cfgBuf = (char *) (cfgPtr->layoutSpecific); 85 RF_StripeNum_t l, SUID; 86 87 SUID = l = 0; 88 numCompleteSpareRegionsPerDisk = 0; 89 90 /* 1. create layout specific structure */ 91 RF_MallocAndAdd(info, sizeof(RF_DeclusteredConfigInfo_t), (RF_DeclusteredConfigInfo_t *), raidPtr->cleanupList); 92 if (info == NULL) 93 return (ENOMEM); 94 layoutPtr->layoutSpecificInfo = (void *) info; 95 info->SpareTable = NULL; 96 97 /* 2. extract parameters from the config structure */ 98 if (layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) { 99 (void)memcpy(info->sparemap_fname, cfgBuf, RF_SPAREMAP_NAME_LEN); 100 } 101 cfgBuf += RF_SPAREMAP_NAME_LEN; 102 103 b = *((int *) cfgBuf); 104 cfgBuf += sizeof(int); 105 v = *((int *) cfgBuf); 106 cfgBuf += sizeof(int); 107 k = *((int *) cfgBuf); 108 cfgBuf += sizeof(int); 109 r = *((int *) cfgBuf); 110 cfgBuf += sizeof(int); 111 lambda = *((int *) cfgBuf); 112 cfgBuf += sizeof(int); 113 raidPtr->noRotate = *((int *) cfgBuf); 114 cfgBuf += sizeof(int); 115 116 /* the sparemaps are generated assuming that parity is rotated, so we 117 * issue a warning if both distributed sparing and no-rotate are on at 118 * the same time */ 119 if ((layoutPtr->map->flags & RF_DISTRIBUTE_SPARE) && raidPtr->noRotate) { 120 RF_ERRORMSG("Warning: distributed sparing specified without parity rotation.\n"); 121 } 122 if (raidPtr->numCol != v) { 123 RF_ERRORMSG2("RAID: config error: table element count (%d) not equal to no. of cols (%d)\n", v, raidPtr->numCol); 124 return (EINVAL); 125 } 126 /* 3. set up the values used in the mapping code */ 127 info->BlocksPerTable = b; 128 info->Lambda = lambda; 129 info->NumParityReps = info->groupSize = k; 130 info->SUsPerTable = b * (k - 1) * layoutPtr->SUsPerPU; /* b blks, k-1 SUs each */ 131 info->SUsPerFullTable = k * info->SUsPerTable; /* rot k times */ 132 info->PUsPerBlock = k - 1; 133 info->SUsPerBlock = info->PUsPerBlock * layoutPtr->SUsPerPU; 134 info->TableDepthInPUs = (b * k) / v; 135 info->FullTableDepthInPUs = info->TableDepthInPUs * k; /* k repetitions */ 136 137 /* used only in distributed sparing case */ 138 info->FullTablesPerSpareRegion = (v - 1) / rf_gcd(r, v - 1); /* (v-1)/gcd fulltables */ 139 info->TablesPerSpareRegion = k * info->FullTablesPerSpareRegion; 140 info->SpareSpaceDepthPerRegionInSUs = (r * info->TablesPerSpareRegion / (v - 1)) * layoutPtr->SUsPerPU; 141 142 /* check to make sure the block design is sufficiently small */ 143 if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) { 144 if (info->FullTableDepthInPUs * layoutPtr->SUsPerPU + info->SpareSpaceDepthPerRegionInSUs > layoutPtr->stripeUnitsPerDisk) { 145 RF_ERRORMSG3("RAID: config error: Full Table depth (%d) + Spare Space (%d) larger than disk size (%d) (BD too big)\n", 146 (int) info->FullTableDepthInPUs, 147 (int) info->SpareSpaceDepthPerRegionInSUs, 148 (int) layoutPtr->stripeUnitsPerDisk); 149 return (EINVAL); 150 } 151 } else { 152 if (info->TableDepthInPUs * layoutPtr->SUsPerPU > layoutPtr->stripeUnitsPerDisk) { 153 RF_ERRORMSG2("RAID: config error: Table depth (%d) larger than disk size (%d) (BD too big)\n", 154 (int) (info->TableDepthInPUs * layoutPtr->SUsPerPU), \ 155 (int) layoutPtr->stripeUnitsPerDisk); 156 return (EINVAL); 157 } 158 } 159 160 161 /* compute the size of each disk, and the number of tables in the last 162 * fulltable (which need not be complete) */ 163 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 164 165 PUsPerDisk = layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU; 166 spareRegionDepthInPUs = (info->TablesPerSpareRegion * info->TableDepthInPUs + 167 (info->TablesPerSpareRegion * info->TableDepthInPUs) / (v - 1)); 168 info->SpareRegionDepthInSUs = spareRegionDepthInPUs * layoutPtr->SUsPerPU; 169 170 numCompleteSpareRegionsPerDisk = PUsPerDisk / spareRegionDepthInPUs; 171 info->NumCompleteSRs = numCompleteSpareRegionsPerDisk; 172 extraPUsPerDisk = PUsPerDisk % spareRegionDepthInPUs; 173 174 /* assume conservatively that we need the full amount of spare 175 * space in one region in order to provide spares for the 176 * partial spare region at the end of the array. We set "i" 177 * to the number of tables in the partial spare region. This 178 * may actually include some fulltables. */ 179 extraPUsPerDisk -= (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU); 180 if (extraPUsPerDisk <= 0) 181 i = 0; 182 else 183 i = extraPUsPerDisk / info->TableDepthInPUs; 184 185 complete_FT_count = raidPtr->numRow * (numCompleteSpareRegionsPerDisk * (info->TablesPerSpareRegion / k) + i / k); 186 info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable; 187 info->ExtraTablesPerDisk = i % k; 188 189 /* note that in the last spare region, the spare space is 190 * complete even though data/parity space is not */ 191 totSparePUsPerDisk = (numCompleteSpareRegionsPerDisk + 1) * (info->SpareSpaceDepthPerRegionInSUs / layoutPtr->SUsPerPU); 192 info->TotSparePUsPerDisk = totSparePUsPerDisk; 193 194 layoutPtr->stripeUnitsPerDisk = 195 ((complete_FT_count / raidPtr->numRow) * info->FullTableDepthInPUs + /* data & parity space */ 196 info->ExtraTablesPerDisk * info->TableDepthInPUs + 197 totSparePUsPerDisk /* spare space */ 198 ) * layoutPtr->SUsPerPU; 199 layoutPtr->dataStripeUnitsPerDisk = 200 (complete_FT_count * info->FullTableDepthInPUs + info->ExtraTablesPerDisk * info->TableDepthInPUs) 201 * layoutPtr->SUsPerPU * (k - 1) / k; 202 203 } else { 204 /* non-dist spare case: force each disk to contain an 205 * integral number of tables */ 206 layoutPtr->stripeUnitsPerDisk /= (info->TableDepthInPUs * layoutPtr->SUsPerPU); 207 layoutPtr->stripeUnitsPerDisk *= (info->TableDepthInPUs * layoutPtr->SUsPerPU); 208 209 /* compute the number of tables in the last fulltable, which 210 * need not be complete */ 211 complete_FT_count = 212 ((layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU) / info->FullTableDepthInPUs) * raidPtr->numRow; 213 214 info->FullTableLimitSUID = complete_FT_count * info->SUsPerFullTable; 215 info->ExtraTablesPerDisk = 216 ((layoutPtr->stripeUnitsPerDisk / layoutPtr->SUsPerPU) / info->TableDepthInPUs) % k; 217 } 218 219 raidPtr->sectorsPerDisk = layoutPtr->stripeUnitsPerDisk * layoutPtr->sectorsPerStripeUnit; 220 221 /* find the disk offset of the stripe unit where the last fulltable 222 * starts */ 223 numCompleteFullTablesPerDisk = complete_FT_count / raidPtr->numRow; 224 diskOffsetOfLastFullTableInSUs = numCompleteFullTablesPerDisk * info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 225 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 226 SpareSpaceInSUs = numCompleteSpareRegionsPerDisk * info->SpareSpaceDepthPerRegionInSUs; 227 diskOffsetOfLastFullTableInSUs += SpareSpaceInSUs; 228 info->DiskOffsetOfLastSpareSpaceChunkInSUs = 229 diskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU; 230 } 231 info->DiskOffsetOfLastFullTableInSUs = diskOffsetOfLastFullTableInSUs; 232 info->numCompleteFullTablesPerDisk = numCompleteFullTablesPerDisk; 233 234 /* 4. create and initialize the lookup tables */ 235 info->LayoutTable = rf_make_2d_array(b, k, raidPtr->cleanupList); 236 if (info->LayoutTable == NULL) 237 return (ENOMEM); 238 info->OffsetTable = rf_make_2d_array(b, k, raidPtr->cleanupList); 239 if (info->OffsetTable == NULL) 240 return (ENOMEM); 241 info->BlockTable = rf_make_2d_array(info->TableDepthInPUs * layoutPtr->SUsPerPU, raidPtr->numCol, raidPtr->cleanupList); 242 if (info->BlockTable == NULL) 243 return (ENOMEM); 244 245 first_avail_slot = rf_make_1d_array(v, NULL); 246 if (first_avail_slot == NULL) 247 return (ENOMEM); 248 249 for (i = 0; i < b; i++) 250 for (j = 0; j < k; j++) 251 info->LayoutTable[i][j] = *cfgBuf++; 252 253 /* initialize offset table */ 254 for (i = 0; i < b; i++) 255 for (j = 0; j < k; j++) { 256 info->OffsetTable[i][j] = first_avail_slot[info->LayoutTable[i][j]]; 257 first_avail_slot[info->LayoutTable[i][j]]++; 258 } 259 260 /* initialize block table */ 261 for (SUID = l = 0; l < layoutPtr->SUsPerPU; l++) { 262 for (i = 0; i < b; i++) { 263 for (j = 0; j < k; j++) { 264 info->BlockTable[(info->OffsetTable[i][j] * layoutPtr->SUsPerPU) + l] 265 [info->LayoutTable[i][j]] = SUID; 266 } 267 SUID++; 268 } 269 } 270 271 rf_free_1d_array(first_avail_slot, v); 272 273 /* 5. set up the remaining redundant-but-useful parameters */ 274 275 raidPtr->totalSectors = (k * complete_FT_count + raidPtr->numRow * info->ExtraTablesPerDisk) * 276 info->SUsPerTable * layoutPtr->sectorsPerStripeUnit; 277 layoutPtr->numStripe = (raidPtr->totalSectors / layoutPtr->sectorsPerStripeUnit) / (k - 1); 278 279 /* strange evaluation order below to try and minimize overflow 280 * problems */ 281 282 layoutPtr->dataSectorsPerStripe = (k - 1) * layoutPtr->sectorsPerStripeUnit; 283 layoutPtr->numDataCol = k - 1; 284 layoutPtr->numParityCol = 1; 285 286 return (0); 287 } 288 /* declustering with distributed sparing */ 289 static void rf_ShutdownDeclusteredDS(RF_ThreadArg_t); 290 static void 291 rf_ShutdownDeclusteredDS(RF_ThreadArg_t arg) 292 { 293 RF_DeclusteredConfigInfo_t *info; 294 RF_Raid_t *raidPtr; 295 296 raidPtr = (RF_Raid_t *) arg; 297 info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 298 if (info->SpareTable) 299 rf_FreeSpareTable(raidPtr); 300 } 301 302 int 303 rf_ConfigureDeclusteredDS(RF_ShutdownList_t **listp, RF_Raid_t *raidPtr, 304 RF_Config_t *cfgPtr) 305 { 306 int rc; 307 308 rc = rf_ConfigureDeclustered(listp, raidPtr, cfgPtr); 309 if (rc) 310 return (rc); 311 rc = rf_ShutdownCreate(listp, rf_ShutdownDeclusteredDS, raidPtr); 312 if (rc) { 313 RF_ERRORMSG1("Got %d adding shutdown event for DeclusteredDS\n", rc); 314 rf_ShutdownDeclusteredDS(raidPtr); 315 return (rc); 316 } 317 return (0); 318 } 319 320 void 321 rf_MapSectorDeclustered(RF_Raid_t *raidPtr, RF_RaidAddr_t raidSector, 322 RF_RowCol_t *row, RF_RowCol_t *col, 323 RF_SectorNum_t *diskSector, int remap) 324 { 325 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 326 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 327 RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit; 328 RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset; 329 RF_StripeNum_t BlockID, BlockOffset, RepIndex; 330 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 331 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 332 RF_StripeNum_t base_suid = 0, outSU, SpareRegion = 0, SpareSpace = 0; 333 334 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 335 336 FullTableID = SUID / sus_per_fulltable; /* fulltable ID within array 337 * (across rows) */ 338 if (raidPtr->numRow == 1) 339 *row = 0; /* avoid a mod and a div in the common case */ 340 else { 341 *row = FullTableID % raidPtr->numRow; 342 FullTableID /= raidPtr->numRow; /* convert to fulltable ID on 343 * this disk */ 344 } 345 if (raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE) { 346 SpareRegion = FullTableID / info->FullTablesPerSpareRegion; 347 SpareSpace = SpareRegion * info->SpareSpaceDepthPerRegionInSUs; 348 } 349 FullTableOffset = SUID % sus_per_fulltable; 350 TableID = FullTableOffset / info->SUsPerTable; 351 TableOffset = FullTableOffset - TableID * info->SUsPerTable; 352 BlockID = TableOffset / info->PUsPerBlock; 353 BlockOffset = TableOffset - BlockID * info->PUsPerBlock; 354 BlockID %= info->BlocksPerTable; 355 RepIndex = info->PUsPerBlock - TableID; 356 if (!raidPtr->noRotate) 357 BlockOffset += ((BlockOffset >= RepIndex) ? 1 : 0); 358 *col = info->LayoutTable[BlockID][BlockOffset]; 359 360 /* remap to distributed spare space if indicated */ 361 if (remap) { 362 RF_ASSERT(raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared || 363 (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal)); 364 rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU); 365 } else { 366 367 outSU = base_suid; 368 outSU += FullTableID * fulltable_depth; /* offs to strt of FT */ 369 outSU += SpareSpace; /* skip rsvd spare space */ 370 outSU += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU; /* offs to strt of tble */ 371 outSU += info->OffsetTable[BlockID][BlockOffset] * layoutPtr->SUsPerPU; /* offs to the PU */ 372 } 373 outSU += TableOffset / (info->BlocksPerTable * info->PUsPerBlock); /* offs to the SU within 374 * a PU */ 375 376 /* convert SUs to sectors, and, if not aligned to SU boundary, add in 377 * offset to sector. */ 378 *diskSector = outSU * layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit); 379 380 RF_ASSERT(*col != -1); 381 } 382 383 384 /* prototyping this inexplicably causes the compile of the layout table (rf_layout.c) to fail */ 385 void 386 rf_MapParityDeclustered(RF_Raid_t *raidPtr, RF_RaidAddr_t raidSector, 387 RF_RowCol_t *row, RF_RowCol_t *col, 388 RF_SectorNum_t *diskSector, int remap) 389 { 390 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 391 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 392 RF_StripeNum_t SUID = raidSector / layoutPtr->sectorsPerStripeUnit; 393 RF_StripeNum_t FullTableID, FullTableOffset, TableID, TableOffset; 394 RF_StripeNum_t BlockID, BlockOffset, RepIndex; 395 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 396 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 397 RF_StripeNum_t base_suid = 0, outSU, SpareRegion = 0, SpareSpace = 0; 398 399 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 400 401 /* compute row & (possibly) spare space exactly as before */ 402 FullTableID = SUID / sus_per_fulltable; 403 if (raidPtr->numRow == 1) 404 *row = 0; /* avoid a mod and a div in the common case */ 405 else { 406 *row = FullTableID % raidPtr->numRow; 407 FullTableID /= raidPtr->numRow; /* convert to fulltable ID on 408 * this disk */ 409 } 410 if ((raidPtr->Layout.map->flags & RF_DISTRIBUTE_SPARE)) { 411 SpareRegion = FullTableID / info->FullTablesPerSpareRegion; 412 SpareSpace = SpareRegion * info->SpareSpaceDepthPerRegionInSUs; 413 } 414 /* compute BlockID and RepIndex exactly as before */ 415 FullTableOffset = SUID % sus_per_fulltable; 416 TableID = FullTableOffset / info->SUsPerTable; 417 TableOffset = FullTableOffset - TableID * info->SUsPerTable; 418 /* TableOffset = FullTableOffset % info->SUsPerTable; */ 419 /* BlockID = (TableOffset / info->PUsPerBlock) % 420 * info->BlocksPerTable; */ 421 BlockID = TableOffset / info->PUsPerBlock; 422 /* BlockOffset = TableOffset % info->PUsPerBlock; */ 423 BlockOffset = TableOffset - BlockID * info->PUsPerBlock; 424 BlockID %= info->BlocksPerTable; 425 426 /* the parity block is in the position indicated by RepIndex */ 427 RepIndex = (raidPtr->noRotate) ? info->PUsPerBlock : info->PUsPerBlock - TableID; 428 *col = info->LayoutTable[BlockID][RepIndex]; 429 430 if (remap) { 431 RF_ASSERT(raidPtr->Disks[*row][*col].status == rf_ds_reconstructing || raidPtr->Disks[*row][*col].status == rf_ds_dist_spared || 432 (rf_copyback_in_progress && raidPtr->Disks[*row][*col].status == rf_ds_optimal)); 433 rf_remap_to_spare_space(layoutPtr, info, *row, FullTableID, TableID, BlockID, (base_suid) ? 1 : 0, SpareRegion, col, &outSU); 434 } else { 435 436 /* compute sector as before, except use RepIndex instead of 437 * BlockOffset */ 438 outSU = base_suid; 439 outSU += FullTableID * fulltable_depth; 440 outSU += SpareSpace; /* skip rsvd spare space */ 441 outSU += TableID * info->TableDepthInPUs * layoutPtr->SUsPerPU; 442 outSU += info->OffsetTable[BlockID][RepIndex] * layoutPtr->SUsPerPU; 443 } 444 445 outSU += TableOffset / (info->BlocksPerTable * info->PUsPerBlock); 446 *diskSector = outSU * layoutPtr->sectorsPerStripeUnit + (raidSector % layoutPtr->sectorsPerStripeUnit); 447 448 RF_ASSERT(*col != -1); 449 } 450 /* returns an array of ints identifying the disks that comprise the stripe containing the indicated address. 451 * the caller must _never_ attempt to modify this array. 452 */ 453 void 454 rf_IdentifyStripeDeclustered(RF_Raid_t *raidPtr, RF_RaidAddr_t addr, 455 RF_RowCol_t **diskids, RF_RowCol_t *outRow) 456 { 457 RF_RaidLayout_t *layoutPtr = &(raidPtr->Layout); 458 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 459 RF_StripeCount_t sus_per_fulltable = info->SUsPerFullTable; 460 RF_StripeCount_t fulltable_depth = info->FullTableDepthInPUs * layoutPtr->SUsPerPU; 461 RF_StripeNum_t base_suid = 0; 462 RF_StripeNum_t SUID = rf_RaidAddressToStripeUnitID(layoutPtr, addr); 463 RF_StripeNum_t stripeID, FullTableID; 464 int tableOffset; 465 466 rf_decluster_adjust_params(layoutPtr, &SUID, &sus_per_fulltable, &fulltable_depth, &base_suid); 467 FullTableID = SUID / sus_per_fulltable; /* fulltable ID within array 468 * (across rows) */ 469 *outRow = FullTableID % raidPtr->numRow; 470 stripeID = rf_StripeUnitIDToStripeID(layoutPtr, SUID); /* find stripe offset 471 * into array */ 472 tableOffset = (stripeID % info->BlocksPerTable); /* find offset into 473 * block design table */ 474 *diskids = info->LayoutTable[tableOffset]; 475 } 476 /* This returns the default head-separation limit, which is measured 477 * in "required units for reconstruction". Each time a disk fetches 478 * a unit, it bumps a counter. The head-sep code prohibits any disk 479 * from getting more than headSepLimit counter values ahead of any 480 * other. 481 * 482 * We assume here that the number of floating recon buffers is already 483 * set. There are r stripes to be reconstructed in each table, and so 484 * if we have a total of B buffers, we can have at most B/r tables 485 * under recon at any one time. In each table, lambda units are required 486 * from each disk, so given B buffers, the head sep limit has to be 487 * (lambda*B)/r units. We subtract one to avoid weird boundary cases. 488 * 489 * for example, suppose were given 50 buffers, r=19, and lambda=4 as in 490 * the 20.5 design. There are 19 stripes/table to be reconstructed, so 491 * we can have 50/19 tables concurrently under reconstruction, which means 492 * we can allow the fastest disk to get 50/19 tables ahead of the slower 493 * disk. There are lambda "required units" for each disk, so the fastest 494 * disk can get 4*50/19 = 10 counter values ahead of the slowest. 495 * 496 * If numBufsToAccumulate is not 1, we need to limit the head sep further 497 * because multiple bufs will be required for each stripe under recon. 498 */ 499 RF_HeadSepLimit_t 500 rf_GetDefaultHeadSepLimitDeclustered(RF_Raid_t *raidPtr) 501 { 502 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 503 504 return (info->Lambda * raidPtr->numFloatingReconBufs / info->TableDepthInPUs / rf_numBufsToAccumulate); 505 } 506 /* returns the default number of recon buffers to use. The value 507 * is somewhat arbitrary...it's intended to be large enough to allow 508 * for a reasonably large head-sep limit, but small enough that you 509 * don't use up all your system memory with buffers. 510 */ 511 int 512 rf_GetDefaultNumFloatingReconBuffersDeclustered(RF_Raid_t * raidPtr) 513 { 514 return (100 * rf_numBufsToAccumulate); 515 } 516 /* sectors in the last fulltable of the array need to be handled 517 * specially since this fulltable can be incomplete. this function 518 * changes the values of certain params to handle this. 519 * 520 * the idea here is that MapSector et. al. figure out which disk the 521 * addressed unit lives on by computing the modulos of the unit number 522 * with the number of units per fulltable, table, etc. In the last 523 * fulltable, there are fewer units per fulltable, so we need to adjust 524 * the number of user data units per fulltable to reflect this. 525 * 526 * so, we (1) convert the fulltable size and depth parameters to 527 * the size of the partial fulltable at the end, (2) compute the 528 * disk sector offset where this fulltable starts, and (3) convert 529 * the users stripe unit number from an offset into the array to 530 * an offset into the last fulltable. 531 */ 532 void 533 rf_decluster_adjust_params(RF_RaidLayout_t *layoutPtr, 534 RF_StripeNum_t *SUID, 535 RF_StripeCount_t *sus_per_fulltable, 536 RF_StripeCount_t *fulltable_depth, 537 RF_StripeNum_t *base_suid) 538 { 539 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 540 541 if (*SUID >= info->FullTableLimitSUID) { 542 /* new full table size is size of last full table on disk */ 543 *sus_per_fulltable = info->ExtraTablesPerDisk * info->SUsPerTable; 544 545 /* new full table depth is corresponding depth */ 546 *fulltable_depth = info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU; 547 548 /* set up the new base offset */ 549 *base_suid = info->DiskOffsetOfLastFullTableInSUs; 550 551 /* convert users array address to an offset into the last 552 * fulltable */ 553 *SUID -= info->FullTableLimitSUID; 554 } 555 } 556 /* 557 * map a stripe ID to a parity stripe ID. 558 * See comment above RaidAddressToParityStripeID in layout.c. 559 */ 560 void 561 rf_MapSIDToPSIDDeclustered(RF_RaidLayout_t *layoutPtr, 562 RF_StripeNum_t stripeID, 563 RF_StripeNum_t *psID, 564 RF_ReconUnitNum_t *which_ru) 565 { 566 RF_DeclusteredConfigInfo_t *info; 567 568 info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 569 570 *psID = (stripeID / (layoutPtr->SUsPerPU * info->BlocksPerTable)) 571 * info->BlocksPerTable + (stripeID % info->BlocksPerTable); 572 *which_ru = (stripeID % (info->BlocksPerTable * layoutPtr->SUsPerPU)) 573 / info->BlocksPerTable; 574 RF_ASSERT((*which_ru) < layoutPtr->SUsPerPU / layoutPtr->SUsPerRU); 575 } 576 /* 577 * Called from MapSector and MapParity to retarget an access at the spare unit. 578 * Modifies the "col" and "outSU" parameters only. 579 */ 580 void 581 rf_remap_to_spare_space(RF_RaidLayout_t *layoutPtr, 582 RF_DeclusteredConfigInfo_t *info, 583 RF_RowCol_t row, 584 RF_StripeNum_t FullTableID, 585 RF_StripeNum_t TableID, 586 RF_SectorNum_t BlockID, 587 RF_StripeNum_t base_suid, 588 RF_StripeNum_t SpareRegion, 589 RF_RowCol_t *outCol, 590 RF_StripeNum_t *outSU) 591 { 592 RF_StripeNum_t ftID, spareTableStartSU, TableInSpareRegion, lastSROffset, 593 which_ft; 594 595 /* 596 * note that FullTableID and hence SpareRegion may have gotten 597 * tweaked by rf_decluster_adjust_params. We detect this by 598 * noticing that base_suid is not 0. 599 */ 600 if (base_suid == 0) { 601 ftID = FullTableID; 602 } else { 603 /* 604 * There may be > 1.0 full tables in the last (i.e. partial) 605 * spare region. find out which of these we're in. 606 */ 607 lastSROffset = info->NumCompleteSRs * info->SpareRegionDepthInSUs; 608 which_ft = (info->DiskOffsetOfLastFullTableInSUs - lastSROffset) / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU); 609 610 /* compute the actual full table ID */ 611 ftID = info->DiskOffsetOfLastFullTableInSUs / (info->FullTableDepthInPUs * layoutPtr->SUsPerPU) + which_ft; 612 SpareRegion = info->NumCompleteSRs; 613 } 614 TableInSpareRegion = (ftID * info->NumParityReps + TableID) % info->TablesPerSpareRegion; 615 616 *outCol = info->SpareTable[TableInSpareRegion][BlockID].spareDisk; 617 RF_ASSERT(*outCol != -1); 618 619 spareTableStartSU = (SpareRegion == info->NumCompleteSRs) ? 620 info->DiskOffsetOfLastFullTableInSUs + info->ExtraTablesPerDisk * info->TableDepthInPUs * layoutPtr->SUsPerPU : 621 (SpareRegion + 1) * info->SpareRegionDepthInSUs - info->SpareSpaceDepthPerRegionInSUs; 622 *outSU = spareTableStartSU + info->SpareTable[TableInSpareRegion][BlockID].spareBlockOffsetInSUs; 623 if (*outSU >= layoutPtr->stripeUnitsPerDisk) { 624 printf("rf_remap_to_spare_space: invalid remapped disk SU offset %ld\n", (long) *outSU); 625 } 626 } 627 628 #endif /* (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) */ 629 630 #if (RF_INCLUDE_PARITY_DECLUSTERING_DS > 0) 631 int 632 rf_InstallSpareTable(RF_Raid_t *raidPtr, RF_RowCol_t frow, RF_RowCol_t fcol) 633 { 634 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 635 RF_SparetWait_t *req; 636 int retcode; 637 638 RF_Malloc(req, sizeof(*req), (RF_SparetWait_t *)); 639 req->C = raidPtr->numCol; 640 req->G = raidPtr->Layout.numDataCol + raidPtr->Layout.numParityCol; 641 req->fcol = fcol; 642 req->SUsPerPU = raidPtr->Layout.SUsPerPU; 643 req->TablesPerSpareRegion = info->TablesPerSpareRegion; 644 req->BlocksPerTable = info->BlocksPerTable; 645 req->TableDepthInPUs = info->TableDepthInPUs; 646 req->SpareSpaceDepthPerRegionInSUs = info->SpareSpaceDepthPerRegionInSUs; 647 648 retcode = rf_GetSpareTableFromDaemon(req); 649 RF_ASSERT(!retcode); /* XXX -- fix this to recover gracefully -- 650 * XXX */ 651 return (retcode); 652 } 653 #endif 654 #if (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) 655 /* 656 * Invoked via ioctl to install a spare table in the kernel. 657 */ 658 int 659 rf_SetSpareTable(RF_Raid_t *raidPtr, void *data) 660 { 661 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) raidPtr->Layout.layoutSpecificInfo; 662 RF_SpareTableEntry_t **ptrs; 663 int i, retcode; 664 665 /* what we need to copyin is a 2-d array, so first copyin the user 666 * pointers to the rows in the table */ 667 RF_Malloc(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **)); 668 retcode = copyin((caddr_t) data, (caddr_t) ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 669 670 if (retcode) 671 return (retcode); 672 673 /* now allocate kernel space for the row pointers */ 674 RF_Malloc(info->SpareTable, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *), (RF_SpareTableEntry_t **)); 675 676 /* now allocate kernel space for each row in the table, and copy it in 677 * from user space */ 678 for (i = 0; i < info->TablesPerSpareRegion; i++) { 679 RF_Malloc(info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t), (RF_SpareTableEntry_t *)); 680 retcode = copyin(ptrs[i], info->SpareTable[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t)); 681 if (retcode) { 682 info->SpareTable = NULL; /* blow off the memory 683 * we've allocated */ 684 return (retcode); 685 } 686 } 687 688 /* free up the temporary array we used */ 689 RF_Free(ptrs, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 690 691 return (0); 692 } 693 694 RF_ReconUnitCount_t 695 rf_GetNumSpareRUsDeclustered(RF_Raid_t *raidPtr) 696 { 697 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 698 699 return (((RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo)->TotSparePUsPerDisk); 700 } 701 #endif /* (RF_INCLUDE_PARITY_DECLUSTERING > 0) || (RF_INCLUDE_PARITY_DECLUSTERING_PQ > 0) */ 702 703 void 704 rf_FreeSpareTable(RF_Raid_t *raidPtr) 705 { 706 long i; 707 RF_RaidLayout_t *layoutPtr = &raidPtr->Layout; 708 RF_DeclusteredConfigInfo_t *info = (RF_DeclusteredConfigInfo_t *) layoutPtr->layoutSpecificInfo; 709 RF_SpareTableEntry_t **table = info->SpareTable; 710 711 for (i = 0; i < info->TablesPerSpareRegion; i++) { 712 RF_Free(table[i], info->BlocksPerTable * sizeof(RF_SpareTableEntry_t)); 713 } 714 RF_Free(table, info->TablesPerSpareRegion * sizeof(RF_SpareTableEntry_t *)); 715 info->SpareTable = (RF_SpareTableEntry_t **) NULL; 716 } 717