1 /* $NetBSD: rf_dagfuncs.c,v 1.32 2020/06/19 19:29:39 jdolecek Exp $ */ 2 /* 3 * Copyright (c) 1995 Carnegie-Mellon University. 4 * All rights reserved. 5 * 6 * Author: Mark Holland, William V. Courtright II 7 * 8 * Permission to use, copy, modify and distribute this software and 9 * its documentation is hereby granted, provided that both the copyright 10 * notice and this permission notice appear in all copies of the 11 * software, derivative works or modified versions, and any portions 12 * thereof, and that both notices appear in supporting documentation. 13 * 14 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 15 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 16 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 17 * 18 * Carnegie Mellon requests users of this software to return to 19 * 20 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 21 * School of Computer Science 22 * Carnegie Mellon University 23 * Pittsburgh PA 15213-3890 24 * 25 * any improvements or extensions that they make and grant Carnegie the 26 * rights to redistribute these changes. 27 */ 28 29 /* 30 * dagfuncs.c -- DAG node execution routines 31 * 32 * Rules: 33 * 1. Every DAG execution function must eventually cause node->status to 34 * get set to "good" or "bad", and "FinishNode" to be called. In the 35 * case of nodes that complete immediately (xor, NullNodeFunc, etc), 36 * the node execution function can do these two things directly. In 37 * the case of nodes that have to wait for some event (a disk read to 38 * complete, a lock to be released, etc) to occur before they can 39 * complete, this is typically achieved by having whatever module 40 * is doing the operation call GenericWakeupFunc upon completion. 41 * 2. DAG execution functions should check the status in the DAG header 42 * and NOP out their operations if the status is not "enable". However, 43 * execution functions that release resources must be sure to release 44 * them even when they NOP out the function that would use them. 45 * Functions that acquire resources should go ahead and acquire them 46 * even when they NOP, so that a downstream release node will not have 47 * to check to find out whether or not the acquire was suppressed. 48 */ 49 50 #include <sys/cdefs.h> 51 __KERNEL_RCSID(0, "$NetBSD: rf_dagfuncs.c,v 1.32 2020/06/19 19:29:39 jdolecek Exp $"); 52 53 #include <sys/param.h> 54 #include <sys/ioctl.h> 55 56 #include "rf_archs.h" 57 #include "rf_raid.h" 58 #include "rf_dag.h" 59 #include "rf_layout.h" 60 #include "rf_etimer.h" 61 #include "rf_acctrace.h" 62 #include "rf_diskqueue.h" 63 #include "rf_dagfuncs.h" 64 #include "rf_general.h" 65 #include "rf_engine.h" 66 #include "rf_dagutils.h" 67 68 #include "rf_kintf.h" 69 70 #if RF_INCLUDE_PARITYLOGGING > 0 71 #include "rf_paritylog.h" 72 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 73 74 void (*rf_DiskReadFunc) (RF_DagNode_t *); 75 void (*rf_DiskWriteFunc) (RF_DagNode_t *); 76 void (*rf_DiskReadUndoFunc) (RF_DagNode_t *); 77 void (*rf_DiskWriteUndoFunc) (RF_DagNode_t *); 78 void (*rf_RegularXorUndoFunc) (RF_DagNode_t *); 79 void (*rf_SimpleXorUndoFunc) (RF_DagNode_t *); 80 void (*rf_RecoveryXorUndoFunc) (RF_DagNode_t *); 81 82 /***************************************************************************** 83 * main (only) configuration routine for this module 84 ****************************************************************************/ 85 int 86 rf_ConfigureDAGFuncs(RF_ShutdownList_t **listp) 87 { 88 RF_ASSERT(((sizeof(long) == 8) && RF_LONGSHIFT == 3) || 89 ((sizeof(long) == 4) && RF_LONGSHIFT == 2)); 90 rf_DiskReadFunc = rf_DiskReadFuncForThreads; 91 rf_DiskReadUndoFunc = rf_DiskUndoFunc; 92 rf_DiskWriteFunc = rf_DiskWriteFuncForThreads; 93 rf_DiskWriteUndoFunc = rf_DiskUndoFunc; 94 rf_RegularXorUndoFunc = rf_NullNodeUndoFunc; 95 rf_SimpleXorUndoFunc = rf_NullNodeUndoFunc; 96 rf_RecoveryXorUndoFunc = rf_NullNodeUndoFunc; 97 return (0); 98 } 99 100 101 102 /***************************************************************************** 103 * the execution function associated with a terminate node 104 ****************************************************************************/ 105 void 106 rf_TerminateFunc(RF_DagNode_t *node) 107 { 108 RF_ASSERT(node->dagHdr->numCommits == node->dagHdr->numCommitNodes); 109 node->status = rf_good; 110 rf_FinishNode(node, RF_THREAD_CONTEXT); 111 } 112 113 void 114 rf_TerminateUndoFunc(RF_DagNode_t *node) 115 { 116 } 117 118 119 /***************************************************************************** 120 * execution functions associated with a mirror node 121 * 122 * parameters: 123 * 124 * 0 - physical disk addres of data 125 * 1 - buffer for holding read data 126 * 2 - parity stripe ID 127 * 3 - flags 128 * 4 - physical disk address of mirror (parity) 129 * 130 ****************************************************************************/ 131 132 void 133 rf_DiskReadMirrorIdleFunc(RF_DagNode_t *node) 134 { 135 /* select the mirror copy with the shortest queue and fill in node 136 * parameters with physical disk address */ 137 138 rf_SelectMirrorDiskIdle(node); 139 rf_DiskReadFunc(node); 140 } 141 142 #if (RF_INCLUDE_CHAINDECLUSTER > 0) || (RF_INCLUDE_INTERDECLUSTER > 0) || (RF_DEBUG_VALIDATE_DAG > 0) 143 void 144 rf_DiskReadMirrorPartitionFunc(RF_DagNode_t *node) 145 { 146 /* select the mirror copy with the shortest queue and fill in node 147 * parameters with physical disk address */ 148 149 rf_SelectMirrorDiskPartition(node); 150 rf_DiskReadFunc(node); 151 } 152 #endif 153 154 void 155 rf_DiskReadMirrorUndoFunc(RF_DagNode_t *node) 156 { 157 } 158 159 160 161 #if RF_INCLUDE_PARITYLOGGING > 0 162 /***************************************************************************** 163 * the execution function associated with a parity log update node 164 ****************************************************************************/ 165 void 166 rf_ParityLogUpdateFunc(RF_DagNode_t *node) 167 { 168 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 169 void *bf = (void *) node->params[1].p; 170 RF_ParityLogData_t *logData; 171 #if RF_ACC_TRACE > 0 172 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 173 RF_Etimer_t timer; 174 #endif 175 176 if (node->dagHdr->status == rf_enable) { 177 #if RF_ACC_TRACE > 0 178 RF_ETIMER_START(timer); 179 #endif 180 logData = rf_CreateParityLogData(RF_UPDATE, pda, bf, 181 (RF_Raid_t *) (node->dagHdr->raidPtr), 182 node->wakeFunc, node, 183 node->dagHdr->tracerec, timer); 184 if (logData) 185 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE); 186 else { 187 #if RF_ACC_TRACE > 0 188 RF_ETIMER_STOP(timer); 189 RF_ETIMER_EVAL(timer); 190 tracerec->plog_us += RF_ETIMER_VAL_US(timer); 191 #endif 192 (node->wakeFunc) (node, ENOMEM); 193 } 194 } 195 } 196 197 198 /***************************************************************************** 199 * the execution function associated with a parity log overwrite node 200 ****************************************************************************/ 201 void 202 rf_ParityLogOverwriteFunc(RF_DagNode_t *node) 203 { 204 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 205 void *bf = (void *) node->params[1].p; 206 RF_ParityLogData_t *logData; 207 #if RF_ACC_TRACE > 0 208 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 209 RF_Etimer_t timer; 210 #endif 211 212 if (node->dagHdr->status == rf_enable) { 213 #if RF_ACC_TRACE > 0 214 RF_ETIMER_START(timer); 215 #endif 216 logData = rf_CreateParityLogData(RF_OVERWRITE, pda, bf, 217 (RF_Raid_t *) (node->dagHdr->raidPtr), 218 node->wakeFunc, node, node->dagHdr->tracerec, timer); 219 if (logData) 220 rf_ParityLogAppend(logData, RF_FALSE, NULL, RF_FALSE); 221 else { 222 #if RF_ACC_TRACE > 0 223 RF_ETIMER_STOP(timer); 224 RF_ETIMER_EVAL(timer); 225 tracerec->plog_us += RF_ETIMER_VAL_US(timer); 226 #endif 227 (node->wakeFunc) (node, ENOMEM); 228 } 229 } 230 } 231 232 void 233 rf_ParityLogUpdateUndoFunc(RF_DagNode_t *node) 234 { 235 } 236 237 void 238 rf_ParityLogOverwriteUndoFunc(RF_DagNode_t *node) 239 { 240 } 241 #endif /* RF_INCLUDE_PARITYLOGGING > 0 */ 242 243 /***************************************************************************** 244 * the execution function associated with a NOP node 245 ****************************************************************************/ 246 void 247 rf_NullNodeFunc(RF_DagNode_t *node) 248 { 249 node->status = rf_good; 250 rf_FinishNode(node, RF_THREAD_CONTEXT); 251 } 252 253 void 254 rf_NullNodeUndoFunc(RF_DagNode_t *node) 255 { 256 node->status = rf_undone; 257 rf_FinishNode(node, RF_THREAD_CONTEXT); 258 } 259 260 261 /***************************************************************************** 262 * the execution function associated with a disk-read node 263 ****************************************************************************/ 264 void 265 rf_DiskReadFuncForThreads(RF_DagNode_t *node) 266 { 267 RF_DiskQueueData_t *req; 268 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 269 void *bf = (void *) node->params[1].p; 270 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v; 271 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v); 272 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v); 273 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_READ : RF_IO_TYPE_NOP; 274 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues; 275 276 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector, 277 bf, parityStripeID, which_ru, node->wakeFunc, node, 278 #if RF_ACC_TRACE > 0 279 node->dagHdr->tracerec, 280 #else 281 NULL, 282 #endif 283 (void *) (node->dagHdr->raidPtr), 0, node->dagHdr->bp, PR_NOWAIT); 284 if (!req) { 285 (node->wakeFunc) (node, ENOMEM); 286 } else { 287 node->dagFuncData = (void *) req; 288 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority); 289 } 290 } 291 292 293 /***************************************************************************** 294 * the execution function associated with a disk-write node 295 ****************************************************************************/ 296 void 297 rf_DiskWriteFuncForThreads(RF_DagNode_t *node) 298 { 299 RF_DiskQueueData_t *req; 300 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 301 void *bf = (void *) node->params[1].p; 302 RF_StripeNum_t parityStripeID = (RF_StripeNum_t) node->params[2].v; 303 unsigned priority = RF_EXTRACT_PRIORITY(node->params[3].v); 304 unsigned which_ru = RF_EXTRACT_RU(node->params[3].v); 305 RF_IoType_t iotype = (node->dagHdr->status == rf_enable) ? RF_IO_TYPE_WRITE : RF_IO_TYPE_NOP; 306 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues; 307 308 /* normal processing (rollaway or forward recovery) begins here */ 309 req = rf_CreateDiskQueueData(iotype, pda->startSector, pda->numSector, 310 bf, parityStripeID, which_ru, node->wakeFunc, node, 311 #if RF_ACC_TRACE > 0 312 node->dagHdr->tracerec, 313 #else 314 NULL, 315 #endif 316 (void *) (node->dagHdr->raidPtr), 317 0, node->dagHdr->bp, PR_NOWAIT); 318 319 if (!req) { 320 (node->wakeFunc) (node, ENOMEM); 321 } else { 322 node->dagFuncData = (void *) req; 323 rf_DiskIOEnqueue(&(dqs[pda->col]), req, priority); 324 } 325 } 326 /***************************************************************************** 327 * the undo function for disk nodes 328 * Note: this is not a proper undo of a write node, only locks are released. 329 * old data is not restored to disk! 330 ****************************************************************************/ 331 void 332 rf_DiskUndoFunc(RF_DagNode_t *node) 333 { 334 RF_DiskQueueData_t *req; 335 RF_PhysDiskAddr_t *pda = (RF_PhysDiskAddr_t *) node->params[0].p; 336 RF_DiskQueue_t *dqs = ((RF_Raid_t *) (node->dagHdr->raidPtr))->Queues; 337 338 req = rf_CreateDiskQueueData(RF_IO_TYPE_NOP, 339 0L, 0, NULL, 0L, 0, node->wakeFunc, node, 340 #if RF_ACC_TRACE > 0 341 node->dagHdr->tracerec, 342 #else 343 NULL, 344 #endif 345 (void *) (node->dagHdr->raidPtr), 346 0, NULL, PR_NOWAIT); 347 if (!req) 348 (node->wakeFunc) (node, ENOMEM); 349 else { 350 node->dagFuncData = (void *) req; 351 rf_DiskIOEnqueue(&(dqs[pda->col]), req, RF_IO_NORMAL_PRIORITY); 352 } 353 } 354 355 /***************************************************************************** 356 * Callback routine for DiskRead and DiskWrite nodes. When the disk 357 * op completes, the routine is called to set the node status and 358 * inform the execution engine that the node has fired. 359 ****************************************************************************/ 360 void 361 rf_GenericWakeupFunc(void *v, int status) 362 { 363 RF_DagNode_t *node = v; 364 365 switch (node->status) { 366 case rf_fired: 367 if (status) 368 node->status = rf_bad; 369 else 370 node->status = rf_good; 371 break; 372 case rf_recover: 373 /* probably should never reach this case */ 374 if (status) 375 node->status = rf_panic; 376 else 377 node->status = rf_undone; 378 break; 379 default: 380 printf("rf_GenericWakeupFunc:"); 381 printf("node->status is %d,", node->status); 382 printf("status is %d \n", status); 383 RF_PANIC(); 384 break; 385 } 386 if (node->dagFuncData) 387 rf_FreeDiskQueueData((RF_DiskQueueData_t *) node->dagFuncData); 388 rf_FinishNode(node, RF_INTR_CONTEXT); 389 } 390 391 392 /***************************************************************************** 393 * there are three distinct types of xor nodes: 394 395 * A "regular xor" is used in the fault-free case where the access 396 * spans a complete stripe unit. It assumes that the result buffer is 397 * one full stripe unit in size, and uses the stripe-unit-offset 398 * values that it computes from the PDAs to determine where within the 399 * stripe unit to XOR each argument buffer. 400 * 401 * A "simple xor" is used in the fault-free case where the access 402 * touches only a portion of one (or two, in some cases) stripe 403 * unit(s). It assumes that all the argument buffers are of the same 404 * size and have the same stripe unit offset. 405 * 406 * A "recovery xor" is used in the degraded-mode case. It's similar 407 * to the regular xor function except that it takes the failed PDA as 408 * an additional parameter, and uses it to determine what portions of 409 * the argument buffers need to be xor'd into the result buffer, and 410 * where in the result buffer they should go. 411 ****************************************************************************/ 412 413 /* xor the params together and store the result in the result field. 414 * assume the result field points to a buffer that is the size of one 415 * SU, and use the pda params to determine where within the buffer to 416 * XOR the input buffers. */ 417 void 418 rf_RegularXorFunc(RF_DagNode_t *node) 419 { 420 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p; 421 #if RF_ACC_TRACE > 0 422 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 423 RF_Etimer_t timer; 424 #endif 425 int i, retcode; 426 427 retcode = 0; 428 if (node->dagHdr->status == rf_enable) { 429 /* don't do the XOR if the input is the same as the output */ 430 #if RF_ACC_TRACE > 0 431 RF_ETIMER_START(timer); 432 #endif 433 for (i = 0; i < node->numParams - 1; i += 2) 434 if (node->params[i + 1].p != node->results[0]) { 435 retcode = rf_XorIntoBuffer(raidPtr, (RF_PhysDiskAddr_t *) node->params[i].p, 436 (char *) node->params[i + 1].p, (char *) node->results[0]); 437 } 438 #if RF_ACC_TRACE > 0 439 RF_ETIMER_STOP(timer); 440 RF_ETIMER_EVAL(timer); 441 tracerec->xor_us += RF_ETIMER_VAL_US(timer); 442 #endif 443 } 444 rf_GenericWakeupFunc(node, retcode); /* call wake func 445 * explicitly since no 446 * I/O in this node */ 447 } 448 /* xor the inputs into the result buffer, ignoring placement issues */ 449 void 450 rf_SimpleXorFunc(RF_DagNode_t *node) 451 { 452 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p; 453 int i, retcode = 0; 454 #if RF_ACC_TRACE > 0 455 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 456 RF_Etimer_t timer; 457 #endif 458 459 if (node->dagHdr->status == rf_enable) { 460 #if RF_ACC_TRACE > 0 461 RF_ETIMER_START(timer); 462 #endif 463 /* don't do the XOR if the input is the same as the output */ 464 for (i = 0; i < node->numParams - 1; i += 2) 465 if (node->params[i + 1].p != node->results[0]) { 466 retcode = rf_bxor((char *) node->params[i + 1].p, (char *) node->results[0], 467 rf_RaidAddressToByte(raidPtr, ((RF_PhysDiskAddr_t *) node->params[i].p)->numSector)); 468 } 469 #if RF_ACC_TRACE > 0 470 RF_ETIMER_STOP(timer); 471 RF_ETIMER_EVAL(timer); 472 tracerec->xor_us += RF_ETIMER_VAL_US(timer); 473 #endif 474 } 475 rf_GenericWakeupFunc(node, retcode); /* call wake func 476 * explicitly since no 477 * I/O in this node */ 478 } 479 /* this xor is used by the degraded-mode dag functions to recover lost 480 * data. the second-to-last parameter is the PDA for the failed 481 * portion of the access. the code here looks at this PDA and assumes 482 * that the xor target buffer is equal in size to the number of 483 * sectors in the failed PDA. It then uses the other PDAs in the 484 * parameter list to determine where within the target buffer the 485 * corresponding data should be xored. */ 486 void 487 rf_RecoveryXorFunc(RF_DagNode_t *node) 488 { 489 RF_Raid_t *raidPtr = (RF_Raid_t *) node->params[node->numParams - 1].p; 490 RF_RaidLayout_t *layoutPtr = (RF_RaidLayout_t *) & raidPtr->Layout; 491 RF_PhysDiskAddr_t *failedPDA = (RF_PhysDiskAddr_t *) node->params[node->numParams - 2].p; 492 int i, retcode = 0; 493 RF_PhysDiskAddr_t *pda; 494 int suoffset, failedSUOffset = rf_StripeUnitOffset(layoutPtr, failedPDA->startSector); 495 char *srcbuf, *destbuf; 496 #if RF_ACC_TRACE > 0 497 RF_AccTraceEntry_t *tracerec = node->dagHdr->tracerec; 498 RF_Etimer_t timer; 499 #endif 500 501 if (node->dagHdr->status == rf_enable) { 502 #if RF_ACC_TRACE > 0 503 RF_ETIMER_START(timer); 504 #endif 505 for (i = 0; i < node->numParams - 2; i += 2) 506 if (node->params[i + 1].p != node->results[0]) { 507 pda = (RF_PhysDiskAddr_t *) node->params[i].p; 508 srcbuf = (char *) node->params[i + 1].p; 509 suoffset = rf_StripeUnitOffset(layoutPtr, pda->startSector); 510 destbuf = ((char *) node->results[0]) + rf_RaidAddressToByte(raidPtr, suoffset - failedSUOffset); 511 retcode = rf_bxor(srcbuf, destbuf, rf_RaidAddressToByte(raidPtr, pda->numSector)); 512 } 513 #if RF_ACC_TRACE > 0 514 RF_ETIMER_STOP(timer); 515 RF_ETIMER_EVAL(timer); 516 tracerec->xor_us += RF_ETIMER_VAL_US(timer); 517 #endif 518 } 519 rf_GenericWakeupFunc(node, retcode); 520 } 521 /***************************************************************************** 522 * The next three functions are utilities used by the above 523 * xor-execution functions. 524 ****************************************************************************/ 525 526 527 /* 528 * this is just a glorified buffer xor. targbuf points to a buffer 529 * that is one full stripe unit in size. srcbuf points to a buffer 530 * that may be less than 1 SU, but never more. When the access 531 * described by pda is one SU in size (which by implication means it's 532 * SU-aligned), all that happens is (targbuf) <- (srcbuf ^ targbuf). 533 * When the access is less than one SU in size the XOR occurs on only 534 * the portion of targbuf identified in the pda. */ 535 536 int 537 rf_XorIntoBuffer(RF_Raid_t *raidPtr, RF_PhysDiskAddr_t *pda, 538 char *srcbuf, char *targbuf) 539 { 540 char *targptr; 541 int sectPerSU = raidPtr->Layout.sectorsPerStripeUnit; 542 int SUOffset = pda->startSector % sectPerSU; 543 int length, retcode = 0; 544 545 RF_ASSERT(pda->numSector <= sectPerSU); 546 547 targptr = targbuf + rf_RaidAddressToByte(raidPtr, SUOffset); 548 length = rf_RaidAddressToByte(raidPtr, pda->numSector); 549 retcode = rf_bxor(srcbuf, targptr, length); 550 return (retcode); 551 } 552 /* it really should be the case that the buffer pointers (returned by 553 * malloc) are aligned to the natural word size of the machine, so 554 * this is the only case we optimize for. The length should always be 555 * a multiple of the sector size, so there should be no problem with 556 * leftover bytes at the end. */ 557 int 558 rf_bxor(char *src, char *dest, int len) 559 { 560 unsigned mask = sizeof(long) - 1, retcode = 0; 561 562 if (!(((unsigned long) src) & mask) && 563 !(((unsigned long) dest) & mask) && !(len & mask)) { 564 retcode = rf_longword_bxor((unsigned long *) src, 565 (unsigned long *) dest, 566 len >> RF_LONGSHIFT); 567 } else { 568 RF_ASSERT(0); 569 } 570 return (retcode); 571 } 572 573 /* When XORing in kernel mode, we need to map each user page to kernel 574 * space before we can access it. We don't want to assume anything 575 * about which input buffers are in kernel/user space, nor about their 576 * alignment, so in each loop we compute the maximum number of bytes 577 * that we can xor without crossing any page boundaries, and do only 578 * this many bytes before the next remap. 579 * 580 * len - is in longwords 581 */ 582 int 583 rf_longword_bxor(unsigned long *src, unsigned long *dest, int len) 584 { 585 unsigned long *end = src + len; 586 unsigned long d0, d1, d2, d3, s0, s1, s2, s3; /* temps */ 587 unsigned long *pg_src, *pg_dest; /* per-page source/dest pointers */ 588 int longs_this_time;/* # longwords to xor in the current iteration */ 589 590 pg_src = src; 591 pg_dest = dest; 592 if (!pg_src || !pg_dest) 593 return (EFAULT); 594 595 while (len >= 4) { 596 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(pg_src), RF_BLIP(pg_dest)) >> RF_LONGSHIFT); /* note len in longwords */ 597 src += longs_this_time; 598 dest += longs_this_time; 599 len -= longs_this_time; 600 while (longs_this_time >= 4) { 601 d0 = pg_dest[0]; 602 d1 = pg_dest[1]; 603 d2 = pg_dest[2]; 604 d3 = pg_dest[3]; 605 s0 = pg_src[0]; 606 s1 = pg_src[1]; 607 s2 = pg_src[2]; 608 s3 = pg_src[3]; 609 pg_dest[0] = d0 ^ s0; 610 pg_dest[1] = d1 ^ s1; 611 pg_dest[2] = d2 ^ s2; 612 pg_dest[3] = d3 ^ s3; 613 pg_src += 4; 614 pg_dest += 4; 615 longs_this_time -= 4; 616 } 617 while (longs_this_time > 0) { /* cannot cross any page 618 * boundaries here */ 619 *pg_dest++ ^= *pg_src++; 620 longs_this_time--; 621 } 622 623 /* either we're done, or we've reached a page boundary on one 624 * (or possibly both) of the pointers */ 625 if (len) { 626 if (RF_PAGE_ALIGNED(src)) 627 pg_src = src; 628 if (RF_PAGE_ALIGNED(dest)) 629 pg_dest = dest; 630 if (!pg_src || !pg_dest) 631 return (EFAULT); 632 } 633 } 634 while (src < end) { 635 *pg_dest++ ^= *pg_src++; 636 src++; 637 dest++; 638 len--; 639 if (RF_PAGE_ALIGNED(src)) 640 pg_src = src; 641 if (RF_PAGE_ALIGNED(dest)) 642 pg_dest = dest; 643 } 644 RF_ASSERT(len == 0); 645 return (0); 646 } 647 648 #if 0 649 /* 650 dst = a ^ b ^ c; 651 a may equal dst 652 see comment above longword_bxor 653 len is length in longwords 654 */ 655 int 656 rf_longword_bxor3(unsigned long *dst, unsigned long *a, unsigned long *b, 657 unsigned long *c, int len, void *bp) 658 { 659 unsigned long a0, a1, a2, a3, b0, b1, b2, b3; 660 unsigned long *pg_a, *pg_b, *pg_c, *pg_dst; /* per-page source/dest 661 * pointers */ 662 int longs_this_time;/* # longs to xor in the current iteration */ 663 char dst_is_a = 0; 664 665 pg_a = a; 666 pg_b = b; 667 pg_c = c; 668 if (a == dst) { 669 pg_dst = pg_a; 670 dst_is_a = 1; 671 } else { 672 pg_dst = dst; 673 } 674 675 /* align dest to cache line. Can't cross a pg boundary on dst here. */ 676 while ((((unsigned long) pg_dst) & 0x1f)) { 677 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++; 678 dst++; 679 a++; 680 b++; 681 c++; 682 if (RF_PAGE_ALIGNED(a)) { 683 pg_a = a; 684 if (!pg_a) 685 return (EFAULT); 686 } 687 if (RF_PAGE_ALIGNED(b)) { 688 pg_b = a; 689 if (!pg_b) 690 return (EFAULT); 691 } 692 if (RF_PAGE_ALIGNED(c)) { 693 pg_c = a; 694 if (!pg_c) 695 return (EFAULT); 696 } 697 len--; 698 } 699 700 while (len > 4) { 701 longs_this_time = RF_MIN(len, RF_MIN(RF_BLIP(a), RF_MIN(RF_BLIP(b), RF_MIN(RF_BLIP(c), RF_BLIP(dst)))) >> RF_LONGSHIFT); 702 a += longs_this_time; 703 b += longs_this_time; 704 c += longs_this_time; 705 dst += longs_this_time; 706 len -= longs_this_time; 707 while (longs_this_time >= 4) { 708 a0 = pg_a[0]; 709 longs_this_time -= 4; 710 711 a1 = pg_a[1]; 712 a2 = pg_a[2]; 713 714 a3 = pg_a[3]; 715 pg_a += 4; 716 717 b0 = pg_b[0]; 718 b1 = pg_b[1]; 719 720 b2 = pg_b[2]; 721 b3 = pg_b[3]; 722 /* start dual issue */ 723 a0 ^= b0; 724 b0 = pg_c[0]; 725 726 pg_b += 4; 727 a1 ^= b1; 728 729 a2 ^= b2; 730 a3 ^= b3; 731 732 b1 = pg_c[1]; 733 a0 ^= b0; 734 735 b2 = pg_c[2]; 736 a1 ^= b1; 737 738 b3 = pg_c[3]; 739 a2 ^= b2; 740 741 pg_dst[0] = a0; 742 a3 ^= b3; 743 pg_dst[1] = a1; 744 pg_c += 4; 745 pg_dst[2] = a2; 746 pg_dst[3] = a3; 747 pg_dst += 4; 748 } 749 while (longs_this_time > 0) { /* cannot cross any page 750 * boundaries here */ 751 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++; 752 longs_this_time--; 753 } 754 755 if (len) { 756 if (RF_PAGE_ALIGNED(a)) { 757 pg_a = a; 758 if (!pg_a) 759 return (EFAULT); 760 if (dst_is_a) 761 pg_dst = pg_a; 762 } 763 if (RF_PAGE_ALIGNED(b)) { 764 pg_b = b; 765 if (!pg_b) 766 return (EFAULT); 767 } 768 if (RF_PAGE_ALIGNED(c)) { 769 pg_c = c; 770 if (!pg_c) 771 return (EFAULT); 772 } 773 if (!dst_is_a) 774 if (RF_PAGE_ALIGNED(dst)) { 775 pg_dst = dst; 776 if (!pg_dst) 777 return (EFAULT); 778 } 779 } 780 } 781 while (len) { 782 *pg_dst++ = *pg_a++ ^ *pg_b++ ^ *pg_c++; 783 dst++; 784 a++; 785 b++; 786 c++; 787 if (RF_PAGE_ALIGNED(a)) { 788 pg_a = a; 789 if (!pg_a) 790 return (EFAULT); 791 if (dst_is_a) 792 pg_dst = pg_a; 793 } 794 if (RF_PAGE_ALIGNED(b)) { 795 pg_b = b; 796 if (!pg_b) 797 return (EFAULT); 798 } 799 if (RF_PAGE_ALIGNED(c)) { 800 pg_c = c; 801 if (!pg_c) 802 return (EFAULT); 803 } 804 if (!dst_is_a) 805 if (RF_PAGE_ALIGNED(dst)) { 806 pg_dst = dst; 807 if (!pg_dst) 808 return (EFAULT); 809 } 810 len--; 811 } 812 return (0); 813 } 814 815 int 816 rf_bxor3(unsigned char *dst, unsigned char *a, unsigned char *b, 817 unsigned char *c, unsigned long len, void *bp) 818 { 819 RF_ASSERT(((RF_UL(dst) | RF_UL(a) | RF_UL(b) | RF_UL(c) | len) & 0x7) == 0); 820 821 return (rf_longword_bxor3((unsigned long *) dst, (unsigned long *) a, 822 (unsigned long *) b, (unsigned long *) c, len >> RF_LONGSHIFT, bp)); 823 } 824 #endif 825