1dnl PowerPC-64 mpn_mul_basecase. 2 3dnl Copyright 1999-2001, 2003-2006, 2008 Free Software Foundation, Inc. 4 5dnl This file is part of the GNU MP Library. 6dnl 7dnl The GNU MP Library is free software; you can redistribute it and/or modify 8dnl it under the terms of either: 9dnl 10dnl * the GNU Lesser General Public License as published by the Free 11dnl Software Foundation; either version 3 of the License, or (at your 12dnl option) any later version. 13dnl 14dnl or 15dnl 16dnl * the GNU General Public License as published by the Free Software 17dnl Foundation; either version 2 of the License, or (at your option) any 18dnl later version. 19dnl 20dnl or both in parallel, as here. 21dnl 22dnl The GNU MP Library is distributed in the hope that it will be useful, but 23dnl WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 24dnl or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 25dnl for more details. 26dnl 27dnl You should have received copies of the GNU General Public License and the 28dnl GNU Lesser General Public License along with the GNU MP Library. If not, 29dnl see https://www.gnu.org/licenses/. 30 31include(`../config.m4') 32 33C cycles/limb 34C POWER3/PPC630 6-18 35C POWER4/PPC970 8 36C POWER5 8 37C POWER6 24 38 39C INPUT PARAMETERS 40define(`rp', `r3') 41define(`up', `r4') 42define(`un', `r5') 43define(`vp', `r6') 44define(`vn', `r7') 45 46define(`v0', `r25') 47define(`outer_rp', `r22') 48define(`outer_up', `r23') 49 50ASM_START() 51PROLOGUE(mpn_mul_basecase) 52 53C Special code for un <= 2, for efficiency of these important cases, 54C and since it simplifies the default code. 55 cmpdi cr0, un, 2 56 bgt cr0, L(un_gt2) 57 cmpdi cr6, vn, 1 58 ld r7, 0(vp) 59 ld r5, 0(up) 60 mulld r8, r5, r7 C weight 0 61 mulhdu r9, r5, r7 C weight 1 62 std r8, 0(rp) 63 beq cr0, L(2x) 64 std r9, 8(rp) 65 blr 66 ALIGN(16) 67L(2x): ld r0, 8(up) 68 mulld r8, r0, r7 C weight 1 69 mulhdu r10, r0, r7 C weight 2 70 addc r9, r9, r8 71 addze r10, r10 72 bne cr6, L(2x2) 73 std r9, 8(rp) 74 std r10, 16(rp) 75 blr 76 ALIGN(16) 77L(2x2): ld r6, 8(vp) 78 nop 79 mulld r8, r5, r6 C weight 1 80 mulhdu r11, r5, r6 C weight 2 81 addc r9, r9, r8 82 std r9, 8(rp) 83 adde r11, r11, r10 84 mulld r12, r0, r6 C weight 2 85 mulhdu r0, r0, r6 C weight 3 86 addze r0, r0 87 addc r11, r11, r12 88 addze r0, r0 89 std r11, 16(rp) 90 std r0, 24(rp) 91 blr 92 93L(un_gt2): 94 std r31, -8(r1) 95 std r30, -16(r1) 96 std r29, -24(r1) 97 std r28, -32(r1) 98 std r27, -40(r1) 99 std r26, -48(r1) 100 std r25, -56(r1) 101 std r24, -64(r1) 102 std r23, -72(r1) 103 std r22, -80(r1) 104 105 mr outer_rp, rp 106 mr outer_up, up 107 108 ld v0, 0(vp) C new v limb 109 addi vp, vp, 8 110 ld r26, 0(up) 111 112 rldicl. r0, un, 0,62 C r0 = n & 3, set cr0 113 cmpdi cr6, r0, 2 114 addi un, un, 1 C compute count... 115 srdi un, un, 2 C ...for ctr 116 mtctr un C copy inner loop count into ctr 117 beq cr0, L(b0) 118 blt cr6, L(b1) 119 beq cr6, L(b2) 120 121 122 ALIGN(16) 123L(b3): mulld r0, r26, v0 124 mulhdu r12, r26, v0 125 addic r0, r0, 0 126 std r0, 0(rp) 127 ld r26, 8(up) 128 ld r27, 16(up) 129 bdz L(end_m_3) 130 131 ALIGN(16) 132L(lo_m_3): 133 mulld r0, r26, v0 134 mulhdu r31, r26, v0 135 ld r26, 24(up) 136 nop 137 mulld r24, r27, v0 138 mulhdu r8, r27, v0 139 ld r27, 32(up) 140 nop 141 adde r0, r0, r12 142 adde r24, r24, r31 143 mulld r9, r26, v0 144 mulhdu r10, r26, v0 145 ld r26, 40(up) 146 nop 147 mulld r11, r27, v0 148 mulhdu r12, r27, v0 149 ld r27, 48(up) 150 std r0, 8(rp) 151 adde r9, r9, r8 152 std r24, 16(rp) 153 adde r11, r11, r10 154 std r9, 24(rp) 155 addi up, up, 32 156 std r11, 32(rp) 157 addi rp, rp, 32 158 bdnz L(lo_m_3) 159 160 ALIGN(16) 161L(end_m_3): 162 mulld r0, r26, v0 163 mulhdu r31, r26, v0 164 165 mulld r24, r27, v0 166 mulhdu r8, r27, v0 167 168 adde r0, r0, r12 169 adde r24, r24, r31 170 171 std r0, 8(rp) 172 std r24, 16(rp) 173 addze r8, r8 174 std r8, 24(rp) 175 addic. vn, vn, -1 176 beq L(ret) 177 178 ALIGN(16) 179L(outer_lo_3): 180 mtctr un C copy inner loop count into ctr 181 addi rp, outer_rp, 8 182 mr up, outer_up 183 addi outer_rp, outer_rp, 8 184 ld v0, 0(vp) C new v limb 185 addi vp, vp, 8 186 ld r26, 0(up) 187 ld r28, 0(rp) 188 mulld r0, r26, v0 189 mulhdu r12, r26, v0 190 addc r0, r0, r28 191 std r0, 0(rp) 192 ld r26, 8(up) 193 ld r27, 16(up) 194 bdz L(end_3) 195 196 ALIGN(16) C registers dying 197L(lo_3): 198 mulld r0, r26, v0 C 199 mulhdu r10, r26, v0 C 26 200 ld r26, 24(up) C 201 ld r28, 8(rp) C 202 mulld r24, r27, v0 C 203 mulhdu r8, r27, v0 C 27 204 ld r27, 32(up) C 205 ld r29, 16(rp) C 206 adde r0, r0, r12 C 0 12 207 adde r24, r24, r10 C 24 10 208 mulld r9, r26, v0 C 209 mulhdu r10, r26, v0 C 26 210 ld r26, 40(up) C 211 ld r30, 24(rp) C 212 mulld r11, r27, v0 C 213 mulhdu r12, r27, v0 C 27 214 ld r27, 48(up) C 215 ld r31, 32(rp) C 216 adde r9, r9, r8 C 8 9 217 adde r11, r11, r10 C 10 11 218 addze r12, r12 C 12 219 addc r0, r0, r28 C 0 28 220 std r0, 8(rp) C 0 221 adde r24, r24, r29 C 7 29 222 std r24, 16(rp) C 7 223 adde r9, r9, r30 C 9 30 224 std r9, 24(rp) C 9 225 adde r11, r11, r31 C 11 31 226 std r11, 32(rp) C 11 227 addi up, up, 32 C 228 addi rp, rp, 32 C 229 bdnz L(lo_3) C 230 231 ALIGN(16) 232L(end_3): 233 mulld r0, r26, v0 234 mulhdu r10, r26, v0 235 ld r28, 8(rp) 236 nop 237 mulld r24, r27, v0 238 mulhdu r8, r27, v0 239 ld r29, 16(rp) 240 nop 241 adde r0, r0, r12 242 adde r24, r24, r10 243 addze r8, r8 244 addc r0, r0, r28 245 std r0, 8(rp) 246 adde r24, r24, r29 247 std r24, 16(rp) 248 addze r8, r8 249 std r8, 24(rp) 250 251 addic. vn, vn, -1 252 bne L(outer_lo_3) 253 b L(ret) 254 255 256 ALIGN(16) 257L(b0): ld r27, 8(up) 258 addi up, up, 8 259 mulld r0, r26, v0 260 mulhdu r10, r26, v0 261 mulld r24, r27, v0 262 mulhdu r8, r27, v0 263 addc r24, r24, r10 264 addze r12, r8 265 std r0, 0(rp) 266 std r24, 8(rp) 267 addi rp, rp, 8 268 ld r26, 8(up) 269 ld r27, 16(up) 270 bdz L(end_m_0) 271 272 ALIGN(16) 273L(lo_m_0): 274 mulld r0, r26, v0 275 mulhdu r31, r26, v0 276 ld r26, 24(up) 277 nop 278 mulld r24, r27, v0 279 mulhdu r8, r27, v0 280 ld r27, 32(up) 281 nop 282 adde r0, r0, r12 283 adde r24, r24, r31 284 mulld r9, r26, v0 285 mulhdu r10, r26, v0 286 ld r26, 40(up) 287 nop 288 mulld r11, r27, v0 289 mulhdu r12, r27, v0 290 ld r27, 48(up) 291 std r0, 8(rp) 292 adde r9, r9, r8 293 std r24, 16(rp) 294 adde r11, r11, r10 295 std r9, 24(rp) 296 addi up, up, 32 297 std r11, 32(rp) 298 addi rp, rp, 32 299 bdnz L(lo_m_0) 300 301 ALIGN(16) 302L(end_m_0): 303 mulld r0, r26, v0 304 mulhdu r31, r26, v0 305 306 mulld r24, r27, v0 307 mulhdu r8, r27, v0 308 309 adde r0, r0, r12 310 adde r24, r24, r31 311 312 std r0, 8(rp) 313 addze r8, r8 314 std r24, 16(rp) 315 addic. vn, vn, -1 316 std r8, 24(rp) 317 nop 318 beq L(ret) 319 320 ALIGN(16) 321L(outer_lo_0): 322 mtctr un C copy inner loop count into ctr 323 addi rp, outer_rp, 16 324 addi up, outer_up, 8 325 addi outer_rp, outer_rp, 8 326 ld v0, 0(vp) C new v limb 327 addi vp, vp, 8 328 ld r26, -8(up) 329 ld r27, 0(up) 330 ld r28, -8(rp) 331 ld r29, 0(rp) 332 nop 333 nop 334 mulld r0, r26, v0 335 mulhdu r10, r26, v0 336 mulld r24, r27, v0 337 mulhdu r8, r27, v0 338 addc r24, r24, r10 339 addze r12, r8 340 addc r0, r0, r28 341 std r0, -8(rp) 342 adde r24, r24, r29 343 std r24, 0(rp) 344 ld r26, 8(up) 345 ld r27, 16(up) 346 bdz L(end_0) 347 348 ALIGN(16) C registers dying 349L(lo_0): 350 mulld r0, r26, v0 C 351 mulhdu r10, r26, v0 C 26 352 ld r26, 24(up) C 353 ld r28, 8(rp) C 354 mulld r24, r27, v0 C 355 mulhdu r8, r27, v0 C 27 356 ld r27, 32(up) C 357 ld r29, 16(rp) C 358 adde r0, r0, r12 C 0 12 359 adde r24, r24, r10 C 24 10 360 mulld r9, r26, v0 C 361 mulhdu r10, r26, v0 C 26 362 ld r26, 40(up) C 363 ld r30, 24(rp) C 364 mulld r11, r27, v0 C 365 mulhdu r12, r27, v0 C 27 366 ld r27, 48(up) C 367 ld r31, 32(rp) C 368 adde r9, r9, r8 C 8 9 369 adde r11, r11, r10 C 10 11 370 addze r12, r12 C 12 371 addc r0, r0, r28 C 0 28 372 std r0, 8(rp) C 0 373 adde r24, r24, r29 C 7 29 374 std r24, 16(rp) C 7 375 adde r9, r9, r30 C 9 30 376 std r9, 24(rp) C 9 377 adde r11, r11, r31 C 11 31 378 std r11, 32(rp) C 11 379 addi up, up, 32 C 380 addi rp, rp, 32 C 381 bdnz L(lo_0) C 382 383 ALIGN(16) 384L(end_0): 385 mulld r0, r26, v0 386 mulhdu r10, r26, v0 387 ld r28, 8(rp) 388 nop 389 mulld r24, r27, v0 390 mulhdu r8, r27, v0 391 ld r29, 16(rp) 392 nop 393 adde r0, r0, r12 394 adde r24, r24, r10 395 addze r8, r8 396 addic. vn, vn, -1 397 addc r0, r0, r28 398 std r0, 8(rp) 399 adde r24, r24, r29 400 std r24, 16(rp) 401 addze r8, r8 402 std r8, 24(rp) 403 bne L(outer_lo_0) 404 b L(ret) 405 406 407 ALIGN(16) 408L(b1): ld r27, 8(up) 409 nop 410 mulld r0, r26, v0 411 mulhdu r31, r26, v0 412 ld r26, 16(up) 413 mulld r24, r27, v0 414 mulhdu r8, r27, v0 415 mulld r9, r26, v0 416 mulhdu r10, r26, v0 417 addc r24, r24, r31 418 adde r9, r9, r8 419 addze r12, r10 420 std r0, 0(rp) 421 std r24, 8(rp) 422 std r9, 16(rp) 423 addi up, up, 16 424 addi rp, rp, 16 425 ld r26, 8(up) 426 ld r27, 16(up) 427 bdz L(end_m_1) 428 429 ALIGN(16) 430L(lo_m_1): 431 mulld r0, r26, v0 432 mulhdu r31, r26, v0 433 ld r26, 24(up) 434 nop 435 mulld r24, r27, v0 436 mulhdu r8, r27, v0 437 ld r27, 32(up) 438 nop 439 adde r0, r0, r12 440 adde r24, r24, r31 441 mulld r9, r26, v0 442 mulhdu r10, r26, v0 443 ld r26, 40(up) 444 nop 445 mulld r11, r27, v0 446 mulhdu r12, r27, v0 447 ld r27, 48(up) 448 std r0, 8(rp) 449 adde r9, r9, r8 450 std r24, 16(rp) 451 adde r11, r11, r10 452 std r9, 24(rp) 453 addi up, up, 32 454 std r11, 32(rp) 455 addi rp, rp, 32 456 bdnz L(lo_m_1) 457 458 ALIGN(16) 459L(end_m_1): 460 mulld r0, r26, v0 461 mulhdu r31, r26, v0 462 463 mulld r24, r27, v0 464 mulhdu r8, r27, v0 465 466 adde r0, r0, r12 467 adde r24, r24, r31 468 469 std r0, 8(rp) 470 addze r8, r8 471 std r24, 16(rp) 472 addic. vn, vn, -1 473 std r8, 24(rp) 474 nop 475 beq L(ret) 476 477 ALIGN(16) 478L(outer_lo_1): 479 mtctr un C copy inner loop count into ctr 480 addi rp, outer_rp, 24 481 addi up, outer_up, 16 482 addi outer_rp, outer_rp, 8 483 ld v0, 0(vp) C new v limb 484 addi vp, vp, 8 485 ld r26, -16(up) 486 ld r27, -8(up) 487 mulld r0, r26, v0 488 mulhdu r31, r26, v0 489 ld r26, 0(up) 490 ld r28, -16(rp) 491 mulld r24, r27, v0 492 mulhdu r8, r27, v0 493 ld r29, -8(rp) 494 ld r30, 0(rp) 495 mulld r9, r26, v0 496 mulhdu r10, r26, v0 497 addc r24, r24, r31 498 adde r9, r9, r8 499 addze r12, r10 500 addc r0, r0, r28 501 std r0, -16(rp) 502 adde r24, r24, r29 503 std r24, -8(rp) 504 adde r9, r9, r30 505 std r9, 0(rp) 506 ld r26, 8(up) 507 ld r27, 16(up) 508 bdz L(end_1) 509 510 ALIGN(16) C registers dying 511L(lo_1): 512 mulld r0, r26, v0 C 513 mulhdu r10, r26, v0 C 26 514 ld r26, 24(up) C 515 ld r28, 8(rp) C 516 mulld r24, r27, v0 C 517 mulhdu r8, r27, v0 C 27 518 ld r27, 32(up) C 519 ld r29, 16(rp) C 520 adde r0, r0, r12 C 0 12 521 adde r24, r24, r10 C 24 10 522 mulld r9, r26, v0 C 523 mulhdu r10, r26, v0 C 26 524 ld r26, 40(up) C 525 ld r30, 24(rp) C 526 mulld r11, r27, v0 C 527 mulhdu r12, r27, v0 C 27 528 ld r27, 48(up) C 529 ld r31, 32(rp) C 530 adde r9, r9, r8 C 8 9 531 adde r11, r11, r10 C 10 11 532 addze r12, r12 C 12 533 addc r0, r0, r28 C 0 28 534 std r0, 8(rp) C 0 535 adde r24, r24, r29 C 7 29 536 std r24, 16(rp) C 7 537 adde r9, r9, r30 C 9 30 538 std r9, 24(rp) C 9 539 adde r11, r11, r31 C 11 31 540 std r11, 32(rp) C 11 541 addi up, up, 32 C 542 addi rp, rp, 32 C 543 bdnz L(lo_1) C 544 545 ALIGN(16) 546L(end_1): 547 mulld r0, r26, v0 548 mulhdu r10, r26, v0 549 ld r28, 8(rp) 550 nop 551 mulld r24, r27, v0 552 mulhdu r8, r27, v0 553 ld r29, 16(rp) 554 nop 555 adde r0, r0, r12 556 adde r24, r24, r10 557 addze r8, r8 558 addic. vn, vn, -1 559 addc r0, r0, r28 560 std r0, 8(rp) 561 adde r24, r24, r29 562 std r24, 16(rp) 563 addze r8, r8 564 std r8, 24(rp) 565 bne L(outer_lo_1) 566 b L(ret) 567 568 569 ALIGN(16) 570L(b2): ld r27, 8(up) 571 addi up, up, -8 572 addi rp, rp, -8 573 li r12, 0 574 addic r12, r12, 0 575 576 ALIGN(16) 577L(lo_m_2): 578 mulld r0, r26, v0 579 mulhdu r31, r26, v0 580 ld r26, 24(up) 581 nop 582 mulld r24, r27, v0 583 mulhdu r8, r27, v0 584 ld r27, 32(up) 585 nop 586 adde r0, r0, r12 587 adde r24, r24, r31 588 mulld r9, r26, v0 589 mulhdu r10, r26, v0 590 ld r26, 40(up) 591 nop 592 mulld r11, r27, v0 593 mulhdu r12, r27, v0 594 ld r27, 48(up) 595 std r0, 8(rp) 596 adde r9, r9, r8 597 std r24, 16(rp) 598 adde r11, r11, r10 599 std r9, 24(rp) 600 addi up, up, 32 601 std r11, 32(rp) 602 603 addi rp, rp, 32 604 bdnz L(lo_m_2) 605 606 ALIGN(16) 607L(end_m_2): 608 mulld r0, r26, v0 609 mulhdu r31, r26, v0 610 611 mulld r24, r27, v0 612 mulhdu r8, r27, v0 613 614 adde r0, r0, r12 615 adde r24, r24, r31 616 617 std r0, 8(rp) 618 addze r8, r8 619 std r24, 16(rp) 620 addic. vn, vn, -1 621 std r8, 24(rp) 622 nop 623 beq L(ret) 624 625 ALIGN(16) 626L(outer_lo_2): 627 mtctr un C copy inner loop count into ctr 628 addi rp, outer_rp, 0 629 addi up, outer_up, -8 630 addi outer_rp, outer_rp, 8 631 ld v0, 0(vp) C new v limb 632 addi vp, vp, 8 633 ld r26, 8(up) 634 ld r27, 16(up) 635 li r12, 0 636 addic r12, r12, 0 637 638 ALIGN(16) C registers dying 639L(lo_2): 640 mulld r0, r26, v0 C 641 mulhdu r10, r26, v0 C 26 642 ld r26, 24(up) C 643 ld r28, 8(rp) C 644 mulld r24, r27, v0 C 645 mulhdu r8, r27, v0 C 27 646 ld r27, 32(up) C 647 ld r29, 16(rp) C 648 adde r0, r0, r12 C 0 12 649 adde r24, r24, r10 C 24 10 650 mulld r9, r26, v0 C 651 mulhdu r10, r26, v0 C 26 652 ld r26, 40(up) C 653 ld r30, 24(rp) C 654 mulld r11, r27, v0 C 655 mulhdu r12, r27, v0 C 27 656 ld r27, 48(up) C 657 ld r31, 32(rp) C 658 adde r9, r9, r8 C 8 9 659 adde r11, r11, r10 C 10 11 660 addze r12, r12 C 12 661 addc r0, r0, r28 C 0 28 662 std r0, 8(rp) C 0 663 adde r24, r24, r29 C 7 29 664 std r24, 16(rp) C 7 665 adde r9, r9, r30 C 9 30 666 std r9, 24(rp) C 9 667 adde r11, r11, r31 C 11 31 668 std r11, 32(rp) C 11 669 addi up, up, 32 C 670 addi rp, rp, 32 C 671 bdnz L(lo_2) C 672 673 ALIGN(16) 674L(end_2): 675 mulld r0, r26, v0 676 mulhdu r10, r26, v0 677 ld r28, 8(rp) 678 nop 679 mulld r24, r27, v0 680 mulhdu r8, r27, v0 681 ld r29, 16(rp) 682 nop 683 adde r0, r0, r12 684 adde r24, r24, r10 685 addze r8, r8 686 addic. vn, vn, -1 687 addc r0, r0, r28 688 std r0, 8(rp) 689 adde r24, r24, r29 690 std r24, 16(rp) 691 addze r8, r8 692 std r8, 24(rp) 693 bne L(outer_lo_2) 694 b L(ret) 695 696 697L(ret): ld r31, -8(r1) 698 ld r30, -16(r1) 699 ld r29, -24(r1) 700 ld r28, -32(r1) 701 ld r27, -40(r1) 702 ld r26, -48(r1) 703 ld r25, -56(r1) 704 ld r24, -64(r1) 705 ld r23, -72(r1) 706 ld r22, -80(r1) 707 blr 708EPILOGUE() 709