Lines Matching +full:0 +full:x86
1 //===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9 // This file contains the X86 implementation of the TargetInstrInfo class.
14 #include "X86.h"
52 #define DEBUG_TYPE "x86-instr-info"
64 " fuse, but the X86 backend currently can't"),
86 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64
87 : X86::ADJCALLSTACKDOWN32),
88 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
89 : X86::ADJCALLSTACKUP32),
90 X86::CATCHRET, (STI.is64Bit() ? X86::RET64 : X86::RET32)),
109 case X86::GR8RegClassID:
110 return &X86::GR8_NOREX2RegClass;
111 case X86::GR16RegClassID:
112 return &X86::GR16_NOREX2RegClass;
113 case X86::GR32RegClassID:
114 return &X86::GR32_NOREX2RegClass;
115 case X86::GR64RegClassID:
116 return &X86::GR64_NOREX2RegClass;
117 case X86::GR32_NOSPRegClassID:
118 return &X86::GR32_NOREX2_NOSPRegClass;
119 case X86::GR64_NOSPRegClassID:
120 return &X86::GR64_NOREX2_NOSPRegClass;
130 case X86::MOVSX16rr8:
131 case X86::MOVZX16rr8:
132 case X86::MOVSX32rr8:
133 case X86::MOVZX32rr8:
134 case X86::MOVSX64rr8:
140 case X86::MOVSX32rr16:
141 case X86::MOVZX32rr16:
142 case X86::MOVSX64rr16:
143 case X86::MOVSX64rr32: {
144 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
148 DstReg = MI.getOperand(0).getReg();
152 case X86::MOVSX16rr8:
153 case X86::MOVZX16rr8:
154 case X86::MOVSX32rr8:
155 case X86::MOVZX32rr8:
156 case X86::MOVSX64rr8:
157 SubIdx = X86::sub_8bit;
159 case X86::MOVSX32rr16:
160 case X86::MOVZX32rr16:
161 case X86::MOVSX64rr16:
162 SubIdx = X86::sub_16bit;
164 case X86::MOVSX64rr32:
165 SubIdx = X86::sub_32bit;
184 using namespace X86;
185 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
192 // operations like popcnt, but are believed to be constant time on x86.
247 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
250 case X86::IMUL16rm:
251 case X86::IMUL16rmi:
252 case X86::IMUL32rm:
253 case X86::IMUL32rmi:
254 case X86::IMUL64rm:
255 case X86::IMUL64rmi32:
259 // operations like popcnt, but are believed to be constant time on x86.
261 case X86::BSF16rm:
262 case X86::BSF32rm:
263 case X86::BSF64rm:
264 case X86::BSR16rm:
265 case X86::BSR32rm:
266 case X86::BSR64rm:
267 case X86::LZCNT16rm:
268 case X86::LZCNT32rm:
269 case X86::LZCNT64rm:
270 case X86::POPCNT16rm:
271 case X86::POPCNT32rm:
272 case X86::POPCNT64rm:
273 case X86::TZCNT16rm:
274 case X86::TZCNT32rm:
275 case X86::TZCNT64rm:
280 case X86::BLCFILL32rm:
281 case X86::BLCFILL64rm:
282 case X86::BLCI32rm:
283 case X86::BLCI64rm:
284 case X86::BLCIC32rm:
285 case X86::BLCIC64rm:
286 case X86::BLCMSK32rm:
287 case X86::BLCMSK64rm:
288 case X86::BLCS32rm:
289 case X86::BLCS64rm:
290 case X86::BLSFILL32rm:
291 case X86::BLSFILL64rm:
292 case X86::BLSI32rm:
293 case X86::BLSI64rm:
294 case X86::BLSIC32rm:
295 case X86::BLSIC64rm:
296 case X86::BLSMSK32rm:
297 case X86::BLSMSK64rm:
298 case X86::BLSR32rm:
299 case X86::BLSR64rm:
300 case X86::TZMSK32rm:
301 case X86::TZMSK64rm:
305 case X86::BEXTR32rm:
306 case X86::BEXTR64rm:
307 case X86::BEXTRI32mi:
308 case X86::BEXTRI64mi:
309 case X86::BZHI32rm:
310 case X86::BZHI64rm:
313 case X86::ADC8rm:
314 case X86::ADC16rm:
315 case X86::ADC32rm:
316 case X86::ADC64rm:
317 case X86::ADD8rm:
318 case X86::ADD16rm:
319 case X86::ADD32rm:
320 case X86::ADD64rm:
321 case X86::AND8rm:
322 case X86::AND16rm:
323 case X86::AND32rm:
324 case X86::AND64rm:
325 case X86::ANDN32rm:
326 case X86::ANDN64rm:
327 case X86::OR8rm:
328 case X86::OR16rm:
329 case X86::OR32rm:
330 case X86::OR64rm:
331 case X86::SBB8rm:
332 case X86::SBB16rm:
333 case X86::SBB32rm:
334 case X86::SBB64rm:
335 case X86::SUB8rm:
336 case X86::SUB16rm:
337 case X86::SUB32rm:
338 case X86::SUB64rm:
339 case X86::XOR8rm:
340 case X86::XOR16rm:
341 case X86::XOR32rm:
342 case X86::XOR64rm:
345 // time on x86. Called out separately as this is among the most surprising
347 case X86::MULX32rm:
348 case X86::MULX64rm:
351 case X86::RORX32mi:
352 case X86::RORX64mi:
353 case X86::SARX32rm:
354 case X86::SARX64rm:
355 case X86::SHLX32rm:
356 case X86::SHLX64rm:
357 case X86::SHRX32rm:
358 case X86::SHRX64rm:
361 case X86::CVTTSD2SI64rm:
362 case X86::VCVTTSD2SI64rm:
363 case X86::VCVTTSD2SI64Zrm:
364 case X86::CVTTSD2SIrm:
365 case X86::VCVTTSD2SIrm:
366 case X86::VCVTTSD2SIZrm:
367 case X86::CVTTSS2SI64rm:
368 case X86::VCVTTSS2SI64rm:
369 case X86::VCVTTSS2SI64Zrm:
370 case X86::CVTTSS2SIrm:
371 case X86::VCVTTSS2SIrm:
372 case X86::VCVTTSS2SIZrm:
373 case X86::CVTSI2SDrm:
374 case X86::VCVTSI2SDrm:
375 case X86::VCVTSI2SDZrm:
376 case X86::CVTSI2SSrm:
377 case X86::VCVTSI2SSrm:
378 case X86::VCVTSI2SSZrm:
379 case X86::CVTSI642SDrm:
380 case X86::VCVTSI642SDrm:
381 case X86::VCVTSI642SDZrm:
382 case X86::CVTSI642SSrm:
383 case X86::VCVTSI642SSrm:
384 case X86::VCVTSI642SSZrm:
385 case X86::CVTSS2SDrm:
386 case X86::VCVTSS2SDrm:
387 case X86::VCVTSS2SDZrm:
388 case X86::CVTSD2SSrm:
389 case X86::VCVTSD2SSrm:
390 case X86::VCVTSD2SSZrm:
392 case X86::VCVTTSD2USI64Zrm:
393 case X86::VCVTTSD2USIZrm:
394 case X86::VCVTTSS2USI64Zrm:
395 case X86::VCVTTSS2USIZrm:
396 case X86::VCVTUSI2SDZrm:
397 case X86::VCVTUSI642SDZrm:
398 case X86::VCVTUSI2SSZrm:
399 case X86::VCVTUSI642SSZrm:
402 case X86::MOV8rm:
403 case X86::MOV8rm_NOREX:
404 case X86::MOV16rm:
405 case X86::MOV32rm:
406 case X86::MOV64rm:
407 case X86::MOVSX16rm8:
408 case X86::MOVSX32rm16:
409 case X86::MOVSX32rm8:
410 case X86::MOVSX32rm8_NOREX:
411 case X86::MOVSX64rm16:
412 case X86::MOVSX64rm32:
413 case X86::MOVSX64rm8:
414 case X86::MOVZX16rm8:
415 case X86::MOVZX32rm16:
416 case X86::MOVZX32rm8:
417 case X86::MOVZX32rm8_NOREX:
418 case X86::MOVZX64rm16:
419 case X86::MOVZX64rm8:
450 return 0;
459 return 0;
460 case X86::PUSH32r:
461 case X86::PUSH32rmm:
462 case X86::PUSH32rmr:
463 case X86::PUSH32i:
465 case X86::PUSH64r:
466 case X86::PUSH64rmm:
467 case X86::PUSH64rmr:
468 case X86::PUSH64i32:
477 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() &&
478 MI.getOperand(Op + X86::AddrScaleAmt).isImm() &&
479 MI.getOperand(Op + X86::AddrIndexReg).isReg() &&
480 MI.getOperand(Op + X86::AddrDisp).isImm() &&
481 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 &&
482 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 &&
483 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) {
484 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex();
494 case X86::MOV8rm:
495 case X86::KMOVBkm:
496 case X86::KMOVBkm_EVEX:
499 case X86::MOV16rm:
500 case X86::KMOVWkm:
501 case X86::KMOVWkm_EVEX:
502 case X86::VMOVSHZrm:
503 case X86::VMOVSHZrm_alt:
506 case X86::MOV32rm:
507 case X86::MOVSSrm:
508 case X86::MOVSSrm_alt:
509 case X86::VMOVSSrm:
510 case X86::VMOVSSrm_alt:
511 case X86::VMOVSSZrm:
512 case X86::VMOVSSZrm_alt:
513 case X86::KMOVDkm:
514 case X86::KMOVDkm_EVEX:
517 case X86::MOV64rm:
518 case X86::LD_Fp64m:
519 case X86::MOVSDrm:
520 case X86::MOVSDrm_alt:
521 case X86::VMOVSDrm:
522 case X86::VMOVSDrm_alt:
523 case X86::VMOVSDZrm:
524 case X86::VMOVSDZrm_alt:
525 case X86::MMX_MOVD64rm:
526 case X86::MMX_MOVQ64rm:
527 case X86::KMOVQkm:
528 case X86::KMOVQkm_EVEX:
531 case X86::MOVAPSrm:
532 case X86::MOVUPSrm:
533 case X86::MOVAPDrm:
534 case X86::MOVUPDrm:
535 case X86::MOVDQArm:
536 case X86::MOVDQUrm:
537 case X86::VMOVAPSrm:
538 case X86::VMOVUPSrm:
539 case X86::VMOVAPDrm:
540 case X86::VMOVUPDrm:
541 case X86::VMOVDQArm:
542 case X86::VMOVDQUrm:
543 case X86::VMOVAPSZ128rm:
544 case X86::VMOVUPSZ128rm:
545 case X86::VMOVAPSZ128rm_NOVLX:
546 case X86::VMOVUPSZ128rm_NOVLX:
547 case X86::VMOVAPDZ128rm:
548 case X86::VMOVUPDZ128rm:
549 case X86::VMOVDQU8Z128rm:
550 case X86::VMOVDQU16Z128rm:
551 case X86::VMOVDQA32Z128rm:
552 case X86::VMOVDQU32Z128rm:
553 case X86::VMOVDQA64Z128rm:
554 case X86::VMOVDQU64Z128rm:
557 case X86::VMOVAPSYrm:
558 case X86::VMOVUPSYrm:
559 case X86::VMOVAPDYrm:
560 case X86::VMOVUPDYrm:
561 case X86::VMOVDQAYrm:
562 case X86::VMOVDQUYrm:
563 case X86::VMOVAPSZ256rm:
564 case X86::VMOVUPSZ256rm:
565 case X86::VMOVAPSZ256rm_NOVLX:
566 case X86::VMOVUPSZ256rm_NOVLX:
567 case X86::VMOVAPDZ256rm:
568 case X86::VMOVUPDZ256rm:
569 case X86::VMOVDQU8Z256rm:
570 case X86::VMOVDQU16Z256rm:
571 case X86::VMOVDQA32Z256rm:
572 case X86::VMOVDQU32Z256rm:
573 case X86::VMOVDQA64Z256rm:
574 case X86::VMOVDQU64Z256rm:
577 case X86::VMOVAPSZrm:
578 case X86::VMOVUPSZrm:
579 case X86::VMOVAPDZrm:
580 case X86::VMOVUPDZrm:
581 case X86::VMOVDQU8Zrm:
582 case X86::VMOVDQU16Zrm:
583 case X86::VMOVDQA32Zrm:
584 case X86::VMOVDQU32Zrm:
585 case X86::VMOVDQA64Zrm:
586 case X86::VMOVDQU64Zrm:
596 case X86::MOV8mr:
597 case X86::KMOVBmk:
598 case X86::KMOVBmk_EVEX:
601 case X86::MOV16mr:
602 case X86::KMOVWmk:
603 case X86::KMOVWmk_EVEX:
604 case X86::VMOVSHZmr:
607 case X86::MOV32mr:
608 case X86::MOVSSmr:
609 case X86::VMOVSSmr:
610 case X86::VMOVSSZmr:
611 case X86::KMOVDmk:
612 case X86::KMOVDmk_EVEX:
615 case X86::MOV64mr:
616 case X86::ST_FpP64m:
617 case X86::MOVSDmr:
618 case X86::VMOVSDmr:
619 case X86::VMOVSDZmr:
620 case X86::MMX_MOVD64mr:
621 case X86::MMX_MOVQ64mr:
622 case X86::MMX_MOVNTQmr:
623 case X86::KMOVQmk:
624 case X86::KMOVQmk_EVEX:
627 case X86::MOVAPSmr:
628 case X86::MOVUPSmr:
629 case X86::MOVAPDmr:
630 case X86::MOVUPDmr:
631 case X86::MOVDQAmr:
632 case X86::MOVDQUmr:
633 case X86::VMOVAPSmr:
634 case X86::VMOVUPSmr:
635 case X86::VMOVAPDmr:
636 case X86::VMOVUPDmr:
637 case X86::VMOVDQAmr:
638 case X86::VMOVDQUmr:
639 case X86::VMOVUPSZ128mr:
640 case X86::VMOVAPSZ128mr:
641 case X86::VMOVUPSZ128mr_NOVLX:
642 case X86::VMOVAPSZ128mr_NOVLX:
643 case X86::VMOVUPDZ128mr:
644 case X86::VMOVAPDZ128mr:
645 case X86::VMOVDQA32Z128mr:
646 case X86::VMOVDQU32Z128mr:
647 case X86::VMOVDQA64Z128mr:
648 case X86::VMOVDQU64Z128mr:
649 case X86::VMOVDQU8Z128mr:
650 case X86::VMOVDQU16Z128mr:
653 case X86::VMOVUPSYmr:
654 case X86::VMOVAPSYmr:
655 case X86::VMOVUPDYmr:
656 case X86::VMOVAPDYmr:
657 case X86::VMOVDQUYmr:
658 case X86::VMOVDQAYmr:
659 case X86::VMOVUPSZ256mr:
660 case X86::VMOVAPSZ256mr:
661 case X86::VMOVUPSZ256mr_NOVLX:
662 case X86::VMOVAPSZ256mr_NOVLX:
663 case X86::VMOVUPDZ256mr:
664 case X86::VMOVAPDZ256mr:
665 case X86::VMOVDQU8Z256mr:
666 case X86::VMOVDQU16Z256mr:
667 case X86::VMOVDQA32Z256mr:
668 case X86::VMOVDQU32Z256mr:
669 case X86::VMOVDQA64Z256mr:
670 case X86::VMOVDQU64Z256mr:
673 case X86::VMOVUPSZmr:
674 case X86::VMOVAPSZmr:
675 case X86::VMOVUPDZmr:
676 case X86::VMOVAPDZmr:
677 case X86::VMOVDQU8Zmr:
678 case X86::VMOVDQU16Zmr:
679 case X86::VMOVDQA32Zmr:
680 case X86::VMOVDQU32Zmr:
681 case X86::VMOVDQA64Zmr:
682 case X86::VMOVDQU64Zmr:
699 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex))
700 return MI.getOperand(0).getReg();
701 return 0;
717 return MI.getOperand(0).getReg();
720 return 0;
733 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 &&
734 isFrameOperand(MI, 0, FrameIndex))
735 return MI.getOperand(X86::AddrNumOperands).getReg();
736 return 0;
752 return MI.getOperand(X86::AddrNumOperands).getReg();
755 return 0;
758 /// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
765 if (DefMI.getOpcode() != X86::MOVPC32r)
781 case X86::IMPLICIT_DEF:
784 case X86::LOAD_STACK_GUARD:
785 case X86::LD_Fp032:
786 case X86::LD_Fp064:
787 case X86::LD_Fp080:
788 case X86::LD_Fp132:
789 case X86::LD_Fp164:
790 case X86::LD_Fp180:
791 case X86::AVX1_SETALLONES:
792 case X86::AVX2_SETALLONES:
793 case X86::AVX512_128_SET0:
794 case X86::AVX512_256_SET0:
795 case X86::AVX512_512_SET0:
796 case X86::AVX512_512_SETALLONES:
797 case X86::AVX512_FsFLD0SD:
798 case X86::AVX512_FsFLD0SH:
799 case X86::AVX512_FsFLD0SS:
800 case X86::AVX512_FsFLD0F128:
801 case X86::AVX_SET0:
802 case X86::FsFLD0SD:
803 case X86::FsFLD0SS:
804 case X86::FsFLD0SH:
805 case X86::FsFLD0F128:
806 case X86::KSET0D:
807 case X86::KSET0Q:
808 case X86::KSET0W:
809 case X86::KSET1D:
810 case X86::KSET1Q:
811 case X86::KSET1W:
812 case X86::MMX_SET0:
813 case X86::MOV32ImmSExti8:
814 case X86::MOV32r0:
815 case X86::MOV32r1:
816 case X86::MOV32r_1:
817 case X86::MOV32ri64:
818 case X86::MOV64ImmSExti8:
819 case X86::V_SET0:
820 case X86::V_SETALLONES:
821 case X86::MOV16ri:
822 case X86::MOV32ri:
823 case X86::MOV64ri:
824 case X86::MOV64ri32:
825 case X86::MOV8ri:
826 case X86::PTILEZEROV:
829 case X86::MOV8rm:
830 case X86::MOV8rm_NOREX:
831 case X86::MOV16rm:
832 case X86::MOV32rm:
833 case X86::MOV64rm:
834 case X86::MOVSSrm:
835 case X86::MOVSSrm_alt:
836 case X86::MOVSDrm:
837 case X86::MOVSDrm_alt:
838 case X86::MOVAPSrm:
839 case X86::MOVUPSrm:
840 case X86::MOVAPDrm:
841 case X86::MOVUPDrm:
842 case X86::MOVDQArm:
843 case X86::MOVDQUrm:
844 case X86::VMOVSSrm:
845 case X86::VMOVSSrm_alt:
846 case X86::VMOVSDrm:
847 case X86::VMOVSDrm_alt:
848 case X86::VMOVAPSrm:
849 case X86::VMOVUPSrm:
850 case X86::VMOVAPDrm:
851 case X86::VMOVUPDrm:
852 case X86::VMOVDQArm:
853 case X86::VMOVDQUrm:
854 case X86::VMOVAPSYrm:
855 case X86::VMOVUPSYrm:
856 case X86::VMOVAPDYrm:
857 case X86::VMOVUPDYrm:
858 case X86::VMOVDQAYrm:
859 case X86::VMOVDQUYrm:
860 case X86::MMX_MOVD64rm:
861 case X86::MMX_MOVQ64rm:
862 case X86::VBROADCASTSSrm:
863 case X86::VBROADCASTSSYrm:
864 case X86::VBROADCASTSDYrm:
866 case X86::VPBROADCASTBZ128rm:
867 case X86::VPBROADCASTBZ256rm:
868 case X86::VPBROADCASTBZrm:
869 case X86::VBROADCASTF32X2Z256rm:
870 case X86::VBROADCASTF32X2Zrm:
871 case X86::VBROADCASTI32X2Z128rm:
872 case X86::VBROADCASTI32X2Z256rm:
873 case X86::VBROADCASTI32X2Zrm:
874 case X86::VPBROADCASTWZ128rm:
875 case X86::VPBROADCASTWZ256rm:
876 case X86::VPBROADCASTWZrm:
877 case X86::VPBROADCASTDZ128rm:
878 case X86::VPBROADCASTDZ256rm:
879 case X86::VPBROADCASTDZrm:
880 case X86::VBROADCASTSSZ128rm:
881 case X86::VBROADCASTSSZ256rm:
882 case X86::VBROADCASTSSZrm:
883 case X86::VPBROADCASTQZ128rm:
884 case X86::VPBROADCASTQZ256rm:
885 case X86::VPBROADCASTQZrm:
886 case X86::VBROADCASTSDZ256rm:
887 case X86::VBROADCASTSDZrm:
888 case X86::VMOVSSZrm:
889 case X86::VMOVSSZrm_alt:
890 case X86::VMOVSDZrm:
891 case X86::VMOVSDZrm_alt:
892 case X86::VMOVSHZrm:
893 case X86::VMOVSHZrm_alt:
894 case X86::VMOVAPDZ128rm:
895 case X86::VMOVAPDZ256rm:
896 case X86::VMOVAPDZrm:
897 case X86::VMOVAPSZ128rm:
898 case X86::VMOVAPSZ256rm:
899 case X86::VMOVAPSZ128rm_NOVLX:
900 case X86::VMOVAPSZ256rm_NOVLX:
901 case X86::VMOVAPSZrm:
902 case X86::VMOVDQA32Z128rm:
903 case X86::VMOVDQA32Z256rm:
904 case X86::VMOVDQA32Zrm:
905 case X86::VMOVDQA64Z128rm:
906 case X86::VMOVDQA64Z256rm:
907 case X86::VMOVDQA64Zrm:
908 case X86::VMOVDQU16Z128rm:
909 case X86::VMOVDQU16Z256rm:
910 case X86::VMOVDQU16Zrm:
911 case X86::VMOVDQU32Z128rm:
912 case X86::VMOVDQU32Z256rm:
913 case X86::VMOVDQU32Zrm:
914 case X86::VMOVDQU64Z128rm:
915 case X86::VMOVDQU64Z256rm:
916 case X86::VMOVDQU64Zrm:
917 case X86::VMOVDQU8Z128rm:
918 case X86::VMOVDQU8Z256rm:
919 case X86::VMOVDQU8Zrm:
920 case X86::VMOVUPDZ128rm:
921 case X86::VMOVUPDZ256rm:
922 case X86::VMOVUPDZrm:
923 case X86::VMOVUPSZ128rm:
924 case X86::VMOVUPSZ256rm:
925 case X86::VMOVUPSZ128rm_NOVLX:
926 case X86::VMOVUPSZ256rm_NOVLX:
927 case X86::VMOVUPSZrm: {
929 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() &&
930 MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
931 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
932 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
934 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
935 if (BaseReg == 0 || BaseReg == X86::RIP)
938 if (!(!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal())) {
948 case X86::LEA32r:
949 case X86::LEA64r: {
950 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
951 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
952 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
953 !MI.getOperand(1 + X86::AddrDisp).isReg()) {
955 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg())
957 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
958 if (BaseReg == 0)
977 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI);
978 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) !=
984 case X86::MOV32r0:
985 Value = 0;
987 case X86::MOV32r1:
990 case X86::MOV32r_1:
998 BuildMI(MBB, I, DL, get(X86::MOV32ri))
999 .add(Orig.getOperand(0))
1007 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1013 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS &&
1038 return ShAmt < 4 && ShAmt > 0;
1047 if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG &&
1048 CmpInstr.getOpcode() == X86::TEST64rr) &&
1049 !(CmpValDefInstr.getOpcode() == X86::COPY &&
1050 CmpInstr.getOpcode() == X86::TEST16rr))
1056 assert((CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) &&
1066 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) &&
1070 if (CmpInstr.getOpcode() == X86::TEST16rr) {
1079 if (!((VregDefInstr->getOpcode() == X86::AND32ri ||
1080 VregDefInstr->getOpcode() == X86::AND64ri32) &&
1085 if (CmpInstr.getOpcode() == X86::TEST64rr) {
1086 // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is
1087 // typically 0.
1088 if (CmpValDefInstr.getOperand(1).getImm() != 0)
1091 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically
1093 if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit)
1109 if (X86::isAND(VregDefInstr->getOpcode())) {
1113 // %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit
1129 if (Instr.modifiesRegister(X86::EFLAGS, TRI))
1144 // and is known to be 0 as a result of `TEST64rr`.
1167 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1169 RC = Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1176 if (Opc != X86::LEA64_32r) {
1201 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit)
1234 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&
1239 // Opcode = X86::LEA32r;
1240 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1242 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass)
1243 // : RegInfo.createVirtualRegister(&X86::GR32RegClass);
1247 unsigned Opcode = X86::LEA64_32r;
1248 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1249 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass);
1258 // least on modern x86 machines).
1260 Register Dest = MI.getOperand(0).getReg();
1263 bool IsDead = MI.getOperand(0).isDead();
1265 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1268 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA);
1281 case X86::SHL8ri:
1282 case X86::SHL16ri: {
1284 MIB.addReg(0)
1287 .addImm(0)
1288 .addReg(0);
1291 case X86::INC8r:
1292 case X86::INC16r:
1295 case X86::DEC8r:
1296 case X86::DEC16r:
1299 case X86::ADD8ri:
1300 case X86::ADD8ri_DB:
1301 case X86::ADD16ri:
1302 case X86::ADD16ri_DB:
1305 case X86::ADD8rr:
1306 case X86::ADD8rr_DB:
1307 case X86::ADD16rr:
1308 case X86::ADD16rr_DB: {
1318 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1320 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1323 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF),
1399 /// three-address instruction on demand. This allows the X86 target (for
1417 const MachineOperand &Dest = MI.getOperand(0);
1440 case X86::SHL64ri: {
1448 Src.getReg(), &X86::GR64_NOSPRegClass))
1451 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
1453 .addReg(0)
1456 .addImm(0)
1457 .addReg(0);
1460 case X86::SHL32ri: {
1466 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1470 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1477 .addReg(0)
1480 .addImm(0)
1481 .addReg(0);
1482 if (ImplicitOp.getReg() != 0)
1491 case X86::SHL8ri:
1494 case X86::SHL16ri: {
1501 case X86::INC64r:
1502 case X86::INC32r: {
1504 unsigned Opc = MIOpc == X86::INC64r
1505 ? X86::LEA64r
1506 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1508 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1516 if (ImplicitOp.getReg() != 0)
1526 case X86::DEC64r:
1527 case X86::DEC32r: {
1529 unsigned Opc = MIOpc == X86::DEC64r
1530 ? X86::LEA64r
1531 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1534 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1542 if (ImplicitOp.getReg() != 0)
1552 case X86::DEC8r:
1553 case X86::INC8r:
1556 case X86::DEC16r:
1557 case X86::INC16r:
1559 case X86::ADD64rr:
1560 case X86::ADD64rr_DB:
1561 case X86::ADD32rr:
1562 case X86::ADD32rr_DB: {
1565 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
1566 Opc = X86::LEA64r;
1568 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1572 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false);
1578 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1591 if (ImplicitOp.getReg() != 0)
1593 if (ImplicitOp2.getReg() != 0)
1608 case X86::ADD8rr:
1609 case X86::ADD8rr_DB:
1612 case X86::ADD16rr:
1613 case X86::ADD16rr_DB:
1615 case X86::ADD64ri32:
1616 case X86::ADD64ri32_DB:
1619 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
1622 case X86::ADD32ri:
1623 case X86::ADD32ri_DB: {
1625 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1628 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1636 if (ImplicitOp.getReg() != 0)
1646 case X86::ADD8ri:
1647 case X86::ADD8ri_DB:
1650 case X86::ADD16ri:
1651 case X86::ADD16ri_DB:
1653 case X86::SUB8ri:
1654 case X86::SUB16ri:
1657 case X86::SUB32ri: {
1665 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1668 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1676 if (ImplicitOp.getReg() != 0)
1687 case X86::SUB64ri32: {
1697 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
1702 case X86::VMOVDQU8Z128rmk:
1703 case X86::VMOVDQU8Z256rmk:
1704 case X86::VMOVDQU8Zrmk:
1705 case X86::VMOVDQU16Z128rmk:
1706 case X86::VMOVDQU16Z256rmk:
1707 case X86::VMOVDQU16Zrmk:
1708 case X86::VMOVDQU32Z128rmk:
1709 case X86::VMOVDQA32Z128rmk:
1710 case X86::VMOVDQU32Z256rmk:
1711 case X86::VMOVDQA32Z256rmk:
1712 case X86::VMOVDQU32Zrmk:
1713 case X86::VMOVDQA32Zrmk:
1714 case X86::VMOVDQU64Z128rmk:
1715 case X86::VMOVDQA64Z128rmk:
1716 case X86::VMOVDQU64Z256rmk:
1717 case X86::VMOVDQA64Z256rmk:
1718 case X86::VMOVDQU64Zrmk:
1719 case X86::VMOVDQA64Zrmk:
1720 case X86::VMOVUPDZ128rmk:
1721 case X86::VMOVAPDZ128rmk:
1722 case X86::VMOVUPDZ256rmk:
1723 case X86::VMOVAPDZ256rmk:
1724 case X86::VMOVUPDZrmk:
1725 case X86::VMOVAPDZrmk:
1726 case X86::VMOVUPSZ128rmk:
1727 case X86::VMOVAPSZ128rmk:
1728 case X86::VMOVUPSZ256rmk:
1729 case X86::VMOVAPSZ256rmk:
1730 case X86::VMOVUPSZrmk:
1731 case X86::VMOVAPSZrmk:
1732 case X86::VBROADCASTSDZ256rmk:
1733 case X86::VBROADCASTSDZrmk:
1734 case X86::VBROADCASTSSZ128rmk:
1735 case X86::VBROADCASTSSZ256rmk:
1736 case X86::VBROADCASTSSZrmk:
1737 case X86::VPBROADCASTDZ128rmk:
1738 case X86::VPBROADCASTDZ256rmk:
1739 case X86::VPBROADCASTDZrmk:
1740 case X86::VPBROADCASTQZ128rmk:
1741 case X86::VPBROADCASTQZ256rmk:
1742 case X86::VPBROADCASTQZrmk: {
1747 case X86::VMOVDQU8Z128rmk:
1748 Opc = X86::VPBLENDMBZ128rmk;
1750 case X86::VMOVDQU8Z256rmk:
1751 Opc = X86::VPBLENDMBZ256rmk;
1753 case X86::VMOVDQU8Zrmk:
1754 Opc = X86::VPBLENDMBZrmk;
1756 case X86::VMOVDQU16Z128rmk:
1757 Opc = X86::VPBLENDMWZ128rmk;
1759 case X86::VMOVDQU16Z256rmk:
1760 Opc = X86::VPBLENDMWZ256rmk;
1762 case X86::VMOVDQU16Zrmk:
1763 Opc = X86::VPBLENDMWZrmk;
1765 case X86::VMOVDQU32Z128rmk:
1766 Opc = X86::VPBLENDMDZ128rmk;
1768 case X86::VMOVDQU32Z256rmk:
1769 Opc = X86::VPBLENDMDZ256rmk;
1771 case X86::VMOVDQU32Zrmk:
1772 Opc = X86::VPBLENDMDZrmk;
1774 case X86::VMOVDQU64Z128rmk:
1775 Opc = X86::VPBLENDMQZ128rmk;
1777 case X86::VMOVDQU64Z256rmk:
1778 Opc = X86::VPBLENDMQZ256rmk;
1780 case X86::VMOVDQU64Zrmk:
1781 Opc = X86::VPBLENDMQZrmk;
1783 case X86::VMOVUPDZ128rmk:
1784 Opc = X86::VBLENDMPDZ128rmk;
1786 case X86::VMOVUPDZ256rmk:
1787 Opc = X86::VBLENDMPDZ256rmk;
1789 case X86::VMOVUPDZrmk:
1790 Opc = X86::VBLENDMPDZrmk;
1792 case X86::VMOVUPSZ128rmk:
1793 Opc = X86::VBLENDMPSZ128rmk;
1795 case X86::VMOVUPSZ256rmk:
1796 Opc = X86::VBLENDMPSZ256rmk;
1798 case X86::VMOVUPSZrmk:
1799 Opc = X86::VBLENDMPSZrmk;
1801 case X86::VMOVDQA32Z128rmk:
1802 Opc = X86::VPBLENDMDZ128rmk;
1804 case X86::VMOVDQA32Z256rmk:
1805 Opc = X86::VPBLENDMDZ256rmk;
1807 case X86::VMOVDQA32Zrmk:
1808 Opc = X86::VPBLENDMDZrmk;
1810 case X86::VMOVDQA64Z128rmk:
1811 Opc = X86::VPBLENDMQZ128rmk;
1813 case X86::VMOVDQA64Z256rmk:
1814 Opc = X86::VPBLENDMQZ256rmk;
1816 case X86::VMOVDQA64Zrmk:
1817 Opc = X86::VPBLENDMQZrmk;
1819 case X86::VMOVAPDZ128rmk:
1820 Opc = X86::VBLENDMPDZ128rmk;
1822 case X86::VMOVAPDZ256rmk:
1823 Opc = X86::VBLENDMPDZ256rmk;
1825 case X86::VMOVAPDZrmk:
1826 Opc = X86::VBLENDMPDZrmk;
1828 case X86::VMOVAPSZ128rmk:
1829 Opc = X86::VBLENDMPSZ128rmk;
1831 case X86::VMOVAPSZ256rmk:
1832 Opc = X86::VBLENDMPSZ256rmk;
1834 case X86::VMOVAPSZrmk:
1835 Opc = X86::VBLENDMPSZrmk;
1837 case X86::VBROADCASTSDZ256rmk:
1838 Opc = X86::VBLENDMPDZ256rmbk;
1840 case X86::VBROADCASTSDZrmk:
1841 Opc = X86::VBLENDMPDZrmbk;
1843 case X86::VBROADCASTSSZ128rmk:
1844 Opc = X86::VBLENDMPSZ128rmbk;
1846 case X86::VBROADCASTSSZ256rmk:
1847 Opc = X86::VBLENDMPSZ256rmbk;
1849 case X86::VBROADCASTSSZrmk:
1850 Opc = X86::VBLENDMPSZrmbk;
1852 case X86::VPBROADCASTDZ128rmk:
1853 Opc = X86::VPBLENDMDZ128rmbk;
1855 case X86::VPBROADCASTDZ256rmk:
1856 Opc = X86::VPBLENDMDZ256rmbk;
1858 case X86::VPBROADCASTDZrmk:
1859 Opc = X86::VPBLENDMDZrmbk;
1861 case X86::VPBROADCASTQZ128rmk:
1862 Opc = X86::VPBLENDMQZ128rmbk;
1864 case X86::VPBROADCASTQZ256rmk:
1865 Opc = X86::VPBLENDMQZ256rmbk;
1867 case X86::VPBROADCASTQZrmk:
1868 Opc = X86::VPBLENDMQZrmbk;
1885 case X86::VMOVDQU8Z128rrk:
1886 case X86::VMOVDQU8Z256rrk:
1887 case X86::VMOVDQU8Zrrk:
1888 case X86::VMOVDQU16Z128rrk:
1889 case X86::VMOVDQU16Z256rrk:
1890 case X86::VMOVDQU16Zrrk:
1891 case X86::VMOVDQU32Z128rrk:
1892 case X86::VMOVDQA32Z128rrk:
1893 case X86::VMOVDQU32Z256rrk:
1894 case X86::VMOVDQA32Z256rrk:
1895 case X86::VMOVDQU32Zrrk:
1896 case X86::VMOVDQA32Zrrk:
1897 case X86::VMOVDQU64Z128rrk:
1898 case X86::VMOVDQA64Z128rrk:
1899 case X86::VMOVDQU64Z256rrk:
1900 case X86::VMOVDQA64Z256rrk:
1901 case X86::VMOVDQU64Zrrk:
1902 case X86::VMOVDQA64Zrrk:
1903 case X86::VMOVUPDZ128rrk:
1904 case X86::VMOVAPDZ128rrk:
1905 case X86::VMOVUPDZ256rrk:
1906 case X86::VMOVAPDZ256rrk:
1907 case X86::VMOVUPDZrrk:
1908 case X86::VMOVAPDZrrk:
1909 case X86::VMOVUPSZ128rrk:
1910 case X86::VMOVAPSZ128rrk:
1911 case X86::VMOVUPSZ256rrk:
1912 case X86::VMOVAPSZ256rrk:
1913 case X86::VMOVUPSZrrk:
1914 case X86::VMOVAPSZrrk: {
1919 case X86::VMOVDQU8Z128rrk:
1920 Opc = X86::VPBLENDMBZ128rrk;
1922 case X86::VMOVDQU8Z256rrk:
1923 Opc = X86::VPBLENDMBZ256rrk;
1925 case X86::VMOVDQU8Zrrk:
1926 Opc = X86::VPBLENDMBZrrk;
1928 case X86::VMOVDQU16Z128rrk:
1929 Opc = X86::VPBLENDMWZ128rrk;
1931 case X86::VMOVDQU16Z256rrk:
1932 Opc = X86::VPBLENDMWZ256rrk;
1934 case X86::VMOVDQU16Zrrk:
1935 Opc = X86::VPBLENDMWZrrk;
1937 case X86::VMOVDQU32Z128rrk:
1938 Opc = X86::VPBLENDMDZ128rrk;
1940 case X86::VMOVDQU32Z256rrk:
1941 Opc = X86::VPBLENDMDZ256rrk;
1943 case X86::VMOVDQU32Zrrk:
1944 Opc = X86::VPBLENDMDZrrk;
1946 case X86::VMOVDQU64Z128rrk:
1947 Opc = X86::VPBLENDMQZ128rrk;
1949 case X86::VMOVDQU64Z256rrk:
1950 Opc = X86::VPBLENDMQZ256rrk;
1952 case X86::VMOVDQU64Zrrk:
1953 Opc = X86::VPBLENDMQZrrk;
1955 case X86::VMOVUPDZ128rrk:
1956 Opc = X86::VBLENDMPDZ128rrk;
1958 case X86::VMOVUPDZ256rrk:
1959 Opc = X86::VBLENDMPDZ256rrk;
1961 case X86::VMOVUPDZrrk:
1962 Opc = X86::VBLENDMPDZrrk;
1964 case X86::VMOVUPSZ128rrk:
1965 Opc = X86::VBLENDMPSZ128rrk;
1967 case X86::VMOVUPSZ256rrk:
1968 Opc = X86::VBLENDMPSZ256rrk;
1970 case X86::VMOVUPSZrrk:
1971 Opc = X86::VBLENDMPSZrrk;
1973 case X86::VMOVDQA32Z128rrk:
1974 Opc = X86::VPBLENDMDZ128rrk;
1976 case X86::VMOVDQA32Z256rrk:
1977 Opc = X86::VPBLENDMDZ256rrk;
1979 case X86::VMOVDQA32Zrrk:
1980 Opc = X86::VPBLENDMDZrrk;
1982 case X86::VMOVDQA64Z128rrk:
1983 Opc = X86::VPBLENDMQZ128rrk;
1985 case X86::VMOVDQA64Z256rrk:
1986 Opc = X86::VPBLENDMQZ256rrk;
1988 case X86::VMOVDQA64Zrrk:
1989 Opc = X86::VPBLENDMQZrrk;
1991 case X86::VMOVAPDZ128rrk:
1992 Opc = X86::VBLENDMPDZ128rrk;
1994 case X86::VMOVAPDZ256rrk:
1995 Opc = X86::VBLENDMPDZ256rrk;
1997 case X86::VMOVAPDZrrk:
1998 Opc = X86::VBLENDMPDZrrk;
2000 case X86::VMOVAPSZ128rrk:
2001 Opc = X86::VBLENDMPSZ128rrk;
2003 case X86::VMOVAPSZ256rrk:
2004 Opc = X86::VBLENDMPSZ256rrk;
2006 case X86::VMOVAPSZrrk:
2007 Opc = X86::VBLENDMPSZrrk;
2025 for (unsigned I = 0; I < NumRegOperands; ++I) {
2050 /// Case 0 - Possible to commute the first and second operands.
2066 return 0;
2083 // not implemented yet. So, just return 0 in that case.
2097 const unsigned Form132Index = 0;
2101 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2;
2118 FMAForms[0] = FMA3Group.get132Opcode();
2123 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
2139 {0x04, 0x10, 0x08, 0x20}, // Swap bits 2/4 and 3/5.
2140 {0x02, 0x10, 0x08, 0x40}, // Swap bits 1/4 and 3/6.
2141 {0x02, 0x04, 0x20, 0x40}, // Swap bits 1/2 and 5/6.
2146 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
2149 if (Imm & SwapMasks[Case][0])
2152 NewImm |= SwapMasks[Case][0];
2164 case X86::VPERMI2##Suffix##Z128rr: \
2165 case X86::VPERMT2##Suffix##Z128rr: \
2166 case X86::VPERMI2##Suffix##Z256rr: \
2167 case X86::VPERMT2##Suffix##Z256rr: \
2168 case X86::VPERMI2##Suffix##Zrr: \
2169 case X86::VPERMT2##Suffix##Zrr: \
2170 case X86::VPERMI2##Suffix##Z128rm: \
2171 case X86::VPERMT2##Suffix##Z128rm: \
2172 case X86::VPERMI2##Suffix##Z256rm: \
2173 case X86::VPERMT2##Suffix##Z256rm: \
2174 case X86::VPERMI2##Suffix##Zrm: \
2175 case X86::VPERMT2##Suffix##Zrm: \
2176 case X86::VPERMI2##Suffix##Z128rrkz: \
2177 case X86::VPERMT2##Suffix##Z128rrkz: \
2178 case X86::VPERMI2##Suffix##Z256rrkz: \
2179 case X86::VPERMT2##Suffix##Z256rrkz: \
2180 case X86::VPERMI2##Suffix##Zrrkz: \
2181 case X86::VPERMT2##Suffix##Zrrkz: \
2182 case X86::VPERMI2##Suffix##Z128rmkz: \
2183 case X86::VPERMT2##Suffix##Z128rmkz: \
2184 case X86::VPERMI2##Suffix##Z256rmkz: \
2185 case X86::VPERMT2##Suffix##Z256rmkz: \
2186 case X86::VPERMI2##Suffix##Zrmkz: \
2187 case X86::VPERMT2##Suffix##Zrmkz:
2191 case X86::VPERMI2##Suffix##Z128rmb: \
2192 case X86::VPERMT2##Suffix##Z128rmb: \
2193 case X86::VPERMI2##Suffix##Z256rmb: \
2194 case X86::VPERMT2##Suffix##Z256rmb: \
2195 case X86::VPERMI2##Suffix##Zrmb: \
2196 case X86::VPERMT2##Suffix##Zrmb: \
2197 case X86::VPERMI2##Suffix##Z128rmbkz: \
2198 case X86::VPERMT2##Suffix##Z128rmbkz: \
2199 case X86::VPERMI2##Suffix##Z256rmbkz: \
2200 case X86::VPERMT2##Suffix##Z256rmbkz: \
2201 case X86::VPERMI2##Suffix##Zrmbkz: \
2202 case X86::VPERMT2##Suffix##Zrmbkz:
2223 case X86::Orig##Z128rr: \
2224 return X86::New##Z128rr; \
2225 case X86::Orig##Z128rrkz: \
2226 return X86::New##Z128rrkz; \
2227 case X86::Orig##Z128rm: \
2228 return X86::New##Z128rm; \
2229 case X86::Orig##Z128rmkz: \
2230 return X86::New##Z128rmkz; \
2231 case X86::Orig##Z256rr: \
2232 return X86::New##Z256rr; \
2233 case X86::Orig##Z256rrkz: \
2234 return X86::New##Z256rrkz; \
2235 case X86::Orig##Z256rm: \
2236 return X86::New##Z256rm; \
2237 case X86::Orig##Z256rmkz: \
2238 return X86::New##Z256rmkz; \
2239 case X86::Orig##Zrr: \
2240 return X86::New##Zrr; \
2241 case X86::Orig##Zrrkz: \
2242 return X86::New##Zrrkz; \
2243 case X86::Orig##Zrm: \
2244 return X86::New##Zrm; \
2245 case X86::Orig##Zrmkz: \
2246 return X86::New##Zrmkz;
2250 case X86::Orig##Z128rmb: \
2251 return X86::New##Z128rmb; \
2252 case X86::Orig##Z128rmbkz: \
2253 return X86::New##Z128rmbkz; \
2254 case X86::Orig##Z256rmb: \
2255 return X86::New##Z256rmb; \
2256 case X86::Orig##Z256rmbkz: \
2257 return X86::New##Z256rmbkz; \
2258 case X86::Orig##Zrmb: \
2259 return X86::New##Zrmb; \
2260 case X86::Orig##Zrmbkz: \
2261 return X86::New##Zrmbkz;
2295 case X86::OP: \
2296 case X86::OP##_ND:
2311 case X86::A: \
2312 Opc = X86::B; \
2315 case X86::A##_ND: \
2316 Opc = X86::B##_ND; \
2319 case X86::B: \
2320 Opc = X86::A; \
2323 case X86::B##_ND: \
2324 Opc = X86::A##_ND; \
2338 case X86::PFSUBrr:
2339 case X86::PFSUBRrr:
2344 get(X86::PFSUBRrr == Opc ? X86::PFSUBrr : X86::PFSUBRrr));
2346 case X86::BLENDPDrri:
2347 case X86::BLENDPSrri:
2348 case X86::VBLENDPDrri:
2349 case X86::VBLENDPSrri:
2352 unsigned Mask = (Opc == X86::BLENDPDrri || Opc == X86::VBLENDPDrri) ? 0x03: 0x0F;
2355 case X86::FROM: \
2356 Opc = X86::TO; \
2374 case X86::PBLENDWrri:
2375 case X86::VBLENDPDYrri:
2376 case X86::VBLENDPSYrri:
2377 case X86::VPBLENDDrri:
2378 case X86::VPBLENDWrri:
2379 case X86::VPBLENDDYrri:
2380 case X86::VPBLENDWYrri: {
2385 case X86::BLENDPDrri:
2386 Mask = (int8_t)0x03;
2388 case X86::BLENDPSrri:
2389 Mask = (int8_t)0x0F;
2391 case X86::PBLENDWrri:
2392 Mask = (int8_t)0xFF;
2394 case X86::VBLENDPDrri:
2395 Mask = (int8_t)0x03;
2397 case X86::VBLENDPSrri:
2398 Mask = (int8_t)0x0F;
2400 case X86::VBLENDPDYrri:
2401 Mask = (int8_t)0x0F;
2403 case X86::VBLENDPSYrri:
2404 Mask = (int8_t)0xFF;
2406 case X86::VPBLENDDrri:
2407 Mask = (int8_t)0x0F;
2409 case X86::VPBLENDWrri:
2410 Mask = (int8_t)0xFF;
2412 case X86::VPBLENDDYrri:
2413 Mask = (int8_t)0xFF;
2415 case X86::VPBLENDWYrri:
2416 Mask = (int8_t)0xFF;
2427 case X86::INSERTPSrr:
2428 case X86::VINSERTPSrr:
2429 case X86::VINSERTPSZrr: {
2437 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2448 case X86::MOVSDrr:
2449 case X86::MOVSSrr:
2450 case X86::VMOVSDrr:
2451 case X86::VMOVSSrr: {
2458 case X86::MOVSDrr:
2459 Opc = X86::BLENDPDrri;
2460 Mask = 0x02;
2462 case X86::MOVSSrr:
2463 Opc = X86::BLENDPSrri;
2464 Mask = 0x0E;
2466 case X86::VMOVSDrr:
2467 Opc = X86::VBLENDPDrri;
2468 Mask = 0x02;
2470 case X86::VMOVSSrr:
2471 Opc = X86::VBLENDPSrri;
2472 Mask = 0x0E;
2483 WorkingMI->setDesc(get(X86::SHUFPDrri));
2484 WorkingMI->addOperand(MachineOperand::CreateImm(0x02));
2487 case X86::SHUFPDrri: {
2489 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!");
2491 WorkingMI->setDesc(get(X86::MOVSDrr));
2495 case X86::PCLMULQDQrri:
2496 case X86::VPCLMULQDQrri:
2497 case X86::VPCLMULQDQYrri:
2498 case X86::VPCLMULQDQZrri:
2499 case X86::VPCLMULQDQZ128rri:
2500 case X86::VPCLMULQDQZ256rri: {
2501 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0]
2502 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0]
2504 unsigned Src1Hi = Imm & 0x01;
2505 unsigned Src2Hi = Imm & 0x10;
2510 case X86::VPCMPBZ128rri:
2511 case X86::VPCMPUBZ128rri:
2512 case X86::VPCMPBZ256rri:
2513 case X86::VPCMPUBZ256rri:
2514 case X86::VPCMPBZrri:
2515 case X86::VPCMPUBZrri:
2516 case X86::VPCMPDZ128rri:
2517 case X86::VPCMPUDZ128rri:
2518 case X86::VPCMPDZ256rri:
2519 case X86::VPCMPUDZ256rri:
2520 case X86::VPCMPDZrri:
2521 case X86::VPCMPUDZrri:
2522 case X86::VPCMPQZ128rri:
2523 case X86::VPCMPUQZ128rri:
2524 case X86::VPCMPQZ256rri:
2525 case X86::VPCMPUQZ256rri:
2526 case X86::VPCMPQZrri:
2527 case X86::VPCMPUQZrri:
2528 case X86::VPCMPWZ128rri:
2529 case X86::VPCMPUWZ128rri:
2530 case X86::VPCMPWZ256rri:
2531 case X86::VPCMPUWZ256rri:
2532 case X86::VPCMPWZrri:
2533 case X86::VPCMPUWZrri:
2534 case X86::VPCMPBZ128rrik:
2535 case X86::VPCMPUBZ128rrik:
2536 case X86::VPCMPBZ256rrik:
2537 case X86::VPCMPUBZ256rrik:
2538 case X86::VPCMPBZrrik:
2539 case X86::VPCMPUBZrrik:
2540 case X86::VPCMPDZ128rrik:
2541 case X86::VPCMPUDZ128rrik:
2542 case X86::VPCMPDZ256rrik:
2543 case X86::VPCMPUDZ256rrik:
2544 case X86::VPCMPDZrrik:
2545 case X86::VPCMPUDZrrik:
2546 case X86::VPCMPQZ128rrik:
2547 case X86::VPCMPUQZ128rrik:
2548 case X86::VPCMPQZ256rrik:
2549 case X86::VPCMPUQZ256rrik:
2550 case X86::VPCMPQZrrik:
2551 case X86::VPCMPUQZrrik:
2552 case X86::VPCMPWZ128rrik:
2553 case X86::VPCMPUWZ128rrik:
2554 case X86::VPCMPWZ256rrik:
2555 case X86::VPCMPUWZ256rrik:
2556 case X86::VPCMPWZrrik:
2557 case X86::VPCMPUWZrrik:
2561 .setImm(X86::getSwappedVPCMPImm(
2562 MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7));
2564 case X86::VPCOMBri:
2565 case X86::VPCOMUBri:
2566 case X86::VPCOMDri:
2567 case X86::VPCOMUDri:
2568 case X86::VPCOMQri:
2569 case X86::VPCOMUQri:
2570 case X86::VPCOMWri:
2571 case X86::VPCOMUWri:
2575 X86::getSwappedVPCOMImm(MI.getOperand(3).getImm() & 0x7));
2577 case X86::VCMPSDZrri:
2578 case X86::VCMPSSZrri:
2579 case X86::VCMPPDZrri:
2580 case X86::VCMPPSZrri:
2581 case X86::VCMPSHZrri:
2582 case X86::VCMPPHZrri:
2583 case X86::VCMPPHZ128rri:
2584 case X86::VCMPPHZ256rri:
2585 case X86::VCMPPDZ128rri:
2586 case X86::VCMPPSZ128rri:
2587 case X86::VCMPPDZ256rri:
2588 case X86::VCMPPSZ256rri:
2589 case X86::VCMPPDZrrik:
2590 case X86::VCMPPSZrrik:
2591 case X86::VCMPPDZ128rrik:
2592 case X86::VCMPPSZ128rrik:
2593 case X86::VCMPPDZ256rrik:
2594 case X86::VCMPPSZ256rrik:
2597 .setImm(X86::getSwappedVCMPImm(
2598 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f));
2600 case X86::VPERM2F128rr:
2601 case X86::VPERM2I128rr:
2603 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi.
2604 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi.
2606 WorkingMI->getOperand(3).setImm((MI.getOperand(3).getImm() & 0xFF) ^ 0x22);
2608 case X86::MOVHLPSrr:
2609 case X86::UNPCKHPDrr:
2610 case X86::VMOVHLPSrr:
2611 case X86::VUNPCKHPDrr:
2612 case X86::VMOVHLPSZrr:
2613 case X86::VUNPCKHPDZ128rr:
2619 case X86::MOVHLPSrr:
2620 Opc = X86::UNPCKHPDrr;
2622 case X86::UNPCKHPDrr:
2623 Opc = X86::MOVHLPSrr;
2625 case X86::VMOVHLPSrr:
2626 Opc = X86::VUNPCKHPDrr;
2628 case X86::VUNPCKHPDrr:
2629 Opc = X86::VMOVHLPSrr;
2631 case X86::VMOVHLPSZrr:
2632 Opc = X86::VUNPCKHPDZ128rr;
2634 case X86::VUNPCKHPDZ128rr:
2635 Opc = X86::VMOVHLPSZrr;
2646 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
2647 WorkingMI->getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC));
2650 case X86::VPTERNLOGDZrri:
2651 case X86::VPTERNLOGDZrmi:
2652 case X86::VPTERNLOGDZ128rri:
2653 case X86::VPTERNLOGDZ128rmi:
2654 case X86::VPTERNLOGDZ256rri:
2655 case X86::VPTERNLOGDZ256rmi:
2656 case X86::VPTERNLOGQZrri:
2657 case X86::VPTERNLOGQZrmi:
2658 case X86::VPTERNLOGQZ128rri:
2659 case X86::VPTERNLOGQZ128rmi:
2660 case X86::VPTERNLOGQZ256rri:
2661 case X86::VPTERNLOGQZ256rmi:
2662 case X86::VPTERNLOGDZrrik:
2663 case X86::VPTERNLOGDZ128rrik:
2664 case X86::VPTERNLOGDZ256rrik:
2665 case X86::VPTERNLOGQZrrik:
2666 case X86::VPTERNLOGQZ128rrik:
2667 case X86::VPTERNLOGQZ256rrik:
2668 case X86::VPTERNLOGDZrrikz:
2669 case X86::VPTERNLOGDZrmikz:
2670 case X86::VPTERNLOGDZ128rrikz:
2671 case X86::VPTERNLOGDZ128rmikz:
2672 case X86::VPTERNLOGDZ256rrikz:
2673 case X86::VPTERNLOGDZ256rmikz:
2674 case X86::VPTERNLOGQZrrikz:
2675 case X86::VPTERNLOGQZrmikz:
2676 case X86::VPTERNLOGQZ128rrikz:
2677 case X86::VPTERNLOGQZ128rmikz:
2678 case X86::VPTERNLOGQZ256rrikz:
2679 case X86::VPTERNLOGQZ256rmikz:
2680 case X86::VPTERNLOGDZ128rmbi:
2681 case X86::VPTERNLOGDZ256rmbi:
2682 case X86::VPTERNLOGDZrmbi:
2683 case X86::VPTERNLOGQZ128rmbi:
2684 case X86::VPTERNLOGQZ256rmbi:
2685 case X86::VPTERNLOGQZrmbi:
2686 case X86::VPTERNLOGDZ128rmbikz:
2687 case X86::VPTERNLOGDZ256rmbikz:
2688 case X86::VPTERNLOGDZrmbikz:
2689 case X86::VPTERNLOGQZ128rmbikz:
2690 case X86::VPTERNLOGQZ256rmbikz:
2691 case X86::VPTERNLOGQZrmbikz: {
2727 // in the k-mask operand is set to 0, are copied to the result of the
2742 // which the corresponding bit in the k-mask is set to 0.
2824 case X86::CMPSDrri:
2825 case X86::CMPSSrri:
2826 case X86::CMPPDrri:
2827 case X86::CMPPSrri:
2828 case X86::VCMPSDrri:
2829 case X86::VCMPSSrri:
2830 case X86::VCMPPDrri:
2831 case X86::VCMPPSrri:
2832 case X86::VCMPPDYrri:
2833 case X86::VCMPPSYrri:
2834 case X86::VCMPSDZrri:
2835 case X86::VCMPSSZrri:
2836 case X86::VCMPPDZrri:
2837 case X86::VCMPPSZrri:
2838 case X86::VCMPSHZrri:
2839 case X86::VCMPPHZrri:
2840 case X86::VCMPPHZ128rri:
2841 case X86::VCMPPHZ256rri:
2842 case X86::VCMPPDZ128rri:
2843 case X86::VCMPPSZ128rri:
2844 case X86::VCMPPDZ256rri:
2845 case X86::VCMPPSZ256rri:
2846 case X86::VCMPPDZrrik:
2847 case X86::VCMPPSZrrik:
2848 case X86::VCMPPDZ128rrik:
2849 case X86::VCMPPSZ128rrik:
2850 case X86::VCMPPDZ256rrik:
2851 case X86::VCMPPSZ256rrik: {
2852 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0;
2856 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7;
2863 case 0x00: // EQUAL
2864 case 0x03: // UNORDERED
2865 case 0x04: // NOT EQUAL
2866 case 0x07: // ORDERED
2876 case X86::MOVSSrr:
2877 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
2883 case X86::SHUFPDrri:
2885 if (MI.getOperand(3).getImm() == 0x02)
2888 case X86::MOVHLPSrr:
2889 case X86::UNPCKHPDrr:
2890 case X86::VMOVHLPSrr:
2891 case X86::VUNPCKHPDrr:
2892 case X86::VMOVHLPSZrr:
2893 case X86::VUNPCKHPDZ128rr:
2897 case X86::VPTERNLOGDZrri:
2898 case X86::VPTERNLOGDZrmi:
2899 case X86::VPTERNLOGDZ128rri:
2900 case X86::VPTERNLOGDZ128rmi:
2901 case X86::VPTERNLOGDZ256rri:
2902 case X86::VPTERNLOGDZ256rmi:
2903 case X86::VPTERNLOGQZrri:
2904 case X86::VPTERNLOGQZrmi:
2905 case X86::VPTERNLOGQZ128rri:
2906 case X86::VPTERNLOGQZ128rmi:
2907 case X86::VPTERNLOGQZ256rri:
2908 case X86::VPTERNLOGQZ256rmi:
2909 case X86::VPTERNLOGDZrrik:
2910 case X86::VPTERNLOGDZ128rrik:
2911 case X86::VPTERNLOGDZ256rrik:
2912 case X86::VPTERNLOGQZrrik:
2913 case X86::VPTERNLOGQZ128rrik:
2914 case X86::VPTERNLOGQZ256rrik:
2915 case X86::VPTERNLOGDZrrikz:
2916 case X86::VPTERNLOGDZrmikz:
2917 case X86::VPTERNLOGDZ128rrikz:
2918 case X86::VPTERNLOGDZ128rmikz:
2919 case X86::VPTERNLOGDZ256rrikz:
2920 case X86::VPTERNLOGDZ256rmikz:
2921 case X86::VPTERNLOGQZrrikz:
2922 case X86::VPTERNLOGQZrmikz:
2923 case X86::VPTERNLOGQZ128rrikz:
2924 case X86::VPTERNLOGQZ128rmikz:
2925 case X86::VPTERNLOGQZ256rrikz:
2926 case X86::VPTERNLOGQZ256rmikz:
2927 case X86::VPTERNLOGDZ128rmbi:
2928 case X86::VPTERNLOGDZ256rmbi:
2929 case X86::VPTERNLOGDZrmbi:
2930 case X86::VPTERNLOGQZ128rmbi:
2931 case X86::VPTERNLOGQZ256rmbi:
2932 case X86::VPTERNLOGQZrmbi:
2933 case X86::VPTERNLOGDZ128rmbikz:
2934 case X86::VPTERNLOGDZ256rmbikz:
2935 case X86::VPTERNLOGDZrmbikz:
2936 case X86::VPTERNLOGQZ128rmbikz:
2937 case X86::VPTERNLOGQZ256rmbikz:
2938 case X86::VPTERNLOGQZrmbikz:
2940 case X86::VPDPWSSDYrr:
2941 case X86::VPDPWSSDrr:
2942 case X86::VPDPWSSDSYrr:
2943 case X86::VPDPWSSDSrr:
2944 case X86::VPDPWUUDrr:
2945 case X86::VPDPWUUDYrr:
2946 case X86::VPDPWUUDSrr:
2947 case X86::VPDPWUUDSYrr:
2948 case X86::VPDPBSSDSrr:
2949 case X86::VPDPBSSDSYrr:
2950 case X86::VPDPBSSDrr:
2951 case X86::VPDPBSSDYrr:
2952 case X86::VPDPBUUDSrr:
2953 case X86::VPDPBUUDSYrr:
2954 case X86::VPDPBUUDrr:
2955 case X86::VPDPBUUDYrr:
2956 case X86::VPDPWSSDZ128r:
2957 case X86::VPDPWSSDZ128rk:
2958 case X86::VPDPWSSDZ128rkz:
2959 case X86::VPDPWSSDZ256r:
2960 case X86::VPDPWSSDZ256rk:
2961 case X86::VPDPWSSDZ256rkz:
2962 case X86::VPDPWSSDZr:
2963 case X86::VPDPWSSDZrk:
2964 case X86::VPDPWSSDZrkz:
2965 case X86::VPDPWSSDSZ128r:
2966 case X86::VPDPWSSDSZ128rk:
2967 case X86::VPDPWSSDSZ128rkz:
2968 case X86::VPDPWSSDSZ256r:
2969 case X86::VPDPWSSDSZ256rk:
2970 case X86::VPDPWSSDSZ256rkz:
2971 case X86::VPDPWSSDSZr:
2972 case X86::VPDPWSSDSZrk:
2973 case X86::VPDPWSSDSZrkz:
2974 case X86::VPMADD52HUQrr:
2975 case X86::VPMADD52HUQYrr:
2976 case X86::VPMADD52HUQZ128r:
2977 case X86::VPMADD52HUQZ128rk:
2978 case X86::VPMADD52HUQZ128rkz:
2979 case X86::VPMADD52HUQZ256r:
2980 case X86::VPMADD52HUQZ256rk:
2981 case X86::VPMADD52HUQZ256rkz:
2982 case X86::VPMADD52HUQZr:
2983 case X86::VPMADD52HUQZrk:
2984 case X86::VPMADD52HUQZrkz:
2985 case X86::VPMADD52LUQrr:
2986 case X86::VPMADD52LUQYrr:
2987 case X86::VPMADD52LUQZ128r:
2988 case X86::VPMADD52LUQZ128rk:
2989 case X86::VPMADD52LUQZ128rkz:
2990 case X86::VPMADD52LUQZ256r:
2991 case X86::VPMADD52LUQZ256rk:
2992 case X86::VPMADD52LUQZ256rkz:
2993 case X86::VPMADD52LUQZr:
2994 case X86::VPMADD52LUQZrk:
2995 case X86::VPMADD52LUQZrkz:
2996 case X86::VFMADDCPHZr:
2997 case X86::VFMADDCPHZrk:
2998 case X86::VFMADDCPHZrkz:
2999 case X86::VFMADDCPHZ128r:
3000 case X86::VFMADDCPHZ128rk:
3001 case X86::VFMADDCPHZ128rkz:
3002 case X86::VFMADDCPHZ256r:
3003 case X86::VFMADDCPHZ256rk:
3004 case X86::VFMADDCPHZ256rkz:
3005 case X86::VFMADDCSHZr:
3006 case X86::VFMADDCSHZrk:
3007 case X86::VFMADDCSHZrkz: {
3072 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
3073 Opcode != X86::LEA64_32r)
3076 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt);
3077 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp);
3078 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg);
3080 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 ||
3096 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
3122 int X86::getCondSrcNoFromDesc(const MCInstrDesc &MCID) {
3124 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode) ||
3125 X86::isCFCMOVCC(Opcode) || X86::isCCMPCC(Opcode) ||
3126 X86::isCTESTCC(Opcode)))
3133 X86::CondCode X86::getCondFromMI(const MachineInstr &MI) {
3136 if (CondNo < 0)
3137 return X86::COND_INVALID;
3139 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm());
3142 X86::CondCode X86::getCondFromBranch(const MachineInstr &MI) {
3143 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3144 : X86::COND_INVALID;
3147 X86::CondCode X86::getCondFromSETCC(const MachineInstr &MI) {
3148 return X86::isSETCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3149 : X86::COND_INVALID;
3152 X86::CondCode X86::getCondFromCMov(const MachineInstr &MI) {
3153 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3154 : X86::COND_INVALID;
3157 X86::CondCode X86::getCondFromCFCMov(const MachineInstr &MI) {
3158 return X86::isCFCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3159 : X86::COND_INVALID;
3162 X86::CondCode X86::getCondFromCCMP(const MachineInstr &MI) {
3163 return X86::isCCMPCC(MI.getOpcode()) || X86::isCTESTCC(MI.getOpcode())
3164 ? X86::getCondFromMI(MI)
3165 : X86::COND_INVALID;
3168 int X86::getCCMPCondFlagsFromCondCode(X86::CondCode CC) {
3185 // AF = 0 (Auxiliary Carry Flag)
3199 case X86::COND_NO:
3200 case X86::COND_NE:
3201 case X86::COND_GE:
3202 case X86::COND_G:
3203 case X86::COND_AE:
3204 case X86::COND_A:
3205 case X86::COND_NS:
3206 case X86::COND_NP:
3207 return 0;
3208 case X86::COND_O:
3210 case X86::COND_B:
3211 case X86::COND_BE:
3214 case X86::COND_E:
3215 case X86::COND_LE:
3217 case X86::COND_S:
3218 case X86::COND_L:
3220 case X86::COND_P:
3232 return (I == Table.end() || I->OldOpc != Opc) ? 0U : I->NewOpc;
3234 unsigned X86::getNFVariant(unsigned Opc) {
3238 unsigned X86::getNonNDVariant(unsigned Opc) {
3244 X86::CondCode X86::GetOppositeBranchCondition(X86::CondCode CC) {
3248 case X86::COND_E:
3249 return X86::COND_NE;
3250 case X86::COND_NE:
3251 return X86::COND_E;
3252 case X86::COND_L:
3253 return X86::COND_GE;
3254 case X86::COND_LE:
3255 return X86::COND_G;
3256 case X86::COND_G:
3257 return X86::COND_LE;
3258 case X86::COND_GE:
3259 return X86::COND_L;
3260 case X86::COND_B:
3261 return X86::COND_AE;
3262 case X86::COND_BE:
3263 return X86::COND_A;
3264 case X86::COND_A:
3265 return X86::COND_BE;
3266 case X86::COND_AE:
3267 return X86::COND_B;
3268 case X86::COND_S:
3269 return X86::COND_NS;
3270 case X86::COND_NS:
3271 return X86::COND_S;
3272 case X86::COND_P:
3273 return X86::COND_NP;
3274 case X86::COND_NP:
3275 return X86::COND_P;
3276 case X86::COND_O:
3277 return X86::COND_NO;
3278 case X86::COND_NO:
3279 return X86::COND_O;
3280 case X86::COND_NE_OR_P:
3281 return X86::COND_E_AND_NP;
3282 case X86::COND_E_AND_NP:
3283 return X86::COND_NE_OR_P;
3289 static X86::CondCode getSwappedCondition(X86::CondCode CC) {
3292 return X86::COND_INVALID;
3293 case X86::COND_E:
3294 return X86::COND_E;
3295 case X86::COND_NE:
3296 return X86::COND_NE;
3297 case X86::COND_L:
3298 return X86::COND_G;
3299 case X86::COND_LE:
3300 return X86::COND_GE;
3301 case X86::COND_G:
3302 return X86::COND_L;
3303 case X86::COND_GE:
3304 return X86::COND_LE;
3305 case X86::COND_B:
3306 return X86::COND_A;
3307 case X86::COND_BE:
3308 return X86::COND_AE;
3309 case X86::COND_A:
3310 return X86::COND_B;
3311 case X86::COND_AE:
3312 return X86::COND_BE;
3316 std::pair<X86::CondCode, bool>
3317 X86::getX86ConditionCode(CmpInst::Predicate Predicate) {
3318 X86::CondCode CC = X86::COND_INVALID;
3325 CC = X86::COND_E;
3331 CC = X86::COND_A;
3337 CC = X86::COND_AE;
3343 CC = X86::COND_B;
3349 CC = X86::COND_BE;
3352 CC = X86::COND_NE;
3355 CC = X86::COND_P;
3358 CC = X86::COND_NP;
3363 CC = X86::COND_INVALID;
3368 CC = X86::COND_E;
3371 CC = X86::COND_NE;
3374 CC = X86::COND_A;
3377 CC = X86::COND_AE;
3380 CC = X86::COND_B;
3383 CC = X86::COND_BE;
3386 CC = X86::COND_G;
3389 CC = X86::COND_GE;
3392 CC = X86::COND_L;
3395 CC = X86::COND_LE;
3403 unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand,
3410 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV16rm)
3411 : GET_ND_IF_ENABLED(X86::CMOV16rr);
3413 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV32rm)
3414 : GET_ND_IF_ENABLED(X86::CMOV32rr);
3416 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV64rm)
3417 : GET_ND_IF_ENABLED(X86::CMOV64rr);
3422 unsigned X86::getVPCMPImmForCond(ISD::CondCode CC) {
3429 return 0;
3446 unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
3450 case 0x01:
3451 Imm = 0x06;
3453 case 0x02:
3454 Imm = 0x05;
3456 case 0x05:
3457 Imm = 0x02;
3459 case 0x06:
3460 Imm = 0x01;
3462 case 0x00: // EQ
3463 case 0x03: // FALSE
3464 case 0x04: // NE
3465 case 0x07: // TRUE
3473 unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
3477 case 0x00:
3478 Imm = 0x02;
3480 case 0x01:
3481 Imm = 0x03;
3483 case 0x02:
3484 Imm = 0x00;
3486 case 0x03:
3487 Imm = 0x01;
3489 case 0x04: // EQ
3490 case 0x05: // NE
3491 case 0x06: // FALSE
3492 case 0x07: // TRUE
3500 unsigned X86::getSwappedVCMPImm(unsigned Imm) {
3502 switch (Imm & 0x3) {
3505 case 0x00:
3506 case 0x03:
3509 case 0x01:
3510 case 0x02:
3511 // Need to toggle bits 3:0. Bit 4 stays the same.
3512 Imm ^= 0xf;
3519 unsigned X86::getVectorRegisterWidth(const MCOperandInfo &Info) {
3520 if (Info.RegClass == X86::VR128RegClassID ||
3521 Info.RegClass == X86::VR128XRegClassID)
3523 if (Info.RegClass == X86::VR256RegClassID ||
3524 Info.RegClass == X86::VR256XRegClassID)
3526 if (Info.RegClass == X86::VR512RegClassID)
3533 return (Reg == X86::FPCW || Reg == X86::FPSW ||
3534 (Reg >= X86::ST0 && Reg <= X86::ST7));
3538 bool X86::isX87Instruction(MachineInstr &MI) {
3553 int X86::getFirstAddrOperandIdx(const MachineInstr &MI) {
3564 if (MemRefIdx >= 0)
3577 if (NumOps < X86::AddrNumOperands) {
3588 for (unsigned I = 0, E = NumOps - X86::AddrNumOperands; I != E; ++I) {
3592 Desc.operands().begin() + I + X86::AddrNumOperands,
3604 const Constant *X86::getConstantFromPool(const MachineInstr &MI,
3606 assert(MI.getNumOperands() >= (OpNo + X86::AddrNumOperands) &&
3609 const MachineOperand &Index = MI.getOperand(OpNo + X86::AddrIndexReg);
3610 if (!Index.isReg() || Index.getReg() != X86::NoRegister)
3613 const MachineOperand &Disp = MI.getOperand(OpNo + X86::AddrDisp);
3614 if (!Disp.isCPI() || Disp.getOffset() != 0)
3631 case X86::TCRETURNdi:
3632 case X86::TCRETURNri:
3633 case X86::TCRETURNmi:
3634 case X86::TCRETURNdi64:
3635 case X86::TCRETURNri64:
3636 case X86::TCRETURNmi64:
3651 const MachineOperand &Target = TailCall.getOperand(0);
3660 if (TailCall.getOpcode() != X86::TCRETURNdi &&
3661 TailCall.getOpcode() != X86::TCRETURNdi64) {
3672 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) {
3678 if (X86FI->getTCReturnAddrDelta() != 0 ||
3679 TailCall.getOperand(1).getImm() != 0) {
3698 assert(0 && "Can't find the branch to replace!");
3700 X86::CondCode CC = X86::getCondFromBranch(*I);
3702 if (CC != BranchCond[0].getImm())
3708 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3709 : X86::TCRETURNdi64cc;
3712 MIB->addOperand(TailCall.getOperand(0)); // Destination.
3713 MIB.addImm(0); // Stack offset (not used).
3714 MIB->addOperand(BranchCond[0]); // Condition.
3777 if (I->getOpcode() == X86::JMP_1) {
3781 TBB = I->getOperand(0).getMBB();
3792 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
3801 TBB = I->getOperand(0).getMBB();
3806 X86::CondCode BranchCode = X86::getCondFromBranch(*I);
3807 if (BranchCode == X86::COND_INVALID)
3812 if (I->findRegisterUseOperand(X86::EFLAGS, /*TRI=*/nullptr)->isUndef())
3818 TBB = I->getOperand(0).getMBB();
3831 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
3832 auto NewTBB = I->getOperand(0).getMBB();
3840 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) ||
3841 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) {
3842 BranchCode = X86::COND_NE_OR_P;
3843 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) ||
3844 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) {
3848 // X86::COND_E_AND_NP usually has two different branch destinations.
3867 BranchCode = X86::COND_E_AND_NP;
3872 Cond[0].setImm(BranchCode);
3891 assert(MemRefBegin >= 0 && "instr should have memory operand");
3894 const MachineOperand &MO = MI.getOperand(MemRefBegin + X86::AddrDisp);
3909 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r)
3918 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) {
3922 // %0 = LEA64r $rip, 1, $noreg, %jump-table.X
3923 // %1 = MOVSX64rm32 %0, 4, XX, 0, $noreg
3924 // %2 = ADD64rr %1, %0
3926 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) {
3927 Register Reg = MI.getOperand(0).getReg();
3935 if (Add->getOpcode() != X86::ADD64rr && Add->getOpcode() != X86::ADD32rr)
3938 if (JTI1 >= 0)
3941 if (JTI2 >= 0)
3972 if (MI.modifiesRegister(X86::EFLAGS, TRI)) {
3977 if (MI.readsRegister(X86::EFLAGS, TRI))
3986 if (Succ->isLiveIn(X86::EFLAGS))
3999 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
4003 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) &&
4004 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
4005 MBP.LHS = ConditionDef->getOperand(0);
4006 MBP.RHS = MachineOperand::CreateImm(0);
4007 MBP.Predicate = Cond[0].getImm() == X86::COND_NE
4021 unsigned Count = 0;
4027 if (I->getOpcode() != X86::JMP_1 &&
4028 X86::getCondFromBranch(*I) == X86::COND_INVALID)
4046 assert((Cond.size() == 1 || Cond.size() == 0) &&
4047 "X86 branch conditions have one component!");
4053 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB);
4061 unsigned Count = 0;
4062 X86::CondCode CC = (X86::CondCode)Cond[0].getImm();
4064 case X86::COND_NE_OR_P:
4066 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE);
4068 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P);
4071 case X86::COND_E_AND_NP:
4079 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE);
4081 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP);
4085 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC);
4091 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB);
4108 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND)
4119 if (X86::GR16RegClass.hasSubClassEq(RC) ||
4120 X86::GR32RegClass.hasSubClassEq(RC) ||
4121 X86::GR64RegClass.hasSubClassEq(RC)) {
4144 X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8,
4149 .addImm(Cond[0].getImm());
4154 return X86::GR8_ABCD_HRegClass.contains(Reg);
4169 if (X86::VK16RegClass.contains(SrcReg)) {
4170 if (X86::GR64RegClass.contains(DestReg)) {
4172 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk;
4174 if (X86::GR32RegClass.contains(DestReg))
4175 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk)
4176 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk);
4184 if (X86::VK16RegClass.contains(DestReg)) {
4185 if (X86::GR64RegClass.contains(SrcReg)) {
4187 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr;
4189 if (X86::GR32RegClass.contains(SrcReg))
4190 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr)
4191 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr);
4199 if (X86::GR64RegClass.contains(DestReg)) {
4200 if (X86::VR128XRegClass.contains(SrcReg))
4202 return HasAVX512 ? X86::VMOVPQIto64Zrr
4203 : HasAVX ? X86::VMOVPQIto64rr
4204 : X86::MOVPQIto64rr;
4205 if (X86::VR64RegClass.contains(SrcReg))
4207 return X86::MMX_MOVD64from64rr;
4208 } else if (X86::GR64RegClass.contains(SrcReg)) {
4210 if (X86::VR128XRegClass.contains(DestReg))
4211 return HasAVX512 ? X86::VMOV64toPQIZrr
4212 : HasAVX ? X86::VMOV64toPQIrr
4213 : X86::MOV64toPQIrr;
4215 if (X86::VR64RegClass.contains(DestReg))
4216 return X86::MMX_MOVD64to64rr;
4222 if (X86::GR32RegClass.contains(DestReg) &&
4223 X86::VR128XRegClass.contains(SrcReg))
4225 return HasAVX512 ? X86::VMOVPDI2DIZrr
4226 : HasAVX ? X86::VMOVPDI2DIrr
4227 : X86::MOVPDI2DIrr;
4229 if (X86::VR128XRegClass.contains(DestReg) &&
4230 X86::GR32RegClass.contains(SrcReg))
4232 return HasAVX512 ? X86::VMOVDI2PDIZrr
4233 : HasAVX ? X86::VMOVDI2PDIrr
4234 : X86::MOVDI2PDIrr;
4235 return 0;
4246 unsigned Opc = 0;
4247 if (X86::GR64RegClass.contains(DestReg, SrcReg))
4248 Opc = X86::MOV64rr;
4249 else if (X86::GR32RegClass.contains(DestReg, SrcReg))
4250 Opc = X86::MOV32rr;
4251 else if (X86::GR16RegClass.contains(DestReg, SrcReg))
4252 Opc = X86::MOV16rr;
4253 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) {
4254 // Copying to or from a physical H register on x86-64 requires a NOREX
4257 Opc = X86::MOV8rr_NOREX;
4259 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
4262 Opc = X86::MOV8rr;
4263 } else if (X86::VR64RegClass.contains(DestReg, SrcReg))
4264 Opc = X86::MMX_MOVQ64rr;
4265 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) {
4267 Opc = X86::VMOVAPSZ128rr;
4268 else if (X86::VR128RegClass.contains(DestReg, SrcReg))
4269 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
4273 Opc = X86::VMOVAPSZrr;
4276 TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, &X86::VR512RegClass);
4278 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
4280 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) {
4282 Opc = X86::VMOVAPSZ256rr;
4283 else if (X86::VR256RegClass.contains(DestReg, SrcReg))
4284 Opc = X86::VMOVAPSYrr;
4288 Opc = X86::VMOVAPSZrr;
4291 TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, &X86::VR512RegClass);
4293 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
4295 } else if (X86::VR512RegClass.contains(DestReg, SrcReg))
4296 Opc = X86::VMOVAPSZrr;
4299 else if (X86::VK16RegClass.contains(DestReg, SrcReg))
4300 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk)
4301 : (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVWkk);
4311 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
4329 // were asserted as 0 are now undef.
4330 if (MI.getOperand(0).isUndef() && MI.getOperand(0).getSubReg())
4333 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
4340 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
4342 return STI.hasAVX512() ? X86::VMOVSSZrm
4343 : STI.hasAVX() ? X86::VMOVSSrm
4344 : X86::MOVSSrm;
4346 return STI.hasAVX512() ? X86::VMOVSSZmr
4347 : STI.hasAVX() ? X86::VMOVSSmr
4348 : X86::MOVSSmr;
4365 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
4367 // Copying to or from a physical H register on x86-64 requires a NOREX
4369 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
4370 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
4371 return Load ? X86::MOV8rm : X86::MOV8mr;
4373 if (X86::VK16RegClass.hasSubClassEq(RC))
4374 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)
4375 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk);
4376 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
4377 return Load ? X86::MOV16rm : X86::MOV16mr;
4379 if (X86::GR32RegClass.hasSubClassEq(RC))
4380 return Load ? X86::MOV32rm : X86::MOV32mr;
4381 if (X86::FR32XRegClass.hasSubClassEq(RC))
4382 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt
4383 : HasAVX ? X86::VMOVSSrm_alt
4384 : X86::MOVSSrm_alt)
4385 : (HasAVX512 ? X86::VMOVSSZmr
4386 : HasAVX ? X86::VMOVSSmr
4387 : X86::MOVSSmr);
4388 if (X86::RFP32RegClass.hasSubClassEq(RC))
4389 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
4390 if (X86::VK32RegClass.hasSubClassEq(RC)) {
4392 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm)
4393 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk);
4397 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
4398 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
4399 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
4400 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
4401 X86::VK16PAIRRegClass.hasSubClassEq(RC))
4402 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
4403 if (X86::FR16RegClass.hasSubClassEq(RC) ||
4404 X86::FR16XRegClass.hasSubClassEq(RC))
4408 if (X86::GR64RegClass.hasSubClassEq(RC))
4409 return Load ? X86::MOV64rm : X86::MOV64mr;
4410 if (X86::FR64XRegClass.hasSubClassEq(RC))
4411 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt
4412 : HasAVX ? X86::VMOVSDrm_alt
4413 : X86::MOVSDrm_alt)
4414 : (HasAVX512 ? X86::VMOVSDZmr
4415 : HasAVX ? X86::VMOVSDmr
4416 : X86::MOVSDmr);
4417 if (X86::VR64RegClass.hasSubClassEq(RC))
4418 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
4419 if (X86::RFP64RegClass.hasSubClassEq(RC))
4420 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
4421 if (X86::VK64RegClass.hasSubClassEq(RC)) {
4423 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm)
4424 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk);
4428 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
4429 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
4431 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
4434 return Load ? (HasVLX ? X86::VMOVAPSZ128rm
4435 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX
4436 : HasAVX ? X86::VMOVAPSrm
4437 : X86::MOVAPSrm)
4438 : (HasVLX ? X86::VMOVAPSZ128mr
4439 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX
4440 : HasAVX ? X86::VMOVAPSmr
4441 : X86::MOVAPSmr);
4443 return Load ? (HasVLX ? X86::VMOVUPSZ128rm
4444 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX
4445 : HasAVX ? X86::VMOVUPSrm
4446 : X86::MOVUPSrm)
4447 : (HasVLX ? X86::VMOVUPSZ128mr
4448 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX
4449 : HasAVX ? X86::VMOVUPSmr
4450 : X86::MOVUPSmr);
4455 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
4458 return Load ? (HasVLX ? X86::VMOVAPSZ256rm
4459 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
4460 : X86::VMOVAPSYrm)
4461 : (HasVLX ? X86::VMOVAPSZ256mr
4462 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
4463 : X86::VMOVAPSYmr);
4465 return Load ? (HasVLX ? X86::VMOVUPSZ256rm
4466 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
4467 : X86::VMOVUPSYrm)
4468 : (HasVLX ? X86::VMOVUPSZ256mr
4469 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
4470 : X86::VMOVUPSYmr);
4472 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
4475 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
4477 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
4479 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
4482 return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
4483 : GET_EGPR_IF_ENABLED(X86::TILESTORED);
4493 if (MemRefBegin < 0)
4498 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg);
4502 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp);
4509 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg();
4510 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm();
4523 if (AM.ScaledReg != X86::NoRegister) {
4551 // instruction. It is quite common for x86-64.
4555 // %6:gr64 = SUBREG_TO_REG 0, killed %8:gr32, %subreg.sub_32bit
4561 if (SubIdx != X86::sub_32bit || FillBits != 0)
4569 if (MovMI->getOpcode() == X86::MOV32r0 &&
4570 MovMI->getOperand(0).getReg() == MovReg) {
4571 ImmVal = 0;
4575 if (MovMI->getOpcode() != X86::MOV32ri &&
4576 MovMI->getOpcode() != X86::MOV64ri &&
4577 MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri)
4580 if (!MovMI->getOperand(1).isImm() || MovMI->getOperand(0).getReg() != MovReg)
4594 case X86::SHR64ri:
4595 case X86::SHR32ri:
4596 case X86::SHL64ri:
4597 case X86::SHL32ri:
4598 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() &&
4600 return MI->getOperand(0).getReg() == NullValueReg &&
4604 case X86::MOV32rr:
4620 if (MemRefBegin < 0)
4626 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg);
4630 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
4633 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() !=
4634 X86::NoRegister)
4637 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp);
4650 // with X86 maintainers, and fix it accordingly. For now, it is ok, since
4651 // there is no use of `Width` for X86 back-end at the moment.
4653 !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0;
4675 case X86::TILELOADD:
4676 case X86::TILESTORED:
4677 case X86::TILELOADD_EVEX:
4678 case X86::TILESTORED_EVEX:
4690 case X86::TILESTORED:
4691 case X86::TILESTORED_EVEX: {
4694 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4695 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4699 MachineOperand &MO = NewMI->getOperand(X86::AddrIndexReg);
4704 case X86::TILELOADD:
4705 case X86::TILELOADD_EVEX: {
4708 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4709 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4712 MachineOperand &MO = NewMI->getOperand(1 + X86::AddrIndexReg);
4771 case X86::CMP64ri32:
4772 case X86::CMP32ri:
4773 case X86::CMP16ri:
4774 case X86::CMP8ri:
4775 SrcReg = MI.getOperand(0).getReg();
4776 SrcReg2 = 0;
4778 CmpMask = ~0;
4781 CmpMask = CmpValue = 0;
4790 SrcReg2 = 0;
4791 CmpMask = 0;
4792 CmpValue = 0;
4800 CmpMask = 0;
4801 CmpValue = 0;
4808 SrcReg2 = 0;
4810 CmpMask = ~0;
4813 CmpMask = CmpValue = 0;
4816 case X86::CMP64rr:
4817 case X86::CMP32rr:
4818 case X86::CMP16rr:
4819 case X86::CMP8rr:
4820 SrcReg = MI.getOperand(0).getReg();
4822 CmpMask = 0;
4823 CmpValue = 0;
4825 case X86::TEST8rr:
4826 case X86::TEST16rr:
4827 case X86::TEST32rr:
4828 case X86::TEST64rr:
4829 SrcReg = MI.getOperand(0).getReg();
4833 SrcReg2 = 0;
4834 CmpMask = ~0;
4835 CmpValue = 0;
4847 case X86::CMP64rr:
4848 case X86::CMP32rr:
4849 case X86::CMP16rr:
4850 case X86::CMP8rr:
4872 case X86::CMP64ri32:
4873 case X86::CMP32ri:
4874 case X86::CMP16ri:
4875 case X86::CMP8ri:
4880 case X86::TEST64rr:
4881 case X86::TEST32rr:
4882 case X86::TEST16rr:
4883 case X86::TEST8rr: {
4884 if (ImmMask != 0) {
4892 *ImmDelta = 0;
4921 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and
4925 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) {
4946 return getTruncatedShiftCount(MI, 2) != 0;
4957 return ShAmt != 0;
4966 return getTruncatedShiftCount(MI, 3) != 0;
5028 case X86::LZCNT16rr:
5029 case X86::LZCNT16rm:
5030 case X86::LZCNT32rr:
5031 case X86::LZCNT32rm:
5032 case X86::LZCNT64rr:
5033 case X86::LZCNT64rm:
5034 case X86::POPCNT16rr:
5035 case X86::POPCNT16rm:
5036 case X86::POPCNT32rr:
5037 case X86::POPCNT32rm:
5038 case X86::POPCNT64rr:
5039 case X86::POPCNT64rm:
5040 case X86::TZCNT16rr:
5041 case X86::TZCNT16rm:
5042 case X86::TZCNT32rr:
5043 case X86::TZCNT32rm:
5044 case X86::TZCNT64rr:
5045 case X86::TZCNT64rm:
5083 case X86::ANDN32rr:
5084 case X86::ANDN32rm:
5085 case X86::ANDN64rr:
5086 case X86::ANDN64rm:
5087 case X86::BLSI32rr:
5088 case X86::BLSI32rm:
5089 case X86::BLSI64rr:
5090 case X86::BLSI64rm:
5091 case X86::BLSMSK32rr:
5092 case X86::BLSMSK32rm:
5093 case X86::BLSMSK64rr:
5094 case X86::BLSMSK64rm:
5095 case X86::BLSR32rr:
5096 case X86::BLSR32rm:
5097 case X86::BLSR64rr:
5098 case X86::BLSR64rm:
5099 case X86::BLCFILL32rr:
5100 case X86::BLCFILL32rm:
5101 case X86::BLCFILL64rr:
5102 case X86::BLCFILL64rm:
5103 case X86::BLCI32rr:
5104 case X86::BLCI32rm:
5105 case X86::BLCI64rr:
5106 case X86::BLCI64rm:
5107 case X86::BLCIC32rr:
5108 case X86::BLCIC32rm:
5109 case X86::BLCIC64rr:
5110 case X86::BLCIC64rm:
5111 case X86::BLCMSK32rr:
5112 case X86::BLCMSK32rm:
5113 case X86::BLCMSK64rr:
5114 case X86::BLCMSK64rm:
5115 case X86::BLCS32rr:
5116 case X86::BLCS32rm:
5117 case X86::BLCS64rr:
5118 case X86::BLCS64rm:
5119 case X86::BLSFILL32rr:
5120 case X86::BLSFILL32rm:
5121 case X86::BLSFILL64rr:
5122 case X86::BLSFILL64rm:
5123 case X86::BLSIC32rr:
5124 case X86::BLSIC32rm:
5125 case X86::BLSIC64rr:
5126 case X86::BLSIC64rm:
5127 case X86::BZHI32rr:
5128 case X86::BZHI32rm:
5129 case X86::BZHI64rr:
5130 case X86::BZHI64rm:
5131 case X86::T1MSKC32rr:
5132 case X86::T1MSKC32rm:
5133 case X86::T1MSKC64rr:
5134 case X86::T1MSKC64rm:
5135 case X86::TZMSK32rr:
5136 case X86::TZMSK32rm:
5137 case X86::TZMSK64rr:
5138 case X86::TZMSK64rm:
5144 case X86::BEXTR32rr:
5145 case X86::BEXTR64rr:
5146 case X86::BEXTR32rm:
5147 case X86::BEXTR64rm:
5148 case X86::BEXTRI32ri:
5149 case X86::BEXTRI32mi:
5150 case X86::BEXTRI64ri:
5151 case X86::BEXTRI64mi:
5160 static X86::CondCode isUseDefConvertible(const MachineInstr &MI) {
5163 return X86::COND_INVALID;
5168 return X86::COND_AE;
5169 case X86::LZCNT16rr:
5170 case X86::LZCNT32rr:
5171 case X86::LZCNT64rr:
5172 return X86::COND_B;
5173 case X86::POPCNT16rr:
5174 case X86::POPCNT32rr:
5175 case X86::POPCNT64rr:
5176 return X86::COND_E;
5177 case X86::TZCNT16rr:
5178 case X86::TZCNT32rr:
5179 case X86::TZCNT64rr:
5180 return X86::COND_B;
5181 case X86::BSF16rr:
5182 case X86::BSF32rr:
5183 case X86::BSF64rr:
5184 case X86::BSR16rr:
5185 case X86::BSR32rr:
5186 case X86::BSR64rr:
5187 return X86::COND_E;
5188 case X86::BLSI32rr:
5189 case X86::BLSI64rr:
5190 return X86::COND_AE;
5191 case X86::BLSR32rr:
5192 case X86::BLSR64rr:
5193 case X86::BLSMSK32rr:
5194 case X86::BLSMSK64rr:
5195 return X86::COND_B;
5223 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
5226 unsigned NewOpcode = 0;
5228 CASE_ND(A) NewOpcode = X86::B; \
5248 CmpInstr.removeOperand(0);
5252 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
5253 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
5261 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
5276 X86::CondCode NewCC = X86::COND_INVALID;
5277 int64_t ImmDelta = 0;
5308 // %src_reg = subreg_to_reg 0, %reg, %subreg.sub_index
5314 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode()));
5322 if (Inst.modifiesRegister(X86::EFLAGS, TRI)) {
5332 if (NewCC != X86::COND_INVALID && Inst.getOperand(1).isReg() &&
5354 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 &&
5355 Inst.registerDefIsDead(X86::EFLAGS, TRI)) {
5381 SmallVector<std::pair<MachineInstr *, X86::CondCode>, 4> OpsToUpdate;
5385 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI);
5386 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI);
5397 X86::CondCode OldCC = X86::getCondFromMI(Instr);
5398 if ((MI || IsSwapped || ImmDelta != 0) && OldCC == X86::COND_INVALID)
5401 X86::CondCode ReplacementCC = X86::COND_INVALID;
5406 case X86::COND_A:
5407 case X86::COND_AE:
5408 case X86::COND_B:
5409 case X86::COND_BE:
5412 case X86::COND_G:
5413 case X86::COND_GE:
5414 case X86::COND_L:
5415 case X86::COND_LE:
5421 case X86::COND_O:
5422 case X86::COND_NO:
5427 case X86::COND_S:
5428 case X86::COND_NS:
5442 case X86::COND_E:
5445 case X86::COND_NE:
5454 if (ReplacementCC == X86::COND_INVALID)
5457 } else if (ImmDelta != 0) {
5462 case X86::COND_L: // x <s (C + 1) --> x <=s C
5465 ReplacementCC = X86::COND_LE;
5467 case X86::COND_B: // x <u (C + 1) --> x <=u C
5468 if (ImmDelta != 1 || CmpValue == 0)
5470 ReplacementCC = X86::COND_BE;
5472 case X86::COND_GE: // x >=s (C + 1) --> x >s C
5475 ReplacementCC = X86::COND_G;
5477 case X86::COND_AE: // x >=u (C + 1) --> x >u C
5478 if (ImmDelta != 1 || CmpValue == 0)
5480 ReplacementCC = X86::COND_A;
5482 case X86::COND_G: // x >s (C - 1) --> x >=s C
5485 ReplacementCC = X86::COND_GE;
5487 case X86::COND_A: // x >u (C - 1) --> x >=u C
5490 ReplacementCC = X86::COND_AE;
5492 case X86::COND_LE: // x <=s (C - 1) --> x <s C
5495 ReplacementCC = X86::COND_L;
5497 case X86::COND_BE: // x <=u (C - 1) --> x <u C
5500 ReplacementCC = X86::COND_B;
5514 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS, TRI)) {
5525 if (Successor->isLiveIn(X86::EFLAGS))
5544 if (!Instr->readsRegister(X86::EFLAGS, TRI) &&
5545 Instr->modifiesRegister(X86::EFLAGS, TRI)) {
5558 Sub->findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
5573 if (!MBB->isLiveIn(X86::EFLAGS))
5574 MBB->addLiveIn(X86::EFLAGS);
5596 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
5613 FoldAsLoadDefReg = 0;
5620 /// \returns true if the instruction can be changed to COPY when imm is 0.
5642 return 0;
5644 case X86::FROM: \
5645 return X86::TO; \
5646 case X86::FROM##_ND: \
5647 return X86::TO##_ND;
5678 case X86::FROM: \
5679 return X86::TO;
5709 if ((Reg.isPhysical() && X86::GR64RegClass.contains(Reg)) ||
5710 (Reg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC))) {
5726 Register ToReg = UseMI.getOperand(0).getReg();
5730 bool GR32Reg = (ToReg.isVirtual() && X86::GR32RegClass.hasSubClassEq(RC)) ||
5731 (ToReg.isPhysical() && X86::GR32RegClass.contains(ToReg));
5732 bool GR64Reg = (ToReg.isVirtual() && X86::GR64RegClass.hasSubClassEq(RC)) ||
5733 (ToReg.isPhysical() && X86::GR64RegClass.contains(ToReg));
5734 bool GR8Reg = (ToReg.isVirtual() && X86::GR8RegClass.hasSubClassEq(RC)) ||
5735 (ToReg.isPhysical() && X86::GR8RegClass.contains(ToReg));
5737 if (ImmVal == 0) {
5745 NewOpc = X86::MOV32ri64;
5747 NewOpc = X86::MOV64ri;
5749 NewOpc = X86::MOV32ri;
5750 if (ImmVal == 0) {
5754 TRI, X86::EFLAGS, UseMI) != MachineBasicBlock::LQR_Dead)
5761 UseMI.setDesc(get(X86::MOV32r0));
5764 UseMI.addOperand(MachineOperand::CreateReg(X86::EFLAGS, /*isDef=*/true,
5771 NewOpc = X86::MOV8ri;
5781 if ((NewOpc == X86::SUB64ri32 || NewOpc == X86::SUB32ri ||
5782 NewOpc == X86::SBB64ri32 || NewOpc == X86::SBB32ri ||
5783 NewOpc == X86::SUB64ri32_ND || NewOpc == X86::SUB32ri_ND ||
5784 NewOpc == X86::SBB64ri32_ND || NewOpc == X86::SBB32ri_ND) &&
5788 if (((NewOpc == X86::CMP64ri32 || NewOpc == X86::CMP32ri) ||
5789 (NewOpc == X86::CCMP64ri32 || NewOpc == X86::CCMP32ri)) &&
5793 using namespace X86;
5801 assert(Reg == X86::CL);
5819 if (ImmVal == 0 && canConvert2Copy(NewOpc) &&
5820 UseMI.registerDefIsDead(X86::EFLAGS, /*TRI=*/nullptr)) {
5821 // %100 = add %101, 0
5828 UseMI.findRegisterDefOperandIdx(X86::EFLAGS, /*TRI=*/nullptr));
5829 UseMI.untieRegOperand(0);
5835 if (!UseMI.getOperand(0).isDef()) {
5836 Op1 = 0; // TEST, CMP, CTEST, CCMP
5878 Register Reg = MIB.getReg(0);
5907 Register Reg = MIB.getReg(0);
5910 BuildMI(MBB, MIB.getInstr(), DL, TII.get(X86::XOR32rr), Reg)
5915 MIB->setDesc(TII.get(MinusOne ? X86::DEC32r : X86::INC32r));
5927 assert(Imm != 0 && "Using push/pop for 0 is not efficient.");
5933 assert(MIB->getOpcode() == X86::MOV64ImmSExti8 ||
5934 MIB->getOpcode() == X86::MOV32ImmSExti8);
5940 MIB->setDesc(TII.get(MIB->getOpcode() == X86::MOV32ImmSExti8
5941 ? X86::MOV32ri
5942 : X86::MOV64ri));
5949 BuildMI(MBB, I, DL, TII.get(X86::PUSH64i32)).addImm(Imm);
5950 MIB->setDesc(TII.get(X86::POP64r));
5951 MIB->getOperand(0).setReg(getX86SubSuperRegister(MIB.getReg(0), 64));
5953 assert(MIB->getOpcode() == X86::MOV32ImmSExti8);
5955 BuildMI(MBB, I, DL, TII.get(X86::PUSH32i)).addImm(Imm);
5956 MIB->setDesc(TII.get(X86::POP32r));
5985 Register Reg = MIB.getReg(0);
5995 BuildMI(MBB, I, DL, TII.get(X86::MOV64rm), Reg)
5996 .addReg(X86::RIP)
5998 .addReg(0)
5999 .addGlobalAddress(GV, 0, X86II::MO_GOTPCREL)
6000 .addReg(0)
6003 MIB->setDesc(TII.get(X86::MOV64rm));
6004 MIB.addReg(Reg, RegState::Kill).addImm(1).addReg(0).addImm(0).addReg(0);
6013 MIB->getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
6026 Register DestReg = MIB.getReg(0);
6035 DestReg = TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass);
6036 MIB->getOperand(0).setReg(DestReg);
6048 Register SrcReg = MIB.getReg(X86::AddrNumOperands);
6057 SrcReg = TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass);
6058 MIB->getOperand(X86::AddrNumOperands).setReg(SrcReg);
6059 MIB.addImm(0x0); // Append immediate to extract from the lower bits.
6081 case X86::MOV32r0:
6082 return Expand2AddrUndef(MIB, get(X86::XOR32rr));
6083 case X86::MOV32r1:
6085 case X86::MOV32r_1:
6087 case X86::MOV32ImmSExti8:
6088 case X86::MOV64ImmSExti8:
6090 case X86::SETB_C32r:
6091 return Expand2AddrUndef(MIB, get(X86::SBB32rr));
6092 case X86::SETB_C64r:
6093 return Expand2AddrUndef(MIB, get(X86::SBB64rr));
6094 case X86::MMX_SET0:
6095 return Expand2AddrUndef(MIB, get(X86::MMX_PXORrr));
6096 case X86::V_SET0:
6097 case X86::FsFLD0SS:
6098 case X86::FsFLD0SD:
6099 case X86::FsFLD0SH:
6100 case X86::FsFLD0F128:
6101 return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr));
6102 case X86::AVX_SET0: {
6105 Register SrcReg = MIB.getReg(0);
6106 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm);
6107 MIB->getOperand(0).setReg(XReg);
6108 Expand2AddrUndef(MIB, get(X86::VXORPSrr));
6112 case X86::AVX512_128_SET0:
6113 case X86::AVX512_FsFLD0SH:
6114 case X86::AVX512_FsFLD0SS:
6115 case X86::AVX512_FsFLD0SD:
6116 case X86::AVX512_FsFLD0F128: {
6118 Register SrcReg = MIB.getReg(0);
6122 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
6125 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
6126 MIB->getOperand(0).setReg(SrcReg);
6127 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
6129 case X86::AVX512_256_SET0:
6130 case X86::AVX512_512_SET0: {
6132 Register SrcReg = MIB.getReg(0);
6135 Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm);
6136 MIB->getOperand(0).setReg(XReg);
6137 Expand2AddrUndef(MIB, get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
6141 if (MI.getOpcode() == X86::AVX512_256_SET0) {
6144 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
6145 MIB->getOperand(0).setReg(ZReg);
6147 return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
6149 case X86::V_SETALLONES:
6151 get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
6152 case X86::AVX2_SETALLONES:
6153 return Expand2AddrUndef(MIB, get(X86::VPCMPEQDYrr));
6154 case X86::AVX1_SETALLONES: {
6155 Register Reg = MIB.getReg(0);
6156 // VCMPPSYrri with an immediate 0xf should produce VCMPTRUEPS.
6157 MIB->setDesc(get(X86::VCMPPSYrri));
6158 MIB.addReg(Reg, RegState::Undef).addReg(Reg, RegState::Undef).addImm(0xf);
6161 case X86::AVX512_512_SETALLONES: {
6162 Register Reg = MIB.getReg(0);
6163 MIB->setDesc(get(X86::VPTERNLOGDZrri));
6165 // 0xff will return 1s for any input.
6169 .addImm(0xff);
6172 case X86::AVX512_512_SEXT_MASK_32:
6173 case X86::AVX512_512_SEXT_MASK_64: {
6174 Register Reg = MIB.getReg(0);
6177 unsigned Opc = (MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64)
6178 ? X86::VPTERNLOGQZrrikz
6179 : X86::VPTERNLOGDZrrikz;
6183 // 0xff will return 1s for any input.
6188 .addImm(0xff);
6191 case X86::VMOVAPSZ128rm_NOVLX:
6192 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSrm),
6193 get(X86::VBROADCASTF32X4rm), X86::sub_xmm);
6194 case X86::VMOVUPSZ128rm_NOVLX:
6195 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSrm),
6196 get(X86::VBROADCASTF32X4rm), X86::sub_xmm);
6197 case X86::VMOVAPSZ256rm_NOVLX:
6198 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVAPSYrm),
6199 get(X86::VBROADCASTF64X4rm), X86::sub_ymm);
6200 case X86::VMOVUPSZ256rm_NOVLX:
6201 return expandNOVLXLoad(MIB, &getRegisterInfo(), get(X86::VMOVUPSYrm),
6202 get(X86::VBROADCASTF64X4rm), X86::sub_ymm);
6203 case X86::VMOVAPSZ128mr_NOVLX:
6204 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSmr),
6205 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm);
6206 case X86::VMOVUPSZ128mr_NOVLX:
6207 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSmr),
6208 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm);
6209 case X86::VMOVAPSZ256mr_NOVLX:
6210 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVAPSYmr),
6211 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm);
6212 case X86::VMOVUPSZ256mr_NOVLX:
6213 return expandNOVLXStore(MIB, &getRegisterInfo(), get(X86::VMOVUPSYmr),
6214 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm);
6215 case X86::MOV32ri64: {
6216 Register Reg = MIB.getReg(0);
6217 Register Reg32 = RI.getSubReg(Reg, X86::sub_32bit);
6218 MI.setDesc(get(X86::MOV32ri));
6219 MIB->getOperand(0).setReg(Reg32);
6224 case X86::RDFLAGS32:
6225 case X86::RDFLAGS64: {
6226 unsigned Is64Bit = MI.getOpcode() == X86::RDFLAGS64;
6230 get(Is64Bit ? X86::PUSHF64 : X86::PUSHF32))
6237 assert(NewMI->getOperand(2).getReg() == X86::EFLAGS &&
6240 assert(NewMI->getOperand(3).getReg() == X86::DF &&
6244 MIB->setDesc(get(Is64Bit ? X86::POP64r : X86::POP32r));
6248 case X86::WRFLAGS32:
6249 case X86::WRFLAGS64: {
6250 unsigned Is64Bit = MI.getOpcode() == X86::WRFLAGS64;
6254 get(Is64Bit ? X86::PUSH64r : X86::PUSH32r))
6255 .addReg(MI.getOperand(0).getReg());
6257 get(Is64Bit ? X86::POPF64 : X86::POPF32));
6269 case X86::KSET0W:
6270 return Expand2AddrKreg(MIB, get(X86::KXORWrr), X86::K0);
6271 case X86::KSET0D:
6272 return Expand2AddrKreg(MIB, get(X86::KXORDrr), X86::K0);
6273 case X86::KSET0Q:
6274 return Expand2AddrKreg(MIB, get(X86::KXORQrr), X86::K0);
6275 case X86::KSET1W:
6276 return Expand2AddrKreg(MIB, get(X86::KXNORWrr), X86::K0);
6277 case X86::KSET1D:
6278 return Expand2AddrKreg(MIB, get(X86::KXNORDrr), X86::K0);
6279 case X86::KSET1Q:
6280 return Expand2AddrKreg(MIB, get(X86::KXNORQrr), X86::K0);
6284 case X86::XOR64_FP:
6285 case X86::XOR32_FP:
6287 case X86::SHLDROT32ri:
6288 return expandSHXDROT(MIB, get(X86::SHLD32rri8));
6289 case X86::SHLDROT64ri:
6290 return expandSHXDROT(MIB, get(X86::SHLD64rri8));
6291 case X86::SHRDROT32ri:
6292 return expandSHXDROT(MIB, get(X86::SHRD32rri8));
6293 case X86::SHRDROT64ri:
6294 return expandSHXDROT(MIB, get(X86::SHRD64rri8));
6295 case X86::ADD8rr_DB:
6296 MIB->setDesc(get(X86::OR8rr));
6298 case X86::ADD16rr_DB:
6299 MIB->setDesc(get(X86::OR16rr));
6301 case X86::ADD32rr_DB:
6302 MIB->setDesc(get(X86::OR32rr));
6304 case X86::ADD64rr_DB:
6305 MIB->setDesc(get(X86::OR64rr));
6307 case X86::ADD8ri_DB:
6308 MIB->setDesc(get(X86::OR8ri));
6310 case X86::ADD16ri_DB:
6311 MIB->setDesc(get(X86::OR16ri));
6313 case X86::ADD32ri_DB:
6314 MIB->setDesc(get(X86::OR32ri));
6316 case X86::ADD64ri32_DB:
6317 MIB->setDesc(get(X86::OR64ri32));
6342 case X86::CVTSI2SSrr:
6343 case X86::CVTSI2SSrm:
6344 case X86::CVTSI642SSrr:
6345 case X86::CVTSI642SSrm:
6346 case X86::CVTSI2SDrr:
6347 case X86::CVTSI2SDrm:
6348 case X86::CVTSI642SDrr:
6349 case X86::CVTSI642SDrm:
6353 case X86::CVTSD2SSrr:
6354 case X86::CVTSD2SSrm:
6355 case X86::CVTSS2SDrr:
6356 case X86::CVTSS2SDrm:
6357 case X86::MOVHPDrm:
6358 case X86::MOVHPSrm:
6359 case X86::MOVLPDrm:
6360 case X86::MOVLPSrm:
6361 case X86::RCPSSr:
6362 case X86::RCPSSm:
6363 case X86::RCPSSr_Int:
6364 case X86::RCPSSm_Int:
6365 case X86::ROUNDSDri:
6366 case X86::ROUNDSDmi:
6367 case X86::ROUNDSSri:
6368 case X86::ROUNDSSmi:
6369 case X86::RSQRTSSr:
6370 case X86::RSQRTSSm:
6371 case X86::RSQRTSSr_Int:
6372 case X86::RSQRTSSm_Int:
6373 case X86::SQRTSSr:
6374 case X86::SQRTSSm:
6375 case X86::SQRTSSr_Int:
6376 case X86::SQRTSSm_Int:
6377 case X86::SQRTSDr:
6378 case X86::SQRTSDm:
6379 case X86::SQRTSDr_Int:
6380 case X86::SQRTSDm_Int:
6382 case X86::VFCMULCPHZ128rm:
6383 case X86::VFCMULCPHZ128rmb:
6384 case X86::VFCMULCPHZ128rmbkz:
6385 case X86::VFCMULCPHZ128rmkz:
6386 case X86::VFCMULCPHZ128rr:
6387 case X86::VFCMULCPHZ128rrkz:
6388 case X86::VFCMULCPHZ256rm:
6389 case X86::VFCMULCPHZ256rmb:
6390 case X86::VFCMULCPHZ256rmbkz:
6391 case X86::VFCMULCPHZ256rmkz:
6392 case X86::VFCMULCPHZ256rr:
6393 case X86::VFCMULCPHZ256rrkz:
6394 case X86::VFCMULCPHZrm:
6395 case X86::VFCMULCPHZrmb:
6396 case X86::VFCMULCPHZrmbkz:
6397 case X86::VFCMULCPHZrmkz:
6398 case X86::VFCMULCPHZrr:
6399 case X86::VFCMULCPHZrrb:
6400 case X86::VFCMULCPHZrrbkz:
6401 case X86::VFCMULCPHZrrkz:
6402 case X86::VFMULCPHZ128rm:
6403 case X86::VFMULCPHZ128rmb:
6404 case X86::VFMULCPHZ128rmbkz:
6405 case X86::VFMULCPHZ128rmkz:
6406 case X86::VFMULCPHZ128rr:
6407 case X86::VFMULCPHZ128rrkz:
6408 case X86::VFMULCPHZ256rm:
6409 case X86::VFMULCPHZ256rmb:
6410 case X86::VFMULCPHZ256rmbkz:
6411 case X86::VFMULCPHZ256rmkz:
6412 case X86::VFMULCPHZ256rr:
6413 case X86::VFMULCPHZ256rrkz:
6414 case X86::VFMULCPHZrm:
6415 case X86::VFMULCPHZrmb:
6416 case X86::VFMULCPHZrmbkz:
6417 case X86::VFMULCPHZrmkz:
6418 case X86::VFMULCPHZrr:
6419 case X86::VFMULCPHZrrb:
6420 case X86::VFMULCPHZrrbkz:
6421 case X86::VFMULCPHZrrkz:
6422 case X86::VFCMULCSHZrm:
6423 case X86::VFCMULCSHZrmkz:
6424 case X86::VFCMULCSHZrr:
6425 case X86::VFCMULCSHZrrb:
6426 case X86::VFCMULCSHZrrbkz:
6427 case X86::VFCMULCSHZrrkz:
6428 case X86::VFMULCSHZrm:
6429 case X86::VFMULCSHZrmkz:
6430 case X86::VFMULCSHZrr:
6431 case X86::VFMULCSHZrrb:
6432 case X86::VFMULCSHZrrbkz:
6433 case X86::VFMULCSHZrrkz:
6435 case X86::VPERMDYrm:
6436 case X86::VPERMDYrr:
6437 case X86::VPERMQYmi:
6438 case X86::VPERMQYri:
6439 case X86::VPERMPSYrm:
6440 case X86::VPERMPSYrr:
6441 case X86::VPERMPDYmi:
6442 case X86::VPERMPDYri:
6443 case X86::VPERMDZ256rm:
6444 case X86::VPERMDZ256rmb:
6445 case X86::VPERMDZ256rmbkz:
6446 case X86::VPERMDZ256rmkz:
6447 case X86::VPERMDZ256rr:
6448 case X86::VPERMDZ256rrkz:
6449 case X86::VPERMDZrm:
6450 case X86::VPERMDZrmb:
6451 case X86::VPERMDZrmbkz:
6452 case X86::VPERMDZrmkz:
6453 case X86::VPERMDZrr:
6454 case X86::VPERMDZrrkz:
6455 case X86::VPERMQZ256mbi:
6456 case X86::VPERMQZ256mbikz:
6457 case X86::VPERMQZ256mi:
6458 case X86::VPERMQZ256mikz:
6459 case X86::VPERMQZ256ri:
6460 case X86::VPERMQZ256rikz:
6461 case X86::VPERMQZ256rm:
6462 case X86::VPERMQZ256rmb:
6463 case X86::VPERMQZ256rmbkz:
6464 case X86::VPERMQZ256rmkz:
6465 case X86::VPERMQZ256rr:
6466 case X86::VPERMQZ256rrkz:
6467 case X86::VPERMQZmbi:
6468 case X86::VPERMQZmbikz:
6469 case X86::VPERMQZmi:
6470 case X86::VPERMQZmikz:
6471 case X86::VPERMQZri:
6472 case X86::VPERMQZrikz:
6473 case X86::VPERMQZrm:
6474 case X86::VPERMQZrmb:
6475 case X86::VPERMQZrmbkz:
6476 case X86::VPERMQZrmkz:
6477 case X86::VPERMQZrr:
6478 case X86::VPERMQZrrkz:
6479 case X86::VPERMPSZ256rm:
6480 case X86::VPERMPSZ256rmb:
6481 case X86::VPERMPSZ256rmbkz:
6482 case X86::VPERMPSZ256rmkz:
6483 case X86::VPERMPSZ256rr:
6484 case X86::VPERMPSZ256rrkz:
6485 case X86::VPERMPSZrm:
6486 case X86::VPERMPSZrmb:
6487 case X86::VPERMPSZrmbkz:
6488 case X86::VPERMPSZrmkz:
6489 case X86::VPERMPSZrr:
6490 case X86::VPERMPSZrrkz:
6491 case X86::VPERMPDZ256mbi:
6492 case X86::VPERMPDZ256mbikz:
6493 case X86::VPERMPDZ256mi:
6494 case X86::VPERMPDZ256mikz:
6495 case X86::VPERMPDZ256ri:
6496 case X86::VPERMPDZ256rikz:
6497 case X86::VPERMPDZ256rm:
6498 case X86::VPERMPDZ256rmb:
6499 case X86::VPERMPDZ256rmbkz:
6500 case X86::VPERMPDZ256rmkz:
6501 case X86::VPERMPDZ256rr:
6502 case X86::VPERMPDZ256rrkz:
6503 case X86::VPERMPDZmbi:
6504 case X86::VPERMPDZmbikz:
6505 case X86::VPERMPDZmi:
6506 case X86::VPERMPDZmikz:
6507 case X86::VPERMPDZri:
6508 case X86::VPERMPDZrikz:
6509 case X86::VPERMPDZrm:
6510 case X86::VPERMPDZrmb:
6511 case X86::VPERMPDZrmbkz:
6512 case X86::VPERMPDZrmkz:
6513 case X86::VPERMPDZrr:
6514 case X86::VPERMPDZrrkz:
6516 case X86::VRANGEPDZ128rmbi:
6517 case X86::VRANGEPDZ128rmbikz:
6518 case X86::VRANGEPDZ128rmi:
6519 case X86::VRANGEPDZ128rmikz:
6520 case X86::VRANGEPDZ128rri:
6521 case X86::VRANGEPDZ128rrikz:
6522 case X86::VRANGEPDZ256rmbi:
6523 case X86::VRANGEPDZ256rmbikz:
6524 case X86::VRANGEPDZ256rmi:
6525 case X86::VRANGEPDZ256rmikz:
6526 case X86::VRANGEPDZ256rri:
6527 case X86::VRANGEPDZ256rrikz:
6528 case X86::VRANGEPDZrmbi:
6529 case X86::VRANGEPDZrmbikz:
6530 case X86::VRANGEPDZrmi:
6531 case X86::VRANGEPDZrmikz:
6532 case X86::VRANGEPDZrri:
6533 case X86::VRANGEPDZrrib:
6534 case X86::VRANGEPDZrribkz:
6535 case X86::VRANGEPDZrrikz:
6536 case X86::VRANGEPSZ128rmbi:
6537 case X86::VRANGEPSZ128rmbikz:
6538 case X86::VRANGEPSZ128rmi:
6539 case X86::VRANGEPSZ128rmikz:
6540 case X86::VRANGEPSZ128rri:
6541 case X86::VRANGEPSZ128rrikz:
6542 case X86::VRANGEPSZ256rmbi:
6543 case X86::VRANGEPSZ256rmbikz:
6544 case X86::VRANGEPSZ256rmi:
6545 case X86::VRANGEPSZ256rmikz:
6546 case X86::VRANGEPSZ256rri:
6547 case X86::VRANGEPSZ256rrikz:
6548 case X86::VRANGEPSZrmbi:
6549 case X86::VRANGEPSZrmbikz:
6550 case X86::VRANGEPSZrmi:
6551 case X86::VRANGEPSZrmikz:
6552 case X86::VRANGEPSZrri:
6553 case X86::VRANGEPSZrrib:
6554 case X86::VRANGEPSZrribkz:
6555 case X86::VRANGEPSZrrikz:
6556 case X86::VRANGESDZrmi:
6557 case X86::VRANGESDZrmikz:
6558 case X86::VRANGESDZrri:
6559 case X86::VRANGESDZrrib:
6560 case X86::VRANGESDZrribkz:
6561 case X86::VRANGESDZrrikz:
6562 case X86::VRANGESSZrmi:
6563 case X86::VRANGESSZrmikz:
6564 case X86::VRANGESSZrri:
6565 case X86::VRANGESSZrrib:
6566 case X86::VRANGESSZrribkz:
6567 case X86::VRANGESSZrrikz:
6569 case X86::VGETMANTSSZrmi:
6570 case X86::VGETMANTSSZrmikz:
6571 case X86::VGETMANTSSZrri:
6572 case X86::VGETMANTSSZrrib:
6573 case X86::VGETMANTSSZrribkz:
6574 case X86::VGETMANTSSZrrikz:
6575 case X86::VGETMANTSDZrmi:
6576 case X86::VGETMANTSDZrmikz:
6577 case X86::VGETMANTSDZrri:
6578 case X86::VGETMANTSDZrrib:
6579 case X86::VGETMANTSDZrribkz:
6580 case X86::VGETMANTSDZrrikz:
6581 case X86::VGETMANTSHZrmi:
6582 case X86::VGETMANTSHZrmikz:
6583 case X86::VGETMANTSHZrri:
6584 case X86::VGETMANTSHZrrib:
6585 case X86::VGETMANTSHZrribkz:
6586 case X86::VGETMANTSHZrrikz:
6587 case X86::VGETMANTPSZ128rmbi:
6588 case X86::VGETMANTPSZ128rmbikz:
6589 case X86::VGETMANTPSZ128rmi:
6590 case X86::VGETMANTPSZ128rmikz:
6591 case X86::VGETMANTPSZ256rmbi:
6592 case X86::VGETMANTPSZ256rmbikz:
6593 case X86::VGETMANTPSZ256rmi:
6594 case X86::VGETMANTPSZ256rmikz:
6595 case X86::VGETMANTPSZrmbi:
6596 case X86::VGETMANTPSZrmbikz:
6597 case X86::VGETMANTPSZrmi:
6598 case X86::VGETMANTPSZrmikz:
6599 case X86::VGETMANTPDZ128rmbi:
6600 case X86::VGETMANTPDZ128rmbikz:
6601 case X86::VGETMANTPDZ128rmi:
6602 case X86::VGETMANTPDZ128rmikz:
6603 case X86::VGETMANTPDZ256rmbi:
6604 case X86::VGETMANTPDZ256rmbikz:
6605 case X86::VGETMANTPDZ256rmi:
6606 case X86::VGETMANTPDZ256rmikz:
6607 case X86::VGETMANTPDZrmbi:
6608 case X86::VGETMANTPDZrmbikz:
6609 case X86::VGETMANTPDZrmi:
6610 case X86::VGETMANTPDZrmikz:
6612 case X86::VPMULLQZ128rm:
6613 case X86::VPMULLQZ128rmb:
6614 case X86::VPMULLQZ128rmbkz:
6615 case X86::VPMULLQZ128rmkz:
6616 case X86::VPMULLQZ128rr:
6617 case X86::VPMULLQZ128rrkz:
6618 case X86::VPMULLQZ256rm:
6619 case X86::VPMULLQZ256rmb:
6620 case X86::VPMULLQZ256rmbkz:
6621 case X86::VPMULLQZ256rmkz:
6622 case X86::VPMULLQZ256rr:
6623 case X86::VPMULLQZ256rrkz:
6624 case X86::VPMULLQZrm:
6625 case X86::VPMULLQZrmb:
6626 case X86::VPMULLQZrmbkz:
6627 case X86::VPMULLQZrmkz:
6628 case X86::VPMULLQZrr:
6629 case X86::VPMULLQZrrkz:
6632 case X86::POPCNT32rm:
6633 case X86::POPCNT32rr:
6634 case X86::POPCNT64rm:
6635 case X86::POPCNT64rr:
6637 case X86::LZCNT32rm:
6638 case X86::LZCNT32rr:
6639 case X86::LZCNT64rm:
6640 case X86::LZCNT64rr:
6641 case X86::TZCNT32rm:
6642 case X86::TZCNT32rr:
6643 case X86::TZCNT64rm:
6644 case X86::TZCNT64rr:
6656 if (OpNum != 0 || !hasPartialRegUpdate(MI.getOpcode(), Subtarget))
6657 return 0;
6660 const MachineOperand &MO = MI.getOperand(0);
6664 return 0;
6667 return 0;
6684 case X86::MMX_PUNPCKHBWrr:
6685 case X86::MMX_PUNPCKHWDrr:
6686 case X86::MMX_PUNPCKHDQrr:
6687 case X86::MMX_PUNPCKLBWrr:
6688 case X86::MMX_PUNPCKLWDrr:
6689 case X86::MMX_PUNPCKLDQrr:
6690 case X86::MOVHLPSrr:
6691 case X86::PACKSSWBrr:
6692 case X86::PACKUSWBrr:
6693 case X86::PACKSSDWrr:
6694 case X86::PACKUSDWrr:
6695 case X86::PUNPCKHBWrr:
6696 case X86::PUNPCKLBWrr:
6697 case X86::PUNPCKHWDrr:
6698 case X86::PUNPCKLWDrr:
6699 case X86::PUNPCKHDQrr:
6700 case X86::PUNPCKLDQrr:
6701 case X86::PUNPCKHQDQrr:
6702 case X86::PUNPCKLQDQrr:
6703 case X86::SHUFPDrri:
6704 case X86::SHUFPSrri:
6712 case X86::VMOVLHPSrr:
6713 case X86::VMOVLHPSZrr:
6714 case X86::VPACKSSWBrr:
6715 case X86::VPACKUSWBrr:
6716 case X86::VPACKSSDWrr:
6717 case X86::VPACKUSDWrr:
6718 case X86::VPACKSSWBZ128rr:
6719 case X86::VPACKUSWBZ128rr:
6720 case X86::VPACKSSDWZ128rr:
6721 case X86::VPACKUSDWZ128rr:
6722 case X86::VPERM2F128rr:
6723 case X86::VPERM2I128rr:
6724 case X86::VSHUFF32X4Z256rri:
6725 case X86::VSHUFF32X4Zrri:
6726 case X86::VSHUFF64X2Z256rri:
6727 case X86::VSHUFF64X2Zrri:
6728 case X86::VSHUFI32X4Z256rri:
6729 case X86::VSHUFI32X4Zrri:
6730 case X86::VSHUFI64X2Z256rri:
6731 case X86::VSHUFI64X2Zrri:
6732 case X86::VPUNPCKHBWrr:
6733 case X86::VPUNPCKLBWrr:
6734 case X86::VPUNPCKHBWYrr:
6735 case X86::VPUNPCKLBWYrr:
6736 case X86::VPUNPCKHBWZ128rr:
6737 case X86::VPUNPCKLBWZ128rr:
6738 case X86::VPUNPCKHBWZ256rr:
6739 case X86::VPUNPCKLBWZ256rr:
6740 case X86::VPUNPCKHBWZrr:
6741 case X86::VPUNPCKLBWZrr:
6742 case X86::VPUNPCKHWDrr:
6743 case X86::VPUNPCKLWDrr:
6744 case X86::VPUNPCKHWDYrr:
6745 case X86::VPUNPCKLWDYrr:
6746 case X86::VPUNPCKHWDZ128rr:
6747 case X86::VPUNPCKLWDZ128rr:
6748 case X86::VPUNPCKHWDZ256rr:
6749 case X86::VPUNPCKLWDZ256rr:
6750 case X86::VPUNPCKHWDZrr:
6751 case X86::VPUNPCKLWDZrr:
6752 case X86::VPUNPCKHDQrr:
6753 case X86::VPUNPCKLDQrr:
6754 case X86::VPUNPCKHDQYrr:
6755 case X86::VPUNPCKLDQYrr:
6756 case X86::VPUNPCKHDQZ128rr:
6757 case X86::VPUNPCKLDQZ128rr:
6758 case X86::VPUNPCKHDQZ256rr:
6759 case X86::VPUNPCKLDQZ256rr:
6760 case X86::VPUNPCKHDQZrr:
6761 case X86::VPUNPCKLDQZrr:
6762 case X86::VPUNPCKHQDQrr:
6763 case X86::VPUNPCKLQDQrr:
6764 case X86::VPUNPCKHQDQYrr:
6765 case X86::VPUNPCKLQDQYrr:
6766 case X86::VPUNPCKHQDQZ128rr:
6767 case X86::VPUNPCKLQDQZ128rr:
6768 case X86::VPUNPCKHQDQZ256rr:
6769 case X86::VPUNPCKLQDQZ256rr:
6770 case X86::VPUNPCKHQDQZrr:
6771 case X86::VPUNPCKLQDQZrr:
6777 case X86::VCVTSI2SSrr:
6778 case X86::VCVTSI2SSrm:
6779 case X86::VCVTSI2SSrr_Int:
6780 case X86::VCVTSI2SSrm_Int:
6781 case X86::VCVTSI642SSrr:
6782 case X86::VCVTSI642SSrm:
6783 case X86::VCVTSI642SSrr_Int:
6784 case X86::VCVTSI642SSrm_Int:
6785 case X86::VCVTSI2SDrr:
6786 case X86::VCVTSI2SDrm:
6787 case X86::VCVTSI2SDrr_Int:
6788 case X86::VCVTSI2SDrm_Int:
6789 case X86::VCVTSI642SDrr:
6790 case X86::VCVTSI642SDrm:
6791 case X86::VCVTSI642SDrr_Int:
6792 case X86::VCVTSI642SDrm_Int:
6794 case X86::VCVTSI2SSZrr:
6795 case X86::VCVTSI2SSZrm:
6796 case X86::VCVTSI2SSZrr_Int:
6797 case X86::VCVTSI2SSZrrb_Int:
6798 case X86::VCVTSI2SSZrm_Int:
6799 case X86::VCVTSI642SSZrr:
6800 case X86::VCVTSI642SSZrm:
6801 case X86::VCVTSI642SSZrr_Int:
6802 case X86::VCVTSI642SSZrrb_Int:
6803 case X86::VCVTSI642SSZrm_Int:
6804 case X86::VCVTSI2SDZrr:
6805 case X86::VCVTSI2SDZrm:
6806 case X86::VCVTSI2SDZrr_Int:
6807 case X86::VCVTSI2SDZrm_Int:
6808 case X86::VCVTSI642SDZrr:
6809 case X86::VCVTSI642SDZrm:
6810 case X86::VCVTSI642SDZrr_Int:
6811 case X86::VCVTSI642SDZrrb_Int:
6812 case X86::VCVTSI642SDZrm_Int:
6813 case X86::VCVTUSI2SSZrr:
6814 case X86::VCVTUSI2SSZrm:
6815 case X86::VCVTUSI2SSZrr_Int:
6816 case X86::VCVTUSI2SSZrrb_Int:
6817 case X86::VCVTUSI2SSZrm_Int:
6818 case X86::VCVTUSI642SSZrr:
6819 case X86::VCVTUSI642SSZrm:
6820 case X86::VCVTUSI642SSZrr_Int:
6821 case X86::VCVTUSI642SSZrrb_Int:
6822 case X86::VCVTUSI642SSZrm_Int:
6823 case X86::VCVTUSI2SDZrr:
6824 case X86::VCVTUSI2SDZrm:
6825 case X86::VCVTUSI2SDZrr_Int:
6826 case X86::VCVTUSI2SDZrm_Int:
6827 case X86::VCVTUSI642SDZrr:
6828 case X86::VCVTUSI642SDZrm:
6829 case X86::VCVTUSI642SDZrr_Int:
6830 case X86::VCVTUSI642SDZrrb_Int:
6831 case X86::VCVTUSI642SDZrm_Int:
6832 case X86::VCVTSI2SHZrr:
6833 case X86::VCVTSI2SHZrm:
6834 case X86::VCVTSI2SHZrr_Int:
6835 case X86::VCVTSI2SHZrrb_Int:
6836 case X86::VCVTSI2SHZrm_Int:
6837 case X86::VCVTSI642SHZrr:
6838 case X86::VCVTSI642SHZrm:
6839 case X86::VCVTSI642SHZrr_Int:
6840 case X86::VCVTSI642SHZrrb_Int:
6841 case X86::VCVTSI642SHZrm_Int:
6842 case X86::VCVTUSI2SHZrr:
6843 case X86::VCVTUSI2SHZrm:
6844 case X86::VCVTUSI2SHZrr_Int:
6845 case X86::VCVTUSI2SHZrrb_Int:
6846 case X86::VCVTUSI2SHZrm_Int:
6847 case X86::VCVTUSI642SHZrr:
6848 case X86::VCVTUSI642SHZrm:
6849 case X86::VCVTUSI642SHZrr_Int:
6850 case X86::VCVTUSI642SHZrrb_Int:
6851 case X86::VCVTUSI642SHZrm_Int:
6855 case X86::VCVTSD2SSrr:
6856 case X86::VCVTSD2SSrm:
6857 case X86::VCVTSD2SSrr_Int:
6858 case X86::VCVTSD2SSrm_Int:
6859 case X86::VCVTSS2SDrr:
6860 case X86::VCVTSS2SDrm:
6861 case X86::VCVTSS2SDrr_Int:
6862 case X86::VCVTSS2SDrm_Int:
6863 case X86::VRCPSSr:
6864 case X86::VRCPSSr_Int:
6865 case X86::VRCPSSm:
6866 case X86::VRCPSSm_Int:
6867 case X86::VROUNDSDri:
6868 case X86::VROUNDSDmi:
6869 case X86::VROUNDSDri_Int:
6870 case X86::VROUNDSDmi_Int:
6871 case X86::VROUNDSSri:
6872 case X86::VROUNDSSmi:
6873 case X86::VROUNDSSri_Int:
6874 case X86::VROUNDSSmi_Int:
6875 case X86::VRSQRTSSr:
6876 case X86::VRSQRTSSr_Int:
6877 case X86::VRSQRTSSm:
6878 case X86::VRSQRTSSm_Int:
6879 case X86::VSQRTSSr:
6880 case X86::VSQRTSSr_Int:
6881 case X86::VSQRTSSm:
6882 case X86::VSQRTSSm_Int:
6883 case X86::VSQRTSDr:
6884 case X86::VSQRTSDr_Int:
6885 case X86::VSQRTSDm:
6886 case X86::VSQRTSDm_Int:
6888 case X86::VCVTSD2SSZrr:
6889 case X86::VCVTSD2SSZrr_Int:
6890 case X86::VCVTSD2SSZrrb_Int:
6891 case X86::VCVTSD2SSZrm:
6892 case X86::VCVTSD2SSZrm_Int:
6893 case X86::VCVTSS2SDZrr:
6894 case X86::VCVTSS2SDZrr_Int:
6895 case X86::VCVTSS2SDZrrb_Int:
6896 case X86::VCVTSS2SDZrm:
6897 case X86::VCVTSS2SDZrm_Int:
6898 case X86::VGETEXPSDZr:
6899 case X86::VGETEXPSDZrb:
6900 case X86::VGETEXPSDZm:
6901 case X86::VGETEXPSSZr:
6902 case X86::VGETEXPSSZrb:
6903 case X86::VGETEXPSSZm:
6904 case X86::VGETMANTSDZrri:
6905 case X86::VGETMANTSDZrrib:
6906 case X86::VGETMANTSDZrmi:
6907 case X86::VGETMANTSSZrri:
6908 case X86::VGETMANTSSZrrib:
6909 case X86::VGETMANTSSZrmi:
6910 case X86::VRNDSCALESDZr:
6911 case X86::VRNDSCALESDZr_Int:
6912 case X86::VRNDSCALESDZrb_Int:
6913 case X86::VRNDSCALESDZm:
6914 case X86::VRNDSCALESDZm_Int:
6915 case X86::VRNDSCALESSZr:
6916 case X86::VRNDSCALESSZr_Int:
6917 case X86::VRNDSCALESSZrb_Int:
6918 case X86::VRNDSCALESSZm:
6919 case X86::VRNDSCALESSZm_Int:
6920 case X86::VRCP14SDZrr:
6921 case X86::VRCP14SDZrm:
6922 case X86::VRCP14SSZrr:
6923 case X86::VRCP14SSZrm:
6924 case X86::VRCPSHZrr:
6925 case X86::VRCPSHZrm:
6926 case X86::VRSQRTSHZrr:
6927 case X86::VRSQRTSHZrm:
6928 case X86::VREDUCESHZrmi:
6929 case X86::VREDUCESHZrri:
6930 case X86::VREDUCESHZrrib:
6931 case X86::VGETEXPSHZr:
6932 case X86::VGETEXPSHZrb:
6933 case X86::VGETEXPSHZm:
6934 case X86::VGETMANTSHZrri:
6935 case X86::VGETMANTSHZrrib:
6936 case X86::VGETMANTSHZrmi:
6937 case X86::VRNDSCALESHZr:
6938 case X86::VRNDSCALESHZr_Int:
6939 case X86::VRNDSCALESHZrb_Int:
6940 case X86::VRNDSCALESHZm:
6941 case X86::VRNDSCALESHZm_Int:
6942 case X86::VSQRTSHZr:
6943 case X86::VSQRTSHZr_Int:
6944 case X86::VSQRTSHZrb_Int:
6945 case X86::VSQRTSHZm:
6946 case X86::VSQRTSHZm_Int:
6947 case X86::VRCP28SDZr:
6948 case X86::VRCP28SDZrb:
6949 case X86::VRCP28SDZm:
6950 case X86::VRCP28SSZr:
6951 case X86::VRCP28SSZrb:
6952 case X86::VRCP28SSZm:
6953 case X86::VREDUCESSZrmi:
6954 case X86::VREDUCESSZrri:
6955 case X86::VREDUCESSZrrib:
6956 case X86::VRSQRT14SDZrr:
6957 case X86::VRSQRT14SDZrm:
6958 case X86::VRSQRT14SSZrr:
6959 case X86::VRSQRT14SSZrm:
6960 case X86::VRSQRT28SDZr:
6961 case X86::VRSQRT28SDZrb:
6962 case X86::VRSQRT28SDZm:
6963 case X86::VRSQRT28SSZr:
6964 case X86::VRSQRT28SSZrb:
6965 case X86::VRSQRT28SSZm:
6966 case X86::VSQRTSSZr:
6967 case X86::VSQRTSSZr_Int:
6968 case X86::VSQRTSSZrb_Int:
6969 case X86::VSQRTSSZm:
6970 case X86::VSQRTSSZm_Int:
6971 case X86::VSQRTSDZr:
6972 case X86::VSQRTSDZr_Int:
6973 case X86::VSQRTSDZrb_Int:
6974 case X86::VSQRTSDZm:
6975 case X86::VSQRTSDZm_Int:
6976 case X86::VCVTSD2SHZrr:
6977 case X86::VCVTSD2SHZrr_Int:
6978 case X86::VCVTSD2SHZrrb_Int:
6979 case X86::VCVTSD2SHZrm:
6980 case X86::VCVTSD2SHZrm_Int:
6981 case X86::VCVTSS2SHZrr:
6982 case X86::VCVTSS2SHZrr_Int:
6983 case X86::VCVTSS2SHZrrb_Int:
6984 case X86::VCVTSS2SHZrm:
6985 case X86::VCVTSS2SHZrm_Int:
6986 case X86::VCVTSH2SDZrr:
6987 case X86::VCVTSH2SDZrr_Int:
6988 case X86::VCVTSH2SDZrrb_Int:
6989 case X86::VCVTSH2SDZrm:
6990 case X86::VCVTSH2SDZrm_Int:
6991 case X86::VCVTSH2SSZrr:
6992 case X86::VCVTSH2SSZrr_Int:
6993 case X86::VCVTSH2SSZrrb_Int:
6994 case X86::VCVTSH2SSZrm:
6995 case X86::VCVTSH2SSZrm_Int:
6997 case X86::VMOVSSZrrk:
6998 case X86::VMOVSDZrrk:
7000 case X86::VMOVSSZrrkz:
7001 case X86::VMOVSDZrrkz:
7029 return 0;
7039 if (X86::VR128RegClass.contains(Reg)) {
7042 unsigned Opc = Subtarget.hasAVX() ? X86::VXORPSrr : X86::XORPSrr;
7047 } else if (X86::VR256RegClass.contains(Reg)) {
7050 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm);
7051 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VXORPSrr), XReg)
7056 } else if (X86::VR128XRegClass.contains(Reg)) {
7061 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), Reg)
7065 } else if (X86::VR256XRegClass.contains(Reg) ||
7066 X86::VR512RegClass.contains(Reg)) {
7072 Register XReg = TRI->getSubReg(Reg, X86::sub_xmm);
7073 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::VPXORDZ128rr), XReg)
7078 } else if (X86::GR64RegClass.contains(Reg)) {
7081 Register XReg = TRI->getSubReg(Reg, X86::sub_32bit);
7082 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), XReg)
7087 } else if (X86::GR32RegClass.contains(Reg)) {
7088 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(X86::XOR32rr), Reg)
7096 int PtrOffset = 0) {
7101 for (unsigned i = 0; i != NumAddrOps; ++i)
7108 for (unsigned i = 0; i != NumAddrOps; ++i) {
7110 if (i == 3 && PtrOffset != 0) {
7125 for (int Idx : llvm::seq<int>(0, NewMI.getNumOperands())) {
7159 for (unsigned i = 0; i != NumOps; ++i) {
7178 int PtrOffset = 0) {
7184 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
7213 return MIB.addImm(0);
7221 case X86::INSERTPSrr:
7222 case X86::VINSERTPSrr:
7223 case X86::VINSERTPSZrr:
7235 if ((Size == 0 || Size >= 16) && RCSize >= 16 &&
7236 (MI.getOpcode() != X86::INSERTPSrr || Alignment >= Align(4))) {
7240 (MI.getOpcode() == X86::VINSERTPSZrr) ? X86::VINSERTPSZrm
7241 : (MI.getOpcode() == X86::VINSERTPSrr) ? X86::VINSERTPSrm
7242 : X86::INSERTPSrm;
7250 case X86::MOVHLPSrr:
7251 case X86::VMOVHLPSrr:
7252 case X86::VMOVHLPSZrr:
7260 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment >= Align(8)) {
7262 (MI.getOpcode() == X86::VMOVHLPSZrr) ? X86::VMOVLPSZ128rm
7263 : (MI.getOpcode() == X86::VMOVHLPSrr) ? X86::VMOVLPSrm
7264 : X86::MOVLPSrm;
7271 case X86::UNPCKLPDrr:
7279 if ((Size == 0 || Size >= 16) && RCSize >= 16 && Alignment < Align(16)) {
7281 fuseInst(MF, X86::MOVHPDrm, OpNum, MOs, InsertPt, MI, *this);
7286 case X86::MOV32r0:
7288 makeM0Inst(*this, (Size == 4) ? X86::MOV32mi : X86::MOV64mi32, MOs,
7323 Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register();
7326 bool Tied1 = 0 == MI.getDesc().getOperandConstraint(Idx1, MCOI::TIED_TO);
7327 bool Tied2 = 0 == MI.getDesc().getOperandConstraint(Idx2, MCOI::TIED_TO);
7353 (Opc == X86::CALL32r || Opc == X86::CALL64r || Opc == X86::PUSH16r ||
7354 Opc == X86::PUSH32r || Opc == X86::PUSH64r))
7364 bool IsTwoAddr = NumOps > 1 && OpNum < 2 && MI.getOperand(0).isReg() &&
7366 MI.getOperand(0).getReg() == MI.getOperand(1).getReg();
7370 if (Opc == X86::ADD32ri &&
7377 if (MOs.size() == X86::AddrNumOperands &&
7378 MOs[X86::AddrDisp].getTargetFlags() == X86II::MO_GOTTPOFF &&
7379 Opc != X86::ADD64rr)
7397 unsigned NonNDOpc = Subtarget.hasNDD() ? X86::getNonNDVariant(Opc) : 0U;
7420 if (Opcode != X86::MOV64rm || RCSize != 8 || Size != 4)
7422 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
7424 Opcode = X86::MOV32rm;
7441 Register DstReg = NewMI->getOperand(0).getReg();
7443 NewMI->getOperand(0).setReg(RI.getSubReg(DstReg, X86::sub_32bit));
7445 NewMI->getOperand(0).setSubReg(X86::sub_32bit);
7491 if (MI.getOpcode() == X86::MOV32r0 && SubReg == X86::sub_32bit)
7493 if (SubReg && (MO.isDef() || SubReg == X86::sub_8bit_hi))
7507 return foldMemoryOperandImpl(MF, MI, Ops[0],
7511 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
7512 unsigned NewOpc = 0;
7513 unsigned RCSize = 0;
7518 return (Subtarget.hasNDD() ? X86::getNonNDVariant(Opc) : 0U) ? Impl()
7520 case X86::TEST8rr:
7521 NewOpc = X86::CMP8ri;
7524 case X86::TEST16rr:
7525 NewOpc = X86::CMP16ri;
7528 case X86::TEST32rr:
7529 NewOpc = X86::CMP32ri;
7532 case X86::TEST64rr:
7533 NewOpc = X86::CMP64ri32;
7541 // Change to CMPXXri r, 0 first.
7543 MI.getOperand(1).ChangeToImmediate(0);
7571 MF.getRegInfo().getRegClass(LoadMI.getOperand(0).getReg());
7574 if ((Opc == X86::MOVSSrm || Opc == X86::VMOVSSrm || Opc == X86::VMOVSSZrm ||
7575 Opc == X86::MOVSSrm_alt || Opc == X86::VMOVSSrm_alt ||
7576 Opc == X86::VMOVSSZrm_alt) &&
7582 case X86::CVTSS2SDrr_Int:
7583 case X86::VCVTSS2SDrr_Int:
7584 case X86::VCVTSS2SDZrr_Int:
7585 case X86::VCVTSS2SDZrr_Intk:
7586 case X86::VCVTSS2SDZrr_Intkz:
7587 case X86::CVTSS2SIrr_Int:
7588 case X86::CVTSS2SI64rr_Int:
7589 case X86::VCVTSS2SIrr_Int:
7590 case X86::VCVTSS2SI64rr_Int:
7591 case X86::VCVTSS2SIZrr_Int:
7592 case X86::VCVTSS2SI64Zrr_Int:
7593 case X86::CVTTSS2SIrr_Int:
7594 case X86::CVTTSS2SI64rr_Int:
7595 case X86::VCVTTSS2SIrr_Int:
7596 case X86::VCVTTSS2SI64rr_Int:
7597 case X86::VCVTTSS2SIZrr_Int:
7598 case X86::VCVTTSS2SI64Zrr_Int:
7599 case X86::VCVTSS2USIZrr_Int:
7600 case X86::VCVTSS2USI64Zrr_Int:
7601 case X86::VCVTTSS2USIZrr_Int:
7602 case X86::VCVTTSS2USI64Zrr_Int:
7603 case X86::RCPSSr_Int:
7604 case X86::VRCPSSr_Int:
7605 case X86::RSQRTSSr_Int:
7606 case X86::VRSQRTSSr_Int:
7607 case X86::ROUNDSSri_Int:
7608 case X86::VROUNDSSri_Int:
7609 case X86::COMISSrr_Int:
7610 case X86::VCOMISSrr_Int:
7611 case X86::VCOMISSZrr_Int:
7612 case X86::UCOMISSrr_Int:
7613 case X86::VUCOMISSrr_Int:
7614 case X86::VUCOMISSZrr_Int:
7615 case X86::ADDSSrr_Int:
7616 case X86::VADDSSrr_Int:
7617 case X86::VADDSSZrr_Int:
7618 case X86::CMPSSrri_Int:
7619 case X86::VCMPSSrri_Int:
7620 case X86::VCMPSSZrri_Int:
7621 case X86::DIVSSrr_Int:
7622 case X86::VDIVSSrr_Int:
7623 case X86::VDIVSSZrr_Int:
7624 case X86::MAXSSrr_Int:
7625 case X86::VMAXSSrr_Int:
7626 case X86::VMAXSSZrr_Int:
7627 case X86::MINSSrr_Int:
7628 case X86::VMINSSrr_Int:
7629 case X86::VMINSSZrr_Int:
7630 case X86::MULSSrr_Int:
7631 case X86::VMULSSrr_Int:
7632 case X86::VMULSSZrr_Int:
7633 case X86::SQRTSSr_Int:
7634 case X86::VSQRTSSr_Int:
7635 case X86::VSQRTSSZr_Int:
7636 case X86::SUBSSrr_Int:
7637 case X86::VSUBSSrr_Int:
7638 case X86::VSUBSSZrr_Int:
7639 case X86::VADDSSZrr_Intk:
7640 case X86::VADDSSZrr_Intkz:
7641 case X86::VCMPSSZrri_Intk:
7642 case X86::VDIVSSZrr_Intk:
7643 case X86::VDIVSSZrr_Intkz:
7644 case X86::VMAXSSZrr_Intk:
7645 case X86::VMAXSSZrr_Intkz:
7646 case X86::VMINSSZrr_Intk:
7647 case X86::VMINSSZrr_Intkz:
7648 case X86::VMULSSZrr_Intk:
7649 case X86::VMULSSZrr_Intkz:
7650 case X86::VSQRTSSZr_Intk:
7651 case X86::VSQRTSSZr_Intkz:
7652 case X86::VSUBSSZrr_Intk:
7653 case X86::VSUBSSZrr_Intkz:
7654 case X86::VFMADDSS4rr_Int:
7655 case X86::VFNMADDSS4rr_Int:
7656 case X86::VFMSUBSS4rr_Int:
7657 case X86::VFNMSUBSS4rr_Int:
7658 case X86::VFMADD132SSr_Int:
7659 case X86::VFNMADD132SSr_Int:
7660 case X86::VFMADD213SSr_Int:
7661 case X86::VFNMADD213SSr_Int:
7662 case X86::VFMADD231SSr_Int:
7663 case X86::VFNMADD231SSr_Int:
7664 case X86::VFMSUB132SSr_Int:
7665 case X86::VFNMSUB132SSr_Int:
7666 case X86::VFMSUB213SSr_Int:
7667 case X86::VFNMSUB213SSr_Int:
7668 case X86::VFMSUB231SSr_Int:
7669 case X86::VFNMSUB231SSr_Int:
7670 case X86::VFMADD132SSZr_Int:
7671 case X86::VFNMADD132SSZr_Int:
7672 case X86::VFMADD213SSZr_Int:
7673 case X86::VFNMADD213SSZr_Int:
7674 case X86::VFMADD231SSZr_Int:
7675 case X86::VFNMADD231SSZr_Int:
7676 case X86::VFMSUB132SSZr_Int:
7677 case X86::VFNMSUB132SSZr_Int:
7678 case X86::VFMSUB213SSZr_Int:
7679 case X86::VFNMSUB213SSZr_Int:
7680 case X86::VFMSUB231SSZr_Int:
7681 case X86::VFNMSUB231SSZr_Int:
7682 case X86::VFMADD132SSZr_Intk:
7683 case X86::VFNMADD132SSZr_Intk:
7684 case X86::VFMADD213SSZr_Intk:
7685 case X86::VFNMADD213SSZr_Intk:
7686 case X86::VFMADD231SSZr_Intk:
7687 case X86::VFNMADD231SSZr_Intk:
7688 case X86::VFMSUB132SSZr_Intk:
7689 case X86::VFNMSUB132SSZr_Intk:
7690 case X86::VFMSUB213SSZr_Intk:
7691 case X86::VFNMSUB213SSZr_Intk:
7692 case X86::VFMSUB231SSZr_Intk:
7693 case X86::VFNMSUB231SSZr_Intk:
7694 case X86::VFMADD132SSZr_Intkz:
7695 case X86::VFNMADD132SSZr_Intkz:
7696 case X86::VFMADD213SSZr_Intkz:
7697 case X86::VFNMADD213SSZr_Intkz:
7698 case X86::VFMADD231SSZr_Intkz:
7699 case X86::VFNMADD231SSZr_Intkz:
7700 case X86::VFMSUB132SSZr_Intkz:
7701 case X86::VFNMSUB132SSZr_Intkz:
7702 case X86::VFMSUB213SSZr_Intkz:
7703 case X86::VFNMSUB213SSZr_Intkz:
7704 case X86::VFMSUB231SSZr_Intkz:
7705 case X86::VFNMSUB231SSZr_Intkz:
7706 case X86::VFIXUPIMMSSZrri:
7707 case X86::VFIXUPIMMSSZrrik:
7708 case X86::VFIXUPIMMSSZrrikz:
7709 case X86::VFPCLASSSSZrr:
7710 case X86::VFPCLASSSSZrrk:
7711 case X86::VGETEXPSSZr:
7712 case X86::VGETEXPSSZrk:
7713 case X86::VGETEXPSSZrkz:
7714 case X86::VGETMANTSSZrri:
7715 case X86::VGETMANTSSZrrik:
7716 case X86::VGETMANTSSZrrikz:
7717 case X86::VRANGESSZrri:
7718 case X86::VRANGESSZrrik:
7719 case X86::VRANGESSZrrikz:
7720 case X86::VRCP14SSZrr:
7721 case X86::VRCP14SSZrrk:
7722 case X86::VRCP14SSZrrkz:
7723 case X86::VRCP28SSZr:
7724 case X86::VRCP28SSZrk:
7725 case X86::VRCP28SSZrkz:
7726 case X86::VREDUCESSZrri:
7727 case X86::VREDUCESSZrrik:
7728 case X86::VREDUCESSZrrikz:
7729 case X86::VRNDSCALESSZr_Int:
7730 case X86::VRNDSCALESSZr_Intk:
7731 case X86::VRNDSCALESSZr_Intkz:
7732 case X86::VRSQRT14SSZrr:
7733 case X86::VRSQRT14SSZrrk:
7734 case X86::VRSQRT14SSZrrkz:
7735 case X86::VRSQRT28SSZr:
7736 case X86::VRSQRT28SSZrk:
7737 case X86::VRSQRT28SSZrkz:
7738 case X86::VSCALEFSSZrr:
7739 case X86::VSCALEFSSZrrk:
7740 case X86::VSCALEFSSZrrkz:
7747 if ((Opc == X86::MOVSDrm || Opc == X86::VMOVSDrm || Opc == X86::VMOVSDZrm ||
7748 Opc == X86::MOVSDrm_alt || Opc == X86::VMOVSDrm_alt ||
7749 Opc == X86::VMOVSDZrm_alt) &&
7755 case X86::CVTSD2SSrr_Int:
7756 case X86::VCVTSD2SSrr_Int:
7757 case X86::VCVTSD2SSZrr_Int:
7758 case X86::VCVTSD2SSZrr_Intk:
7759 case X86::VCVTSD2SSZrr_Intkz:
7760 case X86::CVTSD2SIrr_Int:
7761 case X86::CVTSD2SI64rr_Int:
7762 case X86::VCVTSD2SIrr_Int:
7763 case X86::VCVTSD2SI64rr_Int:
7764 case X86::VCVTSD2SIZrr_Int:
7765 case X86::VCVTSD2SI64Zrr_Int:
7766 case X86::CVTTSD2SIrr_Int:
7767 case X86::CVTTSD2SI64rr_Int:
7768 case X86::VCVTTSD2SIrr_Int:
7769 case X86::VCVTTSD2SI64rr_Int:
7770 case X86::VCVTTSD2SIZrr_Int:
7771 case X86::VCVTTSD2SI64Zrr_Int:
7772 case X86::VCVTSD2USIZrr_Int:
7773 case X86::VCVTSD2USI64Zrr_Int:
7774 case X86::VCVTTSD2USIZrr_Int:
7775 case X86::VCVTTSD2USI64Zrr_Int:
7776 case X86::ROUNDSDri_Int:
7777 case X86::VROUNDSDri_Int:
7778 case X86::COMISDrr_Int:
7779 case X86::VCOMISDrr_Int:
7780 case X86::VCOMISDZrr_Int:
7781 case X86::UCOMISDrr_Int:
7782 case X86::VUCOMISDrr_Int:
7783 case X86::VUCOMISDZrr_Int:
7784 case X86::ADDSDrr_Int:
7785 case X86::VADDSDrr_Int:
7786 case X86::VADDSDZrr_Int:
7787 case X86::CMPSDrri_Int:
7788 case X86::VCMPSDrri_Int:
7789 case X86::VCMPSDZrri_Int:
7790 case X86::DIVSDrr_Int:
7791 case X86::VDIVSDrr_Int:
7792 case X86::VDIVSDZrr_Int:
7793 case X86::MAXSDrr_Int:
7794 case X86::VMAXSDrr_Int:
7795 case X86::VMAXSDZrr_Int:
7796 case X86::MINSDrr_Int:
7797 case X86::VMINSDrr_Int:
7798 case X86::VMINSDZrr_Int:
7799 case X86::MULSDrr_Int:
7800 case X86::VMULSDrr_Int:
7801 case X86::VMULSDZrr_Int:
7802 case X86::SQRTSDr_Int:
7803 case X86::VSQRTSDr_Int:
7804 case X86::VSQRTSDZr_Int:
7805 case X86::SUBSDrr_Int:
7806 case X86::VSUBSDrr_Int:
7807 case X86::VSUBSDZrr_Int:
7808 case X86::VADDSDZrr_Intk:
7809 case X86::VADDSDZrr_Intkz:
7810 case X86::VCMPSDZrri_Intk:
7811 case X86::VDIVSDZrr_Intk:
7812 case X86::VDIVSDZrr_Intkz:
7813 case X86::VMAXSDZrr_Intk:
7814 case X86::VMAXSDZrr_Intkz:
7815 case X86::VMINSDZrr_Intk:
7816 case X86::VMINSDZrr_Intkz:
7817 case X86::VMULSDZrr_Intk:
7818 case X86::VMULSDZrr_Intkz:
7819 case X86::VSQRTSDZr_Intk:
7820 case X86::VSQRTSDZr_Intkz:
7821 case X86::VSUBSDZrr_Intk:
7822 case X86::VSUBSDZrr_Intkz:
7823 case X86::VFMADDSD4rr_Int:
7824 case X86::VFNMADDSD4rr_Int:
7825 case X86::VFMSUBSD4rr_Int:
7826 case X86::VFNMSUBSD4rr_Int:
7827 case X86::VFMADD132SDr_Int:
7828 case X86::VFNMADD132SDr_Int:
7829 case X86::VFMADD213SDr_Int:
7830 case X86::VFNMADD213SDr_Int:
7831 case X86::VFMADD231SDr_Int:
7832 case X86::VFNMADD231SDr_Int:
7833 case X86::VFMSUB132SDr_Int:
7834 case X86::VFNMSUB132SDr_Int:
7835 case X86::VFMSUB213SDr_Int:
7836 case X86::VFNMSUB213SDr_Int:
7837 case X86::VFMSUB231SDr_Int:
7838 case X86::VFNMSUB231SDr_Int:
7839 case X86::VFMADD132SDZr_Int:
7840 case X86::VFNMADD132SDZr_Int:
7841 case X86::VFMADD213SDZr_Int:
7842 case X86::VFNMADD213SDZr_Int:
7843 case X86::VFMADD231SDZr_Int:
7844 case X86::VFNMADD231SDZr_Int:
7845 case X86::VFMSUB132SDZr_Int:
7846 case X86::VFNMSUB132SDZr_Int:
7847 case X86::VFMSUB213SDZr_Int:
7848 case X86::VFNMSUB213SDZr_Int:
7849 case X86::VFMSUB231SDZr_Int:
7850 case X86::VFNMSUB231SDZr_Int:
7851 case X86::VFMADD132SDZr_Intk:
7852 case X86::VFNMADD132SDZr_Intk:
7853 case X86::VFMADD213SDZr_Intk:
7854 case X86::VFNMADD213SDZr_Intk:
7855 case X86::VFMADD231SDZr_Intk:
7856 case X86::VFNMADD231SDZr_Intk:
7857 case X86::VFMSUB132SDZr_Intk:
7858 case X86::VFNMSUB132SDZr_Intk:
7859 case X86::VFMSUB213SDZr_Intk:
7860 case X86::VFNMSUB213SDZr_Intk:
7861 case X86::VFMSUB231SDZr_Intk:
7862 case X86::VFNMSUB231SDZr_Intk:
7863 case X86::VFMADD132SDZr_Intkz:
7864 case X86::VFNMADD132SDZr_Intkz:
7865 case X86::VFMADD213SDZr_Intkz:
7866 case X86::VFNMADD213SDZr_Intkz:
7867 case X86::VFMADD231SDZr_Intkz:
7868 case X86::VFNMADD231SDZr_Intkz:
7869 case X86::VFMSUB132SDZr_Intkz:
7870 case X86::VFNMSUB132SDZr_Intkz:
7871 case X86::VFMSUB213SDZr_Intkz:
7872 case X86::VFNMSUB213SDZr_Intkz:
7873 case X86::VFMSUB231SDZr_Intkz:
7874 case X86::VFNMSUB231SDZr_Intkz:
7875 case X86::VFIXUPIMMSDZrri:
7876 case X86::VFIXUPIMMSDZrrik:
7877 case X86::VFIXUPIMMSDZrrikz:
7878 case X86::VFPCLASSSDZrr:
7879 case X86::VFPCLASSSDZrrk:
7880 case X86::VGETEXPSDZr:
7881 case X86::VGETEXPSDZrk:
7882 case X86::VGETEXPSDZrkz:
7883 case X86::VGETMANTSDZrri:
7884 case X86::VGETMANTSDZrrik:
7885 case X86::VGETMANTSDZrrikz:
7886 case X86::VRANGESDZrri:
7887 case X86::VRANGESDZrrik:
7888 case X86::VRANGESDZrrikz:
7889 case X86::VRCP14SDZrr:
7890 case X86::VRCP14SDZrrk:
7891 case X86::VRCP14SDZrrkz:
7892 case X86::VRCP28SDZr:
7893 case X86::VRCP28SDZrk:
7894 case X86::VRCP28SDZrkz:
7895 case X86::VREDUCESDZrri:
7896 case X86::VREDUCESDZrrik:
7897 case X86::VREDUCESDZrrikz:
7898 case X86::VRNDSCALESDZr_Int:
7899 case X86::VRNDSCALESDZr_Intk:
7900 case X86::VRNDSCALESDZr_Intkz:
7901 case X86::VRSQRT14SDZrr:
7902 case X86::VRSQRT14SDZrrk:
7903 case X86::VRSQRT14SDZrrkz:
7904 case X86::VRSQRT28SDZr:
7905 case X86::VRSQRT28SDZrk:
7906 case X86::VRSQRT28SDZrkz:
7907 case X86::VSCALEFSDZrr:
7908 case X86::VSCALEFSDZrrk:
7909 case X86::VSCALEFSDZrrkz:
7916 if ((Opc == X86::VMOVSHZrm || Opc == X86::VMOVSHZrm_alt) && RegSize > 16) {
7921 case X86::VADDSHZrr_Int:
7922 case X86::VCMPSHZrri_Int:
7923 case X86::VDIVSHZrr_Int:
7924 case X86::VMAXSHZrr_Int:
7925 case X86::VMINSHZrr_Int:
7926 case X86::VMULSHZrr_Int:
7927 case X86::VSUBSHZrr_Int:
7928 case X86::VADDSHZrr_Intk:
7929 case X86::VADDSHZrr_Intkz:
7930 case X86::VCMPSHZrri_Intk:
7931 case X86::VDIVSHZrr_Intk:
7932 case X86::VDIVSHZrr_Intkz:
7933 case X86::VMAXSHZrr_Intk:
7934 case X86::VMAXSHZrr_Intkz:
7935 case X86::VMINSHZrr_Intk:
7936 case X86::VMINSHZrr_Intkz:
7937 case X86::VMULSHZrr_Intk:
7938 case X86::VMULSHZrr_Intkz:
7939 case X86::VSUBSHZrr_Intk:
7940 case X86::VSUBSHZrr_Intkz:
7941 case X86::VFMADD132SHZr_Int:
7942 case X86::VFNMADD132SHZr_Int:
7943 case X86::VFMADD213SHZr_Int:
7944 case X86::VFNMADD213SHZr_Int:
7945 case X86::VFMADD231SHZr_Int:
7946 case X86::VFNMADD231SHZr_Int:
7947 case X86::VFMSUB132SHZr_Int:
7948 case X86::VFNMSUB132SHZr_Int:
7949 case X86::VFMSUB213SHZr_Int:
7950 case X86::VFNMSUB213SHZr_Int:
7951 case X86::VFMSUB231SHZr_Int:
7952 case X86::VFNMSUB231SHZr_Int:
7953 case X86::VFMADD132SHZr_Intk:
7954 case X86::VFNMADD132SHZr_Intk:
7955 case X86::VFMADD213SHZr_Intk:
7956 case X86::VFNMADD213SHZr_Intk:
7957 case X86::VFMADD231SHZr_Intk:
7958 case X86::VFNMADD231SHZr_Intk:
7959 case X86::VFMSUB132SHZr_Intk:
7960 case X86::VFNMSUB132SHZr_Intk:
7961 case X86::VFMSUB213SHZr_Intk:
7962 case X86::VFNMSUB213SHZr_Intk:
7963 case X86::VFMSUB231SHZr_Intk:
7964 case X86::VFNMSUB231SHZr_Intk:
7965 case X86::VFMADD132SHZr_Intkz:
7966 case X86::VFNMADD132SHZr_Intkz:
7967 case X86::VFMADD213SHZr_Intkz:
7968 case X86::VFNMADD213SHZr_Intkz:
7969 case X86::VFMADD231SHZr_Intkz:
7970 case X86::VFNMADD231SHZr_Intkz:
7971 case X86::VFMSUB132SHZr_Intkz:
7972 case X86::VFNMSUB132SHZr_Intkz:
7973 case X86::VFMSUB213SHZr_Intkz:
7974 case X86::VFNMSUB213SHZr_Intkz:
7975 case X86::VFMSUB231SHZr_Intkz:
7976 case X86::VFNMSUB231SHZr_Intkz:
8024 case X86::AVX512_512_SET0:
8025 case X86::AVX512_512_SETALLONES:
8028 case X86::AVX2_SETALLONES:
8029 case X86::AVX1_SETALLONES:
8030 case X86::AVX_SET0:
8031 case X86::AVX512_256_SET0:
8034 case X86::V_SET0:
8035 case X86::V_SETALLONES:
8036 case X86::AVX512_128_SET0:
8037 case X86::FsFLD0F128:
8038 case X86::AVX512_FsFLD0F128:
8041 case X86::MMX_SET0:
8042 case X86::FsFLD0SD:
8043 case X86::AVX512_FsFLD0SD:
8046 case X86::FsFLD0SS:
8047 case X86::AVX512_FsFLD0SS:
8050 case X86::FsFLD0SH:
8051 case X86::AVX512_FsFLD0SH:
8057 if (Ops.size() == 2 && Ops[0] == 0 && Ops[1] == 1) {
8058 unsigned NewOpc = 0;
8062 case X86::TEST8rr:
8063 NewOpc = X86::CMP8ri;
8065 case X86::TEST16rr:
8066 NewOpc = X86::CMP16ri;
8068 case X86::TEST32rr:
8069 NewOpc = X86::CMP32ri;
8071 case X86::TEST64rr:
8072 NewOpc = X86::CMP64ri32;
8075 // Change to CMPXXri r, 0 first.
8077 MI.getOperand(1).ChangeToImmediate(0);
8083 if (LoadMI.getOperand(0).getSubReg() != MI.getOperand(Ops[0]).getSubReg())
8086 SmallVector<MachineOperand, X86::AddrNumOperands> MOs;
8088 case X86::MMX_SET0:
8089 case X86::V_SET0:
8090 case X86::V_SETALLONES:
8091 case X86::AVX2_SETALLONES:
8092 case X86::AVX1_SETALLONES:
8093 case X86::AVX_SET0:
8094 case X86::AVX512_128_SET0:
8095 case X86::AVX512_256_SET0:
8096 case X86::AVX512_512_SET0:
8097 case X86::AVX512_512_SETALLONES:
8098 case X86::FsFLD0SH:
8099 case X86::AVX512_FsFLD0SH:
8100 case X86::FsFLD0SD:
8101 case X86::AVX512_FsFLD0SD:
8102 case X86::FsFLD0SS:
8103 case X86::AVX512_FsFLD0SS:
8104 case X86::FsFLD0F128:
8105 case X86::AVX512_FsFLD0F128: {
8113 // x86-32 PIC requires a PIC base register for constant pools.
8114 unsigned PICBase = 0;
8118 PICBase = X86::RIP;
8132 case X86::FsFLD0SS:
8133 case X86::AVX512_FsFLD0SS:
8136 case X86::FsFLD0SD:
8137 case X86::AVX512_FsFLD0SD:
8140 case X86::FsFLD0F128:
8141 case X86::AVX512_FsFLD0F128:
8144 case X86::FsFLD0SH:
8145 case X86::AVX512_FsFLD0SH:
8148 case X86::AVX512_512_SETALLONES:
8151 case X86::AVX512_512_SET0:
8155 case X86::AVX1_SETALLONES:
8156 case X86::AVX2_SETALLONES:
8159 case X86::AVX512_256_SET0:
8160 case X86::AVX_SET0:
8165 case X86::MMX_SET0:
8169 case X86::V_SETALLONES:
8172 case X86::V_SET0:
8173 case X86::AVX512_128_SET0:
8186 MOs.push_back(MachineOperand::CreateReg(0, false));
8187 MOs.push_back(MachineOperand::CreateCPI(CPI, 0));
8188 MOs.push_back(MachineOperand::CreateReg(0, false));
8191 case X86::VPBROADCASTBZ128rm:
8192 case X86::VPBROADCASTBZ256rm:
8193 case X86::VPBROADCASTBZrm:
8194 case X86::VBROADCASTF32X2Z256rm:
8195 case X86::VBROADCASTF32X2Zrm:
8196 case X86::VBROADCASTI32X2Z128rm:
8197 case X86::VBROADCASTI32X2Z256rm:
8198 case X86::VBROADCASTI32X2Zrm:
8203 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands, \
8205 return foldMemoryBroadcast(MF, MI, Ops[0], MOs, InsertPt, /*Size=*/SIZE, \
8207 case X86::VPBROADCASTWZ128rm:
8208 case X86::VPBROADCASTWZ256rm:
8209 case X86::VPBROADCASTWZrm:
8211 case X86::VPBROADCASTDZ128rm:
8212 case X86::VPBROADCASTDZ256rm:
8213 case X86::VPBROADCASTDZrm:
8214 case X86::VBROADCASTSSZ128rm:
8215 case X86::VBROADCASTSSZ256rm:
8216 case X86::VBROADCASTSSZrm:
8218 case X86::VPBROADCASTQZ128rm:
8219 case X86::VPBROADCASTQZ256rm:
8220 case X86::VPBROADCASTQZrm:
8221 case X86::VBROADCASTSDZ256rm:
8222 case X86::VBROADCASTSDZrm:
8229 MOs.append(LoadMI.operands_begin() + NumOps - X86::AddrNumOperands,
8234 return foldMemoryOperandImpl(MF, MI, Ops[0], MOs, InsertPt,
8235 /*Size=*/0, Alignment, /*AllowCommute=*/true);
8326 return X86::OP16; \
8328 return X86::OP32; \
8330 return X86::OP64; \
8374 if (!MI.hasOneMemOperand() && RC == &X86::VR128RegClass &&
8380 SmallVector<MachineOperand, X86::AddrNumOperands> AddrOps;
8384 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
8386 if (i >= Index && i < Index + X86::AddrNumOperands)
8418 for (unsigned i = 1; i != 1 + X86::AddrNumOperands; ++i) {
8419 MachineOperand &MO = NewMIs[0]->getOperand(i);
8445 // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
8449 case X86::CMP64ri32:
8450 case X86::CMP32ri:
8451 case X86::CMP16ri:
8452 case X86::CMP8ri: {
8453 MachineOperand &MO0 = DataMI->getOperand(0);
8455 if (MO1.isImm() && MO1.getImm() == 0) {
8460 case X86::CMP64ri32:
8461 NewOpc = X86::TEST64rr;
8463 case X86::CMP32ri:
8464 NewOpc = X86::TEST32rr;
8466 case X86::CMP16ri:
8467 NewOpc = X86::TEST16rr;
8469 case X86::CMP8ri:
8470 NewOpc = X86::TEST8rr;
8482 const TargetRegisterClass *DstRC = getRegClass(MCID, 0, &RI, MF);
8521 for (unsigned i = 0; i != NumOps - 1; ++i) {
8523 if (i >= Index - NumDefs && i < Index - NumDefs + X86::AddrNumOperands)
8538 if (MMOs.empty() && RC == &X86::VR128RegClass &&
8551 Opc = getLoadRegOpcode(0, RC, isAligned, Subtarget);
8564 if (MCID.getNumDefs() > 0) {
8565 DstRC = getRegClass(MCID, 0, &RI, MF);
8568 for (unsigned i = 0, e = N->getNumValues(); i != e; ++i) {
8574 BeforeOps.push_back(SDValue(Load, 0));
8576 // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
8580 case X86::CMP64ri32:
8581 case X86::CMP32ri:
8582 case X86::CMP16ri:
8583 case X86::CMP8ri:
8588 case X86::CMP64ri32:
8589 Opc = X86::TEST64rr;
8591 case X86::CMP32ri:
8592 Opc = X86::TEST32rr;
8594 case X86::CMP16ri:
8595 Opc = X86::TEST16rr;
8597 case X86::CMP8ri:
8598 Opc = X86::TEST8rr;
8601 BeforeOps[1] = BeforeOps[0];
8610 AddrOps.push_back(SDValue(NewNode, 0));
8613 if (MMOs.empty() && RC == &X86::VR128RegClass &&
8622 DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget),
8639 return 0;
8643 return 0;
8645 return 0;
8661 case X86::MOV8rm:
8662 case X86::MOV16rm:
8663 case X86::MOV32rm:
8664 case X86::MOV64rm:
8665 case X86::LD_Fp32m:
8666 case X86::LD_Fp64m:
8667 case X86::LD_Fp80m:
8668 case X86::MOVSSrm:
8669 case X86::MOVSSrm_alt:
8670 case X86::MOVSDrm:
8671 case X86::MOVSDrm_alt:
8672 case X86::MMX_MOVD64rm:
8673 case X86::MMX_MOVQ64rm:
8674 case X86::MOVAPSrm:
8675 case X86::MOVUPSrm:
8676 case X86::MOVAPDrm:
8677 case X86::MOVUPDrm:
8678 case X86::MOVDQArm:
8679 case X86::MOVDQUrm:
8681 case X86::VMOVSSrm:
8682 case X86::VMOVSSrm_alt:
8683 case X86::VMOVSDrm:
8684 case X86::VMOVSDrm_alt:
8685 case X86::VMOVAPSrm:
8686 case X86::VMOVUPSrm:
8687 case X86::VMOVAPDrm:
8688 case X86::VMOVUPDrm:
8689 case X86::VMOVDQArm:
8690 case X86::VMOVDQUrm:
8691 case X86::VMOVAPSYrm:
8692 case X86::VMOVUPSYrm:
8693 case X86::VMOVAPDYrm:
8694 case X86::VMOVUPDYrm:
8695 case X86::VMOVDQAYrm:
8696 case X86::VMOVDQUYrm:
8698 case X86::VMOVSSZrm:
8699 case X86::VMOVSSZrm_alt:
8700 case X86::VMOVSDZrm:
8701 case X86::VMOVSDZrm_alt:
8702 case X86::VMOVAPSZ128rm:
8703 case X86::VMOVUPSZ128rm:
8704 case X86::VMOVAPSZ128rm_NOVLX:
8705 case X86::VMOVUPSZ128rm_NOVLX:
8706 case X86::VMOVAPDZ128rm:
8707 case X86::VMOVUPDZ128rm:
8708 case X86::VMOVDQU8Z128rm:
8709 case X86::VMOVDQU16Z128rm:
8710 case X86::VMOVDQA32Z128rm:
8711 case X86::VMOVDQU32Z128rm:
8712 case X86::VMOVDQA64Z128rm:
8713 case X86::VMOVDQU64Z128rm:
8714 case X86::VMOVAPSZ256rm:
8715 case X86::VMOVUPSZ256rm:
8716 case X86::VMOVAPSZ256rm_NOVLX:
8717 case X86::VMOVUPSZ256rm_NOVLX:
8718 case X86::VMOVAPDZ256rm:
8719 case X86::VMOVUPDZ256rm:
8720 case X86::VMOVDQU8Z256rm:
8721 case X86::VMOVDQU16Z256rm:
8722 case X86::VMOVDQA32Z256rm:
8723 case X86::VMOVDQU32Z256rm:
8724 case X86::VMOVDQA64Z256rm:
8725 case X86::VMOVDQU64Z256rm:
8726 case X86::VMOVAPSZrm:
8727 case X86::VMOVUPSZrm:
8728 case X86::VMOVAPDZrm:
8729 case X86::VMOVUPDZrm:
8730 case X86::VMOVDQU8Zrm:
8731 case X86::VMOVDQU16Zrm:
8732 case X86::VMOVDQA32Zrm:
8733 case X86::VMOVDQU32Zrm:
8734 case X86::VMOVDQA64Zrm:
8735 case X86::VMOVDQU64Zrm:
8736 case X86::KMOVBkm:
8737 case X86::KMOVBkm_EVEX:
8738 case X86::KMOVWkm:
8739 case X86::KMOVWkm_EVEX:
8740 case X86::KMOVDkm:
8741 case X86::KMOVDkm_EVEX:
8742 case X86::KMOVQkm:
8743 case X86::KMOVQkm_EVEX:
8758 if (!HasSameOp(X86::AddrBaseReg) || !HasSameOp(X86::AddrScaleAmt) ||
8759 !HasSameOp(X86::AddrIndexReg) || !HasSameOp(X86::AddrSegmentReg))
8767 auto Disp1 = dyn_cast<ConstantSDNode>(Load1->getOperand(X86::AddrDisp));
8768 auto Disp2 = dyn_cast<ConstantSDNode>(Load2->getOperand(X86::AddrDisp));
8792 case X86::LD_Fp32m:
8793 case X86::LD_Fp64m:
8794 case X86::LD_Fp80m:
8795 case X86::MMX_MOVD64rm:
8796 case X86::MMX_MOVQ64rm:
8800 EVT VT = Load1->getValueType(0);
8832 if (Opcode == X86::ENDBR64 || Opcode == X86::ENDBR32 ||
8833 Opcode == X86::PLDTILECFGV)
8846 assert(Cond.size() == 1 && "Invalid X86 branch condition!");
8847 X86::CondCode CC = static_cast<X86::CondCode>(Cond[0].getImm());
8848 Cond[0].setImm(GetOppositeBranchCondition(CC));
8856 return !(RC == &X86::CCRRegClass || RC == &X86::DFCCRRegClass ||
8857 RC == &X86::RFP32RegClass || RC == &X86::RFP64RegClass ||
8858 RC == &X86::RFP80RegClass);
8870 if (GlobalBaseReg != 0)
8877 Subtarget.is64Bit() ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass);
8905 assert(((OldWidth % NewWidth) == 0 || (NewWidth % OldWidth) == 0) &&
8907 unsigned NewMask = 0;
8909 if ((OldWidth % NewWidth) == 0) {
8912 for (unsigned i = 0; i != NewWidth; ++i) {
8916 else if (Sub != 0x0)
8922 for (unsigned i = 0; i != OldWidth; ++i) {
8939 uint16_t validDomains = 0;
8943 validDomains |= 0x2; // PackedSingle
8945 validDomains |= 0x4; // PackedDouble
8947 validDomains |= 0x8; // PackedInt
8953 case X86::BLENDPDrmi:
8954 case X86::BLENDPDrri:
8955 case X86::VBLENDPDrmi:
8956 case X86::VBLENDPDrri:
8958 case X86::VBLENDPDYrmi:
8959 case X86::VBLENDPDYrri:
8961 case X86::BLENDPSrmi:
8962 case X86::BLENDPSrri:
8963 case X86::VBLENDPSrmi:
8964 case X86::VBLENDPSrri:
8965 case X86::VPBLENDDrmi:
8966 case X86::VPBLENDDrri:
8968 case X86::VBLENDPSYrmi:
8969 case X86::VBLENDPSYrri:
8970 case X86::VPBLENDDYrmi:
8971 case X86::VPBLENDDYrri:
8973 case X86::PBLENDWrmi:
8974 case X86::PBLENDWrri:
8975 case X86::VPBLENDWrmi:
8976 case X86::VPBLENDWrri:
8978 case X86::VPBLENDWYrmi:
8979 case X86::VPBLENDWYrri:
8981 case X86::VPANDDZ128rr:
8982 case X86::VPANDDZ128rm:
8983 case X86::VPANDDZ256rr:
8984 case X86::VPANDDZ256rm:
8985 case X86::VPANDQZ128rr:
8986 case X86::VPANDQZ128rm:
8987 case X86::VPANDQZ256rr:
8988 case X86::VPANDQZ256rm:
8989 case X86::VPANDNDZ128rr:
8990 case X86::VPANDNDZ128rm:
8991 case X86::VPANDNDZ256rr:
8992 case X86::VPANDNDZ256rm:
8993 case X86::VPANDNQZ128rr:
8994 case X86::VPANDNQZ128rm:
8995 case X86::VPANDNQZ256rr:
8996 case X86::VPANDNQZ256rm:
8997 case X86::VPORDZ128rr:
8998 case X86::VPORDZ128rm:
8999 case X86::VPORDZ256rr:
9000 case X86::VPORDZ256rm:
9001 case X86::VPORQZ128rr:
9002 case X86::VPORQZ128rm:
9003 case X86::VPORQZ256rr:
9004 case X86::VPORQZ256rm:
9005 case X86::VPXORDZ128rr:
9006 case X86::VPXORDZ128rm:
9007 case X86::VPXORDZ256rr:
9008 case X86::VPXORDZ256rm:
9009 case X86::VPXORQZ128rr:
9010 case X86::VPXORQZ128rm:
9011 case X86::VPXORQZ256rr:
9012 case X86::VPXORQZ256rm:
9016 return 0;
9018 if (RI.getEncodingValue(MI.getOperand(0).getReg()) >= 16)
9019 return 0;
9021 return 0;
9025 return 0;
9028 return 0xe;
9029 case X86::MOVHLPSrr:
9037 MI.getOperand(0).getSubReg() == 0 &&
9038 MI.getOperand(1).getSubReg() == 0 && MI.getOperand(2).getSubReg() == 0)
9039 return 0x6;
9040 return 0;
9041 case X86::SHUFPDrri:
9042 return 0x6;
9044 return 0;
9051 assert(Domain > 0 && Domain < 4 && "Invalid execution domain");
9093 case X86::BLENDPDrmi:
9094 case X86::BLENDPDrri:
9095 case X86::VBLENDPDrmi:
9096 case X86::VBLENDPDrri:
9098 case X86::VBLENDPDYrmi:
9099 case X86::VBLENDPDYrri:
9101 case X86::BLENDPSrmi:
9102 case X86::BLENDPSrri:
9103 case X86::VBLENDPSrmi:
9104 case X86::VBLENDPSrri:
9105 case X86::VPBLENDDrmi:
9106 case X86::VPBLENDDrri:
9108 case X86::VBLENDPSYrmi:
9109 case X86::VBLENDPSYrri:
9110 case X86::VPBLENDDYrmi:
9111 case X86::VPBLENDDYrri:
9113 case X86::PBLENDWrmi:
9114 case X86::PBLENDWrri:
9115 case X86::VPBLENDWrmi:
9116 case X86::VPBLENDWrri:
9118 case X86::VPBLENDWYrmi:
9119 case X86::VPBLENDWYrri:
9121 case X86::VPANDDZ128rr:
9122 case X86::VPANDDZ128rm:
9123 case X86::VPANDDZ256rr:
9124 case X86::VPANDDZ256rm:
9125 case X86::VPANDQZ128rr:
9126 case X86::VPANDQZ128rm:
9127 case X86::VPANDQZ256rr:
9128 case X86::VPANDQZ256rm:
9129 case X86::VPANDNDZ128rr:
9130 case X86::VPANDNDZ128rm:
9131 case X86::VPANDNDZ256rr:
9132 case X86::VPANDNDZ256rm:
9133 case X86::VPANDNQZ128rr:
9134 case X86::VPANDNQZ128rm:
9135 case X86::VPANDNQZ256rr:
9136 case X86::VPANDNQZ256rm:
9137 case X86::VPORDZ128rr:
9138 case X86::VPORDZ128rm:
9139 case X86::VPORDZ256rr:
9140 case X86::VPORDZ256rm:
9141 case X86::VPORQZ128rr:
9142 case X86::VPORQZ128rm:
9143 case X86::VPORQZ256rr:
9144 case X86::VPORQZ256rm:
9145 case X86::VPXORDZ128rr:
9146 case X86::VPXORDZ128rm:
9147 case X86::VPXORDZ256rr:
9148 case X86::VPXORDZ256rm:
9149 case X86::VPXORQZ128rr:
9150 case X86::VPXORQZ128rm:
9151 case X86::VPXORQZ256rr:
9152 case X86::VPXORQZ256rm: {
9167 case X86::UNPCKHPDrr:
9168 case X86::MOVHLPSrr:
9172 MI.getOperand(0).getSubReg() == 0 &&
9173 MI.getOperand(1).getSubReg() == 0 &&
9174 MI.getOperand(2).getSubReg() == 0) {
9179 if (Opcode == X86::MOVHLPSrr)
9182 case X86::SHUFPDrri: {
9185 unsigned NewImm = 0x44;
9187 NewImm |= 0x0a;
9189 NewImm |= 0xa0;
9191 MI.setDesc(get(X86::SHUFPSrri));
9203 uint16_t validDomains = 0;
9211 validDomains = 0xe;
9213 validDomains = Subtarget.hasAVX2() ? 0xe : 0x6;
9215 validDomains = 0x6;
9220 return std::make_pair(0, 0);
9221 validDomains = 0xe;
9223 validDomains = 0xe;
9226 validDomains = 0xe;
9231 validDomains = 0xa;
9233 validDomains = 0xc;
9241 assert(Domain > 0 && Domain < 4 && "Invalid execution domain");
9293 BuildMI(MBB, MI, DL, get(X86::NOOP));
9299 Nop.setOpcode(X86::NOOP);
9307 case X86::DIVPDrm:
9308 case X86::DIVPDrr:
9309 case X86::DIVPSrm:
9310 case X86::DIVPSrr:
9311 case X86::DIVSDrm:
9312 case X86::DIVSDrm_Int:
9313 case X86::DIVSDrr:
9314 case X86::DIVSDrr_Int:
9315 case X86::DIVSSrm:
9316 case X86::DIVSSrm_Int:
9317 case X86::DIVSSrr:
9318 case X86::DIVSSrr_Int:
9319 case X86::SQRTPDm:
9320 case X86::SQRTPDr:
9321 case X86::SQRTPSm:
9322 case X86::SQRTPSr:
9323 case X86::SQRTSDm:
9324 case X86::SQRTSDm_Int:
9325 case X86::SQRTSDr:
9326 case X86::SQRTSDr_Int:
9327 case X86::SQRTSSm:
9328 case X86::SQRTSSm_Int:
9329 case X86::SQRTSSr:
9330 case X86::SQRTSSr_Int:
9332 case X86::VDIVPDrm:
9333 case X86::VDIVPDrr:
9334 case X86::VDIVPDYrm:
9335 case X86::VDIVPDYrr:
9336 case X86::VDIVPSrm:
9337 case X86::VDIVPSrr:
9338 case X86::VDIVPSYrm:
9339 case X86::VDIVPSYrr:
9340 case X86::VDIVSDrm:
9341 case X86::VDIVSDrm_Int:
9342 case X86::VDIVSDrr:
9343 case X86::VDIVSDrr_Int:
9344 case X86::VDIVSSrm:
9345 case X86::VDIVSSrm_Int:
9346 case X86::VDIVSSrr:
9347 case X86::VDIVSSrr_Int:
9348 case X86::VSQRTPDm:
9349 case X86::VSQRTPDr:
9350 case X86::VSQRTPDYm:
9351 case X86::VSQRTPDYr:
9352 case X86::VSQRTPSm:
9353 case X86::VSQRTPSr:
9354 case X86::VSQRTPSYm:
9355 case X86::VSQRTPSYr:
9356 case X86::VSQRTSDm:
9357 case X86::VSQRTSDm_Int:
9358 case X86::VSQRTSDr:
9359 case X86::VSQRTSDr_Int:
9360 case X86::VSQRTSSm:
9361 case X86::VSQRTSSm_Int:
9362 case X86::VSQRTSSr:
9363 case X86::VSQRTSSr_Int:
9365 case X86::VDIVPDZ128rm:
9366 case X86::VDIVPDZ128rmb:
9367 case X86::VDIVPDZ128rmbk:
9368 case X86::VDIVPDZ128rmbkz:
9369 case X86::VDIVPDZ128rmk:
9370 case X86::VDIVPDZ128rmkz:
9371 case X86::VDIVPDZ128rr:
9372 case X86::VDIVPDZ128rrk:
9373 case X86::VDIVPDZ128rrkz:
9374 case X86::VDIVPDZ256rm:
9375 case X86::VDIVPDZ256rmb:
9376 case X86::VDIVPDZ256rmbk:
9377 case X86::VDIVPDZ256rmbkz:
9378 case X86::VDIVPDZ256rmk:
9379 case X86::VDIVPDZ256rmkz:
9380 case X86::VDIVPDZ256rr:
9381 case X86::VDIVPDZ256rrk:
9382 case X86::VDIVPDZ256rrkz:
9383 case X86::VDIVPDZrrb:
9384 case X86::VDIVPDZrrbk:
9385 case X86::VDIVPDZrrbkz:
9386 case X86::VDIVPDZrm:
9387 case X86::VDIVPDZrmb:
9388 case X86::VDIVPDZrmbk:
9389 case X86::VDIVPDZrmbkz:
9390 case X86::VDIVPDZrmk:
9391 case X86::VDIVPDZrmkz:
9392 case X86::VDIVPDZrr:
9393 case X86::VDIVPDZrrk:
9394 case X86::VDIVPDZrrkz:
9395 case X86::VDIVPSZ128rm:
9396 case X86::VDIVPSZ128rmb:
9397 case X86::VDIVPSZ128rmbk:
9398 case X86::VDIVPSZ128rmbkz:
9399 case X86::VDIVPSZ128rmk:
9400 case X86::VDIVPSZ128rmkz:
9401 case X86::VDIVPSZ128rr:
9402 case X86::VDIVPSZ128rrk:
9403 case X86::VDIVPSZ128rrkz:
9404 case X86::VDIVPSZ256rm:
9405 case X86::VDIVPSZ256rmb:
9406 case X86::VDIVPSZ256rmbk:
9407 case X86::VDIVPSZ256rmbkz:
9408 case X86::VDIVPSZ256rmk:
9409 case X86::VDIVPSZ256rmkz:
9410 case X86::VDIVPSZ256rr:
9411 case X86::VDIVPSZ256rrk:
9412 case X86::VDIVPSZ256rrkz:
9413 case X86::VDIVPSZrrb:
9414 case X86::VDIVPSZrrbk:
9415 case X86::VDIVPSZrrbkz:
9416 case X86::VDIVPSZrm:
9417 case X86::VDIVPSZrmb:
9418 case X86::VDIVPSZrmbk:
9419 case X86::VDIVPSZrmbkz:
9420 case X86::VDIVPSZrmk:
9421 case X86::VDIVPSZrmkz:
9422 case X86::VDIVPSZrr:
9423 case X86::VDIVPSZrrk:
9424 case X86::VDIVPSZrrkz:
9425 case X86::VDIVSDZrm:
9426 case X86::VDIVSDZrr:
9427 case X86::VDIVSDZrm_Int:
9428 case X86::VDIVSDZrm_Intk:
9429 case X86::VDIVSDZrm_Intkz:
9430 case X86::VDIVSDZrr_Int:
9431 case X86::VDIVSDZrr_Intk:
9432 case X86::VDIVSDZrr_Intkz:
9433 case X86::VDIVSDZrrb_Int:
9434 case X86::VDIVSDZrrb_Intk:
9435 case X86::VDIVSDZrrb_Intkz:
9436 case X86::VDIVSSZrm:
9437 case X86::VDIVSSZrr:
9438 case X86::VDIVSSZrm_Int:
9439 case X86::VDIVSSZrm_Intk:
9440 case X86::VDIVSSZrm_Intkz:
9441 case X86::VDIVSSZrr_Int:
9442 case X86::VDIVSSZrr_Intk:
9443 case X86::VDIVSSZrr_Intkz:
9444 case X86::VDIVSSZrrb_Int:
9445 case X86::VDIVSSZrrb_Intk:
9446 case X86::VDIVSSZrrb_Intkz:
9447 case X86::VSQRTPDZ128m:
9448 case X86::VSQRTPDZ128mb:
9449 case X86::VSQRTPDZ128mbk:
9450 case X86::VSQRTPDZ128mbkz:
9451 case X86::VSQRTPDZ128mk:
9452 case X86::VSQRTPDZ128mkz:
9453 case X86::VSQRTPDZ128r:
9454 case X86::VSQRTPDZ128rk:
9455 case X86::VSQRTPDZ128rkz:
9456 case X86::VSQRTPDZ256m:
9457 case X86::VSQRTPDZ256mb:
9458 case X86::VSQRTPDZ256mbk:
9459 case X86::VSQRTPDZ256mbkz:
9460 case X86::VSQRTPDZ256mk:
9461 case X86::VSQRTPDZ256mkz:
9462 case X86::VSQRTPDZ256r:
9463 case X86::VSQRTPDZ256rk:
9464 case X86::VSQRTPDZ256rkz:
9465 case X86::VSQRTPDZm:
9466 case X86::VSQRTPDZmb:
9467 case X86::VSQRTPDZmbk:
9468 case X86::VSQRTPDZmbkz:
9469 case X86::VSQRTPDZmk:
9470 case X86::VSQRTPDZmkz:
9471 case X86::VSQRTPDZr:
9472 case X86::VSQRTPDZrb:
9473 case X86::VSQRTPDZrbk:
9474 case X86::VSQRTPDZrbkz:
9475 case X86::VSQRTPDZrk:
9476 case X86::VSQRTPDZrkz:
9477 case X86::VSQRTPSZ128m:
9478 case X86::VSQRTPSZ128mb:
9479 case X86::VSQRTPSZ128mbk:
9480 case X86::VSQRTPSZ128mbkz:
9481 case X86::VSQRTPSZ128mk:
9482 case X86::VSQRTPSZ128mkz:
9483 case X86::VSQRTPSZ128r:
9484 case X86::VSQRTPSZ128rk:
9485 case X86::VSQRTPSZ128rkz:
9486 case X86::VSQRTPSZ256m:
9487 case X86::VSQRTPSZ256mb:
9488 case X86::VSQRTPSZ256mbk:
9489 case X86::VSQRTPSZ256mbkz:
9490 case X86::VSQRTPSZ256mk:
9491 case X86::VSQRTPSZ256mkz:
9492 case X86::VSQRTPSZ256r:
9493 case X86::VSQRTPSZ256rk:
9494 case X86::VSQRTPSZ256rkz:
9495 case X86::VSQRTPSZm:
9496 case X86::VSQRTPSZmb:
9497 case X86::VSQRTPSZmbk:
9498 case X86::VSQRTPSZmbkz:
9499 case X86::VSQRTPSZmk:
9500 case X86::VSQRTPSZmkz:
9501 case X86::VSQRTPSZr:
9502 case X86::VSQRTPSZrb:
9503 case X86::VSQRTPSZrbk:
9504 case X86::VSQRTPSZrbkz:
9505 case X86::VSQRTPSZrk:
9506 case X86::VSQRTPSZrkz:
9507 case X86::VSQRTSDZm:
9508 case X86::VSQRTSDZm_Int:
9509 case X86::VSQRTSDZm_Intk:
9510 case X86::VSQRTSDZm_Intkz:
9511 case X86::VSQRTSDZr:
9512 case X86::VSQRTSDZr_Int:
9513 case X86::VSQRTSDZr_Intk:
9514 case X86::VSQRTSDZr_Intkz:
9515 case X86::VSQRTSDZrb_Int:
9516 case X86::VSQRTSDZrb_Intk:
9517 case X86::VSQRTSDZrb_Intkz:
9518 case X86::VSQRTSSZm:
9519 case X86::VSQRTSSZm_Int:
9520 case X86::VSQRTSSZm_Intk:
9521 case X86::VSQRTSSZm_Intkz:
9522 case X86::VSQRTSSZr:
9523 case X86::VSQRTSSZr_Int:
9524 case X86::VSQRTSSZr_Intk:
9525 case X86::VSQRTSSZr_Intkz:
9526 case X86::VSQRTSSZrb_Int:
9527 case X86::VSQRTSSZrb_Intk:
9528 case X86::VSQRTSSZrb_Intkz:
9530 case X86::VGATHERDPDYrm:
9531 case X86::VGATHERDPDZ128rm:
9532 case X86::VGATHERDPDZ256rm:
9533 case X86::VGATHERDPDZrm:
9534 case X86::VGATHERDPDrm:
9535 case X86::VGATHERDPSYrm:
9536 case X86::VGATHERDPSZ128rm:
9537 case X86::VGATHERDPSZ256rm:
9538 case X86::VGATHERDPSZrm:
9539 case X86::VGATHERDPSrm:
9540 case X86::VGATHERPF0DPDm:
9541 case X86::VGATHERPF0DPSm:
9542 case X86::VGATHERPF0QPDm:
9543 case X86::VGATHERPF0QPSm:
9544 case X86::VGATHERPF1DPDm:
9545 case X86::VGATHERPF1DPSm:
9546 case X86::VGATHERPF1QPDm:
9547 case X86::VGATHERPF1QPSm:
9548 case X86::VGATHERQPDYrm:
9549 case X86::VGATHERQPDZ128rm:
9550 case X86::VGATHERQPDZ256rm:
9551 case X86::VGATHERQPDZrm:
9552 case X86::VGATHERQPDrm:
9553 case X86::VGATHERQPSYrm:
9554 case X86::VGATHERQPSZ128rm:
9555 case X86::VGATHERQPSZ256rm:
9556 case X86::VGATHERQPSZrm:
9557 case X86::VGATHERQPSrm:
9558 case X86::VPGATHERDDYrm:
9559 case X86::VPGATHERDDZ128rm:
9560 case X86::VPGATHERDDZ256rm:
9561 case X86::VPGATHERDDZrm:
9562 case X86::VPGATHERDDrm:
9563 case X86::VPGATHERDQYrm:
9564 case X86::VPGATHERDQZ128rm:
9565 case X86::VPGATHERDQZ256rm:
9566 case X86::VPGATHERDQZrm:
9567 case X86::VPGATHERDQrm:
9568 case X86::VPGATHERQDYrm:
9569 case X86::VPGATHERQDZ128rm:
9570 case X86::VPGATHERQDZ256rm:
9571 case X86::VPGATHERQDZrm:
9572 case X86::VPGATHERQDrm:
9573 case X86::VPGATHERQQYrm:
9574 case X86::VPGATHERQQZ128rm:
9575 case X86::VPGATHERQQZ256rm:
9576 case X86::VPGATHERQQZrm:
9577 case X86::VPGATHERQQrm:
9578 case X86::VSCATTERDPDZ128mr:
9579 case X86::VSCATTERDPDZ256mr:
9580 case X86::VSCATTERDPDZmr:
9581 case X86::VSCATTERDPSZ128mr:
9582 case X86::VSCATTERDPSZ256mr:
9583 case X86::VSCATTERDPSZmr:
9584 case X86::VSCATTERPF0DPDm:
9585 case X86::VSCATTERPF0DPSm:
9586 case X86::VSCATTERPF0QPDm:
9587 case X86::VSCATTERPF0QPSm:
9588 case X86::VSCATTERPF1DPDm:
9589 case X86::VSCATTERPF1DPSm:
9590 case X86::VSCATTERPF1QPDm:
9591 case X86::VSCATTERPF1QPSm:
9592 case X86::VSCATTERQPDZ128mr:
9593 case X86::VSCATTERQPDZ256mr:
9594 case X86::VSCATTERQPDZmr:
9595 case X86::VSCATTERQPSZ128mr:
9596 case X86::VSCATTERQPSZ256mr:
9597 case X86::VSCATTERQPSZmr:
9598 case X86::VPSCATTERDDZ128mr:
9599 case X86::VPSCATTERDDZ256mr:
9600 case X86::VPSCATTERDDZmr:
9601 case X86::VPSCATTERDQZ128mr:
9602 case X86::VPSCATTERDQZ256mr:
9603 case X86::VPSCATTERDQZmr:
9604 case X86::VPSCATTERQDZ128mr:
9605 case X86::VPSCATTERQDZ256mr:
9606 case X86::VPSCATTERQDZmr:
9607 case X86::VPSCATTERQQZ128mr:
9608 case X86::VPSCATTERQQZ256mr:
9609 case X86::VPSCATTERQQZmr:
9635 Inst.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
9671 case X86::PANDrr:
9672 case X86::PORrr:
9673 case X86::PXORrr:
9674 case X86::ANDPDrr:
9675 case X86::ANDPSrr:
9676 case X86::ORPDrr:
9677 case X86::ORPSrr:
9678 case X86::XORPDrr:
9679 case X86::XORPSrr:
9680 case X86::PADDBrr:
9681 case X86::PADDWrr:
9682 case X86::PADDDrr:
9683 case X86::PADDQrr:
9684 case X86::PMULLWrr:
9685 case X86::PMULLDrr:
9686 case X86::PMAXSBrr:
9687 case X86::PMAXSDrr:
9688 case X86::PMAXSWrr:
9689 case X86::PMAXUBrr:
9690 case X86::PMAXUDrr:
9691 case X86::PMAXUWrr:
9692 case X86::PMINSBrr:
9693 case X86::PMINSDrr:
9694 case X86::PMINSWrr:
9695 case X86::PMINUBrr:
9696 case X86::PMINUDrr:
9697 case X86::PMINUWrr:
9698 case X86::VPANDrr:
9699 case X86::VPANDYrr:
9700 case X86::VPANDDZ128rr:
9701 case X86::VPANDDZ256rr:
9702 case X86::VPANDDZrr:
9703 case X86::VPANDQZ128rr:
9704 case X86::VPANDQZ256rr:
9705 case X86::VPANDQZrr:
9706 case X86::VPORrr:
9707 case X86::VPORYrr:
9708 case X86::VPORDZ128rr:
9709 case X86::VPORDZ256rr:
9710 case X86::VPORDZrr:
9711 case X86::VPORQZ128rr:
9712 case X86::VPORQZ256rr:
9713 case X86::VPORQZrr:
9714 case X86::VPXORrr:
9715 case X86::VPXORYrr:
9716 case X86::VPXORDZ128rr:
9717 case X86::VPXORDZ256rr:
9718 case X86::VPXORDZrr:
9719 case X86::VPXORQZ128rr:
9720 case X86::VPXORQZ256rr:
9721 case X86::VPXORQZrr:
9722 case X86::VANDPDrr:
9723 case X86::VANDPSrr:
9724 case X86::VANDPDYrr:
9725 case X86::VANDPSYrr:
9726 case X86::VANDPDZ128rr:
9727 case X86::VANDPSZ128rr:
9728 case X86::VANDPDZ256rr:
9729 case X86::VANDPSZ256rr:
9730 case X86::VANDPDZrr:
9731 case X86::VANDPSZrr:
9732 case X86::VORPDrr:
9733 case X86::VORPSrr:
9734 case X86::VORPDYrr:
9735 case X86::VORPSYrr:
9736 case X86::VORPDZ128rr:
9737 case X86::VORPSZ128rr:
9738 case X86::VORPDZ256rr:
9739 case X86::VORPSZ256rr:
9740 case X86::VORPDZrr:
9741 case X86::VORPSZrr:
9742 case X86::VXORPDrr:
9743 case X86::VXORPSrr:
9744 case X86::VXORPDYrr:
9745 case X86::VXORPSYrr:
9746 case X86::VXORPDZ128rr:
9747 case X86::VXORPSZ128rr:
9748 case X86::VXORPDZ256rr:
9749 case X86::VXORPSZ256rr:
9750 case X86::VXORPDZrr:
9751 case X86::VXORPSZrr:
9752 case X86::KADDBrr:
9753 case X86::KADDWrr:
9754 case X86::KADDDrr:
9755 case X86::KADDQrr:
9756 case X86::KANDBrr:
9757 case X86::KANDWrr:
9758 case X86::KANDDrr:
9759 case X86::KANDQrr:
9760 case X86::KORBrr:
9761 case X86::KORWrr:
9762 case X86::KORDrr:
9763 case X86::KORQrr:
9764 case X86::KXORBrr:
9765 case X86::KXORWrr:
9766 case X86::KXORDrr:
9767 case X86::KXORQrr:
9768 case X86::VPADDBrr:
9769 case X86::VPADDWrr:
9770 case X86::VPADDDrr:
9771 case X86::VPADDQrr:
9772 case X86::VPADDBYrr:
9773 case X86::VPADDWYrr:
9774 case X86::VPADDDYrr:
9775 case X86::VPADDQYrr:
9776 case X86::VPADDBZ128rr:
9777 case X86::VPADDWZ128rr:
9778 case X86::VPADDDZ128rr:
9779 case X86::VPADDQZ128rr:
9780 case X86::VPADDBZ256rr:
9781 case X86::VPADDWZ256rr:
9782 case X86::VPADDDZ256rr:
9783 case X86::VPADDQZ256rr:
9784 case X86::VPADDBZrr:
9785 case X86::VPADDWZrr:
9786 case X86::VPADDDZrr:
9787 case X86::VPADDQZrr:
9788 case X86::VPMULLWrr:
9789 case X86::VPMULLWYrr:
9790 case X86::VPMULLWZ128rr:
9791 case X86::VPMULLWZ256rr:
9792 case X86::VPMULLWZrr:
9793 case X86::VPMULLDrr:
9794 case X86::VPMULLDYrr:
9795 case X86::VPMULLDZ128rr:
9796 case X86::VPMULLDZ256rr:
9797 case X86::VPMULLDZrr:
9798 case X86::VPMULLQZ128rr:
9799 case X86::VPMULLQZ256rr:
9800 case X86::VPMULLQZrr:
9801 case X86::VPMAXSBrr:
9802 case X86::VPMAXSBYrr:
9803 case X86::VPMAXSBZ128rr:
9804 case X86::VPMAXSBZ256rr:
9805 case X86::VPMAXSBZrr:
9806 case X86::VPMAXSDrr:
9807 case X86::VPMAXSDYrr:
9808 case X86::VPMAXSDZ128rr:
9809 case X86::VPMAXSDZ256rr:
9810 case X86::VPMAXSDZrr:
9811 case X86::VPMAXSQZ128rr:
9812 case X86::VPMAXSQZ256rr:
9813 case X86::VPMAXSQZrr:
9814 case X86::VPMAXSWrr:
9815 case X86::VPMAXSWYrr:
9816 case X86::VPMAXSWZ128rr:
9817 case X86::VPMAXSWZ256rr:
9818 case X86::VPMAXSWZrr:
9819 case X86::VPMAXUBrr:
9820 case X86::VPMAXUBYrr:
9821 case X86::VPMAXUBZ128rr:
9822 case X86::VPMAXUBZ256rr:
9823 case X86::VPMAXUBZrr:
9824 case X86::VPMAXUDrr:
9825 case X86::VPMAXUDYrr:
9826 case X86::VPMAXUDZ128rr:
9827 case X86::VPMAXUDZ256rr:
9828 case X86::VPMAXUDZrr:
9829 case X86::VPMAXUQZ128rr:
9830 case X86::VPMAXUQZ256rr:
9831 case X86::VPMAXUQZrr:
9832 case X86::VPMAXUWrr:
9833 case X86::VPMAXUWYrr:
9834 case X86::VPMAXUWZ128rr:
9835 case X86::VPMAXUWZ256rr:
9836 case X86::VPMAXUWZrr:
9837 case X86::VPMINSBrr:
9838 case X86::VPMINSBYrr:
9839 case X86::VPMINSBZ128rr:
9840 case X86::VPMINSBZ256rr:
9841 case X86::VPMINSBZrr:
9842 case X86::VPMINSDrr:
9843 case X86::VPMINSDYrr:
9844 case X86::VPMINSDZ128rr:
9845 case X86::VPMINSDZ256rr:
9846 case X86::VPMINSDZrr:
9847 case X86::VPMINSQZ128rr:
9848 case X86::VPMINSQZ256rr:
9849 case X86::VPMINSQZrr:
9850 case X86::VPMINSWrr:
9851 case X86::VPMINSWYrr:
9852 case X86::VPMINSWZ128rr:
9853 case X86::VPMINSWZ256rr:
9854 case X86::VPMINSWZrr:
9855 case X86::VPMINUBrr:
9856 case X86::VPMINUBYrr:
9857 case X86::VPMINUBZ128rr:
9858 case X86::VPMINUBZ256rr:
9859 case X86::VPMINUBZrr:
9860 case X86::VPMINUDrr:
9861 case X86::VPMINUDYrr:
9862 case X86::VPMINUDZ128rr:
9863 case X86::VPMINUDZ256rr:
9864 case X86::VPMINUDZrr:
9865 case X86::VPMINUQZ128rr:
9866 case X86::VPMINUQZ256rr:
9867 case X86::VPMINUQZrr:
9868 case X86::VPMINUWrr:
9869 case X86::VPMINUWYrr:
9870 case X86::VPMINUWZ128rr:
9871 case X86::VPMINUWZ256rr:
9872 case X86::VPMINUWZrr:
9876 case X86::MAXCPDrr:
9877 case X86::MAXCPSrr:
9878 case X86::MAXCSDrr:
9879 case X86::MAXCSSrr:
9880 case X86::MINCPDrr:
9881 case X86::MINCPSrr:
9882 case X86::MINCSDrr:
9883 case X86::MINCSSrr:
9884 case X86::VMAXCPDrr:
9885 case X86::VMAXCPSrr:
9886 case X86::VMAXCPDYrr:
9887 case X86::VMAXCPSYrr:
9888 case X86::VMAXCPDZ128rr:
9889 case X86::VMAXCPSZ128rr:
9890 case X86::VMAXCPDZ256rr:
9891 case X86::VMAXCPSZ256rr:
9892 case X86::VMAXCPDZrr:
9893 case X86::VMAXCPSZrr:
9894 case X86::VMAXCSDrr:
9895 case X86::VMAXCSSrr:
9896 case X86::VMAXCSDZrr:
9897 case X86::VMAXCSSZrr:
9898 case X86::VMINCPDrr:
9899 case X86::VMINCPSrr:
9900 case X86::VMINCPDYrr:
9901 case X86::VMINCPSYrr:
9902 case X86::VMINCPDZ128rr:
9903 case X86::VMINCPSZ128rr:
9904 case X86::VMINCPDZ256rr:
9905 case X86::VMINCPSZ256rr:
9906 case X86::VMINCPDZrr:
9907 case X86::VMINCPSZrr:
9908 case X86::VMINCSDrr:
9909 case X86::VMINCSSrr:
9910 case X86::VMINCSDZrr:
9911 case X86::VMINCSSZrr:
9912 case X86::VMAXCPHZ128rr:
9913 case X86::VMAXCPHZ256rr:
9914 case X86::VMAXCPHZrr:
9915 case X86::VMAXCSHZrr:
9916 case X86::VMINCPHZ128rr:
9917 case X86::VMINCPHZ256rr:
9918 case X86::VMINCPHZrr:
9919 case X86::VMINCSHZrr:
9921 case X86::ADDPDrr:
9922 case X86::ADDPSrr:
9923 case X86::ADDSDrr:
9924 case X86::ADDSSrr:
9925 case X86::MULPDrr:
9926 case X86::MULPSrr:
9927 case X86::MULSDrr:
9928 case X86::MULSSrr:
9929 case X86::VADDPDrr:
9930 case X86::VADDPSrr:
9931 case X86::VADDPDYrr:
9932 case X86::VADDPSYrr:
9933 case X86::VADDPDZ128rr:
9934 case X86::VADDPSZ128rr:
9935 case X86::VADDPDZ256rr:
9936 case X86::VADDPSZ256rr:
9937 case X86::VADDPDZrr:
9938 case X86::VADDPSZrr:
9939 case X86::VADDSDrr:
9940 case X86::VADDSSrr:
9941 case X86::VADDSDZrr:
9942 case X86::VADDSSZrr:
9943 case X86::VMULPDrr:
9944 case X86::VMULPSrr:
9945 case X86::VMULPDYrr:
9946 case X86::VMULPSYrr:
9947 case X86::VMULPDZ128rr:
9948 case X86::VMULPSZ128rr:
9949 case X86::VMULPDZ256rr:
9950 case X86::VMULPSZ256rr:
9951 case X86::VMULPDZrr:
9952 case X86::VMULPSZrr:
9953 case X86::VMULSDrr:
9954 case X86::VMULSSrr:
9955 case X86::VMULSDZrr:
9956 case X86::VMULSSZrr:
9957 case X86::VADDPHZ128rr:
9958 case X86::VADDPHZ256rr:
9959 case X86::VADDPHZrr:
9960 case X86::VADDSHZrr:
9961 case X86::VMULPHZ128rr:
9962 case X86::VMULPHZ256rr:
9963 case X86::VMULPHZrr:
9964 case X86::VMULSHZrr:
9978 Register DestReg = MI.getOperand(0).getReg();
10000 if (MI.getOpcode() == X86::MOV8rr || MI.getOpcode() == X86::MOV16rr ||
10004 assert(MI.getOpcode() == X86::MOV32rr && "Unexpected super-register case");
10016 case X86::LEA32r:
10017 case X86::LEA64r:
10018 case X86::LEA64_32r: {
10020 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
10031 (Op2.getReg() == X86::NoRegister || Op2.getReg().isPhysical()));
10035 if ((Op1.isReg() && Op1.getReg() == MI.getOperand(0).getReg()) ||
10036 Op2.getReg() == MI.getOperand(0).getReg())
10038 else if ((Op1.isReg() && Op1.getReg() != X86::NoRegister &&
10039 TRI->regsOverlap(Op1.getReg(), MI.getOperand(0).getReg())) ||
10040 (Op2.getReg() != X86::NoRegister &&
10041 TRI->regsOverlap(Op2.getReg(), MI.getOperand(0).getReg())))
10048 if ((Op1.isReg() && Op1.getReg() != X86::NoRegister)) {
10053 if (Op && Op->isReg() && Op->getReg() == Op2.getReg() && Coef > 0) {
10058 if (Op && Op2.getReg() != X86::NoRegister) {
10060 if (dwarfReg < 0)
10064 Ops.push_back(0);
10068 Ops.push_back(0);
10071 assert(Op2.getReg() != X86::NoRegister);
10076 assert(Op2.getReg() != X86::NoRegister);
10082 if (((Op1.isReg() && Op1.getReg() != X86::NoRegister) || Op1.isFI()) &&
10083 Op2.getReg() != X86::NoRegister) {
10093 case X86::MOV8ri:
10094 case X86::MOV16ri:
10097 case X86::MOV32ri:
10098 case X86::MOV64ri:
10099 case X86::MOV64ri32:
10102 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
10105 case X86::MOV8rr:
10106 case X86::MOV16rr:
10107 case X86::MOV32rr:
10108 case X86::MOV64rr:
10110 case X86::XOR32rr: {
10113 if (!TRI->isSuperRegisterEq(MI.getOperand(0).getReg(), Reg))
10116 return ParamLoadedValue(MachineOperand::CreateImm(0), Expr);
10119 case X86::MOVSX64rr32: {
10126 if (!TRI->isSubRegisterEq(MI.getOperand(0).getReg(), Reg))
10136 if (Reg == MI.getOperand(0).getReg())
10139 assert(X86MCRegisterClasses[X86::GR32RegClassID].contains(Reg) &&
10158 OldMI1.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10160 OldMI2.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10172 NewMI1.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10174 NewMI2.findRegisterDefOperand(X86::EFLAGS, /*TRI=*/nullptr);
10189 return std::make_pair(TF, 0u);
10196 {MO_GOT_ABSOLUTE_ADDRESS, "x86-got-absolute-address"},
10197 {MO_PIC_BASE_OFFSET, "x86-pic-base-offset"},
10198 {MO_GOT, "x86-got"},
10199 {MO_GOTOFF, "x86-gotoff"},
10200 {MO_GOTPCREL, "x86-gotpcrel"},
10201 {MO_GOTPCREL_NORELAX, "x86-gotpcrel-norelax"},
10202 {MO_PLT, "x86-plt"},
10203 {MO_TLSGD, "x86-tlsgd"},
10204 {MO_TLSLD, "x86-tlsld"},
10205 {MO_TLSLDM, "x86-tlsldm"},
10206 {MO_GOTTPOFF, "x86-gottpoff"},
10207 {MO_INDNTPOFF, "x86-indntpoff"},
10208 {MO_TPOFF, "x86-tpoff"},
10209 {MO_DTPOFF, "x86-dtpoff"},
10210 {MO_NTPOFF, "x86-ntpoff"},
10211 {MO_GOTNTPOFF, "x86-gotntpoff"},
10212 {MO_DLLIMPORT, "x86-dllimport"},
10213 {MO_DARWIN_NONLAZY, "x86-darwin-nonlazy"},
10214 {MO_DARWIN_NONLAZY_PIC_BASE, "x86-darwin-nonlazy-pic-base"},
10215 {MO_TLVP, "x86-tlvp"},
10216 {MO_TLVP_PIC_BASE, "x86-tlvp-pic-base"},
10217 {MO_SECREL, "x86-secrel"},
10218 {MO_COFFSTUB, "x86-coffstub"}};
10224 /// global base register for x86-32.
10242 if (GlobalBaseReg == 0)
10254 PC = RegInfo.createVirtualRegister(&X86::GR32RegClass);
10266 Register PBReg = RegInfo.createVirtualRegister(&X86::GR64RegClass);
10267 Register GOTReg = RegInfo.createVirtualRegister(&X86::GR64RegClass);
10268 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PBReg)
10269 .addReg(X86::RIP)
10270 .addImm(0)
10271 .addReg(0)
10273 .addReg(0);
10275 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOV64ri), GOTReg)
10278 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD64rr), PC)
10284 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC)
10285 .addReg(X86::RIP)
10286 .addImm(0)
10287 .addReg(0)
10289 .addReg(0);
10294 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::MOVPC32r), PC).addImm(0);
10301 BuildMI(FirstMBB, MBBI, DL, TII->get(X86::ADD32ri), GlobalBaseReg)
10312 return "X86 PIC Global Base Reg Initialization";
10322 char CGBR::ID = 0;
10342 return VisitNode(DT->getRootNode(), 0);
10358 case X86::TLS_base_addr32:
10359 case X86::TLS_base_addr64:
10391 TII->get(TargetOpcode::COPY), is64Bit ? X86::RAX : X86::EAX)
10411 is64Bit ? &X86::GR64RegClass : &X86::GR32RegClass);
10417 .addReg(is64Bit ? X86::RAX : X86::EAX);
10434 char LDTLSCleanup::ID = 0;
10465 /// * Frame construction overhead: 0 (don't need to return)
10472 unsigned SequenceSize = 0;
10473 for (auto &MI : RepeatedSequenceLocs[0]) {
10474 // FIXME: x86 doesn't implement getInstSizeInBytes, so
10484 unsigned CFICount = 0;
10485 for (auto &I : RepeatedSequenceLocs[0]) {
10499 if (CFICount > 0 && CFICount != CFIInstructions.size())
10504 if (RepeatedSequenceLocs[0].back().isTerminator()) {
10509 0, // Number of bytes to emit frame.
10514 if (CFICount > 0)
10566 if (MI.modifiesRegister(X86::RSP, &RI) || MI.readsRegister(X86::RSP, &RI) ||
10567 MI.getDesc().hasImplicitUseOfPhysReg(X86::RSP) ||
10568 MI.getDesc().hasImplicitDefOfPhysReg(X86::RSP))
10572 if (MI.readsRegister(X86::RIP, &RI) ||
10573 MI.getDesc().hasImplicitUseOfPhysReg(X86::RIP) ||
10574 MI.getDesc().hasImplicitDefOfPhysReg(X86::RIP))
10593 MachineInstr *retq = BuildMI(MF, DebugLoc(), get(X86::RET64));
10603 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(X86::TAILJMPd64))
10607 It = MBB.insert(It, BuildMI(MF, DebugLoc(), get(X86::CALL64pcrel32))
10622 if (ST.hasMMX() && X86::VR64RegClass.contains(Reg))
10633 BuildMI(MBB, Iter, DL, get(X86::MOV32ri), Reg).addImm(0);
10635 BuildMI(MBB, Iter, DL, get(X86::XOR32rr), Reg)
10638 } else if (X86::VR128RegClass.contains(Reg)) {
10644 BuildMI(MBB, Iter, DL, get(X86::PXORrr), Reg)
10647 } else if (X86::VR256RegClass.contains(Reg)) {
10653 BuildMI(MBB, Iter, DL, get(X86::VPXORrr), Reg)
10656 } else if (X86::VR512RegClass.contains(Reg)) {
10662 BuildMI(MBB, Iter, DL, get(X86::VPXORYrr), Reg)
10665 } else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) ||
10666 X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) ||
10667 X86::VK16RegClass.contains(Reg)) {
10672 unsigned Op = ST.hasBWI() ? X86::KXORQrr : X86::KXORWrr;
10684 case X86::VPDPWSSDrr:
10685 case X86::VPDPWSSDrm:
10686 case X86::VPDPWSSDYrr:
10687 case X86::VPDPWSSDYrm: {
10694 case X86::VPDPWSSDZ128r:
10695 case X86::VPDPWSSDZ128m:
10696 case X86::VPDPWSSDZ256r:
10697 case X86::VPDPWSSDZ256m:
10698 case X86::VPDPWSSDZr:
10699 case X86::VPDPWSSDZm: {
10720 unsigned AddOpc = 0;
10721 unsigned MaddOpc = 0;
10730 case X86::VPDPWSSDrr:
10731 MaddOpc = X86::VPMADDWDrr;
10732 AddOpc = X86::VPADDDrr;
10734 case X86::VPDPWSSDrm:
10735 MaddOpc = X86::VPMADDWDrm;
10736 AddOpc = X86::VPADDDrr;
10738 case X86::VPDPWSSDZ128r:
10739 MaddOpc = X86::VPMADDWDZ128rr;
10740 AddOpc = X86::VPADDDZ128rr;
10742 case X86::VPDPWSSDZ128m:
10743 MaddOpc = X86::VPMADDWDZ128rm;
10744 AddOpc = X86::VPADDDZ128rr;
10750 case X86::VPDPWSSDYrr:
10751 MaddOpc = X86::VPMADDWDYrr;
10752 AddOpc = X86::VPADDDYrr;
10754 case X86::VPDPWSSDYrm:
10755 MaddOpc = X86::VPMADDWDYrm;
10756 AddOpc = X86::VPADDDYrr;
10758 case X86::VPDPWSSDZ256r:
10759 MaddOpc = X86::VPMADDWDZ256rr;
10760 AddOpc = X86::VPADDDZ256rr;
10762 case X86::VPDPWSSDZ256m:
10763 MaddOpc = X86::VPMADDWDZ256rm;
10764 AddOpc = X86::VPADDDZ256rr;
10770 case X86::VPDPWSSDZr:
10771 MaddOpc = X86::VPMADDWDZrr;
10772 AddOpc = X86::VPADDDZrr;
10774 case X86::VPDPWSSDZm:
10775 MaddOpc = X86::VPMADDWDZrm;
10776 AddOpc = X86::VPADDDZrr;
10781 RegInfo.getRegClass(Root.getOperand(0).getReg());
10787 Madd->getOperand(0).setReg(NewReg);
10788 InstrIdxForVirtReg.insert(std::make_pair(NewReg, 0));
10790 Register DstReg = Root.getOperand(0).getReg();
10795 .addReg(Madd->getOperand(0).getReg(), getKillRegState(true));