1 //===-- X86FixupVectorConstants.cpp - optimize constant generation -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file examines all full size vector constant pool loads and attempts to 10 // replace them with smaller constant pool entries, including: 11 // * Converting AVX512 memory-fold instructions to their broadcast-fold form. 12 // * Using vzload scalar loads. 13 // * Broadcasting of full width loads. 14 // * Sign/Zero extension of full width loads. 15 // 16 //===----------------------------------------------------------------------===// 17 18 #include "X86.h" 19 #include "X86InstrFoldTables.h" 20 #include "X86InstrInfo.h" 21 #include "X86Subtarget.h" 22 #include "llvm/ADT/Statistic.h" 23 #include "llvm/CodeGen/MachineConstantPool.h" 24 25 using namespace llvm; 26 27 #define DEBUG_TYPE "x86-fixup-vector-constants" 28 29 STATISTIC(NumInstChanges, "Number of instructions changes"); 30 31 namespace { 32 class X86FixupVectorConstantsPass : public MachineFunctionPass { 33 public: 34 static char ID; 35 36 X86FixupVectorConstantsPass() : MachineFunctionPass(ID) {} 37 38 StringRef getPassName() const override { 39 return "X86 Fixup Vector Constants"; 40 } 41 42 bool runOnMachineFunction(MachineFunction &MF) override; 43 bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB, 44 MachineInstr &MI); 45 46 // This pass runs after regalloc and doesn't support VReg operands. 47 MachineFunctionProperties getRequiredProperties() const override { 48 return MachineFunctionProperties().set( 49 MachineFunctionProperties::Property::NoVRegs); 50 } 51 52 private: 53 const X86InstrInfo *TII = nullptr; 54 const X86Subtarget *ST = nullptr; 55 const MCSchedModel *SM = nullptr; 56 }; 57 } // end anonymous namespace 58 59 char X86FixupVectorConstantsPass::ID = 0; 60 61 INITIALIZE_PASS(X86FixupVectorConstantsPass, DEBUG_TYPE, DEBUG_TYPE, false, false) 62 63 FunctionPass *llvm::createX86FixupVectorConstants() { 64 return new X86FixupVectorConstantsPass(); 65 } 66 67 /// Normally, we only allow poison in vector splats. However, as this is part 68 /// of the backend, and working with the DAG representation, which currently 69 /// only natively represents undef values, we need to accept undefs here. 70 static Constant *getSplatValueAllowUndef(const ConstantVector *C) { 71 Constant *Res = nullptr; 72 for (Value *Op : C->operands()) { 73 Constant *OpC = cast<Constant>(Op); 74 if (isa<UndefValue>(OpC)) 75 continue; 76 if (!Res) 77 Res = OpC; 78 else if (Res != OpC) 79 return nullptr; 80 } 81 return Res; 82 } 83 84 // Attempt to extract the full width of bits data from the constant. 85 static std::optional<APInt> extractConstantBits(const Constant *C) { 86 unsigned NumBits = C->getType()->getPrimitiveSizeInBits(); 87 88 if (isa<UndefValue>(C)) 89 return APInt::getZero(NumBits); 90 91 if (auto *CInt = dyn_cast<ConstantInt>(C)) 92 return CInt->getValue(); 93 94 if (auto *CFP = dyn_cast<ConstantFP>(C)) 95 return CFP->getValue().bitcastToAPInt(); 96 97 if (auto *CV = dyn_cast<ConstantVector>(C)) { 98 if (auto *CVSplat = getSplatValueAllowUndef(CV)) { 99 if (std::optional<APInt> Bits = extractConstantBits(CVSplat)) { 100 assert((NumBits % Bits->getBitWidth()) == 0 && "Illegal splat"); 101 return APInt::getSplat(NumBits, *Bits); 102 } 103 } 104 105 APInt Bits = APInt::getZero(NumBits); 106 for (unsigned I = 0, E = CV->getNumOperands(); I != E; ++I) { 107 Constant *Elt = CV->getOperand(I); 108 std::optional<APInt> SubBits = extractConstantBits(Elt); 109 if (!SubBits) 110 return std::nullopt; 111 assert(NumBits == (E * SubBits->getBitWidth()) && 112 "Illegal vector element size"); 113 Bits.insertBits(*SubBits, I * SubBits->getBitWidth()); 114 } 115 return Bits; 116 } 117 118 if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) { 119 bool IsInteger = CDS->getElementType()->isIntegerTy(); 120 bool IsFloat = CDS->getElementType()->isHalfTy() || 121 CDS->getElementType()->isBFloatTy() || 122 CDS->getElementType()->isFloatTy() || 123 CDS->getElementType()->isDoubleTy(); 124 if (IsInteger || IsFloat) { 125 APInt Bits = APInt::getZero(NumBits); 126 unsigned EltBits = CDS->getElementType()->getPrimitiveSizeInBits(); 127 for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { 128 if (IsInteger) 129 Bits.insertBits(CDS->getElementAsAPInt(I), I * EltBits); 130 else 131 Bits.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(), 132 I * EltBits); 133 } 134 return Bits; 135 } 136 } 137 138 return std::nullopt; 139 } 140 141 static std::optional<APInt> extractConstantBits(const Constant *C, 142 unsigned NumBits) { 143 if (std::optional<APInt> Bits = extractConstantBits(C)) 144 return Bits->zextOrTrunc(NumBits); 145 return std::nullopt; 146 } 147 148 // Attempt to compute the splat width of bits data by normalizing the splat to 149 // remove undefs. 150 static std::optional<APInt> getSplatableConstant(const Constant *C, 151 unsigned SplatBitWidth) { 152 const Type *Ty = C->getType(); 153 assert((Ty->getPrimitiveSizeInBits() % SplatBitWidth) == 0 && 154 "Illegal splat width"); 155 156 if (std::optional<APInt> Bits = extractConstantBits(C)) 157 if (Bits->isSplat(SplatBitWidth)) 158 return Bits->trunc(SplatBitWidth); 159 160 // Detect general splats with undefs. 161 // TODO: Do we need to handle NumEltsBits > SplatBitWidth splitting? 162 if (auto *CV = dyn_cast<ConstantVector>(C)) { 163 unsigned NumOps = CV->getNumOperands(); 164 unsigned NumEltsBits = Ty->getScalarSizeInBits(); 165 unsigned NumScaleOps = SplatBitWidth / NumEltsBits; 166 if ((SplatBitWidth % NumEltsBits) == 0) { 167 // Collect the elements and ensure that within the repeated splat sequence 168 // they either match or are undef. 169 SmallVector<Constant *, 16> Sequence(NumScaleOps, nullptr); 170 for (unsigned Idx = 0; Idx != NumOps; ++Idx) { 171 if (Constant *Elt = CV->getAggregateElement(Idx)) { 172 if (isa<UndefValue>(Elt)) 173 continue; 174 unsigned SplatIdx = Idx % NumScaleOps; 175 if (!Sequence[SplatIdx] || Sequence[SplatIdx] == Elt) { 176 Sequence[SplatIdx] = Elt; 177 continue; 178 } 179 } 180 return std::nullopt; 181 } 182 // Extract the constant bits forming the splat and insert into the bits 183 // data, leave undef as zero. 184 APInt SplatBits = APInt::getZero(SplatBitWidth); 185 for (unsigned I = 0; I != NumScaleOps; ++I) { 186 if (!Sequence[I]) 187 continue; 188 if (std::optional<APInt> Bits = extractConstantBits(Sequence[I])) { 189 SplatBits.insertBits(*Bits, I * Bits->getBitWidth()); 190 continue; 191 } 192 return std::nullopt; 193 } 194 return SplatBits; 195 } 196 } 197 198 return std::nullopt; 199 } 200 201 // Split raw bits into a constant vector of elements of a specific bit width. 202 // NOTE: We don't always bother converting to scalars if the vector length is 1. 203 static Constant *rebuildConstant(LLVMContext &Ctx, Type *SclTy, 204 const APInt &Bits, unsigned NumSclBits) { 205 unsigned BitWidth = Bits.getBitWidth(); 206 207 if (NumSclBits == 8) { 208 SmallVector<uint8_t> RawBits; 209 for (unsigned I = 0; I != BitWidth; I += 8) 210 RawBits.push_back(Bits.extractBits(8, I).getZExtValue()); 211 return ConstantDataVector::get(Ctx, RawBits); 212 } 213 214 if (NumSclBits == 16) { 215 SmallVector<uint16_t> RawBits; 216 for (unsigned I = 0; I != BitWidth; I += 16) 217 RawBits.push_back(Bits.extractBits(16, I).getZExtValue()); 218 if (SclTy->is16bitFPTy()) 219 return ConstantDataVector::getFP(SclTy, RawBits); 220 return ConstantDataVector::get(Ctx, RawBits); 221 } 222 223 if (NumSclBits == 32) { 224 SmallVector<uint32_t> RawBits; 225 for (unsigned I = 0; I != BitWidth; I += 32) 226 RawBits.push_back(Bits.extractBits(32, I).getZExtValue()); 227 if (SclTy->isFloatTy()) 228 return ConstantDataVector::getFP(SclTy, RawBits); 229 return ConstantDataVector::get(Ctx, RawBits); 230 } 231 232 assert(NumSclBits == 64 && "Unhandled vector element width"); 233 234 SmallVector<uint64_t> RawBits; 235 for (unsigned I = 0; I != BitWidth; I += 64) 236 RawBits.push_back(Bits.extractBits(64, I).getZExtValue()); 237 if (SclTy->isDoubleTy()) 238 return ConstantDataVector::getFP(SclTy, RawBits); 239 return ConstantDataVector::get(Ctx, RawBits); 240 } 241 242 // Attempt to rebuild a normalized splat vector constant of the requested splat 243 // width, built up of potentially smaller scalar values. 244 static Constant *rebuildSplatCst(const Constant *C, unsigned /*NumBits*/, 245 unsigned /*NumElts*/, unsigned SplatBitWidth) { 246 // TODO: Truncate to NumBits once ConvertToBroadcastAVX512 support this. 247 std::optional<APInt> Splat = getSplatableConstant(C, SplatBitWidth); 248 if (!Splat) 249 return nullptr; 250 251 // Determine scalar size to use for the constant splat vector, clamping as we 252 // might have found a splat smaller than the original constant data. 253 Type *SclTy = C->getType()->getScalarType(); 254 unsigned NumSclBits = SclTy->getPrimitiveSizeInBits(); 255 NumSclBits = std::min<unsigned>(NumSclBits, SplatBitWidth); 256 257 // Fallback to i64 / double. 258 NumSclBits = (NumSclBits == 8 || NumSclBits == 16 || NumSclBits == 32) 259 ? NumSclBits 260 : 64; 261 262 // Extract per-element bits. 263 return rebuildConstant(C->getContext(), SclTy, *Splat, NumSclBits); 264 } 265 266 static Constant *rebuildZeroUpperCst(const Constant *C, unsigned NumBits, 267 unsigned /*NumElts*/, 268 unsigned ScalarBitWidth) { 269 Type *SclTy = C->getType()->getScalarType(); 270 unsigned NumSclBits = SclTy->getPrimitiveSizeInBits(); 271 LLVMContext &Ctx = C->getContext(); 272 273 if (NumBits > ScalarBitWidth) { 274 // Determine if the upper bits are all zero. 275 if (std::optional<APInt> Bits = extractConstantBits(C, NumBits)) { 276 if (Bits->countLeadingZeros() >= (NumBits - ScalarBitWidth)) { 277 // If the original constant was made of smaller elements, try to retain 278 // those types. 279 if (ScalarBitWidth > NumSclBits && (ScalarBitWidth % NumSclBits) == 0) 280 return rebuildConstant(Ctx, SclTy, *Bits, NumSclBits); 281 282 // Fallback to raw integer bits. 283 APInt RawBits = Bits->zextOrTrunc(ScalarBitWidth); 284 return ConstantInt::get(Ctx, RawBits); 285 } 286 } 287 } 288 289 return nullptr; 290 } 291 292 static Constant *rebuildExtCst(const Constant *C, bool IsSExt, 293 unsigned NumBits, unsigned NumElts, 294 unsigned SrcEltBitWidth) { 295 unsigned DstEltBitWidth = NumBits / NumElts; 296 assert((NumBits % NumElts) == 0 && (NumBits % SrcEltBitWidth) == 0 && 297 (DstEltBitWidth % SrcEltBitWidth) == 0 && 298 (DstEltBitWidth > SrcEltBitWidth) && "Illegal extension width"); 299 300 if (std::optional<APInt> Bits = extractConstantBits(C, NumBits)) { 301 assert((Bits->getBitWidth() / DstEltBitWidth) == NumElts && 302 (Bits->getBitWidth() % DstEltBitWidth) == 0 && 303 "Unexpected constant extension"); 304 305 // Ensure every vector element can be represented by the src bitwidth. 306 APInt TruncBits = APInt::getZero(NumElts * SrcEltBitWidth); 307 for (unsigned I = 0; I != NumElts; ++I) { 308 APInt Elt = Bits->extractBits(DstEltBitWidth, I * DstEltBitWidth); 309 if ((IsSExt && Elt.getSignificantBits() > SrcEltBitWidth) || 310 (!IsSExt && Elt.getActiveBits() > SrcEltBitWidth)) 311 return nullptr; 312 TruncBits.insertBits(Elt.trunc(SrcEltBitWidth), I * SrcEltBitWidth); 313 } 314 315 Type *Ty = C->getType(); 316 return rebuildConstant(Ty->getContext(), Ty->getScalarType(), TruncBits, 317 SrcEltBitWidth); 318 } 319 320 return nullptr; 321 } 322 static Constant *rebuildSExtCst(const Constant *C, unsigned NumBits, 323 unsigned NumElts, unsigned SrcEltBitWidth) { 324 return rebuildExtCst(C, true, NumBits, NumElts, SrcEltBitWidth); 325 } 326 static Constant *rebuildZExtCst(const Constant *C, unsigned NumBits, 327 unsigned NumElts, unsigned SrcEltBitWidth) { 328 return rebuildExtCst(C, false, NumBits, NumElts, SrcEltBitWidth); 329 } 330 331 bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, 332 MachineBasicBlock &MBB, 333 MachineInstr &MI) { 334 unsigned Opc = MI.getOpcode(); 335 MachineConstantPool *CP = MI.getParent()->getParent()->getConstantPool(); 336 bool HasSSE41 = ST->hasSSE41(); 337 bool HasAVX2 = ST->hasAVX2(); 338 bool HasDQI = ST->hasDQI(); 339 bool HasBWI = ST->hasBWI(); 340 bool HasVLX = ST->hasVLX(); 341 bool MultiDomain = ST->hasAVX512() || ST->hasNoDomainDelayMov(); 342 343 struct FixupEntry { 344 int Op; 345 int NumCstElts; 346 int MemBitWidth; 347 std::function<Constant *(const Constant *, unsigned, unsigned, unsigned)> 348 RebuildConstant; 349 }; 350 auto FixupConstant = [&](ArrayRef<FixupEntry> Fixups, unsigned RegBitWidth, 351 unsigned OperandNo) { 352 #ifdef EXPENSIVE_CHECKS 353 assert(llvm::is_sorted(Fixups, 354 [](const FixupEntry &A, const FixupEntry &B) { 355 return (A.NumCstElts * A.MemBitWidth) < 356 (B.NumCstElts * B.MemBitWidth); 357 }) && 358 "Constant fixup table not sorted in ascending constant size"); 359 #endif 360 assert(MI.getNumOperands() >= (OperandNo + X86::AddrNumOperands) && 361 "Unexpected number of operands!"); 362 if (auto *C = X86::getConstantFromPool(MI, OperandNo)) { 363 RegBitWidth = 364 RegBitWidth ? RegBitWidth : C->getType()->getPrimitiveSizeInBits(); 365 for (const FixupEntry &Fixup : Fixups) { 366 if (Fixup.Op) { 367 // Construct a suitable constant and adjust the MI to use the new 368 // constant pool entry. 369 if (Constant *NewCst = Fixup.RebuildConstant( 370 C, RegBitWidth, Fixup.NumCstElts, Fixup.MemBitWidth)) { 371 unsigned NewCPI = 372 CP->getConstantPoolIndex(NewCst, Align(Fixup.MemBitWidth / 8)); 373 MI.setDesc(TII->get(Fixup.Op)); 374 MI.getOperand(OperandNo + X86::AddrDisp).setIndex(NewCPI); 375 return true; 376 } 377 } 378 } 379 } 380 return false; 381 }; 382 383 // Attempt to detect a suitable vzload/broadcast/vextload from increasing 384 // constant bitwidths. Prefer vzload/broadcast/vextload for same bitwidth: 385 // - vzload shouldn't ever need a shuffle port to zero the upper elements and 386 // the fp/int domain versions are equally available so we don't introduce a 387 // domain crossing penalty. 388 // - broadcast sometimes need a shuffle port (especially for 8/16-bit 389 // variants), AVX1 only has fp domain broadcasts but AVX2+ have good fp/int 390 // domain equivalents. 391 // - vextload always needs a shuffle port and is only ever int domain. 392 switch (Opc) { 393 /* FP Loads */ 394 case X86::MOVAPDrm: 395 case X86::MOVAPSrm: 396 case X86::MOVUPDrm: 397 case X86::MOVUPSrm: 398 // TODO: SSE3 MOVDDUP Handling 399 return FixupConstant({{X86::MOVSSrm, 1, 32, rebuildZeroUpperCst}, 400 {X86::MOVSDrm, 1, 64, rebuildZeroUpperCst}}, 401 128, 1); 402 case X86::VMOVAPDrm: 403 case X86::VMOVAPSrm: 404 case X86::VMOVUPDrm: 405 case X86::VMOVUPSrm: { 406 FixupEntry Fixups[] = { 407 {MultiDomain ? X86::VPMOVSXBQrm : 0, 2, 8, rebuildSExtCst}, 408 {MultiDomain ? X86::VPMOVZXBQrm : 0, 2, 8, rebuildZExtCst}, 409 {X86::VMOVSSrm, 1, 32, rebuildZeroUpperCst}, 410 {X86::VBROADCASTSSrm, 1, 32, rebuildSplatCst}, 411 {MultiDomain ? X86::VPMOVSXBDrm : 0, 4, 8, rebuildSExtCst}, 412 {MultiDomain ? X86::VPMOVZXBDrm : 0, 4, 8, rebuildZExtCst}, 413 {MultiDomain ? X86::VPMOVSXWQrm : 0, 2, 16, rebuildSExtCst}, 414 {MultiDomain ? X86::VPMOVZXWQrm : 0, 2, 16, rebuildZExtCst}, 415 {X86::VMOVSDrm, 1, 64, rebuildZeroUpperCst}, 416 {X86::VMOVDDUPrm, 1, 64, rebuildSplatCst}, 417 {MultiDomain ? X86::VPMOVSXWDrm : 0, 4, 16, rebuildSExtCst}, 418 {MultiDomain ? X86::VPMOVZXWDrm : 0, 4, 16, rebuildZExtCst}, 419 {MultiDomain ? X86::VPMOVSXDQrm : 0, 2, 32, rebuildSExtCst}, 420 {MultiDomain ? X86::VPMOVZXDQrm : 0, 2, 32, rebuildZExtCst}}; 421 return FixupConstant(Fixups, 128, 1); 422 } 423 case X86::VMOVAPDYrm: 424 case X86::VMOVAPSYrm: 425 case X86::VMOVUPDYrm: 426 case X86::VMOVUPSYrm: { 427 FixupEntry Fixups[] = { 428 {X86::VBROADCASTSSYrm, 1, 32, rebuildSplatCst}, 429 {HasAVX2 && MultiDomain ? X86::VPMOVSXBQYrm : 0, 4, 8, rebuildSExtCst}, 430 {HasAVX2 && MultiDomain ? X86::VPMOVZXBQYrm : 0, 4, 8, rebuildZExtCst}, 431 {X86::VBROADCASTSDYrm, 1, 64, rebuildSplatCst}, 432 {HasAVX2 && MultiDomain ? X86::VPMOVSXBDYrm : 0, 8, 8, rebuildSExtCst}, 433 {HasAVX2 && MultiDomain ? X86::VPMOVZXBDYrm : 0, 8, 8, rebuildZExtCst}, 434 {HasAVX2 && MultiDomain ? X86::VPMOVSXWQYrm : 0, 4, 16, rebuildSExtCst}, 435 {HasAVX2 && MultiDomain ? X86::VPMOVZXWQYrm : 0, 4, 16, rebuildZExtCst}, 436 {X86::VBROADCASTF128rm, 1, 128, rebuildSplatCst}, 437 {HasAVX2 && MultiDomain ? X86::VPMOVSXWDYrm : 0, 8, 16, rebuildSExtCst}, 438 {HasAVX2 && MultiDomain ? X86::VPMOVZXWDYrm : 0, 8, 16, rebuildZExtCst}, 439 {HasAVX2 && MultiDomain ? X86::VPMOVSXDQYrm : 0, 4, 32, rebuildSExtCst}, 440 {HasAVX2 && MultiDomain ? X86::VPMOVZXDQYrm : 0, 4, 32, 441 rebuildZExtCst}}; 442 return FixupConstant(Fixups, 256, 1); 443 } 444 case X86::VMOVAPDZ128rm: 445 case X86::VMOVAPSZ128rm: 446 case X86::VMOVUPDZ128rm: 447 case X86::VMOVUPSZ128rm: { 448 FixupEntry Fixups[] = { 449 {MultiDomain ? X86::VPMOVSXBQZ128rm : 0, 2, 8, rebuildSExtCst}, 450 {MultiDomain ? X86::VPMOVZXBQZ128rm : 0, 2, 8, rebuildZExtCst}, 451 {X86::VMOVSSZrm, 1, 32, rebuildZeroUpperCst}, 452 {X86::VBROADCASTSSZ128rm, 1, 32, rebuildSplatCst}, 453 {MultiDomain ? X86::VPMOVSXBDZ128rm : 0, 4, 8, rebuildSExtCst}, 454 {MultiDomain ? X86::VPMOVZXBDZ128rm : 0, 4, 8, rebuildZExtCst}, 455 {MultiDomain ? X86::VPMOVSXWQZ128rm : 0, 2, 16, rebuildSExtCst}, 456 {MultiDomain ? X86::VPMOVZXWQZ128rm : 0, 2, 16, rebuildZExtCst}, 457 {X86::VMOVSDZrm, 1, 64, rebuildZeroUpperCst}, 458 {X86::VMOVDDUPZ128rm, 1, 64, rebuildSplatCst}, 459 {MultiDomain ? X86::VPMOVSXWDZ128rm : 0, 4, 16, rebuildSExtCst}, 460 {MultiDomain ? X86::VPMOVZXWDZ128rm : 0, 4, 16, rebuildZExtCst}, 461 {MultiDomain ? X86::VPMOVSXDQZ128rm : 0, 2, 32, rebuildSExtCst}, 462 {MultiDomain ? X86::VPMOVZXDQZ128rm : 0, 2, 32, rebuildZExtCst}}; 463 return FixupConstant(Fixups, 128, 1); 464 } 465 case X86::VMOVAPDZ256rm: 466 case X86::VMOVAPSZ256rm: 467 case X86::VMOVUPDZ256rm: 468 case X86::VMOVUPSZ256rm: { 469 FixupEntry Fixups[] = { 470 {X86::VBROADCASTSSZ256rm, 1, 32, rebuildSplatCst}, 471 {MultiDomain ? X86::VPMOVSXBQZ256rm : 0, 4, 8, rebuildSExtCst}, 472 {MultiDomain ? X86::VPMOVZXBQZ256rm : 0, 4, 8, rebuildZExtCst}, 473 {X86::VBROADCASTSDZ256rm, 1, 64, rebuildSplatCst}, 474 {MultiDomain ? X86::VPMOVSXBDZ256rm : 0, 8, 8, rebuildSExtCst}, 475 {MultiDomain ? X86::VPMOVZXBDZ256rm : 0, 8, 8, rebuildZExtCst}, 476 {MultiDomain ? X86::VPMOVSXWQZ256rm : 0, 4, 16, rebuildSExtCst}, 477 {MultiDomain ? X86::VPMOVZXWQZ256rm : 0, 4, 16, rebuildZExtCst}, 478 {X86::VBROADCASTF32X4Z256rm, 1, 128, rebuildSplatCst}, 479 {MultiDomain ? X86::VPMOVSXWDZ256rm : 0, 8, 16, rebuildSExtCst}, 480 {MultiDomain ? X86::VPMOVZXWDZ256rm : 0, 8, 16, rebuildZExtCst}, 481 {MultiDomain ? X86::VPMOVSXDQZ256rm : 0, 4, 32, rebuildSExtCst}, 482 {MultiDomain ? X86::VPMOVZXDQZ256rm : 0, 4, 32, rebuildZExtCst}}; 483 return FixupConstant(Fixups, 256, 1); 484 } 485 case X86::VMOVAPDZrm: 486 case X86::VMOVAPSZrm: 487 case X86::VMOVUPDZrm: 488 case X86::VMOVUPSZrm: { 489 FixupEntry Fixups[] = { 490 {X86::VBROADCASTSSZrm, 1, 32, rebuildSplatCst}, 491 {X86::VBROADCASTSDZrm, 1, 64, rebuildSplatCst}, 492 {MultiDomain ? X86::VPMOVSXBQZrm : 0, 8, 8, rebuildSExtCst}, 493 {MultiDomain ? X86::VPMOVZXBQZrm : 0, 8, 8, rebuildZExtCst}, 494 {X86::VBROADCASTF32X4Zrm, 1, 128, rebuildSplatCst}, 495 {MultiDomain ? X86::VPMOVSXBDZrm : 0, 16, 8, rebuildSExtCst}, 496 {MultiDomain ? X86::VPMOVZXBDZrm : 0, 16, 8, rebuildZExtCst}, 497 {MultiDomain ? X86::VPMOVSXWQZrm : 0, 8, 16, rebuildSExtCst}, 498 {MultiDomain ? X86::VPMOVZXWQZrm : 0, 8, 16, rebuildZExtCst}, 499 {X86::VBROADCASTF64X4Zrm, 1, 256, rebuildSplatCst}, 500 {MultiDomain ? X86::VPMOVSXWDZrm : 0, 16, 16, rebuildSExtCst}, 501 {MultiDomain ? X86::VPMOVZXWDZrm : 0, 16, 16, rebuildZExtCst}, 502 {MultiDomain ? X86::VPMOVSXDQZrm : 0, 8, 32, rebuildSExtCst}, 503 {MultiDomain ? X86::VPMOVZXDQZrm : 0, 8, 32, rebuildZExtCst}}; 504 return FixupConstant(Fixups, 512, 1); 505 } 506 /* Integer Loads */ 507 case X86::MOVDQArm: 508 case X86::MOVDQUrm: { 509 FixupEntry Fixups[] = { 510 {HasSSE41 ? X86::PMOVSXBQrm : 0, 2, 8, rebuildSExtCst}, 511 {HasSSE41 ? X86::PMOVZXBQrm : 0, 2, 8, rebuildZExtCst}, 512 {X86::MOVDI2PDIrm, 1, 32, rebuildZeroUpperCst}, 513 {HasSSE41 ? X86::PMOVSXBDrm : 0, 4, 8, rebuildSExtCst}, 514 {HasSSE41 ? X86::PMOVZXBDrm : 0, 4, 8, rebuildZExtCst}, 515 {HasSSE41 ? X86::PMOVSXWQrm : 0, 2, 16, rebuildSExtCst}, 516 {HasSSE41 ? X86::PMOVZXWQrm : 0, 2, 16, rebuildZExtCst}, 517 {X86::MOVQI2PQIrm, 1, 64, rebuildZeroUpperCst}, 518 {HasSSE41 ? X86::PMOVSXBWrm : 0, 8, 8, rebuildSExtCst}, 519 {HasSSE41 ? X86::PMOVZXBWrm : 0, 8, 8, rebuildZExtCst}, 520 {HasSSE41 ? X86::PMOVSXWDrm : 0, 4, 16, rebuildSExtCst}, 521 {HasSSE41 ? X86::PMOVZXWDrm : 0, 4, 16, rebuildZExtCst}, 522 {HasSSE41 ? X86::PMOVSXDQrm : 0, 2, 32, rebuildSExtCst}, 523 {HasSSE41 ? X86::PMOVZXDQrm : 0, 2, 32, rebuildZExtCst}}; 524 return FixupConstant(Fixups, 128, 1); 525 } 526 case X86::VMOVDQArm: 527 case X86::VMOVDQUrm: { 528 FixupEntry Fixups[] = { 529 {HasAVX2 ? X86::VPBROADCASTBrm : 0, 1, 8, rebuildSplatCst}, 530 {HasAVX2 ? X86::VPBROADCASTWrm : 0, 1, 16, rebuildSplatCst}, 531 {X86::VPMOVSXBQrm, 2, 8, rebuildSExtCst}, 532 {X86::VPMOVZXBQrm, 2, 8, rebuildZExtCst}, 533 {X86::VMOVDI2PDIrm, 1, 32, rebuildZeroUpperCst}, 534 {HasAVX2 ? X86::VPBROADCASTDrm : X86::VBROADCASTSSrm, 1, 32, 535 rebuildSplatCst}, 536 {X86::VPMOVSXBDrm, 4, 8, rebuildSExtCst}, 537 {X86::VPMOVZXBDrm, 4, 8, rebuildZExtCst}, 538 {X86::VPMOVSXWQrm, 2, 16, rebuildSExtCst}, 539 {X86::VPMOVZXWQrm, 2, 16, rebuildZExtCst}, 540 {X86::VMOVQI2PQIrm, 1, 64, rebuildZeroUpperCst}, 541 {HasAVX2 ? X86::VPBROADCASTQrm : X86::VMOVDDUPrm, 1, 64, 542 rebuildSplatCst}, 543 {X86::VPMOVSXBWrm, 8, 8, rebuildSExtCst}, 544 {X86::VPMOVZXBWrm, 8, 8, rebuildZExtCst}, 545 {X86::VPMOVSXWDrm, 4, 16, rebuildSExtCst}, 546 {X86::VPMOVZXWDrm, 4, 16, rebuildZExtCst}, 547 {X86::VPMOVSXDQrm, 2, 32, rebuildSExtCst}, 548 {X86::VPMOVZXDQrm, 2, 32, rebuildZExtCst}}; 549 return FixupConstant(Fixups, 128, 1); 550 } 551 case X86::VMOVDQAYrm: 552 case X86::VMOVDQUYrm: { 553 FixupEntry Fixups[] = { 554 {HasAVX2 ? X86::VPBROADCASTBYrm : 0, 1, 8, rebuildSplatCst}, 555 {HasAVX2 ? X86::VPBROADCASTWYrm : 0, 1, 16, rebuildSplatCst}, 556 {HasAVX2 ? X86::VPBROADCASTDYrm : X86::VBROADCASTSSYrm, 1, 32, 557 rebuildSplatCst}, 558 {HasAVX2 ? X86::VPMOVSXBQYrm : 0, 4, 8, rebuildSExtCst}, 559 {HasAVX2 ? X86::VPMOVZXBQYrm : 0, 4, 8, rebuildZExtCst}, 560 {HasAVX2 ? X86::VPBROADCASTQYrm : X86::VBROADCASTSDYrm, 1, 64, 561 rebuildSplatCst}, 562 {HasAVX2 ? X86::VPMOVSXBDYrm : 0, 8, 8, rebuildSExtCst}, 563 {HasAVX2 ? X86::VPMOVZXBDYrm : 0, 8, 8, rebuildZExtCst}, 564 {HasAVX2 ? X86::VPMOVSXWQYrm : 0, 4, 16, rebuildSExtCst}, 565 {HasAVX2 ? X86::VPMOVZXWQYrm : 0, 4, 16, rebuildZExtCst}, 566 {HasAVX2 ? X86::VBROADCASTI128rm : X86::VBROADCASTF128rm, 1, 128, 567 rebuildSplatCst}, 568 {HasAVX2 ? X86::VPMOVSXBWYrm : 0, 16, 8, rebuildSExtCst}, 569 {HasAVX2 ? X86::VPMOVZXBWYrm : 0, 16, 8, rebuildZExtCst}, 570 {HasAVX2 ? X86::VPMOVSXWDYrm : 0, 8, 16, rebuildSExtCst}, 571 {HasAVX2 ? X86::VPMOVZXWDYrm : 0, 8, 16, rebuildZExtCst}, 572 {HasAVX2 ? X86::VPMOVSXDQYrm : 0, 4, 32, rebuildSExtCst}, 573 {HasAVX2 ? X86::VPMOVZXDQYrm : 0, 4, 32, rebuildZExtCst}}; 574 return FixupConstant(Fixups, 256, 1); 575 } 576 case X86::VMOVDQA32Z128rm: 577 case X86::VMOVDQA64Z128rm: 578 case X86::VMOVDQU32Z128rm: 579 case X86::VMOVDQU64Z128rm: { 580 FixupEntry Fixups[] = { 581 {HasBWI ? X86::VPBROADCASTBZ128rm : 0, 1, 8, rebuildSplatCst}, 582 {HasBWI ? X86::VPBROADCASTWZ128rm : 0, 1, 16, rebuildSplatCst}, 583 {X86::VPMOVSXBQZ128rm, 2, 8, rebuildSExtCst}, 584 {X86::VPMOVZXBQZ128rm, 2, 8, rebuildZExtCst}, 585 {X86::VMOVDI2PDIZrm, 1, 32, rebuildZeroUpperCst}, 586 {X86::VPBROADCASTDZ128rm, 1, 32, rebuildSplatCst}, 587 {X86::VPMOVSXBDZ128rm, 4, 8, rebuildSExtCst}, 588 {X86::VPMOVZXBDZ128rm, 4, 8, rebuildZExtCst}, 589 {X86::VPMOVSXWQZ128rm, 2, 16, rebuildSExtCst}, 590 {X86::VPMOVZXWQZ128rm, 2, 16, rebuildZExtCst}, 591 {X86::VMOVQI2PQIZrm, 1, 64, rebuildZeroUpperCst}, 592 {X86::VPBROADCASTQZ128rm, 1, 64, rebuildSplatCst}, 593 {HasBWI ? X86::VPMOVSXBWZ128rm : 0, 8, 8, rebuildSExtCst}, 594 {HasBWI ? X86::VPMOVZXBWZ128rm : 0, 8, 8, rebuildZExtCst}, 595 {X86::VPMOVSXWDZ128rm, 4, 16, rebuildSExtCst}, 596 {X86::VPMOVZXWDZ128rm, 4, 16, rebuildZExtCst}, 597 {X86::VPMOVSXDQZ128rm, 2, 32, rebuildSExtCst}, 598 {X86::VPMOVZXDQZ128rm, 2, 32, rebuildZExtCst}}; 599 return FixupConstant(Fixups, 128, 1); 600 } 601 case X86::VMOVDQA32Z256rm: 602 case X86::VMOVDQA64Z256rm: 603 case X86::VMOVDQU32Z256rm: 604 case X86::VMOVDQU64Z256rm: { 605 FixupEntry Fixups[] = { 606 {HasBWI ? X86::VPBROADCASTBZ256rm : 0, 1, 8, rebuildSplatCst}, 607 {HasBWI ? X86::VPBROADCASTWZ256rm : 0, 1, 16, rebuildSplatCst}, 608 {X86::VPBROADCASTDZ256rm, 1, 32, rebuildSplatCst}, 609 {X86::VPMOVSXBQZ256rm, 4, 8, rebuildSExtCst}, 610 {X86::VPMOVZXBQZ256rm, 4, 8, rebuildZExtCst}, 611 {X86::VPBROADCASTQZ256rm, 1, 64, rebuildSplatCst}, 612 {X86::VPMOVSXBDZ256rm, 8, 8, rebuildSExtCst}, 613 {X86::VPMOVZXBDZ256rm, 8, 8, rebuildZExtCst}, 614 {X86::VPMOVSXWQZ256rm, 4, 16, rebuildSExtCst}, 615 {X86::VPMOVZXWQZ256rm, 4, 16, rebuildZExtCst}, 616 {X86::VBROADCASTI32X4Z256rm, 1, 128, rebuildSplatCst}, 617 {HasBWI ? X86::VPMOVSXBWZ256rm : 0, 16, 8, rebuildSExtCst}, 618 {HasBWI ? X86::VPMOVZXBWZ256rm : 0, 16, 8, rebuildZExtCst}, 619 {X86::VPMOVSXWDZ256rm, 8, 16, rebuildSExtCst}, 620 {X86::VPMOVZXWDZ256rm, 8, 16, rebuildZExtCst}, 621 {X86::VPMOVSXDQZ256rm, 4, 32, rebuildSExtCst}, 622 {X86::VPMOVZXDQZ256rm, 4, 32, rebuildZExtCst}}; 623 return FixupConstant(Fixups, 256, 1); 624 } 625 case X86::VMOVDQA32Zrm: 626 case X86::VMOVDQA64Zrm: 627 case X86::VMOVDQU32Zrm: 628 case X86::VMOVDQU64Zrm: { 629 FixupEntry Fixups[] = { 630 {HasBWI ? X86::VPBROADCASTBZrm : 0, 1, 8, rebuildSplatCst}, 631 {HasBWI ? X86::VPBROADCASTWZrm : 0, 1, 16, rebuildSplatCst}, 632 {X86::VPBROADCASTDZrm, 1, 32, rebuildSplatCst}, 633 {X86::VPBROADCASTQZrm, 1, 64, rebuildSplatCst}, 634 {X86::VPMOVSXBQZrm, 8, 8, rebuildSExtCst}, 635 {X86::VPMOVZXBQZrm, 8, 8, rebuildZExtCst}, 636 {X86::VBROADCASTI32X4Zrm, 1, 128, rebuildSplatCst}, 637 {X86::VPMOVSXBDZrm, 16, 8, rebuildSExtCst}, 638 {X86::VPMOVZXBDZrm, 16, 8, rebuildZExtCst}, 639 {X86::VPMOVSXWQZrm, 8, 16, rebuildSExtCst}, 640 {X86::VPMOVZXWQZrm, 8, 16, rebuildZExtCst}, 641 {X86::VBROADCASTI64X4Zrm, 1, 256, rebuildSplatCst}, 642 {HasBWI ? X86::VPMOVSXBWZrm : 0, 32, 8, rebuildSExtCst}, 643 {HasBWI ? X86::VPMOVZXBWZrm : 0, 32, 8, rebuildZExtCst}, 644 {X86::VPMOVSXWDZrm, 16, 16, rebuildSExtCst}, 645 {X86::VPMOVZXWDZrm, 16, 16, rebuildZExtCst}, 646 {X86::VPMOVSXDQZrm, 8, 32, rebuildSExtCst}, 647 {X86::VPMOVZXDQZrm, 8, 32, rebuildZExtCst}}; 648 return FixupConstant(Fixups, 512, 1); 649 } 650 } 651 652 auto ConvertToBroadcast = [&](unsigned OpSrc, int BW) { 653 if (OpSrc) { 654 if (const X86FoldTableEntry *Mem2Bcst = 655 llvm::lookupBroadcastFoldTableBySize(OpSrc, BW)) { 656 unsigned OpBcst = Mem2Bcst->DstOp; 657 unsigned OpNoBcst = Mem2Bcst->Flags & TB_INDEX_MASK; 658 FixupEntry Fixups[] = {{(int)OpBcst, 1, BW, rebuildSplatCst}}; 659 // TODO: Add support for RegBitWidth, but currently rebuildSplatCst 660 // doesn't require it (defaults to Constant::getPrimitiveSizeInBits). 661 return FixupConstant(Fixups, 0, OpNoBcst); 662 } 663 } 664 return false; 665 }; 666 667 // Attempt to find a AVX512 mapping from a full width memory-fold instruction 668 // to a broadcast-fold instruction variant. 669 if ((MI.getDesc().TSFlags & X86II::EncodingMask) == X86II::EVEX) 670 return ConvertToBroadcast(Opc, 32) || ConvertToBroadcast(Opc, 64); 671 672 // Reverse the X86InstrInfo::setExecutionDomainCustom EVEX->VEX logic 673 // conversion to see if we can convert to a broadcasted (integer) logic op. 674 if (HasVLX && !HasDQI) { 675 unsigned OpSrc32 = 0, OpSrc64 = 0; 676 switch (Opc) { 677 case X86::VANDPDrm: 678 case X86::VANDPSrm: 679 case X86::VPANDrm: 680 OpSrc32 = X86 ::VPANDDZ128rm; 681 OpSrc64 = X86 ::VPANDQZ128rm; 682 break; 683 case X86::VANDPDYrm: 684 case X86::VANDPSYrm: 685 case X86::VPANDYrm: 686 OpSrc32 = X86 ::VPANDDZ256rm; 687 OpSrc64 = X86 ::VPANDQZ256rm; 688 break; 689 case X86::VANDNPDrm: 690 case X86::VANDNPSrm: 691 case X86::VPANDNrm: 692 OpSrc32 = X86 ::VPANDNDZ128rm; 693 OpSrc64 = X86 ::VPANDNQZ128rm; 694 break; 695 case X86::VANDNPDYrm: 696 case X86::VANDNPSYrm: 697 case X86::VPANDNYrm: 698 OpSrc32 = X86 ::VPANDNDZ256rm; 699 OpSrc64 = X86 ::VPANDNQZ256rm; 700 break; 701 case X86::VORPDrm: 702 case X86::VORPSrm: 703 case X86::VPORrm: 704 OpSrc32 = X86 ::VPORDZ128rm; 705 OpSrc64 = X86 ::VPORQZ128rm; 706 break; 707 case X86::VORPDYrm: 708 case X86::VORPSYrm: 709 case X86::VPORYrm: 710 OpSrc32 = X86 ::VPORDZ256rm; 711 OpSrc64 = X86 ::VPORQZ256rm; 712 break; 713 case X86::VXORPDrm: 714 case X86::VXORPSrm: 715 case X86::VPXORrm: 716 OpSrc32 = X86 ::VPXORDZ128rm; 717 OpSrc64 = X86 ::VPXORQZ128rm; 718 break; 719 case X86::VXORPDYrm: 720 case X86::VXORPSYrm: 721 case X86::VPXORYrm: 722 OpSrc32 = X86 ::VPXORDZ256rm; 723 OpSrc64 = X86 ::VPXORQZ256rm; 724 break; 725 } 726 if (OpSrc32 || OpSrc64) 727 return ConvertToBroadcast(OpSrc32, 32) || ConvertToBroadcast(OpSrc64, 64); 728 } 729 730 return false; 731 } 732 733 bool X86FixupVectorConstantsPass::runOnMachineFunction(MachineFunction &MF) { 734 LLVM_DEBUG(dbgs() << "Start X86FixupVectorConstants\n";); 735 bool Changed = false; 736 ST = &MF.getSubtarget<X86Subtarget>(); 737 TII = ST->getInstrInfo(); 738 SM = &ST->getSchedModel(); 739 740 for (MachineBasicBlock &MBB : MF) { 741 for (MachineInstr &MI : MBB) { 742 if (processInstruction(MF, MBB, MI)) { 743 ++NumInstChanges; 744 Changed = true; 745 } 746 } 747 } 748 LLVM_DEBUG(dbgs() << "End X86FixupVectorConstants\n";); 749 return Changed; 750 } 751