1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // Define several functions to decode x86 specific shuffle semantics using 11 // constants from the constant pool. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "X86ShuffleDecodeConstantPool.h" 16 #include "Utils/X86ShuffleDecode.h" 17 #include "llvm/ADT/APInt.h" 18 #include "llvm/CodeGen/MachineValueType.h" 19 #include "llvm/IR/Constants.h" 20 21 //===----------------------------------------------------------------------===// 22 // Vector Mask Decoding 23 //===----------------------------------------------------------------------===// 24 25 namespace llvm { 26 27 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, 28 APInt &UndefElts, 29 SmallVectorImpl<uint64_t> &RawMask) { 30 // It is not an error for shuffle masks to not be a vector of 31 // MaskEltSizeInBits because the constant pool uniques constants by their 32 // bit representation. 33 // e.g. the following take up the same space in the constant pool: 34 // i128 -170141183420855150465331762880109871104 35 // 36 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160> 37 // 38 // <4 x i32> <i32 -2147483648, i32 -2147483648, 39 // i32 -2147483648, i32 -2147483648> 40 Type *CstTy = C->getType(); 41 if (!CstTy->isVectorTy()) 42 return false; 43 44 Type *CstEltTy = CstTy->getVectorElementType(); 45 if (!CstEltTy->isIntegerTy()) 46 return false; 47 48 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); 49 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits(); 50 unsigned NumCstElts = CstTy->getVectorNumElements(); 51 52 assert((CstSizeInBits % MaskEltSizeInBits) == 0 && 53 "Unaligned shuffle mask size"); 54 55 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits; 56 UndefElts = APInt(NumMaskElts, 0); 57 RawMask.resize(NumMaskElts, 0); 58 59 // Fast path - if the constants match the mask size then copy direct. 60 if (MaskEltSizeInBits == CstEltSizeInBits) { 61 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size"); 62 for (unsigned i = 0; i != NumMaskElts; ++i) { 63 Constant *COp = C->getAggregateElement(i); 64 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) 65 return false; 66 67 if (isa<UndefValue>(COp)) { 68 UndefElts.setBit(i); 69 RawMask[i] = 0; 70 continue; 71 } 72 73 auto *Elt = cast<ConstantInt>(COp); 74 RawMask[i] = Elt->getValue().getZExtValue(); 75 } 76 return true; 77 } 78 79 // Extract all the undef/constant element data and pack into single bitsets. 80 APInt UndefBits(CstSizeInBits, 0); 81 APInt MaskBits(CstSizeInBits, 0); 82 for (unsigned i = 0; i != NumCstElts; ++i) { 83 Constant *COp = C->getAggregateElement(i); 84 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) 85 return false; 86 87 unsigned BitOffset = i * CstEltSizeInBits; 88 89 if (isa<UndefValue>(COp)) { 90 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits); 91 continue; 92 } 93 94 auto *Elt = cast<ConstantInt>(COp); 95 MaskBits |= Elt->getValue().zextOrTrunc(CstSizeInBits).shl(BitOffset); 96 } 97 98 // Now extract the undef/constant bit data into the raw shuffle masks. 99 for (unsigned i = 0; i != NumMaskElts; ++i) { 100 unsigned BitOffset = i * MaskEltSizeInBits; 101 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset); 102 103 // Only treat the element as UNDEF if all bits are UNDEF, otherwise 104 // treat it as zero. 105 if (EltUndef.isAllOnesValue()) { 106 UndefElts.setBit(i); 107 RawMask[i] = 0; 108 continue; 109 } 110 111 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset); 112 RawMask[i] = EltBits.getZExtValue(); 113 } 114 115 return true; 116 } 117 118 void DecodePSHUFBMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) { 119 Type *MaskTy = C->getType(); 120 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 121 (void)MaskTySize; 122 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && 123 "Unexpected vector size."); 124 125 // The shuffle mask requires a byte vector. 126 APInt UndefElts; 127 SmallVector<uint64_t, 64> RawMask; 128 if (!extractConstantMask(C, 8, UndefElts, RawMask)) 129 return; 130 131 unsigned NumElts = RawMask.size(); 132 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) && 133 "Unexpected number of vector elements."); 134 135 for (unsigned i = 0; i != NumElts; ++i) { 136 if (UndefElts[i]) { 137 ShuffleMask.push_back(SM_SentinelUndef); 138 continue; 139 } 140 141 uint64_t Element = RawMask[i]; 142 // If the high bit (7) of the byte is set, the element is zeroed. 143 if (Element & (1 << 7)) 144 ShuffleMask.push_back(SM_SentinelZero); 145 else { 146 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte 147 // lane of the vector we're inside. 148 unsigned Base = i & ~0xf; 149 150 // Only the least significant 4 bits of the byte are used. 151 int Index = Base + (Element & 0xf); 152 ShuffleMask.push_back(Index); 153 } 154 } 155 } 156 157 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, 158 SmallVectorImpl<int> &ShuffleMask) { 159 Type *MaskTy = C->getType(); 160 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 161 (void)MaskTySize; 162 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && 163 "Unexpected vector size."); 164 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size."); 165 166 // The shuffle mask requires elements the same size as the target. 167 APInt UndefElts; 168 SmallVector<uint64_t, 16> RawMask; 169 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 170 return; 171 172 unsigned NumElts = RawMask.size(); 173 unsigned NumEltsPerLane = 128 / ElSize; 174 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) && 175 "Unexpected number of vector elements."); 176 177 for (unsigned i = 0; i != NumElts; ++i) { 178 if (UndefElts[i]) { 179 ShuffleMask.push_back(SM_SentinelUndef); 180 continue; 181 } 182 183 int Index = i & ~(NumEltsPerLane - 1); 184 uint64_t Element = RawMask[i]; 185 if (ElSize == 64) 186 Index += (Element >> 1) & 0x1; 187 else 188 Index += Element & 0x3; 189 190 ShuffleMask.push_back(Index); 191 } 192 } 193 194 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize, 195 SmallVectorImpl<int> &ShuffleMask) { 196 Type *MaskTy = C->getType(); 197 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 198 (void)MaskTySize; 199 assert((MaskTySize == 128 || MaskTySize == 256) && "Unexpected vector size."); 200 201 // The shuffle mask requires elements the same size as the target. 202 APInt UndefElts; 203 SmallVector<uint64_t, 8> RawMask; 204 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 205 return; 206 207 unsigned NumElts = RawMask.size(); 208 unsigned NumEltsPerLane = 128 / ElSize; 209 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) && 210 "Unexpected number of vector elements."); 211 212 for (unsigned i = 0; i != NumElts; ++i) { 213 if (UndefElts[i]) { 214 ShuffleMask.push_back(SM_SentinelUndef); 215 continue; 216 } 217 218 // VPERMIL2 Operation. 219 // Bits[3] - Match Bit. 220 // Bits[2:1] - (Per Lane) PD Shuffle Mask. 221 // Bits[2:0] - (Per Lane) PS Shuffle Mask. 222 uint64_t Selector = RawMask[i]; 223 unsigned MatchBit = (Selector >> 3) & 0x1; 224 225 // M2Z[0:1] MatchBit 226 // 0Xb X Source selected by Selector index. 227 // 10b 0 Source selected by Selector index. 228 // 10b 1 Zero. 229 // 11b 0 Zero. 230 // 11b 1 Source selected by Selector index. 231 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) { 232 ShuffleMask.push_back(SM_SentinelZero); 233 continue; 234 } 235 236 int Index = i & ~(NumEltsPerLane - 1); 237 if (ElSize == 64) 238 Index += (Selector >> 1) & 0x1; 239 else 240 Index += Selector & 0x3; 241 242 int Src = (Selector >> 2) & 0x1; 243 Index += Src * NumElts; 244 ShuffleMask.push_back(Index); 245 } 246 } 247 248 void DecodeVPPERMMask(const Constant *C, SmallVectorImpl<int> &ShuffleMask) { 249 assert(C->getType()->getPrimitiveSizeInBits() == 128 && 250 "Unexpected vector size."); 251 252 // The shuffle mask requires a byte vector. 253 APInt UndefElts; 254 SmallVector<uint64_t, 16> RawMask; 255 if (!extractConstantMask(C, 8, UndefElts, RawMask)) 256 return; 257 258 unsigned NumElts = RawMask.size(); 259 assert(NumElts == 16 && "Unexpected number of vector elements."); 260 261 for (unsigned i = 0; i != NumElts; ++i) { 262 if (UndefElts[i]) { 263 ShuffleMask.push_back(SM_SentinelUndef); 264 continue; 265 } 266 267 // VPPERM Operation 268 // Bits[4:0] - Byte Index (0 - 31) 269 // Bits[7:5] - Permute Operation 270 // 271 // Permute Operation: 272 // 0 - Source byte (no logical operation). 273 // 1 - Invert source byte. 274 // 2 - Bit reverse of source byte. 275 // 3 - Bit reverse of inverted source byte. 276 // 4 - 00h (zero - fill). 277 // 5 - FFh (ones - fill). 278 // 6 - Most significant bit of source byte replicated in all bit positions. 279 // 7 - Invert most significant bit of source byte and replicate in all bit 280 // positions. 281 uint64_t Element = RawMask[i]; 282 uint64_t Index = Element & 0x1F; 283 uint64_t PermuteOp = (Element >> 5) & 0x7; 284 285 if (PermuteOp == 4) { 286 ShuffleMask.push_back(SM_SentinelZero); 287 continue; 288 } 289 if (PermuteOp != 0) { 290 ShuffleMask.clear(); 291 return; 292 } 293 ShuffleMask.push_back((int)Index); 294 } 295 } 296 297 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, 298 SmallVectorImpl<int> &ShuffleMask) { 299 Type *MaskTy = C->getType(); 300 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 301 (void)MaskTySize; 302 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && 303 "Unexpected vector size."); 304 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && 305 "Unexpected vector element size."); 306 307 // The shuffle mask requires elements the same size as the target. 308 APInt UndefElts; 309 SmallVector<uint64_t, 64> RawMask; 310 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 311 return; 312 313 unsigned NumElts = RawMask.size(); 314 315 for (unsigned i = 0; i != NumElts; ++i) { 316 if (UndefElts[i]) { 317 ShuffleMask.push_back(SM_SentinelUndef); 318 continue; 319 } 320 int Index = RawMask[i] & (NumElts - 1); 321 ShuffleMask.push_back(Index); 322 } 323 } 324 325 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, 326 SmallVectorImpl<int> &ShuffleMask) { 327 Type *MaskTy = C->getType(); 328 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 329 (void)MaskTySize; 330 assert((MaskTySize == 128 || MaskTySize == 256 || MaskTySize == 512) && 331 "Unexpected vector size."); 332 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && 333 "Unexpected vector element size."); 334 335 // The shuffle mask requires elements the same size as the target. 336 APInt UndefElts; 337 SmallVector<uint64_t, 64> RawMask; 338 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 339 return; 340 341 unsigned NumElts = RawMask.size(); 342 343 for (unsigned i = 0; i != NumElts; ++i) { 344 if (UndefElts[i]) { 345 ShuffleMask.push_back(SM_SentinelUndef); 346 continue; 347 } 348 int Index = RawMask[i] & (NumElts*2 - 1); 349 ShuffleMask.push_back(Index); 350 } 351 } 352 } // llvm namespace 353