1 //===-- X86ShuffleDecodeConstantPool.cpp - X86 shuffle decode -------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // Define several functions to decode x86 specific shuffle semantics using 11 // constants from the constant pool. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #include "Utils/X86ShuffleDecode.h" 16 #include "llvm/ADT/APInt.h" 17 #include "llvm/IR/Constants.h" 18 19 //===----------------------------------------------------------------------===// 20 // Vector Mask Decoding 21 //===----------------------------------------------------------------------===// 22 23 namespace llvm { 24 25 static bool extractConstantMask(const Constant *C, unsigned MaskEltSizeInBits, 26 APInt &UndefElts, 27 SmallVectorImpl<uint64_t> &RawMask) { 28 // It is not an error for shuffle masks to not be a vector of 29 // MaskEltSizeInBits because the constant pool uniques constants by their 30 // bit representation. 31 // e.g. the following take up the same space in the constant pool: 32 // i128 -170141183420855150465331762880109871104 33 // 34 // <2 x i64> <i64 -9223372034707292160, i64 -9223372034707292160> 35 // 36 // <4 x i32> <i32 -2147483648, i32 -2147483648, 37 // i32 -2147483648, i32 -2147483648> 38 Type *CstTy = C->getType(); 39 if (!CstTy->isVectorTy()) 40 return false; 41 42 Type *CstEltTy = CstTy->getVectorElementType(); 43 if (!CstEltTy->isIntegerTy()) 44 return false; 45 46 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits(); 47 unsigned CstEltSizeInBits = CstTy->getScalarSizeInBits(); 48 unsigned NumCstElts = CstTy->getVectorNumElements(); 49 50 assert((CstSizeInBits % MaskEltSizeInBits) == 0 && 51 "Unaligned shuffle mask size"); 52 53 unsigned NumMaskElts = CstSizeInBits / MaskEltSizeInBits; 54 UndefElts = APInt(NumMaskElts, 0); 55 RawMask.resize(NumMaskElts, 0); 56 57 // Fast path - if the constants match the mask size then copy direct. 58 if (MaskEltSizeInBits == CstEltSizeInBits) { 59 assert(NumCstElts == NumMaskElts && "Unaligned shuffle mask size"); 60 for (unsigned i = 0; i != NumMaskElts; ++i) { 61 Constant *COp = C->getAggregateElement(i); 62 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) 63 return false; 64 65 if (isa<UndefValue>(COp)) { 66 UndefElts.setBit(i); 67 RawMask[i] = 0; 68 continue; 69 } 70 71 auto *Elt = cast<ConstantInt>(COp); 72 RawMask[i] = Elt->getValue().getZExtValue(); 73 } 74 return true; 75 } 76 77 // Extract all the undef/constant element data and pack into single bitsets. 78 APInt UndefBits(CstSizeInBits, 0); 79 APInt MaskBits(CstSizeInBits, 0); 80 for (unsigned i = 0; i != NumCstElts; ++i) { 81 Constant *COp = C->getAggregateElement(i); 82 if (!COp || (!isa<UndefValue>(COp) && !isa<ConstantInt>(COp))) 83 return false; 84 85 unsigned BitOffset = i * CstEltSizeInBits; 86 87 if (isa<UndefValue>(COp)) { 88 UndefBits.setBits(BitOffset, BitOffset + CstEltSizeInBits); 89 continue; 90 } 91 92 MaskBits.insertBits(cast<ConstantInt>(COp)->getValue(), BitOffset); 93 } 94 95 // Now extract the undef/constant bit data into the raw shuffle masks. 96 for (unsigned i = 0; i != NumMaskElts; ++i) { 97 unsigned BitOffset = i * MaskEltSizeInBits; 98 APInt EltUndef = UndefBits.extractBits(MaskEltSizeInBits, BitOffset); 99 100 // Only treat the element as UNDEF if all bits are UNDEF, otherwise 101 // treat it as zero. 102 if (EltUndef.isAllOnesValue()) { 103 UndefElts.setBit(i); 104 RawMask[i] = 0; 105 continue; 106 } 107 108 APInt EltBits = MaskBits.extractBits(MaskEltSizeInBits, BitOffset); 109 RawMask[i] = EltBits.getZExtValue(); 110 } 111 112 return true; 113 } 114 115 void DecodePSHUFBMask(const Constant *C, unsigned Width, 116 SmallVectorImpl<int> &ShuffleMask) { 117 assert((Width == 128 || Width == 256 || Width == 512) && 118 C->getType()->getPrimitiveSizeInBits() >= Width && 119 "Unexpected vector size."); 120 121 // The shuffle mask requires a byte vector. 122 APInt UndefElts; 123 SmallVector<uint64_t, 64> RawMask; 124 if (!extractConstantMask(C, 8, UndefElts, RawMask)) 125 return; 126 127 unsigned NumElts = Width / 8; 128 assert((NumElts == 16 || NumElts == 32 || NumElts == 64) && 129 "Unexpected number of vector elements."); 130 131 for (unsigned i = 0; i != NumElts; ++i) { 132 if (UndefElts[i]) { 133 ShuffleMask.push_back(SM_SentinelUndef); 134 continue; 135 } 136 137 uint64_t Element = RawMask[i]; 138 // If the high bit (7) of the byte is set, the element is zeroed. 139 if (Element & (1 << 7)) 140 ShuffleMask.push_back(SM_SentinelZero); 141 else { 142 // For AVX vectors with 32 bytes the base of the shuffle is the 16-byte 143 // lane of the vector we're inside. 144 unsigned Base = i & ~0xf; 145 146 // Only the least significant 4 bits of the byte are used. 147 int Index = Base + (Element & 0xf); 148 ShuffleMask.push_back(Index); 149 } 150 } 151 } 152 153 void DecodeVPERMILPMask(const Constant *C, unsigned ElSize, unsigned Width, 154 SmallVectorImpl<int> &ShuffleMask) { 155 assert((Width == 128 || Width == 256 || Width == 512) && 156 C->getType()->getPrimitiveSizeInBits() >= Width && 157 "Unexpected vector size."); 158 assert((ElSize == 32 || ElSize == 64) && "Unexpected vector element size."); 159 160 // The shuffle mask requires elements the same size as the target. 161 APInt UndefElts; 162 SmallVector<uint64_t, 16> RawMask; 163 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 164 return; 165 166 unsigned NumElts = Width / ElSize; 167 unsigned NumEltsPerLane = 128 / ElSize; 168 assert((NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) && 169 "Unexpected number of vector elements."); 170 171 for (unsigned i = 0; i != NumElts; ++i) { 172 if (UndefElts[i]) { 173 ShuffleMask.push_back(SM_SentinelUndef); 174 continue; 175 } 176 177 int Index = i & ~(NumEltsPerLane - 1); 178 uint64_t Element = RawMask[i]; 179 if (ElSize == 64) 180 Index += (Element >> 1) & 0x1; 181 else 182 Index += Element & 0x3; 183 184 ShuffleMask.push_back(Index); 185 } 186 } 187 188 void DecodeVPERMIL2PMask(const Constant *C, unsigned M2Z, unsigned ElSize, 189 unsigned Width, 190 SmallVectorImpl<int> &ShuffleMask) { 191 Type *MaskTy = C->getType(); 192 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 193 (void)MaskTySize; 194 assert((MaskTySize == 128 || MaskTySize == 256) && 195 Width >= MaskTySize && "Unexpected vector size."); 196 197 // The shuffle mask requires elements the same size as the target. 198 APInt UndefElts; 199 SmallVector<uint64_t, 8> RawMask; 200 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 201 return; 202 203 unsigned NumElts = Width / ElSize; 204 unsigned NumEltsPerLane = 128 / ElSize; 205 assert((NumElts == 2 || NumElts == 4 || NumElts == 8) && 206 "Unexpected number of vector elements."); 207 208 for (unsigned i = 0; i != NumElts; ++i) { 209 if (UndefElts[i]) { 210 ShuffleMask.push_back(SM_SentinelUndef); 211 continue; 212 } 213 214 // VPERMIL2 Operation. 215 // Bits[3] - Match Bit. 216 // Bits[2:1] - (Per Lane) PD Shuffle Mask. 217 // Bits[2:0] - (Per Lane) PS Shuffle Mask. 218 uint64_t Selector = RawMask[i]; 219 unsigned MatchBit = (Selector >> 3) & 0x1; 220 221 // M2Z[0:1] MatchBit 222 // 0Xb X Source selected by Selector index. 223 // 10b 0 Source selected by Selector index. 224 // 10b 1 Zero. 225 // 11b 0 Zero. 226 // 11b 1 Source selected by Selector index. 227 if ((M2Z & 0x2) != 0u && MatchBit != (M2Z & 0x1)) { 228 ShuffleMask.push_back(SM_SentinelZero); 229 continue; 230 } 231 232 int Index = i & ~(NumEltsPerLane - 1); 233 if (ElSize == 64) 234 Index += (Selector >> 1) & 0x1; 235 else 236 Index += Selector & 0x3; 237 238 int Src = (Selector >> 2) & 0x1; 239 Index += Src * NumElts; 240 ShuffleMask.push_back(Index); 241 } 242 } 243 244 void DecodeVPPERMMask(const Constant *C, unsigned Width, 245 SmallVectorImpl<int> &ShuffleMask) { 246 Type *MaskTy = C->getType(); 247 unsigned MaskTySize = MaskTy->getPrimitiveSizeInBits(); 248 (void)MaskTySize; 249 assert(Width == 128 && Width >= MaskTySize && "Unexpected vector size."); 250 251 // The shuffle mask requires a byte vector. 252 APInt UndefElts; 253 SmallVector<uint64_t, 16> RawMask; 254 if (!extractConstantMask(C, 8, UndefElts, RawMask)) 255 return; 256 257 unsigned NumElts = Width / 8; 258 assert(NumElts == 16 && "Unexpected number of vector elements."); 259 260 for (unsigned i = 0; i != NumElts; ++i) { 261 if (UndefElts[i]) { 262 ShuffleMask.push_back(SM_SentinelUndef); 263 continue; 264 } 265 266 // VPPERM Operation 267 // Bits[4:0] - Byte Index (0 - 31) 268 // Bits[7:5] - Permute Operation 269 // 270 // Permute Operation: 271 // 0 - Source byte (no logical operation). 272 // 1 - Invert source byte. 273 // 2 - Bit reverse of source byte. 274 // 3 - Bit reverse of inverted source byte. 275 // 4 - 00h (zero - fill). 276 // 5 - FFh (ones - fill). 277 // 6 - Most significant bit of source byte replicated in all bit positions. 278 // 7 - Invert most significant bit of source byte and replicate in all bit 279 // positions. 280 uint64_t Element = RawMask[i]; 281 uint64_t Index = Element & 0x1F; 282 uint64_t PermuteOp = (Element >> 5) & 0x7; 283 284 if (PermuteOp == 4) { 285 ShuffleMask.push_back(SM_SentinelZero); 286 continue; 287 } 288 if (PermuteOp != 0) { 289 ShuffleMask.clear(); 290 return; 291 } 292 ShuffleMask.push_back((int)Index); 293 } 294 } 295 296 void DecodeVPERMVMask(const Constant *C, unsigned ElSize, unsigned Width, 297 SmallVectorImpl<int> &ShuffleMask) { 298 assert((Width == 128 || Width == 256 || Width == 512) && 299 C->getType()->getPrimitiveSizeInBits() >= Width && 300 "Unexpected vector size."); 301 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && 302 "Unexpected vector element size."); 303 304 // The shuffle mask requires elements the same size as the target. 305 APInt UndefElts; 306 SmallVector<uint64_t, 64> RawMask; 307 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 308 return; 309 310 unsigned NumElts = Width / ElSize; 311 312 for (unsigned i = 0; i != NumElts; ++i) { 313 if (UndefElts[i]) { 314 ShuffleMask.push_back(SM_SentinelUndef); 315 continue; 316 } 317 int Index = RawMask[i] & (NumElts - 1); 318 ShuffleMask.push_back(Index); 319 } 320 } 321 322 void DecodeVPERMV3Mask(const Constant *C, unsigned ElSize, unsigned Width, 323 SmallVectorImpl<int> &ShuffleMask) { 324 assert((Width == 128 || Width == 256 || Width == 512) && 325 C->getType()->getPrimitiveSizeInBits() >= Width && 326 "Unexpected vector size."); 327 assert((ElSize == 8 || ElSize == 16 || ElSize == 32 || ElSize == 64) && 328 "Unexpected vector element size."); 329 330 // The shuffle mask requires elements the same size as the target. 331 APInt UndefElts; 332 SmallVector<uint64_t, 64> RawMask; 333 if (!extractConstantMask(C, ElSize, UndefElts, RawMask)) 334 return; 335 336 unsigned NumElts = Width / ElSize; 337 338 for (unsigned i = 0; i != NumElts; ++i) { 339 if (UndefElts[i]) { 340 ShuffleMask.push_back(SM_SentinelUndef); 341 continue; 342 } 343 int Index = RawMask[i] & (NumElts*2 - 1); 344 ShuffleMask.push_back(Index); 345 } 346 } 347 } // llvm namespace 348