1 //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file A pass to convert the target-illegal operations created by IR -> MIR 10 /// translation into ones the target expects to be able to select. This may 11 /// occur in multiple phases, for example G_ADD <2 x i8> -> G_ADD <2 x i16> -> 12 /// G_ADD <4 x i16>. 13 /// 14 /// The LegalizerHelper class is where most of the work happens, and is 15 /// designed to be callable from other passes that find themselves with an 16 /// illegal instruction. 17 // 18 //===----------------------------------------------------------------------===// 19 20 #ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 21 #define LLVM_CODEGEN_GLOBALISEL_LEGALIZERHELPER_H 22 23 #include "llvm/CodeGen/GlobalISel/CallLowering.h" 24 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 25 #include "llvm/CodeGen/LowLevelType.h" 26 #include "llvm/CodeGen/MachineFunctionPass.h" 27 #include "llvm/CodeGen/RuntimeLibcalls.h" 28 29 namespace llvm { 30 // Forward declarations. 31 class LegalizerInfo; 32 class Legalizer; 33 class MachineRegisterInfo; 34 class GISelChangeObserver; 35 class TargetLowering; 36 37 class LegalizerHelper { 38 public: 39 /// Expose MIRBuilder so clients can set their own RecordInsertInstruction 40 /// functions 41 MachineIRBuilder &MIRBuilder; 42 43 /// To keep track of changes made by the LegalizerHelper. 44 GISelChangeObserver &Observer; 45 46 private: 47 MachineRegisterInfo &MRI; 48 const LegalizerInfo &LI; 49 const TargetLowering &TLI; 50 51 public: 52 enum LegalizeResult { 53 /// Instruction was already legal and no change was made to the 54 /// MachineFunction. 55 AlreadyLegal, 56 57 /// Instruction has been legalized and the MachineFunction changed. 58 Legalized, 59 60 /// Some kind of error has occurred and we could not legalize this 61 /// instruction. 62 UnableToLegalize, 63 }; 64 65 /// Expose LegalizerInfo so the clients can re-use. getLegalizerInfo()66 const LegalizerInfo &getLegalizerInfo() const { return LI; } getTargetLowering()67 const TargetLowering &getTargetLowering() const { return TLI; } 68 69 LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, 70 MachineIRBuilder &B); 71 LegalizerHelper(MachineFunction &MF, const LegalizerInfo &LI, 72 GISelChangeObserver &Observer, MachineIRBuilder &B); 73 74 /// Replace \p MI by a sequence of legal instructions that can implement the 75 /// same operation. Note that this means \p MI may be deleted, so any iterator 76 /// steps should be performed before calling this function. \p Helper should 77 /// be initialized to the MachineFunction containing \p MI. 78 /// 79 /// Considered as an opaque blob, the legal code will use and define the same 80 /// registers as \p MI. 81 LegalizeResult legalizeInstrStep(MachineInstr &MI); 82 83 /// Legalize an instruction by emiting a runtime library call instead. 84 LegalizeResult libcall(MachineInstr &MI); 85 86 /// Legalize an instruction by reducing the width of the underlying scalar 87 /// type. 88 LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 89 90 /// Legalize an instruction by performing the operation on a wider scalar type 91 /// (for example a 16-bit addition can be safely performed at 32-bits 92 /// precision, ignoring the unused bits). 93 LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 94 95 /// Legalize an instruction by replacing the value type 96 LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 97 98 /// Legalize an instruction by splitting it into simpler parts, hopefully 99 /// understood by the target. 100 LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 101 102 /// Legalize a vector instruction by splitting into multiple components, each 103 /// acting on the same scalar type as the original but with fewer elements. 104 LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, 105 LLT NarrowTy); 106 107 /// Legalize a vector instruction by increasing the number of vector elements 108 /// involved and ignoring the added elements later. 109 LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, 110 LLT MoreTy); 111 112 /// Cast the given value to an LLT::scalar with an equivalent size. Returns 113 /// the register to use if an instruction was inserted. Returns the original 114 /// register if no coercion was necessary. 115 // 116 // This may also fail and return Register() if there is no legal way to cast. 117 Register coerceToScalar(Register Val); 118 119 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 120 /// Use by extending the operand's type to \p WideTy using the specified \p 121 /// ExtOpcode for the extension instruction, and replacing the vreg of the 122 /// operand in place. 123 void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, 124 unsigned ExtOpcode); 125 126 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 127 /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and 128 /// replacing the vreg of the operand in place. 129 void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); 130 131 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 132 /// Def by extending the operand's type to \p WideTy and truncating it back 133 /// with the \p TruncOpcode, and replacing the vreg of the operand in place. 134 void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, 135 unsigned TruncOpcode = TargetOpcode::G_TRUNC); 136 137 // Legalize a single operand \p OpIdx of the machine instruction \p MI as a 138 // Def by truncating the operand's type to \p NarrowTy, replacing in place and 139 // extending back with \p ExtOpcode. 140 void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, 141 unsigned ExtOpcode); 142 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 143 /// Def by performing it with additional vector elements and extracting the 144 /// result elements, and replacing the vreg of the operand in place. 145 void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 146 147 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 148 /// Use by producing a vector with undefined high elements, extracting the 149 /// original vector type, and replacing the vreg of the operand in place. 150 void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); 151 152 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 153 /// use by inserting a G_BITCAST to \p CastTy 154 void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 155 156 /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a 157 /// def by inserting a G_BITCAST from \p CastTy 158 void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx); 159 160 /// Widen \p OrigReg to \p WideTy by merging to a wider type, padding with 161 /// G_IMPLICIT_DEF, and producing dead results. 162 Register widenWithUnmerge(LLT WideTy, Register OrigReg); 163 164 private: 165 LegalizeResult 166 widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 167 LegalizeResult 168 widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 169 LegalizeResult 170 widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 171 LegalizeResult 172 widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); 173 LegalizeResult widenScalarAddSubOverflow(MachineInstr &MI, unsigned TypeIdx, 174 LLT WideTy); 175 LegalizeResult widenScalarAddSubShlSat(MachineInstr &MI, unsigned TypeIdx, 176 LLT WideTy); 177 LegalizeResult widenScalarMulo(MachineInstr &MI, unsigned TypeIdx, 178 LLT WideTy); 179 180 /// Helper function to split a wide generic register into bitwise blocks with 181 /// the given Type (which implies the number of blocks needed). The generic 182 /// registers created are appended to Ops, starting at bit 0 of Reg. 183 void extractParts(Register Reg, LLT Ty, int NumParts, 184 SmallVectorImpl<Register> &VRegs); 185 186 /// Version which handles irregular splits. 187 bool extractParts(Register Reg, LLT RegTy, LLT MainTy, 188 LLT &LeftoverTy, 189 SmallVectorImpl<Register> &VRegs, 190 SmallVectorImpl<Register> &LeftoverVRegs); 191 192 /// Helper function to build a wide generic register \p DstReg of type \p 193 /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, 194 /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate 195 /// for the types. 196 /// 197 /// \p PartRegs must be registers of type \p PartTy. 198 /// 199 /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the 200 /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. 201 void insertParts(Register DstReg, LLT ResultTy, 202 LLT PartTy, ArrayRef<Register> PartRegs, 203 LLT LeftoverTy = LLT(), ArrayRef<Register> LeftoverRegs = {}); 204 205 /// Unmerge \p SrcReg into smaller sized values, and append them to \p 206 /// Parts. The elements of \p Parts will be the greatest common divisor type 207 /// of \p DstTy, \p NarrowTy and the type of \p SrcReg. This will compute and 208 /// return the GCD type. 209 LLT extractGCDType(SmallVectorImpl<Register> &Parts, LLT DstTy, 210 LLT NarrowTy, Register SrcReg); 211 212 /// Unmerge \p SrcReg into \p GCDTy typed registers. This will append all of 213 /// the unpacked registers to \p Parts. This version is if the common unmerge 214 /// type is already known. 215 void extractGCDType(SmallVectorImpl<Register> &Parts, LLT GCDTy, 216 Register SrcReg); 217 218 /// Produce a merge of values in \p VRegs to define \p DstReg. Perform a merge 219 /// from the least common multiple type, and convert as appropriate to \p 220 /// DstReg. 221 /// 222 /// \p VRegs should each have type \p GCDTy. This type should be greatest 223 /// common divisor type of \p DstReg, \p NarrowTy, and an undetermined source 224 /// type. 225 /// 226 /// \p NarrowTy is the desired result merge source type. If the source value 227 /// needs to be widened to evenly cover \p DstReg, inserts high bits 228 /// corresponding to the extension opcode \p PadStrategy. 229 /// 230 /// \p VRegs will be cleared, and the the result \p NarrowTy register pieces 231 /// will replace it. Returns The complete LCMTy that \p VRegs will cover when 232 /// merged. 233 LLT buildLCMMergePieces(LLT DstTy, LLT NarrowTy, LLT GCDTy, 234 SmallVectorImpl<Register> &VRegs, 235 unsigned PadStrategy = TargetOpcode::G_ANYEXT); 236 237 /// Merge the values in \p RemergeRegs to an \p LCMTy typed value. Extract the 238 /// low bits into \p DstReg. This is intended to use the outputs from 239 /// buildLCMMergePieces after processing. 240 void buildWidenedRemergeToDst(Register DstReg, LLT LCMTy, 241 ArrayRef<Register> RemergeRegs); 242 243 /// Perform generic multiplication of values held in multiple registers. 244 /// Generated instructions use only types NarrowTy and i1. 245 /// Destination can be same or two times size of the source. 246 void multiplyRegisters(SmallVectorImpl<Register> &DstRegs, 247 ArrayRef<Register> Src1Regs, 248 ArrayRef<Register> Src2Regs, LLT NarrowTy); 249 250 void changeOpcode(MachineInstr &MI, unsigned NewOpcode); 251 252 LegalizeResult tryNarrowPow2Reduction(MachineInstr &MI, Register SrcReg, 253 LLT SrcTy, LLT NarrowTy, 254 unsigned ScalarOpc); 255 256 public: 257 /// Return the alignment to use for a stack temporary object with the given 258 /// type. 259 Align getStackTemporaryAlignment(LLT Type, Align MinAlign = Align()) const; 260 261 /// Create a stack temporary based on the size in bytes and the alignment 262 MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, 263 MachinePointerInfo &PtrInfo); 264 265 /// Get a pointer to vector element \p Index located in memory for a vector of 266 /// type \p VecTy starting at a base address of \p VecPtr. If \p Index is out 267 /// of bounds the returned pointer is unspecified, but will be within the 268 /// vector bounds. 269 Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index); 270 271 LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, 272 unsigned TypeIdx, LLT NarrowTy); 273 274 /// Legalize a instruction with a vector type where each operand may have a 275 /// different element type. All type indexes must have the same number of 276 /// elements. 277 LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI, 278 unsigned TypeIdx, LLT NarrowTy); 279 280 LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, 281 LLT NarrowTy); 282 283 LegalizeResult 284 fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 285 286 LegalizeResult 287 fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 288 289 LegalizeResult fewerElementsVectorPhi(MachineInstr &MI, 290 unsigned TypeIdx, LLT NarrowTy); 291 292 LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, 293 LLT MoreTy); 294 295 LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, 296 unsigned TypeIdx, 297 LLT NarrowTy); 298 LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, 299 LLT NarrowTy); 300 LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, 301 unsigned TypeIdx, 302 LLT NarrowTy); 303 304 LegalizeResult fewerElementsVectorMulo(MachineInstr &MI, unsigned TypeIdx, 305 LLT NarrowTy); 306 307 LegalizeResult 308 reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); 309 310 /// Legalize an instruction by reducing the operation width, either by 311 /// narrowing the type of the operation or by reducing the number of elements 312 /// of a vector. 313 /// The used strategy (narrow vs. fewerElements) is decided by \p NarrowTy. 314 /// Narrow is used if the scalar type of \p NarrowTy and \p DstTy differ, 315 /// fewerElements is used when the scalar type is the same but the number of 316 /// elements between \p NarrowTy and \p DstTy differ. 317 LegalizeResult reduceOperationWidth(MachineInstr &MI, unsigned TypeIdx, 318 LLT NarrowTy); 319 320 LegalizeResult fewerElementsVectorSextInReg(MachineInstr &MI, unsigned TypeIdx, 321 LLT NarrowTy); 322 323 LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, 324 LLT HalfTy, LLT ShiftAmtTy); 325 326 LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, 327 unsigned TypeIdx, LLT NarrowTy); 328 329 LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 330 LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, 331 LLT NarrowTy); 332 LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); 333 LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 334 LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 335 LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 336 337 LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 338 LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 339 LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 340 LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 341 LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 342 LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); 343 344 /// Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT. 345 LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, 346 LLT CastTy); 347 348 /// Perform Bitcast legalize action on G_INSERT_VECTOR_ELT. 349 LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, 350 LLT CastTy); 351 352 LegalizeResult lowerBitcast(MachineInstr &MI); 353 LegalizeResult lowerLoad(MachineInstr &MI); 354 LegalizeResult lowerStore(MachineInstr &MI); 355 LegalizeResult lowerBitCount(MachineInstr &MI); 356 LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI); 357 LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI); 358 LegalizeResult lowerFunnelShift(MachineInstr &MI); 359 LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI); 360 LegalizeResult lowerRotate(MachineInstr &MI); 361 362 LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); 363 LegalizeResult lowerUITOFP(MachineInstr &MI); 364 LegalizeResult lowerSITOFP(MachineInstr &MI); 365 LegalizeResult lowerFPTOUI(MachineInstr &MI); 366 LegalizeResult lowerFPTOSI(MachineInstr &MI); 367 368 LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI); 369 LegalizeResult lowerFPTRUNC(MachineInstr &MI); 370 LegalizeResult lowerFPOWI(MachineInstr &MI); 371 372 LegalizeResult lowerMinMax(MachineInstr &MI); 373 LegalizeResult lowerFCopySign(MachineInstr &MI); 374 LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); 375 LegalizeResult lowerFMad(MachineInstr &MI); 376 LegalizeResult lowerIntrinsicRound(MachineInstr &MI); 377 LegalizeResult lowerFFloor(MachineInstr &MI); 378 LegalizeResult lowerMergeValues(MachineInstr &MI); 379 LegalizeResult lowerUnmergeValues(MachineInstr &MI); 380 LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI); 381 LegalizeResult lowerShuffleVector(MachineInstr &MI); 382 LegalizeResult lowerDynStackAlloc(MachineInstr &MI); 383 LegalizeResult lowerExtract(MachineInstr &MI); 384 LegalizeResult lowerInsert(MachineInstr &MI); 385 LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI); 386 LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI); 387 LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI); 388 LegalizeResult lowerShlSat(MachineInstr &MI); 389 LegalizeResult lowerBswap(MachineInstr &MI); 390 LegalizeResult lowerBitreverse(MachineInstr &MI); 391 LegalizeResult lowerReadWriteRegister(MachineInstr &MI); 392 LegalizeResult lowerSMULH_UMULH(MachineInstr &MI); 393 LegalizeResult lowerSelect(MachineInstr &MI); 394 LegalizeResult lowerDIVREM(MachineInstr &MI); 395 }; 396 397 /// Helper function that creates a libcall to the given \p Name using the given 398 /// calling convention \p CC. 399 LegalizerHelper::LegalizeResult 400 createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, 401 const CallLowering::ArgInfo &Result, 402 ArrayRef<CallLowering::ArgInfo> Args, CallingConv::ID CC); 403 404 /// Helper function that creates the given libcall. 405 LegalizerHelper::LegalizeResult 406 createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, 407 const CallLowering::ArgInfo &Result, 408 ArrayRef<CallLowering::ArgInfo> Args); 409 410 /// Create a libcall to memcpy et al. 411 LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, 412 MachineRegisterInfo &MRI, 413 MachineInstr &MI); 414 415 } // End namespace llvm. 416 417 #endif 418