1 //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This implements routines for translating from LLVM IR into SelectionDAG IR. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SelectionDAGBuilder.h" 14 #include "SDNodeDbgValue.h" 15 #include "llvm/ADT/APFloat.h" 16 #include "llvm/ADT/APInt.h" 17 #include "llvm/ADT/BitVector.h" 18 #include "llvm/ADT/STLExtras.h" 19 #include "llvm/ADT/SmallPtrSet.h" 20 #include "llvm/ADT/SmallSet.h" 21 #include "llvm/ADT/StringRef.h" 22 #include "llvm/ADT/Twine.h" 23 #include "llvm/Analysis/AliasAnalysis.h" 24 #include "llvm/Analysis/BranchProbabilityInfo.h" 25 #include "llvm/Analysis/ConstantFolding.h" 26 #include "llvm/Analysis/Loads.h" 27 #include "llvm/Analysis/MemoryLocation.h" 28 #include "llvm/Analysis/TargetLibraryInfo.h" 29 #include "llvm/Analysis/ValueTracking.h" 30 #include "llvm/Analysis/VectorUtils.h" 31 #include "llvm/CodeGen/Analysis.h" 32 #include "llvm/CodeGen/AssignmentTrackingAnalysis.h" 33 #include "llvm/CodeGen/CodeGenCommonISel.h" 34 #include "llvm/CodeGen/FunctionLoweringInfo.h" 35 #include "llvm/CodeGen/GCMetadata.h" 36 #include "llvm/CodeGen/MachineBasicBlock.h" 37 #include "llvm/CodeGen/MachineFrameInfo.h" 38 #include "llvm/CodeGen/MachineFunction.h" 39 #include "llvm/CodeGen/MachineInstrBuilder.h" 40 #include "llvm/CodeGen/MachineInstrBundleIterator.h" 41 #include "llvm/CodeGen/MachineMemOperand.h" 42 #include "llvm/CodeGen/MachineModuleInfo.h" 43 #include "llvm/CodeGen/MachineOperand.h" 44 #include "llvm/CodeGen/MachineRegisterInfo.h" 45 #include "llvm/CodeGen/RuntimeLibcalls.h" 46 #include "llvm/CodeGen/SelectionDAG.h" 47 #include "llvm/CodeGen/SelectionDAGTargetInfo.h" 48 #include "llvm/CodeGen/StackMaps.h" 49 #include "llvm/CodeGen/SwiftErrorValueTracking.h" 50 #include "llvm/CodeGen/TargetFrameLowering.h" 51 #include "llvm/CodeGen/TargetInstrInfo.h" 52 #include "llvm/CodeGen/TargetOpcodes.h" 53 #include "llvm/CodeGen/TargetRegisterInfo.h" 54 #include "llvm/CodeGen/TargetSubtargetInfo.h" 55 #include "llvm/CodeGen/WinEHFuncInfo.h" 56 #include "llvm/IR/Argument.h" 57 #include "llvm/IR/Attributes.h" 58 #include "llvm/IR/BasicBlock.h" 59 #include "llvm/IR/CFG.h" 60 #include "llvm/IR/CallingConv.h" 61 #include "llvm/IR/Constant.h" 62 #include "llvm/IR/ConstantRange.h" 63 #include "llvm/IR/Constants.h" 64 #include "llvm/IR/DataLayout.h" 65 #include "llvm/IR/DebugInfo.h" 66 #include "llvm/IR/DebugInfoMetadata.h" 67 #include "llvm/IR/DerivedTypes.h" 68 #include "llvm/IR/DiagnosticInfo.h" 69 #include "llvm/IR/EHPersonalities.h" 70 #include "llvm/IR/Function.h" 71 #include "llvm/IR/GetElementPtrTypeIterator.h" 72 #include "llvm/IR/InlineAsm.h" 73 #include "llvm/IR/InstrTypes.h" 74 #include "llvm/IR/Instructions.h" 75 #include "llvm/IR/IntrinsicInst.h" 76 #include "llvm/IR/Intrinsics.h" 77 #include "llvm/IR/IntrinsicsAArch64.h" 78 #include "llvm/IR/IntrinsicsWebAssembly.h" 79 #include "llvm/IR/LLVMContext.h" 80 #include "llvm/IR/Metadata.h" 81 #include "llvm/IR/Module.h" 82 #include "llvm/IR/Operator.h" 83 #include "llvm/IR/PatternMatch.h" 84 #include "llvm/IR/Statepoint.h" 85 #include "llvm/IR/Type.h" 86 #include "llvm/IR/User.h" 87 #include "llvm/IR/Value.h" 88 #include "llvm/MC/MCContext.h" 89 #include "llvm/Support/AtomicOrdering.h" 90 #include "llvm/Support/Casting.h" 91 #include "llvm/Support/CommandLine.h" 92 #include "llvm/Support/Compiler.h" 93 #include "llvm/Support/Debug.h" 94 #include "llvm/Support/MathExtras.h" 95 #include "llvm/Support/raw_ostream.h" 96 #include "llvm/Target/TargetIntrinsicInfo.h" 97 #include "llvm/Target/TargetMachine.h" 98 #include "llvm/Target/TargetOptions.h" 99 #include "llvm/TargetParser/Triple.h" 100 #include "llvm/Transforms/Utils/Local.h" 101 #include <cstddef> 102 #include <iterator> 103 #include <limits> 104 #include <optional> 105 #include <tuple> 106 107 using namespace llvm; 108 using namespace PatternMatch; 109 using namespace SwitchCG; 110 111 #define DEBUG_TYPE "isel" 112 113 /// LimitFloatPrecision - Generate low-precision inline sequences for 114 /// some float libcalls (6, 8 or 12 bits). 115 static unsigned LimitFloatPrecision; 116 117 static cl::opt<bool> 118 InsertAssertAlign("insert-assert-align", cl::init(true), 119 cl::desc("Insert the experimental `assertalign` node."), 120 cl::ReallyHidden); 121 122 static cl::opt<unsigned, true> 123 LimitFPPrecision("limit-float-precision", 124 cl::desc("Generate low-precision inline sequences " 125 "for some float libcalls"), 126 cl::location(LimitFloatPrecision), cl::Hidden, 127 cl::init(0)); 128 129 static cl::opt<unsigned> SwitchPeelThreshold( 130 "switch-peel-threshold", cl::Hidden, cl::init(66), 131 cl::desc("Set the case probability threshold for peeling the case from a " 132 "switch statement. A value greater than 100 will void this " 133 "optimization")); 134 135 // Limit the width of DAG chains. This is important in general to prevent 136 // DAG-based analysis from blowing up. For example, alias analysis and 137 // load clustering may not complete in reasonable time. It is difficult to 138 // recognize and avoid this situation within each individual analysis, and 139 // future analyses are likely to have the same behavior. Limiting DAG width is 140 // the safe approach and will be especially important with global DAGs. 141 // 142 // MaxParallelChains default is arbitrarily high to avoid affecting 143 // optimization, but could be lowered to improve compile time. Any ld-ld-st-st 144 // sequence over this should have been converted to llvm.memcpy by the 145 // frontend. It is easy to induce this behavior with .ll code such as: 146 // %buffer = alloca [4096 x i8] 147 // %data = load [4096 x i8]* %argPtr 148 // store [4096 x i8] %data, [4096 x i8]* %buffer 149 static const unsigned MaxParallelChains = 64; 150 151 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, 152 const SDValue *Parts, unsigned NumParts, 153 MVT PartVT, EVT ValueVT, const Value *V, 154 std::optional<CallingConv::ID> CC); 155 156 /// getCopyFromParts - Create a value that contains the specified legal parts 157 /// combined into the value they represent. If the parts combine to a type 158 /// larger than ValueVT then AssertOp can be used to specify whether the extra 159 /// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT 160 /// (ISD::AssertSext). 161 static SDValue 162 getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, 163 unsigned NumParts, MVT PartVT, EVT ValueVT, const Value *V, 164 std::optional<CallingConv::ID> CC = std::nullopt, 165 std::optional<ISD::NodeType> AssertOp = std::nullopt) { 166 // Let the target assemble the parts if it wants to 167 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 168 if (SDValue Val = TLI.joinRegisterPartsIntoValue(DAG, DL, Parts, NumParts, 169 PartVT, ValueVT, CC)) 170 return Val; 171 172 if (ValueVT.isVector()) 173 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V, 174 CC); 175 176 assert(NumParts > 0 && "No parts to assemble!"); 177 SDValue Val = Parts[0]; 178 179 if (NumParts > 1) { 180 // Assemble the value from multiple parts. 181 if (ValueVT.isInteger()) { 182 unsigned PartBits = PartVT.getSizeInBits(); 183 unsigned ValueBits = ValueVT.getSizeInBits(); 184 185 // Assemble the power of 2 part. 186 unsigned RoundParts = llvm::bit_floor(NumParts); 187 unsigned RoundBits = PartBits * RoundParts; 188 EVT RoundVT = RoundBits == ValueBits ? 189 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); 190 SDValue Lo, Hi; 191 192 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2); 193 194 if (RoundParts > 2) { 195 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2, 196 PartVT, HalfVT, V); 197 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2, 198 RoundParts / 2, PartVT, HalfVT, V); 199 } else { 200 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]); 201 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]); 202 } 203 204 if (DAG.getDataLayout().isBigEndian()) 205 std::swap(Lo, Hi); 206 207 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi); 208 209 if (RoundParts < NumParts) { 210 // Assemble the trailing non-power-of-2 part. 211 unsigned OddParts = NumParts - RoundParts; 212 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits); 213 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT, 214 OddVT, V, CC); 215 216 // Combine the round and odd parts. 217 Lo = Val; 218 if (DAG.getDataLayout().isBigEndian()) 219 std::swap(Lo, Hi); 220 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 221 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi); 222 Hi = DAG.getNode(ISD::SHL, DL, TotalVT, Hi, 223 DAG.getConstant(Lo.getValueSizeInBits(), DL, 224 TLI.getShiftAmountTy( 225 TotalVT, DAG.getDataLayout()))); 226 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo); 227 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi); 228 } 229 } else if (PartVT.isFloatingPoint()) { 230 // FP split into multiple FP parts (for ppcf128) 231 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && 232 "Unexpected split"); 233 SDValue Lo, Hi; 234 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]); 235 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]); 236 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout())) 237 std::swap(Lo, Hi); 238 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi); 239 } else { 240 // FP split into integer parts (soft fp) 241 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() && 242 !PartVT.isVector() && "Unexpected split"); 243 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); 244 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC); 245 } 246 } 247 248 // There is now one part, held in Val. Correct it to match ValueVT. 249 // PartEVT is the type of the register class that holds the value. 250 // ValueVT is the type of the inline asm operation. 251 EVT PartEVT = Val.getValueType(); 252 253 if (PartEVT == ValueVT) 254 return Val; 255 256 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() && 257 ValueVT.bitsLT(PartEVT)) { 258 // For an FP value in an integer part, we need to truncate to the right 259 // width first. 260 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); 261 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val); 262 } 263 264 // Handle types that have the same size. 265 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits()) 266 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 267 268 // Handle types with different sizes. 269 if (PartEVT.isInteger() && ValueVT.isInteger()) { 270 if (ValueVT.bitsLT(PartEVT)) { 271 // For a truncate, see if we have any information to 272 // indicate whether the truncated bits will always be 273 // zero or sign-extension. 274 if (AssertOp) 275 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val, 276 DAG.getValueType(ValueVT)); 277 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 278 } 279 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); 280 } 281 282 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 283 // FP_ROUND's are always exact here. 284 if (ValueVT.bitsLT(Val.getValueType())) 285 return DAG.getNode( 286 ISD::FP_ROUND, DL, ValueVT, Val, 287 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout()))); 288 289 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); 290 } 291 292 // Handle MMX to a narrower integer type by bitcasting MMX to integer and 293 // then truncating. 294 if (PartEVT == MVT::x86mmx && ValueVT.isInteger() && 295 ValueVT.bitsLT(PartEVT)) { 296 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val); 297 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 298 } 299 300 report_fatal_error("Unknown mismatch in getCopyFromParts!"); 301 } 302 303 static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, 304 const Twine &ErrMsg) { 305 const Instruction *I = dyn_cast_or_null<Instruction>(V); 306 if (!V) 307 return Ctx.emitError(ErrMsg); 308 309 const char *AsmError = ", possible invalid constraint for vector type"; 310 if (const CallInst *CI = dyn_cast<CallInst>(I)) 311 if (CI->isInlineAsm()) 312 return Ctx.emitError(I, ErrMsg + AsmError); 313 314 return Ctx.emitError(I, ErrMsg); 315 } 316 317 /// getCopyFromPartsVector - Create a value that contains the specified legal 318 /// parts combined into the value they represent. If the parts combine to a 319 /// type larger than ValueVT then AssertOp can be used to specify whether the 320 /// extra bits are known to be zero (ISD::AssertZext) or sign extended from 321 /// ValueVT (ISD::AssertSext). 322 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL, 323 const SDValue *Parts, unsigned NumParts, 324 MVT PartVT, EVT ValueVT, const Value *V, 325 std::optional<CallingConv::ID> CallConv) { 326 assert(ValueVT.isVector() && "Not a vector value"); 327 assert(NumParts > 0 && "No parts to assemble!"); 328 const bool IsABIRegCopy = CallConv.has_value(); 329 330 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 331 SDValue Val = Parts[0]; 332 333 // Handle a multi-element vector. 334 if (NumParts > 1) { 335 EVT IntermediateVT; 336 MVT RegisterVT; 337 unsigned NumIntermediates; 338 unsigned NumRegs; 339 340 if (IsABIRegCopy) { 341 NumRegs = TLI.getVectorTypeBreakdownForCallingConv( 342 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, 343 NumIntermediates, RegisterVT); 344 } else { 345 NumRegs = 346 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, 347 NumIntermediates, RegisterVT); 348 } 349 350 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 351 NumParts = NumRegs; // Silence a compiler warning. 352 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 353 assert(RegisterVT.getSizeInBits() == 354 Parts[0].getSimpleValueType().getSizeInBits() && 355 "Part type sizes don't match!"); 356 357 // Assemble the parts into intermediate operands. 358 SmallVector<SDValue, 8> Ops(NumIntermediates); 359 if (NumIntermediates == NumParts) { 360 // If the register was not expanded, truncate or copy the value, 361 // as appropriate. 362 for (unsigned i = 0; i != NumParts; ++i) 363 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1, 364 PartVT, IntermediateVT, V, CallConv); 365 } else if (NumParts > 0) { 366 // If the intermediate type was expanded, build the intermediate 367 // operands from the parts. 368 assert(NumParts % NumIntermediates == 0 && 369 "Must expand into a divisible number of parts!"); 370 unsigned Factor = NumParts / NumIntermediates; 371 for (unsigned i = 0; i != NumIntermediates; ++i) 372 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor, 373 PartVT, IntermediateVT, V, CallConv); 374 } 375 376 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the 377 // intermediate operands. 378 EVT BuiltVectorTy = 379 IntermediateVT.isVector() 380 ? EVT::getVectorVT( 381 *DAG.getContext(), IntermediateVT.getScalarType(), 382 IntermediateVT.getVectorElementCount() * NumParts) 383 : EVT::getVectorVT(*DAG.getContext(), 384 IntermediateVT.getScalarType(), 385 NumIntermediates); 386 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS 387 : ISD::BUILD_VECTOR, 388 DL, BuiltVectorTy, Ops); 389 } 390 391 // There is now one part, held in Val. Correct it to match ValueVT. 392 EVT PartEVT = Val.getValueType(); 393 394 if (PartEVT == ValueVT) 395 return Val; 396 397 if (PartEVT.isVector()) { 398 // Vector/Vector bitcast. 399 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) 400 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 401 402 // If the parts vector has more elements than the value vector, then we 403 // have a vector widening case (e.g. <2 x float> -> <4 x float>). 404 // Extract the elements we want. 405 if (PartEVT.getVectorElementCount() != ValueVT.getVectorElementCount()) { 406 assert((PartEVT.getVectorElementCount().getKnownMinValue() > 407 ValueVT.getVectorElementCount().getKnownMinValue()) && 408 (PartEVT.getVectorElementCount().isScalable() == 409 ValueVT.getVectorElementCount().isScalable()) && 410 "Cannot narrow, it would be a lossy transformation"); 411 PartEVT = 412 EVT::getVectorVT(*DAG.getContext(), PartEVT.getVectorElementType(), 413 ValueVT.getVectorElementCount()); 414 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, PartEVT, Val, 415 DAG.getVectorIdxConstant(0, DL)); 416 if (PartEVT == ValueVT) 417 return Val; 418 if (PartEVT.isInteger() && ValueVT.isFloatingPoint()) 419 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 420 } 421 422 // Promoted vector extract 423 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT); 424 } 425 426 // Trivial bitcast if the types are the same size and the destination 427 // vector type is legal. 428 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() && 429 TLI.isTypeLegal(ValueVT)) 430 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 431 432 if (ValueVT.getVectorNumElements() != 1) { 433 // Certain ABIs require that vectors are passed as integers. For vectors 434 // are the same size, this is an obvious bitcast. 435 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) { 436 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 437 } else if (ValueVT.bitsLT(PartEVT)) { 438 const uint64_t ValueSize = ValueVT.getFixedSizeInBits(); 439 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); 440 // Drop the extra bits. 441 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); 442 return DAG.getBitcast(ValueVT, Val); 443 } 444 445 diagnosePossiblyInvalidConstraint( 446 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion"); 447 return DAG.getUNDEF(ValueVT); 448 } 449 450 // Handle cases such as i8 -> <1 x i1> 451 EVT ValueSVT = ValueVT.getVectorElementType(); 452 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT) { 453 unsigned ValueSize = ValueSVT.getSizeInBits(); 454 if (ValueSize == PartEVT.getSizeInBits()) { 455 Val = DAG.getNode(ISD::BITCAST, DL, ValueSVT, Val); 456 } else if (ValueSVT.isFloatingPoint() && PartEVT.isInteger()) { 457 // It's possible a scalar floating point type gets softened to integer and 458 // then promoted to a larger integer. If PartEVT is the larger integer 459 // we need to truncate it and then bitcast to the FP type. 460 assert(ValueSVT.bitsLT(PartEVT) && "Unexpected types"); 461 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); 462 Val = DAG.getNode(ISD::TRUNCATE, DL, IntermediateType, Val); 463 Val = DAG.getBitcast(ValueSVT, Val); 464 } else { 465 Val = ValueVT.isFloatingPoint() 466 ? DAG.getFPExtendOrRound(Val, DL, ValueSVT) 467 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT); 468 } 469 } 470 471 return DAG.getBuildVector(ValueVT, DL, Val); 472 } 473 474 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl, 475 SDValue Val, SDValue *Parts, unsigned NumParts, 476 MVT PartVT, const Value *V, 477 std::optional<CallingConv::ID> CallConv); 478 479 /// getCopyToParts - Create a series of nodes that contain the specified value 480 /// split into legal parts. If the parts contain more bits than Val, then, for 481 /// integers, ExtendKind can be used to specify how to generate the extra bits. 482 static void 483 getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts, 484 unsigned NumParts, MVT PartVT, const Value *V, 485 std::optional<CallingConv::ID> CallConv = std::nullopt, 486 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) { 487 // Let the target split the parts if it wants to 488 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 489 if (TLI.splitValueIntoRegisterParts(DAG, DL, Val, Parts, NumParts, PartVT, 490 CallConv)) 491 return; 492 EVT ValueVT = Val.getValueType(); 493 494 // Handle the vector case separately. 495 if (ValueVT.isVector()) 496 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V, 497 CallConv); 498 499 unsigned OrigNumParts = NumParts; 500 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && 501 "Copying to an illegal type!"); 502 503 if (NumParts == 0) 504 return; 505 506 assert(!ValueVT.isVector() && "Vector case handled elsewhere"); 507 EVT PartEVT = PartVT; 508 if (PartEVT == ValueVT) { 509 assert(NumParts == 1 && "No-op copy with multiple parts!"); 510 Parts[0] = Val; 511 return; 512 } 513 514 unsigned PartBits = PartVT.getSizeInBits(); 515 if (NumParts * PartBits > ValueVT.getSizeInBits()) { 516 // If the parts cover more bits than the value has, promote the value. 517 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) { 518 assert(NumParts == 1 && "Do not know what to promote to!"); 519 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val); 520 } else { 521 if (ValueVT.isFloatingPoint()) { 522 // FP values need to be bitcast, then extended if they are being put 523 // into a larger container. 524 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits()); 525 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val); 526 } 527 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 528 ValueVT.isInteger() && 529 "Unknown mismatch!"); 530 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 531 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val); 532 if (PartVT == MVT::x86mmx) 533 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 534 } 535 } else if (PartBits == ValueVT.getSizeInBits()) { 536 // Different types of the same size. 537 assert(NumParts == 1 && PartEVT != ValueVT); 538 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 539 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) { 540 // If the parts cover less bits than value has, truncate the value. 541 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) && 542 ValueVT.isInteger() && 543 "Unknown mismatch!"); 544 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 545 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 546 if (PartVT == MVT::x86mmx) 547 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 548 } 549 550 // The value may have changed - recompute ValueVT. 551 ValueVT = Val.getValueType(); 552 assert(NumParts * PartBits == ValueVT.getSizeInBits() && 553 "Failed to tile the value with PartVT!"); 554 555 if (NumParts == 1) { 556 if (PartEVT != ValueVT) { 557 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V, 558 "scalar-to-vector conversion failed"); 559 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 560 } 561 562 Parts[0] = Val; 563 return; 564 } 565 566 // Expand the value into multiple parts. 567 if (NumParts & (NumParts - 1)) { 568 // The number of parts is not a power of 2. Split off and copy the tail. 569 assert(PartVT.isInteger() && ValueVT.isInteger() && 570 "Do not know what to expand to!"); 571 unsigned RoundParts = llvm::bit_floor(NumParts); 572 unsigned RoundBits = RoundParts * PartBits; 573 unsigned OddParts = NumParts - RoundParts; 574 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, 575 DAG.getShiftAmountConstant(RoundBits, ValueVT, DL)); 576 577 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, 578 CallConv); 579 580 if (DAG.getDataLayout().isBigEndian()) 581 // The odd parts were reversed by getCopyToParts - unreverse them. 582 std::reverse(Parts + RoundParts, Parts + NumParts); 583 584 NumParts = RoundParts; 585 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits); 586 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); 587 } 588 589 // The number of parts is a power of 2. Repeatedly bisect the value using 590 // EXTRACT_ELEMENT. 591 Parts[0] = DAG.getNode(ISD::BITCAST, DL, 592 EVT::getIntegerVT(*DAG.getContext(), 593 ValueVT.getSizeInBits()), 594 Val); 595 596 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) { 597 for (unsigned i = 0; i < NumParts; i += StepSize) { 598 unsigned ThisBits = StepSize * PartBits / 2; 599 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits); 600 SDValue &Part0 = Parts[i]; 601 SDValue &Part1 = Parts[i+StepSize/2]; 602 603 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 604 ThisVT, Part0, DAG.getIntPtrConstant(1, DL)); 605 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, 606 ThisVT, Part0, DAG.getIntPtrConstant(0, DL)); 607 608 if (ThisBits == PartBits && ThisVT != PartVT) { 609 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0); 610 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1); 611 } 612 } 613 } 614 615 if (DAG.getDataLayout().isBigEndian()) 616 std::reverse(Parts, Parts + OrigNumParts); 617 } 618 619 static SDValue widenVectorToPartType(SelectionDAG &DAG, SDValue Val, 620 const SDLoc &DL, EVT PartVT) { 621 if (!PartVT.isVector()) 622 return SDValue(); 623 624 EVT ValueVT = Val.getValueType(); 625 ElementCount PartNumElts = PartVT.getVectorElementCount(); 626 ElementCount ValueNumElts = ValueVT.getVectorElementCount(); 627 628 // We only support widening vectors with equivalent element types and 629 // fixed/scalable properties. If a target needs to widen a fixed-length type 630 // to a scalable one, it should be possible to use INSERT_SUBVECTOR below. 631 if (ElementCount::isKnownLE(PartNumElts, ValueNumElts) || 632 PartNumElts.isScalable() != ValueNumElts.isScalable() || 633 PartVT.getVectorElementType() != ValueVT.getVectorElementType()) 634 return SDValue(); 635 636 // Widening a scalable vector to another scalable vector is done by inserting 637 // the vector into a larger undef one. 638 if (PartNumElts.isScalable()) 639 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, PartVT, DAG.getUNDEF(PartVT), 640 Val, DAG.getVectorIdxConstant(0, DL)); 641 642 EVT ElementVT = PartVT.getVectorElementType(); 643 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in 644 // undef elements. 645 SmallVector<SDValue, 16> Ops; 646 DAG.ExtractVectorElements(Val, Ops); 647 SDValue EltUndef = DAG.getUNDEF(ElementVT); 648 Ops.append((PartNumElts - ValueNumElts).getFixedValue(), EltUndef); 649 650 // FIXME: Use CONCAT for 2x -> 4x. 651 return DAG.getBuildVector(PartVT, DL, Ops); 652 } 653 654 /// getCopyToPartsVector - Create a series of nodes that contain the specified 655 /// value split into legal parts. 656 static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL, 657 SDValue Val, SDValue *Parts, unsigned NumParts, 658 MVT PartVT, const Value *V, 659 std::optional<CallingConv::ID> CallConv) { 660 EVT ValueVT = Val.getValueType(); 661 assert(ValueVT.isVector() && "Not a vector"); 662 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 663 const bool IsABIRegCopy = CallConv.has_value(); 664 665 if (NumParts == 1) { 666 EVT PartEVT = PartVT; 667 if (PartEVT == ValueVT) { 668 // Nothing to do. 669 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) { 670 // Bitconvert vector->vector case. 671 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val); 672 } else if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, PartVT)) { 673 Val = Widened; 674 } else if (PartVT.isVector() && 675 PartEVT.getVectorElementType().bitsGE( 676 ValueVT.getVectorElementType()) && 677 PartEVT.getVectorElementCount() == 678 ValueVT.getVectorElementCount()) { 679 680 // Promoted vector extract 681 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); 682 } else if (PartEVT.isVector() && 683 PartEVT.getVectorElementType() != 684 ValueVT.getVectorElementType() && 685 TLI.getTypeAction(*DAG.getContext(), ValueVT) == 686 TargetLowering::TypeWidenVector) { 687 // Combination of widening and promotion. 688 EVT WidenVT = 689 EVT::getVectorVT(*DAG.getContext(), ValueVT.getVectorElementType(), 690 PartVT.getVectorElementCount()); 691 SDValue Widened = widenVectorToPartType(DAG, Val, DL, WidenVT); 692 Val = DAG.getAnyExtOrTrunc(Widened, DL, PartVT); 693 } else { 694 // Don't extract an integer from a float vector. This can happen if the 695 // FP type gets softened to integer and then promoted. The promotion 696 // prevents it from being picked up by the earlier bitcast case. 697 if (ValueVT.getVectorElementCount().isScalar() && 698 (!ValueVT.isFloatingPoint() || !PartVT.isInteger())) { 699 Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val, 700 DAG.getVectorIdxConstant(0, DL)); 701 } else { 702 uint64_t ValueSize = ValueVT.getFixedSizeInBits(); 703 assert(PartVT.getFixedSizeInBits() > ValueSize && 704 "lossy conversion of vector to scalar type"); 705 EVT IntermediateType = EVT::getIntegerVT(*DAG.getContext(), ValueSize); 706 Val = DAG.getBitcast(IntermediateType, Val); 707 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT); 708 } 709 } 710 711 assert(Val.getValueType() == PartVT && "Unexpected vector part value type"); 712 Parts[0] = Val; 713 return; 714 } 715 716 // Handle a multi-element vector. 717 EVT IntermediateVT; 718 MVT RegisterVT; 719 unsigned NumIntermediates; 720 unsigned NumRegs; 721 if (IsABIRegCopy) { 722 NumRegs = TLI.getVectorTypeBreakdownForCallingConv( 723 *DAG.getContext(), *CallConv, ValueVT, IntermediateVT, NumIntermediates, 724 RegisterVT); 725 } else { 726 NumRegs = 727 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT, 728 NumIntermediates, RegisterVT); 729 } 730 731 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!"); 732 NumParts = NumRegs; // Silence a compiler warning. 733 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!"); 734 735 assert(IntermediateVT.isScalableVector() == ValueVT.isScalableVector() && 736 "Mixing scalable and fixed vectors when copying in parts"); 737 738 std::optional<ElementCount> DestEltCnt; 739 740 if (IntermediateVT.isVector()) 741 DestEltCnt = IntermediateVT.getVectorElementCount() * NumIntermediates; 742 else 743 DestEltCnt = ElementCount::getFixed(NumIntermediates); 744 745 EVT BuiltVectorTy = EVT::getVectorVT( 746 *DAG.getContext(), IntermediateVT.getScalarType(), *DestEltCnt); 747 748 if (ValueVT == BuiltVectorTy) { 749 // Nothing to do. 750 } else if (ValueVT.getSizeInBits() == BuiltVectorTy.getSizeInBits()) { 751 // Bitconvert vector->vector case. 752 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val); 753 } else { 754 if (BuiltVectorTy.getVectorElementType().bitsGT( 755 ValueVT.getVectorElementType())) { 756 // Integer promotion. 757 ValueVT = EVT::getVectorVT(*DAG.getContext(), 758 BuiltVectorTy.getVectorElementType(), 759 ValueVT.getVectorElementCount()); 760 Val = DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val); 761 } 762 763 if (SDValue Widened = widenVectorToPartType(DAG, Val, DL, BuiltVectorTy)) { 764 Val = Widened; 765 } 766 } 767 768 assert(Val.getValueType() == BuiltVectorTy && "Unexpected vector value type"); 769 770 // Split the vector into intermediate operands. 771 SmallVector<SDValue, 8> Ops(NumIntermediates); 772 for (unsigned i = 0; i != NumIntermediates; ++i) { 773 if (IntermediateVT.isVector()) { 774 // This does something sensible for scalable vectors - see the 775 // definition of EXTRACT_SUBVECTOR for further details. 776 unsigned IntermediateNumElts = IntermediateVT.getVectorMinNumElements(); 777 Ops[i] = 778 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val, 779 DAG.getVectorIdxConstant(i * IntermediateNumElts, DL)); 780 } else { 781 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val, 782 DAG.getVectorIdxConstant(i, DL)); 783 } 784 } 785 786 // Split the intermediate operands into legal parts. 787 if (NumParts == NumIntermediates) { 788 // If the register was not expanded, promote or copy the value, 789 // as appropriate. 790 for (unsigned i = 0; i != NumParts; ++i) 791 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv); 792 } else if (NumParts > 0) { 793 // If the intermediate type was expanded, split each the value into 794 // legal parts. 795 assert(NumIntermediates != 0 && "division by zero"); 796 assert(NumParts % NumIntermediates == 0 && 797 "Must expand into a divisible number of parts!"); 798 unsigned Factor = NumParts / NumIntermediates; 799 for (unsigned i = 0; i != NumIntermediates; ++i) 800 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V, 801 CallConv); 802 } 803 } 804 805 RegsForValue::RegsForValue(const SmallVector<unsigned, 4> ®s, MVT regvt, 806 EVT valuevt, std::optional<CallingConv::ID> CC) 807 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs), 808 RegCount(1, regs.size()), CallConv(CC) {} 809 810 RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI, 811 const DataLayout &DL, unsigned Reg, Type *Ty, 812 std::optional<CallingConv::ID> CC) { 813 ComputeValueVTs(TLI, DL, Ty, ValueVTs); 814 815 CallConv = CC; 816 817 for (EVT ValueVT : ValueVTs) { 818 unsigned NumRegs = 819 isABIMangled() 820 ? TLI.getNumRegistersForCallingConv(Context, *CC, ValueVT) 821 : TLI.getNumRegisters(Context, ValueVT); 822 MVT RegisterVT = 823 isABIMangled() 824 ? TLI.getRegisterTypeForCallingConv(Context, *CC, ValueVT) 825 : TLI.getRegisterType(Context, ValueVT); 826 for (unsigned i = 0; i != NumRegs; ++i) 827 Regs.push_back(Reg + i); 828 RegVTs.push_back(RegisterVT); 829 RegCount.push_back(NumRegs); 830 Reg += NumRegs; 831 } 832 } 833 834 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG, 835 FunctionLoweringInfo &FuncInfo, 836 const SDLoc &dl, SDValue &Chain, 837 SDValue *Flag, const Value *V) const { 838 // A Value with type {} or [0 x %t] needs no registers. 839 if (ValueVTs.empty()) 840 return SDValue(); 841 842 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 843 844 // Assemble the legal parts into the final values. 845 SmallVector<SDValue, 4> Values(ValueVTs.size()); 846 SmallVector<SDValue, 8> Parts; 847 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 848 // Copy the legal parts from the registers. 849 EVT ValueVT = ValueVTs[Value]; 850 unsigned NumRegs = RegCount[Value]; 851 MVT RegisterVT = isABIMangled() 852 ? TLI.getRegisterTypeForCallingConv( 853 *DAG.getContext(), *CallConv, RegVTs[Value]) 854 : RegVTs[Value]; 855 856 Parts.resize(NumRegs); 857 for (unsigned i = 0; i != NumRegs; ++i) { 858 SDValue P; 859 if (!Flag) { 860 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT); 861 } else { 862 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag); 863 *Flag = P.getValue(2); 864 } 865 866 Chain = P.getValue(1); 867 Parts[i] = P; 868 869 // If the source register was virtual and if we know something about it, 870 // add an assert node. 871 if (!Register::isVirtualRegister(Regs[Part + i]) || 872 !RegisterVT.isInteger()) 873 continue; 874 875 const FunctionLoweringInfo::LiveOutInfo *LOI = 876 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]); 877 if (!LOI) 878 continue; 879 880 unsigned RegSize = RegisterVT.getScalarSizeInBits(); 881 unsigned NumSignBits = LOI->NumSignBits; 882 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros(); 883 884 if (NumZeroBits == RegSize) { 885 // The current value is a zero. 886 // Explicitly express that as it would be easier for 887 // optimizations to kick in. 888 Parts[i] = DAG.getConstant(0, dl, RegisterVT); 889 continue; 890 } 891 892 // FIXME: We capture more information than the dag can represent. For 893 // now, just use the tightest assertzext/assertsext possible. 894 bool isSExt; 895 EVT FromVT(MVT::Other); 896 if (NumZeroBits) { 897 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits); 898 isSExt = false; 899 } else if (NumSignBits > 1) { 900 FromVT = 901 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1); 902 isSExt = true; 903 } else { 904 continue; 905 } 906 // Add an assertion node. 907 assert(FromVT != MVT::Other); 908 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl, 909 RegisterVT, P, DAG.getValueType(FromVT)); 910 } 911 912 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs, 913 RegisterVT, ValueVT, V, CallConv); 914 Part += NumRegs; 915 Parts.clear(); 916 } 917 918 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values); 919 } 920 921 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, 922 const SDLoc &dl, SDValue &Chain, SDValue *Flag, 923 const Value *V, 924 ISD::NodeType PreferredExtendType) const { 925 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 926 ISD::NodeType ExtendKind = PreferredExtendType; 927 928 // Get the list of the values's legal parts. 929 unsigned NumRegs = Regs.size(); 930 SmallVector<SDValue, 8> Parts(NumRegs); 931 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) { 932 unsigned NumParts = RegCount[Value]; 933 934 MVT RegisterVT = isABIMangled() 935 ? TLI.getRegisterTypeForCallingConv( 936 *DAG.getContext(), *CallConv, RegVTs[Value]) 937 : RegVTs[Value]; 938 939 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT)) 940 ExtendKind = ISD::ZERO_EXTEND; 941 942 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part], 943 NumParts, RegisterVT, V, CallConv, ExtendKind); 944 Part += NumParts; 945 } 946 947 // Copy the parts into the registers. 948 SmallVector<SDValue, 8> Chains(NumRegs); 949 for (unsigned i = 0; i != NumRegs; ++i) { 950 SDValue Part; 951 if (!Flag) { 952 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]); 953 } else { 954 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag); 955 *Flag = Part.getValue(1); 956 } 957 958 Chains[i] = Part.getValue(0); 959 } 960 961 if (NumRegs == 1 || Flag) 962 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is 963 // flagged to it. That is the CopyToReg nodes and the user are considered 964 // a single scheduling unit. If we create a TokenFactor and return it as 965 // chain, then the TokenFactor is both a predecessor (operand) of the 966 // user as well as a successor (the TF operands are flagged to the user). 967 // c1, f1 = CopyToReg 968 // c2, f2 = CopyToReg 969 // c3 = TokenFactor c1, c2 970 // ... 971 // = op c3, ..., f2 972 Chain = Chains[NumRegs-1]; 973 else 974 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 975 } 976 977 void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching, 978 unsigned MatchingIdx, const SDLoc &dl, 979 SelectionDAG &DAG, 980 std::vector<SDValue> &Ops) const { 981 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 982 983 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size()); 984 if (HasMatching) 985 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx); 986 else if (!Regs.empty() && Register::isVirtualRegister(Regs.front())) { 987 // Put the register class of the virtual registers in the flag word. That 988 // way, later passes can recompute register class constraints for inline 989 // assembly as well as normal instructions. 990 // Don't do this for tied operands that can use the regclass information 991 // from the def. 992 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 993 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front()); 994 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID()); 995 } 996 997 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32); 998 Ops.push_back(Res); 999 1000 if (Code == InlineAsm::Kind_Clobber) { 1001 // Clobbers should always have a 1:1 mapping with registers, and may 1002 // reference registers that have illegal (e.g. vector) types. Hence, we 1003 // shouldn't try to apply any sort of splitting logic to them. 1004 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && 1005 "No 1:1 mapping from clobbers to regs?"); 1006 Register SP = TLI.getStackPointerRegisterToSaveRestore(); 1007 (void)SP; 1008 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) { 1009 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I])); 1010 assert( 1011 (Regs[I] != SP || 1012 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && 1013 "If we clobbered the stack pointer, MFI should know about it."); 1014 } 1015 return; 1016 } 1017 1018 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) { 1019 MVT RegisterVT = RegVTs[Value]; 1020 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value], 1021 RegisterVT); 1022 for (unsigned i = 0; i != NumRegs; ++i) { 1023 assert(Reg < Regs.size() && "Mismatch in # registers expected"); 1024 unsigned TheReg = Regs[Reg++]; 1025 Ops.push_back(DAG.getRegister(TheReg, RegisterVT)); 1026 } 1027 } 1028 } 1029 1030 SmallVector<std::pair<unsigned, TypeSize>, 4> 1031 RegsForValue::getRegsAndSizes() const { 1032 SmallVector<std::pair<unsigned, TypeSize>, 4> OutVec; 1033 unsigned I = 0; 1034 for (auto CountAndVT : zip_first(RegCount, RegVTs)) { 1035 unsigned RegCount = std::get<0>(CountAndVT); 1036 MVT RegisterVT = std::get<1>(CountAndVT); 1037 TypeSize RegisterSize = RegisterVT.getSizeInBits(); 1038 for (unsigned E = I + RegCount; I != E; ++I) 1039 OutVec.push_back(std::make_pair(Regs[I], RegisterSize)); 1040 } 1041 return OutVec; 1042 } 1043 1044 void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, 1045 AssumptionCache *ac, 1046 const TargetLibraryInfo *li) { 1047 AA = aa; 1048 AC = ac; 1049 GFI = gfi; 1050 LibInfo = li; 1051 Context = DAG.getContext(); 1052 LPadToCallSiteMap.clear(); 1053 SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout()); 1054 } 1055 1056 void SelectionDAGBuilder::clear() { 1057 NodeMap.clear(); 1058 UnusedArgNodeMap.clear(); 1059 PendingLoads.clear(); 1060 PendingExports.clear(); 1061 PendingConstrainedFP.clear(); 1062 PendingConstrainedFPStrict.clear(); 1063 CurInst = nullptr; 1064 HasTailCall = false; 1065 SDNodeOrder = LowestSDNodeOrder; 1066 StatepointLowering.clear(); 1067 } 1068 1069 void SelectionDAGBuilder::clearDanglingDebugInfo() { 1070 DanglingDebugInfoMap.clear(); 1071 } 1072 1073 // Update DAG root to include dependencies on Pending chains. 1074 SDValue SelectionDAGBuilder::updateRoot(SmallVectorImpl<SDValue> &Pending) { 1075 SDValue Root = DAG.getRoot(); 1076 1077 if (Pending.empty()) 1078 return Root; 1079 1080 // Add current root to PendingChains, unless we already indirectly 1081 // depend on it. 1082 if (Root.getOpcode() != ISD::EntryToken) { 1083 unsigned i = 0, e = Pending.size(); 1084 for (; i != e; ++i) { 1085 assert(Pending[i].getNode()->getNumOperands() > 1); 1086 if (Pending[i].getNode()->getOperand(0) == Root) 1087 break; // Don't add the root if we already indirectly depend on it. 1088 } 1089 1090 if (i == e) 1091 Pending.push_back(Root); 1092 } 1093 1094 if (Pending.size() == 1) 1095 Root = Pending[0]; 1096 else 1097 Root = DAG.getTokenFactor(getCurSDLoc(), Pending); 1098 1099 DAG.setRoot(Root); 1100 Pending.clear(); 1101 return Root; 1102 } 1103 1104 SDValue SelectionDAGBuilder::getMemoryRoot() { 1105 return updateRoot(PendingLoads); 1106 } 1107 1108 SDValue SelectionDAGBuilder::getRoot() { 1109 // Chain up all pending constrained intrinsics together with all 1110 // pending loads, by simply appending them to PendingLoads and 1111 // then calling getMemoryRoot(). 1112 PendingLoads.reserve(PendingLoads.size() + 1113 PendingConstrainedFP.size() + 1114 PendingConstrainedFPStrict.size()); 1115 PendingLoads.append(PendingConstrainedFP.begin(), 1116 PendingConstrainedFP.end()); 1117 PendingLoads.append(PendingConstrainedFPStrict.begin(), 1118 PendingConstrainedFPStrict.end()); 1119 PendingConstrainedFP.clear(); 1120 PendingConstrainedFPStrict.clear(); 1121 return getMemoryRoot(); 1122 } 1123 1124 SDValue SelectionDAGBuilder::getControlRoot() { 1125 // We need to emit pending fpexcept.strict constrained intrinsics, 1126 // so append them to the PendingExports list. 1127 PendingExports.append(PendingConstrainedFPStrict.begin(), 1128 PendingConstrainedFPStrict.end()); 1129 PendingConstrainedFPStrict.clear(); 1130 return updateRoot(PendingExports); 1131 } 1132 1133 void SelectionDAGBuilder::visit(const Instruction &I) { 1134 // Set up outgoing PHI node register values before emitting the terminator. 1135 if (I.isTerminator()) { 1136 HandlePHINodesInSuccessorBlocks(I.getParent()); 1137 } 1138 1139 // Add SDDbgValue nodes for any var locs here. Do so before updating 1140 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}. 1141 if (FunctionVarLocs const *FnVarLocs = DAG.getFunctionVarLocs()) { 1142 // Add SDDbgValue nodes for any var locs here. Do so before updating 1143 // SDNodeOrder, as this mapping is {Inst -> Locs BEFORE Inst}. 1144 for (auto It = FnVarLocs->locs_begin(&I), End = FnVarLocs->locs_end(&I); 1145 It != End; ++It) { 1146 auto *Var = FnVarLocs->getDILocalVariable(It->VariableID); 1147 dropDanglingDebugInfo(Var, It->Expr); 1148 SmallVector<Value *> Values(It->Values.location_ops()); 1149 if (!handleDebugValue(Values, Var, It->Expr, It->DL, SDNodeOrder, 1150 It->Values.hasArgList())) 1151 addDanglingDebugInfo(It, SDNodeOrder); 1152 } 1153 } 1154 1155 // Increase the SDNodeOrder if dealing with a non-debug instruction. 1156 if (!isa<DbgInfoIntrinsic>(I)) 1157 ++SDNodeOrder; 1158 1159 CurInst = &I; 1160 1161 // Set inserted listener only if required. 1162 bool NodeInserted = false; 1163 std::unique_ptr<SelectionDAG::DAGNodeInsertedListener> InsertedListener; 1164 MDNode *PCSectionsMD = I.getMetadata(LLVMContext::MD_pcsections); 1165 if (PCSectionsMD) { 1166 InsertedListener = std::make_unique<SelectionDAG::DAGNodeInsertedListener>( 1167 DAG, [&](SDNode *) { NodeInserted = true; }); 1168 } 1169 1170 visit(I.getOpcode(), I); 1171 1172 if (!I.isTerminator() && !HasTailCall && 1173 !isa<GCStatepointInst>(I)) // statepoints handle their exports internally 1174 CopyToExportRegsIfNeeded(&I); 1175 1176 // Handle metadata. 1177 if (PCSectionsMD) { 1178 auto It = NodeMap.find(&I); 1179 if (It != NodeMap.end()) { 1180 DAG.addPCSections(It->second.getNode(), PCSectionsMD); 1181 } else if (NodeInserted) { 1182 // This should not happen; if it does, don't let it go unnoticed so we can 1183 // fix it. Relevant visit*() function is probably missing a setValue(). 1184 errs() << "warning: loosing !pcsections metadata [" 1185 << I.getModule()->getName() << "]\n"; 1186 LLVM_DEBUG(I.dump()); 1187 assert(false); 1188 } 1189 } 1190 1191 CurInst = nullptr; 1192 } 1193 1194 void SelectionDAGBuilder::visitPHI(const PHINode &) { 1195 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!"); 1196 } 1197 1198 void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) { 1199 // Note: this doesn't use InstVisitor, because it has to work with 1200 // ConstantExpr's in addition to instructions. 1201 switch (Opcode) { 1202 default: llvm_unreachable("Unknown instruction type encountered!"); 1203 // Build the switch statement using the Instruction.def file. 1204 #define HANDLE_INST(NUM, OPCODE, CLASS) \ 1205 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break; 1206 #include "llvm/IR/Instruction.def" 1207 } 1208 } 1209 1210 static bool handleDanglingVariadicDebugInfo(SelectionDAG &DAG, 1211 DILocalVariable *Variable, 1212 DebugLoc DL, unsigned Order, 1213 RawLocationWrapper Values, 1214 DIExpression *Expression) { 1215 if (!Values.hasArgList()) 1216 return false; 1217 // For variadic dbg_values we will now insert an undef. 1218 // FIXME: We can potentially recover these! 1219 SmallVector<SDDbgOperand, 2> Locs; 1220 for (const Value *V : Values.location_ops()) { 1221 auto *Undef = UndefValue::get(V->getType()); 1222 Locs.push_back(SDDbgOperand::fromConst(Undef)); 1223 } 1224 SDDbgValue *SDV = DAG.getDbgValueList(Variable, Expression, Locs, {}, 1225 /*IsIndirect=*/false, DL, Order, 1226 /*IsVariadic=*/true); 1227 DAG.AddDbgValue(SDV, /*isParameter=*/false); 1228 return true; 1229 } 1230 1231 void SelectionDAGBuilder::addDanglingDebugInfo(const VarLocInfo *VarLoc, 1232 unsigned Order) { 1233 if (!handleDanglingVariadicDebugInfo( 1234 DAG, 1235 const_cast<DILocalVariable *>(DAG.getFunctionVarLocs() 1236 ->getVariable(VarLoc->VariableID) 1237 .getVariable()), 1238 VarLoc->DL, Order, VarLoc->Values, VarLoc->Expr)) { 1239 DanglingDebugInfoMap[VarLoc->Values.getVariableLocationOp(0)].emplace_back( 1240 VarLoc, Order); 1241 } 1242 } 1243 1244 void SelectionDAGBuilder::addDanglingDebugInfo(const DbgValueInst *DI, 1245 unsigned Order) { 1246 // We treat variadic dbg_values differently at this stage. 1247 if (!handleDanglingVariadicDebugInfo( 1248 DAG, DI->getVariable(), DI->getDebugLoc(), Order, 1249 DI->getWrappedLocation(), DI->getExpression())) { 1250 // TODO: Dangling debug info will eventually either be resolved or produce 1251 // an Undef DBG_VALUE. However in the resolution case, a gap may appear 1252 // between the original dbg.value location and its resolved DBG_VALUE, 1253 // which we should ideally fill with an extra Undef DBG_VALUE. 1254 assert(DI->getNumVariableLocationOps() == 1 && 1255 "DbgValueInst without an ArgList should have a single location " 1256 "operand."); 1257 DanglingDebugInfoMap[DI->getValue(0)].emplace_back(DI, Order); 1258 } 1259 } 1260 1261 void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable, 1262 const DIExpression *Expr) { 1263 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) { 1264 DIVariable *DanglingVariable = DDI.getVariable(DAG.getFunctionVarLocs()); 1265 DIExpression *DanglingExpr = DDI.getExpression(); 1266 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) { 1267 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << printDDI(DDI) 1268 << "\n"); 1269 return true; 1270 } 1271 return false; 1272 }; 1273 1274 for (auto &DDIMI : DanglingDebugInfoMap) { 1275 DanglingDebugInfoVector &DDIV = DDIMI.second; 1276 1277 // If debug info is to be dropped, run it through final checks to see 1278 // whether it can be salvaged. 1279 for (auto &DDI : DDIV) 1280 if (isMatchingDbgValue(DDI)) 1281 salvageUnresolvedDbgValue(DDI); 1282 1283 erase_if(DDIV, isMatchingDbgValue); 1284 } 1285 } 1286 1287 // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, 1288 // generate the debug data structures now that we've seen its definition. 1289 void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, 1290 SDValue Val) { 1291 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V); 1292 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end()) 1293 return; 1294 1295 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second; 1296 for (auto &DDI : DDIV) { 1297 DebugLoc DL = DDI.getDebugLoc(); 1298 unsigned ValSDNodeOrder = Val.getNode()->getIROrder(); 1299 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder(); 1300 DILocalVariable *Variable = DDI.getVariable(DAG.getFunctionVarLocs()); 1301 DIExpression *Expr = DDI.getExpression(); 1302 assert(Variable->isValidLocationForIntrinsic(DL) && 1303 "Expected inlined-at fields to agree"); 1304 SDDbgValue *SDV; 1305 if (Val.getNode()) { 1306 // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a 1307 // FuncArgumentDbgValue (it would be hoisted to the function entry, and if 1308 // we couldn't resolve it directly when examining the DbgValue intrinsic 1309 // in the first place we should not be more successful here). Unless we 1310 // have some test case that prove this to be correct we should avoid 1311 // calling EmitFuncArgumentDbgValue here. 1312 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, DL, 1313 FuncArgumentDbgValueKind::Value, Val)) { 1314 LLVM_DEBUG(dbgs() << "Resolve dangling debug info for " << printDDI(DDI) 1315 << "\n"); 1316 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump()); 1317 // Increase the SDNodeOrder for the DbgValue here to make sure it is 1318 // inserted after the definition of Val when emitting the instructions 1319 // after ISel. An alternative could be to teach 1320 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly. 1321 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() 1322 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to " 1323 << ValSDNodeOrder << "\n"); 1324 SDV = getDbgValue(Val, Variable, Expr, DL, 1325 std::max(DbgSDNodeOrder, ValSDNodeOrder)); 1326 DAG.AddDbgValue(SDV, false); 1327 } else 1328 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " 1329 << printDDI(DDI) << " in EmitFuncArgumentDbgValue\n"); 1330 } else { 1331 LLVM_DEBUG(dbgs() << "Dropping debug info for " << printDDI(DDI) << "\n"); 1332 auto Undef = UndefValue::get(V->getType()); 1333 auto SDV = 1334 DAG.getConstantDbgValue(Variable, Expr, Undef, DL, DbgSDNodeOrder); 1335 DAG.AddDbgValue(SDV, false); 1336 } 1337 } 1338 DDIV.clear(); 1339 } 1340 1341 void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) { 1342 // TODO: For the variadic implementation, instead of only checking the fail 1343 // state of `handleDebugValue`, we need know specifically which values were 1344 // invalid, so that we attempt to salvage only those values when processing 1345 // a DIArgList. 1346 Value *V = DDI.getVariableLocationOp(0); 1347 Value *OrigV = V; 1348 DILocalVariable *Var = DDI.getVariable(DAG.getFunctionVarLocs()); 1349 DIExpression *Expr = DDI.getExpression(); 1350 DebugLoc DL = DDI.getDebugLoc(); 1351 unsigned SDOrder = DDI.getSDNodeOrder(); 1352 1353 // Currently we consider only dbg.value intrinsics -- we tell the salvager 1354 // that DW_OP_stack_value is desired. 1355 bool StackValue = true; 1356 1357 // Can this Value can be encoded without any further work? 1358 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) 1359 return; 1360 1361 // Attempt to salvage back through as many instructions as possible. Bail if 1362 // a non-instruction is seen, such as a constant expression or global 1363 // variable. FIXME: Further work could recover those too. 1364 while (isa<Instruction>(V)) { 1365 Instruction &VAsInst = *cast<Instruction>(V); 1366 // Temporary "0", awaiting real implementation. 1367 SmallVector<uint64_t, 16> Ops; 1368 SmallVector<Value *, 4> AdditionalValues; 1369 V = salvageDebugInfoImpl(VAsInst, Expr->getNumLocationOperands(), Ops, 1370 AdditionalValues); 1371 // If we cannot salvage any further, and haven't yet found a suitable debug 1372 // expression, bail out. 1373 if (!V) 1374 break; 1375 1376 // TODO: If AdditionalValues isn't empty, then the salvage can only be 1377 // represented with a DBG_VALUE_LIST, so we give up. When we have support 1378 // here for variadic dbg_values, remove that condition. 1379 if (!AdditionalValues.empty()) 1380 break; 1381 1382 // New value and expr now represent this debuginfo. 1383 Expr = DIExpression::appendOpsToArg(Expr, Ops, 0, StackValue); 1384 1385 // Some kind of simplification occurred: check whether the operand of the 1386 // salvaged debug expression can be encoded in this DAG. 1387 if (handleDebugValue(V, Var, Expr, DL, SDOrder, /*IsVariadic=*/false)) { 1388 LLVM_DEBUG( 1389 dbgs() << "Salvaged debug location info for:\n " << *Var << "\n" 1390 << *OrigV << "\nBy stripping back to:\n " << *V << "\n"); 1391 return; 1392 } 1393 } 1394 1395 // This was the final opportunity to salvage this debug information, and it 1396 // couldn't be done. Place an undef DBG_VALUE at this location to terminate 1397 // any earlier variable location. 1398 assert(OrigV && "V shouldn't be null"); 1399 auto *Undef = UndefValue::get(OrigV->getType()); 1400 auto *SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder); 1401 DAG.AddDbgValue(SDV, false); 1402 LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << printDDI(DDI) 1403 << "\n"); 1404 } 1405 1406 bool SelectionDAGBuilder::handleDebugValue(ArrayRef<const Value *> Values, 1407 DILocalVariable *Var, 1408 DIExpression *Expr, DebugLoc DbgLoc, 1409 unsigned Order, bool IsVariadic) { 1410 if (Values.empty()) 1411 return true; 1412 SmallVector<SDDbgOperand> LocationOps; 1413 SmallVector<SDNode *> Dependencies; 1414 for (const Value *V : Values) { 1415 // Constant value. 1416 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V) || 1417 isa<ConstantPointerNull>(V)) { 1418 LocationOps.emplace_back(SDDbgOperand::fromConst(V)); 1419 continue; 1420 } 1421 1422 // Look through IntToPtr constants. 1423 if (auto *CE = dyn_cast<ConstantExpr>(V)) 1424 if (CE->getOpcode() == Instruction::IntToPtr) { 1425 LocationOps.emplace_back(SDDbgOperand::fromConst(CE->getOperand(0))); 1426 continue; 1427 } 1428 1429 // If the Value is a frame index, we can create a FrameIndex debug value 1430 // without relying on the DAG at all. 1431 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 1432 auto SI = FuncInfo.StaticAllocaMap.find(AI); 1433 if (SI != FuncInfo.StaticAllocaMap.end()) { 1434 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(SI->second)); 1435 continue; 1436 } 1437 } 1438 1439 // Do not use getValue() in here; we don't want to generate code at 1440 // this point if it hasn't been done yet. 1441 SDValue N = NodeMap[V]; 1442 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map. 1443 N = UnusedArgNodeMap[V]; 1444 if (N.getNode()) { 1445 // Only emit func arg dbg value for non-variadic dbg.values for now. 1446 if (!IsVariadic && 1447 EmitFuncArgumentDbgValue(V, Var, Expr, DbgLoc, 1448 FuncArgumentDbgValueKind::Value, N)) 1449 return true; 1450 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { 1451 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can 1452 // describe stack slot locations. 1453 // 1454 // Consider "int x = 0; int *px = &x;". There are two kinds of 1455 // interesting debug values here after optimization: 1456 // 1457 // dbg.value(i32* %px, !"int *px", !DIExpression()), and 1458 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) 1459 // 1460 // Both describe the direct values of their associated variables. 1461 Dependencies.push_back(N.getNode()); 1462 LocationOps.emplace_back(SDDbgOperand::fromFrameIdx(FISDN->getIndex())); 1463 continue; 1464 } 1465 LocationOps.emplace_back( 1466 SDDbgOperand::fromNode(N.getNode(), N.getResNo())); 1467 continue; 1468 } 1469 1470 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 1471 // Special rules apply for the first dbg.values of parameter variables in a 1472 // function. Identify them by the fact they reference Argument Values, that 1473 // they're parameters, and they are parameters of the current function. We 1474 // need to let them dangle until they get an SDNode. 1475 bool IsParamOfFunc = 1476 isa<Argument>(V) && Var->isParameter() && !DbgLoc.getInlinedAt(); 1477 if (IsParamOfFunc) 1478 return false; 1479 1480 // The value is not used in this block yet (or it would have an SDNode). 1481 // We still want the value to appear for the user if possible -- if it has 1482 // an associated VReg, we can refer to that instead. 1483 auto VMI = FuncInfo.ValueMap.find(V); 1484 if (VMI != FuncInfo.ValueMap.end()) { 1485 unsigned Reg = VMI->second; 1486 // If this is a PHI node, it may be split up into several MI PHI nodes 1487 // (in FunctionLoweringInfo::set). 1488 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, 1489 V->getType(), std::nullopt); 1490 if (RFV.occupiesMultipleRegs()) { 1491 // FIXME: We could potentially support variadic dbg_values here. 1492 if (IsVariadic) 1493 return false; 1494 unsigned Offset = 0; 1495 unsigned BitsToDescribe = 0; 1496 if (auto VarSize = Var->getSizeInBits()) 1497 BitsToDescribe = *VarSize; 1498 if (auto Fragment = Expr->getFragmentInfo()) 1499 BitsToDescribe = Fragment->SizeInBits; 1500 for (const auto &RegAndSize : RFV.getRegsAndSizes()) { 1501 // Bail out if all bits are described already. 1502 if (Offset >= BitsToDescribe) 1503 break; 1504 // TODO: handle scalable vectors. 1505 unsigned RegisterSize = RegAndSize.second; 1506 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe) 1507 ? BitsToDescribe - Offset 1508 : RegisterSize; 1509 auto FragmentExpr = DIExpression::createFragmentExpression( 1510 Expr, Offset, FragmentSize); 1511 if (!FragmentExpr) 1512 continue; 1513 SDDbgValue *SDV = DAG.getVRegDbgValue( 1514 Var, *FragmentExpr, RegAndSize.first, false, DbgLoc, SDNodeOrder); 1515 DAG.AddDbgValue(SDV, false); 1516 Offset += RegisterSize; 1517 } 1518 return true; 1519 } 1520 // We can use simple vreg locations for variadic dbg_values as well. 1521 LocationOps.emplace_back(SDDbgOperand::fromVReg(Reg)); 1522 continue; 1523 } 1524 // We failed to create a SDDbgOperand for V. 1525 return false; 1526 } 1527 1528 // We have created a SDDbgOperand for each Value in Values. 1529 // Should use Order instead of SDNodeOrder? 1530 assert(!LocationOps.empty()); 1531 SDDbgValue *SDV = DAG.getDbgValueList(Var, Expr, LocationOps, Dependencies, 1532 /*IsIndirect=*/false, DbgLoc, 1533 SDNodeOrder, IsVariadic); 1534 DAG.AddDbgValue(SDV, /*isParameter=*/false); 1535 return true; 1536 } 1537 1538 void SelectionDAGBuilder::resolveOrClearDbgInfo() { 1539 // Try to fixup any remaining dangling debug info -- and drop it if we can't. 1540 for (auto &Pair : DanglingDebugInfoMap) 1541 for (auto &DDI : Pair.second) 1542 salvageUnresolvedDbgValue(DDI); 1543 clearDanglingDebugInfo(); 1544 } 1545 1546 /// getCopyFromRegs - If there was virtual register allocated for the value V 1547 /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. 1548 SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { 1549 DenseMap<const Value *, Register>::iterator It = FuncInfo.ValueMap.find(V); 1550 SDValue Result; 1551 1552 if (It != FuncInfo.ValueMap.end()) { 1553 Register InReg = It->second; 1554 1555 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), 1556 DAG.getDataLayout(), InReg, Ty, 1557 std::nullopt); // This is not an ABI copy. 1558 SDValue Chain = DAG.getEntryNode(); 1559 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, 1560 V); 1561 resolveDanglingDebugInfo(V, Result); 1562 } 1563 1564 return Result; 1565 } 1566 1567 /// getValue - Return an SDValue for the given Value. 1568 SDValue SelectionDAGBuilder::getValue(const Value *V) { 1569 // If we already have an SDValue for this value, use it. It's important 1570 // to do this first, so that we don't create a CopyFromReg if we already 1571 // have a regular SDValue. 1572 SDValue &N = NodeMap[V]; 1573 if (N.getNode()) return N; 1574 1575 // If there's a virtual register allocated and initialized for this 1576 // value, use it. 1577 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType())) 1578 return copyFromReg; 1579 1580 // Otherwise create a new SDValue and remember it. 1581 SDValue Val = getValueImpl(V); 1582 NodeMap[V] = Val; 1583 resolveDanglingDebugInfo(V, Val); 1584 return Val; 1585 } 1586 1587 /// getNonRegisterValue - Return an SDValue for the given Value, but 1588 /// don't look in FuncInfo.ValueMap for a virtual register. 1589 SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) { 1590 // If we already have an SDValue for this value, use it. 1591 SDValue &N = NodeMap[V]; 1592 if (N.getNode()) { 1593 if (isIntOrFPConstant(N)) { 1594 // Remove the debug location from the node as the node is about to be used 1595 // in a location which may differ from the original debug location. This 1596 // is relevant to Constant and ConstantFP nodes because they can appear 1597 // as constant expressions inside PHI nodes. 1598 N->setDebugLoc(DebugLoc()); 1599 } 1600 return N; 1601 } 1602 1603 // Otherwise create a new SDValue and remember it. 1604 SDValue Val = getValueImpl(V); 1605 NodeMap[V] = Val; 1606 resolveDanglingDebugInfo(V, Val); 1607 return Val; 1608 } 1609 1610 /// getValueImpl - Helper function for getValue and getNonRegisterValue. 1611 /// Create an SDValue for the given value. 1612 SDValue SelectionDAGBuilder::getValueImpl(const Value *V) { 1613 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 1614 1615 if (const Constant *C = dyn_cast<Constant>(V)) { 1616 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true); 1617 1618 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C)) 1619 return DAG.getConstant(*CI, getCurSDLoc(), VT); 1620 1621 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C)) 1622 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT); 1623 1624 if (isa<ConstantPointerNull>(C)) { 1625 unsigned AS = V->getType()->getPointerAddressSpace(); 1626 return DAG.getConstant(0, getCurSDLoc(), 1627 TLI.getPointerTy(DAG.getDataLayout(), AS)); 1628 } 1629 1630 if (match(C, m_VScale())) 1631 return DAG.getVScale(getCurSDLoc(), VT, APInt(VT.getSizeInBits(), 1)); 1632 1633 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C)) 1634 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT); 1635 1636 if (isa<UndefValue>(C) && !V->getType()->isAggregateType()) 1637 return DAG.getUNDEF(VT); 1638 1639 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) { 1640 visit(CE->getOpcode(), *CE); 1641 SDValue N1 = NodeMap[V]; 1642 assert(N1.getNode() && "visit didn't populate the NodeMap!"); 1643 return N1; 1644 } 1645 1646 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) { 1647 SmallVector<SDValue, 4> Constants; 1648 for (const Use &U : C->operands()) { 1649 SDNode *Val = getValue(U).getNode(); 1650 // If the operand is an empty aggregate, there are no values. 1651 if (!Val) continue; 1652 // Add each leaf value from the operand to the Constants list 1653 // to form a flattened list of all the values. 1654 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1655 Constants.push_back(SDValue(Val, i)); 1656 } 1657 1658 return DAG.getMergeValues(Constants, getCurSDLoc()); 1659 } 1660 1661 if (const ConstantDataSequential *CDS = 1662 dyn_cast<ConstantDataSequential>(C)) { 1663 SmallVector<SDValue, 4> Ops; 1664 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) { 1665 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode(); 1666 // Add each leaf value from the operand to the Constants list 1667 // to form a flattened list of all the values. 1668 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i) 1669 Ops.push_back(SDValue(Val, i)); 1670 } 1671 1672 if (isa<ArrayType>(CDS->getType())) 1673 return DAG.getMergeValues(Ops, getCurSDLoc()); 1674 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); 1675 } 1676 1677 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) { 1678 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && 1679 "Unknown struct or array constant!"); 1680 1681 SmallVector<EVT, 4> ValueVTs; 1682 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs); 1683 unsigned NumElts = ValueVTs.size(); 1684 if (NumElts == 0) 1685 return SDValue(); // empty struct 1686 SmallVector<SDValue, 4> Constants(NumElts); 1687 for (unsigned i = 0; i != NumElts; ++i) { 1688 EVT EltVT = ValueVTs[i]; 1689 if (isa<UndefValue>(C)) 1690 Constants[i] = DAG.getUNDEF(EltVT); 1691 else if (EltVT.isFloatingPoint()) 1692 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT); 1693 else 1694 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT); 1695 } 1696 1697 return DAG.getMergeValues(Constants, getCurSDLoc()); 1698 } 1699 1700 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C)) 1701 return DAG.getBlockAddress(BA, VT); 1702 1703 if (const auto *Equiv = dyn_cast<DSOLocalEquivalent>(C)) 1704 return getValue(Equiv->getGlobalValue()); 1705 1706 if (const auto *NC = dyn_cast<NoCFIValue>(C)) 1707 return getValue(NC->getGlobalValue()); 1708 1709 VectorType *VecTy = cast<VectorType>(V->getType()); 1710 1711 // Now that we know the number and type of the elements, get that number of 1712 // elements into the Ops array based on what kind of constant it is. 1713 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) { 1714 SmallVector<SDValue, 16> Ops; 1715 unsigned NumElements = cast<FixedVectorType>(VecTy)->getNumElements(); 1716 for (unsigned i = 0; i != NumElements; ++i) 1717 Ops.push_back(getValue(CV->getOperand(i))); 1718 1719 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops); 1720 } 1721 1722 if (isa<ConstantAggregateZero>(C)) { 1723 EVT EltVT = 1724 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType()); 1725 1726 SDValue Op; 1727 if (EltVT.isFloatingPoint()) 1728 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT); 1729 else 1730 Op = DAG.getConstant(0, getCurSDLoc(), EltVT); 1731 1732 return NodeMap[V] = DAG.getSplat(VT, getCurSDLoc(), Op); 1733 } 1734 1735 llvm_unreachable("Unknown vector constant"); 1736 } 1737 1738 // If this is a static alloca, generate it as the frameindex instead of 1739 // computation. 1740 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) { 1741 DenseMap<const AllocaInst*, int>::iterator SI = 1742 FuncInfo.StaticAllocaMap.find(AI); 1743 if (SI != FuncInfo.StaticAllocaMap.end()) 1744 return DAG.getFrameIndex( 1745 SI->second, TLI.getValueType(DAG.getDataLayout(), AI->getType())); 1746 } 1747 1748 // If this is an instruction which fast-isel has deferred, select it now. 1749 if (const Instruction *Inst = dyn_cast<Instruction>(V)) { 1750 Register InReg = FuncInfo.InitializeRegForValue(Inst); 1751 1752 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg, 1753 Inst->getType(), std::nullopt); 1754 SDValue Chain = DAG.getEntryNode(); 1755 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V); 1756 } 1757 1758 if (const MetadataAsValue *MD = dyn_cast<MetadataAsValue>(V)) 1759 return DAG.getMDNode(cast<MDNode>(MD->getMetadata())); 1760 1761 if (const auto *BB = dyn_cast<BasicBlock>(V)) 1762 return DAG.getBasicBlock(FuncInfo.MBBMap[BB]); 1763 1764 llvm_unreachable("Can't get register for value!"); 1765 } 1766 1767 void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) { 1768 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); 1769 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX; 1770 bool IsCoreCLR = Pers == EHPersonality::CoreCLR; 1771 bool IsSEH = isAsynchronousEHPersonality(Pers); 1772 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB; 1773 if (!IsSEH) 1774 CatchPadMBB->setIsEHScopeEntry(); 1775 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues. 1776 if (IsMSVCCXX || IsCoreCLR) 1777 CatchPadMBB->setIsEHFuncletEntry(); 1778 } 1779 1780 void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) { 1781 // Update machine-CFG edge. 1782 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()]; 1783 FuncInfo.MBB->addSuccessor(TargetMBB); 1784 TargetMBB->setIsEHCatchretTarget(true); 1785 DAG.getMachineFunction().setHasEHCatchret(true); 1786 1787 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); 1788 bool IsSEH = isAsynchronousEHPersonality(Pers); 1789 if (IsSEH) { 1790 // If this is not a fall-through branch or optimizations are switched off, 1791 // emit the branch. 1792 if (TargetMBB != NextBlock(FuncInfo.MBB) || 1793 TM.getOptLevel() == CodeGenOpt::None) 1794 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, 1795 getControlRoot(), DAG.getBasicBlock(TargetMBB))); 1796 return; 1797 } 1798 1799 // Figure out the funclet membership for the catchret's successor. 1800 // This will be used by the FuncletLayout pass to determine how to order the 1801 // BB's. 1802 // A 'catchret' returns to the outer scope's color. 1803 Value *ParentPad = I.getCatchSwitchParentPad(); 1804 const BasicBlock *SuccessorColor; 1805 if (isa<ConstantTokenNone>(ParentPad)) 1806 SuccessorColor = &FuncInfo.Fn->getEntryBlock(); 1807 else 1808 SuccessorColor = cast<Instruction>(ParentPad)->getParent(); 1809 assert(SuccessorColor && "No parent funclet for catchret!"); 1810 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor]; 1811 assert(SuccessorColorMBB && "No MBB for SuccessorColor!"); 1812 1813 // Create the terminator node. 1814 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other, 1815 getControlRoot(), DAG.getBasicBlock(TargetMBB), 1816 DAG.getBasicBlock(SuccessorColorMBB)); 1817 DAG.setRoot(Ret); 1818 } 1819 1820 void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) { 1821 // Don't emit any special code for the cleanuppad instruction. It just marks 1822 // the start of an EH scope/funclet. 1823 FuncInfo.MBB->setIsEHScopeEntry(); 1824 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); 1825 if (Pers != EHPersonality::Wasm_CXX) { 1826 FuncInfo.MBB->setIsEHFuncletEntry(); 1827 FuncInfo.MBB->setIsCleanupFuncletEntry(); 1828 } 1829 } 1830 1831 // In wasm EH, even though a catchpad may not catch an exception if a tag does 1832 // not match, it is OK to add only the first unwind destination catchpad to the 1833 // successors, because there will be at least one invoke instruction within the 1834 // catch scope that points to the next unwind destination, if one exists, so 1835 // CFGSort cannot mess up with BB sorting order. 1836 // (All catchpads with 'catch (type)' clauses have a 'llvm.rethrow' intrinsic 1837 // call within them, and catchpads only consisting of 'catch (...)' have a 1838 // '__cxa_end_catch' call within them, both of which generate invokes in case 1839 // the next unwind destination exists, i.e., the next unwind destination is not 1840 // the caller.) 1841 // 1842 // Having at most one EH pad successor is also simpler and helps later 1843 // transformations. 1844 // 1845 // For example, 1846 // current: 1847 // invoke void @foo to ... unwind label %catch.dispatch 1848 // catch.dispatch: 1849 // %0 = catchswitch within ... [label %catch.start] unwind label %next 1850 // catch.start: 1851 // ... 1852 // ... in this BB or some other child BB dominated by this BB there will be an 1853 // invoke that points to 'next' BB as an unwind destination 1854 // 1855 // next: ; We don't need to add this to 'current' BB's successor 1856 // ... 1857 static void findWasmUnwindDestinations( 1858 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, 1859 BranchProbability Prob, 1860 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> 1861 &UnwindDests) { 1862 while (EHPadBB) { 1863 const Instruction *Pad = EHPadBB->getFirstNonPHI(); 1864 if (isa<CleanupPadInst>(Pad)) { 1865 // Stop on cleanup pads. 1866 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); 1867 UnwindDests.back().first->setIsEHScopeEntry(); 1868 break; 1869 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { 1870 // Add the catchpad handlers to the possible destinations. We don't 1871 // continue to the unwind destination of the catchswitch for wasm. 1872 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { 1873 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); 1874 UnwindDests.back().first->setIsEHScopeEntry(); 1875 } 1876 break; 1877 } else { 1878 continue; 1879 } 1880 } 1881 } 1882 1883 /// When an invoke or a cleanupret unwinds to the next EH pad, there are 1884 /// many places it could ultimately go. In the IR, we have a single unwind 1885 /// destination, but in the machine CFG, we enumerate all the possible blocks. 1886 /// This function skips over imaginary basic blocks that hold catchswitch 1887 /// instructions, and finds all the "real" machine 1888 /// basic block destinations. As those destinations may not be successors of 1889 /// EHPadBB, here we also calculate the edge probability to those destinations. 1890 /// The passed-in Prob is the edge probability to EHPadBB. 1891 static void findUnwindDestinations( 1892 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, 1893 BranchProbability Prob, 1894 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>> 1895 &UnwindDests) { 1896 EHPersonality Personality = 1897 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); 1898 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX; 1899 bool IsCoreCLR = Personality == EHPersonality::CoreCLR; 1900 bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; 1901 bool IsSEH = isAsynchronousEHPersonality(Personality); 1902 1903 if (IsWasmCXX) { 1904 findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests); 1905 assert(UnwindDests.size() <= 1 && 1906 "There should be at most one unwind destination for wasm"); 1907 return; 1908 } 1909 1910 while (EHPadBB) { 1911 const Instruction *Pad = EHPadBB->getFirstNonPHI(); 1912 BasicBlock *NewEHPadBB = nullptr; 1913 if (isa<LandingPadInst>(Pad)) { 1914 // Stop on landingpads. They are not funclets. 1915 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); 1916 break; 1917 } else if (isa<CleanupPadInst>(Pad)) { 1918 // Stop on cleanup pads. Cleanups are always funclet entries for all known 1919 // personalities. 1920 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); 1921 UnwindDests.back().first->setIsEHScopeEntry(); 1922 UnwindDests.back().first->setIsEHFuncletEntry(); 1923 break; 1924 } else if (const auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) { 1925 // Add the catchpad handlers to the possible destinations. 1926 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { 1927 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); 1928 // For MSVC++ and the CLR, catchblocks are funclets and need prologues. 1929 if (IsMSVCCXX || IsCoreCLR) 1930 UnwindDests.back().first->setIsEHFuncletEntry(); 1931 if (!IsSEH) 1932 UnwindDests.back().first->setIsEHScopeEntry(); 1933 } 1934 NewEHPadBB = CatchSwitch->getUnwindDest(); 1935 } else { 1936 continue; 1937 } 1938 1939 BranchProbabilityInfo *BPI = FuncInfo.BPI; 1940 if (BPI && NewEHPadBB) 1941 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB); 1942 EHPadBB = NewEHPadBB; 1943 } 1944 } 1945 1946 void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) { 1947 // Update successor info. 1948 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; 1949 auto UnwindDest = I.getUnwindDest(); 1950 BranchProbabilityInfo *BPI = FuncInfo.BPI; 1951 BranchProbability UnwindDestProb = 1952 (BPI && UnwindDest) 1953 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest) 1954 : BranchProbability::getZero(); 1955 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests); 1956 for (auto &UnwindDest : UnwindDests) { 1957 UnwindDest.first->setIsEHPad(); 1958 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second); 1959 } 1960 FuncInfo.MBB->normalizeSuccProbs(); 1961 1962 // Create the terminator node. 1963 SDValue Ret = 1964 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot()); 1965 DAG.setRoot(Ret); 1966 } 1967 1968 void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) { 1969 report_fatal_error("visitCatchSwitch not yet implemented!"); 1970 } 1971 1972 void SelectionDAGBuilder::visitRet(const ReturnInst &I) { 1973 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 1974 auto &DL = DAG.getDataLayout(); 1975 SDValue Chain = getControlRoot(); 1976 SmallVector<ISD::OutputArg, 8> Outs; 1977 SmallVector<SDValue, 8> OutVals; 1978 1979 // Calls to @llvm.experimental.deoptimize don't generate a return value, so 1980 // lower 1981 // 1982 // %val = call <ty> @llvm.experimental.deoptimize() 1983 // ret <ty> %val 1984 // 1985 // differently. 1986 if (I.getParent()->getTerminatingDeoptimizeCall()) { 1987 LowerDeoptimizingReturn(); 1988 return; 1989 } 1990 1991 if (!FuncInfo.CanLowerReturn) { 1992 unsigned DemoteReg = FuncInfo.DemoteRegister; 1993 const Function *F = I.getParent()->getParent(); 1994 1995 // Emit a store of the return value through the virtual register. 1996 // Leave Outs empty so that LowerReturn won't try to load return 1997 // registers the usual way. 1998 SmallVector<EVT, 1> PtrValueVTs; 1999 ComputeValueVTs(TLI, DL, 2000 F->getReturnType()->getPointerTo( 2001 DAG.getDataLayout().getAllocaAddrSpace()), 2002 PtrValueVTs); 2003 2004 SDValue RetPtr = 2005 DAG.getCopyFromReg(Chain, getCurSDLoc(), DemoteReg, PtrValueVTs[0]); 2006 SDValue RetOp = getValue(I.getOperand(0)); 2007 2008 SmallVector<EVT, 4> ValueVTs, MemVTs; 2009 SmallVector<uint64_t, 4> Offsets; 2010 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs, 2011 &Offsets); 2012 unsigned NumValues = ValueVTs.size(); 2013 2014 SmallVector<SDValue, 4> Chains(NumValues); 2015 Align BaseAlign = DL.getPrefTypeAlign(I.getOperand(0)->getType()); 2016 for (unsigned i = 0; i != NumValues; ++i) { 2017 // An aggregate return value cannot wrap around the address space, so 2018 // offsets to its parts don't wrap either. 2019 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, 2020 TypeSize::Fixed(Offsets[i])); 2021 2022 SDValue Val = RetOp.getValue(RetOp.getResNo() + i); 2023 if (MemVTs[i] != ValueVTs[i]) 2024 Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]); 2025 Chains[i] = DAG.getStore( 2026 Chain, getCurSDLoc(), Val, 2027 // FIXME: better loc info would be nice. 2028 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), 2029 commonAlignment(BaseAlign, Offsets[i])); 2030 } 2031 2032 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), 2033 MVT::Other, Chains); 2034 } else if (I.getNumOperands() != 0) { 2035 SmallVector<EVT, 4> ValueVTs; 2036 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs); 2037 unsigned NumValues = ValueVTs.size(); 2038 if (NumValues) { 2039 SDValue RetOp = getValue(I.getOperand(0)); 2040 2041 const Function *F = I.getParent()->getParent(); 2042 2043 bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( 2044 I.getOperand(0)->getType(), F->getCallingConv(), 2045 /*IsVarArg*/ false, DL); 2046 2047 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 2048 if (F->getAttributes().hasRetAttr(Attribute::SExt)) 2049 ExtendKind = ISD::SIGN_EXTEND; 2050 else if (F->getAttributes().hasRetAttr(Attribute::ZExt)) 2051 ExtendKind = ISD::ZERO_EXTEND; 2052 2053 LLVMContext &Context = F->getContext(); 2054 bool RetInReg = F->getAttributes().hasRetAttr(Attribute::InReg); 2055 2056 for (unsigned j = 0; j != NumValues; ++j) { 2057 EVT VT = ValueVTs[j]; 2058 2059 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger()) 2060 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind); 2061 2062 CallingConv::ID CC = F->getCallingConv(); 2063 2064 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT); 2065 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT); 2066 SmallVector<SDValue, 4> Parts(NumParts); 2067 getCopyToParts(DAG, getCurSDLoc(), 2068 SDValue(RetOp.getNode(), RetOp.getResNo() + j), 2069 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind); 2070 2071 // 'inreg' on function refers to return value 2072 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 2073 if (RetInReg) 2074 Flags.setInReg(); 2075 2076 if (I.getOperand(0)->getType()->isPointerTy()) { 2077 Flags.setPointer(); 2078 Flags.setPointerAddrSpace( 2079 cast<PointerType>(I.getOperand(0)->getType())->getAddressSpace()); 2080 } 2081 2082 if (NeedsRegBlock) { 2083 Flags.setInConsecutiveRegs(); 2084 if (j == NumValues - 1) 2085 Flags.setInConsecutiveRegsLast(); 2086 } 2087 2088 // Propagate extension type if any 2089 if (ExtendKind == ISD::SIGN_EXTEND) 2090 Flags.setSExt(); 2091 else if (ExtendKind == ISD::ZERO_EXTEND) 2092 Flags.setZExt(); 2093 2094 for (unsigned i = 0; i < NumParts; ++i) { 2095 Outs.push_back(ISD::OutputArg(Flags, 2096 Parts[i].getValueType().getSimpleVT(), 2097 VT, /*isfixed=*/true, 0, 0)); 2098 OutVals.push_back(Parts[i]); 2099 } 2100 } 2101 } 2102 } 2103 2104 // Push in swifterror virtual register as the last element of Outs. This makes 2105 // sure swifterror virtual register will be returned in the swifterror 2106 // physical register. 2107 const Function *F = I.getParent()->getParent(); 2108 if (TLI.supportSwiftError() && 2109 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { 2110 assert(SwiftError.getFunctionArg() && "Need a swift error argument"); 2111 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); 2112 Flags.setSwiftError(); 2113 Outs.push_back(ISD::OutputArg( 2114 Flags, /*vt=*/TLI.getPointerTy(DL), /*argvt=*/EVT(TLI.getPointerTy(DL)), 2115 /*isfixed=*/true, /*origidx=*/1, /*partOffs=*/0)); 2116 // Create SDNode for the swifterror virtual register. 2117 OutVals.push_back( 2118 DAG.getRegister(SwiftError.getOrCreateVRegUseAt( 2119 &I, FuncInfo.MBB, SwiftError.getFunctionArg()), 2120 EVT(TLI.getPointerTy(DL)))); 2121 } 2122 2123 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg(); 2124 CallingConv::ID CallConv = 2125 DAG.getMachineFunction().getFunction().getCallingConv(); 2126 Chain = DAG.getTargetLoweringInfo().LowerReturn( 2127 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG); 2128 2129 // Verify that the target's LowerReturn behaved as expected. 2130 assert(Chain.getNode() && Chain.getValueType() == MVT::Other && 2131 "LowerReturn didn't return a valid chain!"); 2132 2133 // Update the DAG with the new chain value resulting from return lowering. 2134 DAG.setRoot(Chain); 2135 } 2136 2137 /// CopyToExportRegsIfNeeded - If the given value has virtual registers 2138 /// created for it, emit nodes to copy the value into the virtual 2139 /// registers. 2140 void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) { 2141 // Skip empty types 2142 if (V->getType()->isEmptyTy()) 2143 return; 2144 2145 DenseMap<const Value *, Register>::iterator VMI = FuncInfo.ValueMap.find(V); 2146 if (VMI != FuncInfo.ValueMap.end()) { 2147 assert((!V->use_empty() || isa<CallBrInst>(V)) && 2148 "Unused value assigned virtual registers!"); 2149 CopyValueToVirtualRegister(V, VMI->second); 2150 } 2151 } 2152 2153 /// ExportFromCurrentBlock - If this condition isn't known to be exported from 2154 /// the current basic block, add it to ValueMap now so that we'll get a 2155 /// CopyTo/FromReg. 2156 void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) { 2157 // No need to export constants. 2158 if (!isa<Instruction>(V) && !isa<Argument>(V)) return; 2159 2160 // Already exported? 2161 if (FuncInfo.isExportedInst(V)) return; 2162 2163 Register Reg = FuncInfo.InitializeRegForValue(V); 2164 CopyValueToVirtualRegister(V, Reg); 2165 } 2166 2167 bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V, 2168 const BasicBlock *FromBB) { 2169 // The operands of the setcc have to be in this block. We don't know 2170 // how to export them from some other block. 2171 if (const Instruction *VI = dyn_cast<Instruction>(V)) { 2172 // Can export from current BB. 2173 if (VI->getParent() == FromBB) 2174 return true; 2175 2176 // Is already exported, noop. 2177 return FuncInfo.isExportedInst(V); 2178 } 2179 2180 // If this is an argument, we can export it if the BB is the entry block or 2181 // if it is already exported. 2182 if (isa<Argument>(V)) { 2183 if (FromBB->isEntryBlock()) 2184 return true; 2185 2186 // Otherwise, can only export this if it is already exported. 2187 return FuncInfo.isExportedInst(V); 2188 } 2189 2190 // Otherwise, constants can always be exported. 2191 return true; 2192 } 2193 2194 /// Return branch probability calculated by BranchProbabilityInfo for IR blocks. 2195 BranchProbability 2196 SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src, 2197 const MachineBasicBlock *Dst) const { 2198 BranchProbabilityInfo *BPI = FuncInfo.BPI; 2199 const BasicBlock *SrcBB = Src->getBasicBlock(); 2200 const BasicBlock *DstBB = Dst->getBasicBlock(); 2201 if (!BPI) { 2202 // If BPI is not available, set the default probability as 1 / N, where N is 2203 // the number of successors. 2204 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1); 2205 return BranchProbability(1, SuccSize); 2206 } 2207 return BPI->getEdgeProbability(SrcBB, DstBB); 2208 } 2209 2210 void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src, 2211 MachineBasicBlock *Dst, 2212 BranchProbability Prob) { 2213 if (!FuncInfo.BPI) 2214 Src->addSuccessorWithoutProb(Dst); 2215 else { 2216 if (Prob.isUnknown()) 2217 Prob = getEdgeProbability(Src, Dst); 2218 Src->addSuccessor(Dst, Prob); 2219 } 2220 } 2221 2222 static bool InBlock(const Value *V, const BasicBlock *BB) { 2223 if (const Instruction *I = dyn_cast<Instruction>(V)) 2224 return I->getParent() == BB; 2225 return true; 2226 } 2227 2228 /// EmitBranchForMergedCondition - Helper method for FindMergedConditions. 2229 /// This function emits a branch and is used at the leaves of an OR or an 2230 /// AND operator tree. 2231 void 2232 SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, 2233 MachineBasicBlock *TBB, 2234 MachineBasicBlock *FBB, 2235 MachineBasicBlock *CurBB, 2236 MachineBasicBlock *SwitchBB, 2237 BranchProbability TProb, 2238 BranchProbability FProb, 2239 bool InvertCond) { 2240 const BasicBlock *BB = CurBB->getBasicBlock(); 2241 2242 // If the leaf of the tree is a comparison, merge the condition into 2243 // the caseblock. 2244 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) { 2245 // The operands of the cmp have to be in this block. We don't know 2246 // how to export them from some other block. If this is the first block 2247 // of the sequence, no exporting is needed. 2248 if (CurBB == SwitchBB || 2249 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) && 2250 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) { 2251 ISD::CondCode Condition; 2252 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) { 2253 ICmpInst::Predicate Pred = 2254 InvertCond ? IC->getInversePredicate() : IC->getPredicate(); 2255 Condition = getICmpCondCode(Pred); 2256 } else { 2257 const FCmpInst *FC = cast<FCmpInst>(Cond); 2258 FCmpInst::Predicate Pred = 2259 InvertCond ? FC->getInversePredicate() : FC->getPredicate(); 2260 Condition = getFCmpCondCode(Pred); 2261 if (TM.Options.NoNaNsFPMath) 2262 Condition = getFCmpCodeWithoutNaN(Condition); 2263 } 2264 2265 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, 2266 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); 2267 SL->SwitchCases.push_back(CB); 2268 return; 2269 } 2270 } 2271 2272 // Create a CaseBlock record representing this branch. 2273 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; 2274 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), 2275 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); 2276 SL->SwitchCases.push_back(CB); 2277 } 2278 2279 void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, 2280 MachineBasicBlock *TBB, 2281 MachineBasicBlock *FBB, 2282 MachineBasicBlock *CurBB, 2283 MachineBasicBlock *SwitchBB, 2284 Instruction::BinaryOps Opc, 2285 BranchProbability TProb, 2286 BranchProbability FProb, 2287 bool InvertCond) { 2288 // Skip over not part of the tree and remember to invert op and operands at 2289 // next level. 2290 Value *NotCond; 2291 if (match(Cond, m_OneUse(m_Not(m_Value(NotCond)))) && 2292 InBlock(NotCond, CurBB->getBasicBlock())) { 2293 FindMergedConditions(NotCond, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb, 2294 !InvertCond); 2295 return; 2296 } 2297 2298 const Instruction *BOp = dyn_cast<Instruction>(Cond); 2299 const Value *BOpOp0, *BOpOp1; 2300 // Compute the effective opcode for Cond, taking into account whether it needs 2301 // to be inverted, e.g. 2302 // and (not (or A, B)), C 2303 // gets lowered as 2304 // and (and (not A, not B), C) 2305 Instruction::BinaryOps BOpc = (Instruction::BinaryOps)0; 2306 if (BOp) { 2307 BOpc = match(BOp, m_LogicalAnd(m_Value(BOpOp0), m_Value(BOpOp1))) 2308 ? Instruction::And 2309 : (match(BOp, m_LogicalOr(m_Value(BOpOp0), m_Value(BOpOp1))) 2310 ? Instruction::Or 2311 : (Instruction::BinaryOps)0); 2312 if (InvertCond) { 2313 if (BOpc == Instruction::And) 2314 BOpc = Instruction::Or; 2315 else if (BOpc == Instruction::Or) 2316 BOpc = Instruction::And; 2317 } 2318 } 2319 2320 // If this node is not part of the or/and tree, emit it as a branch. 2321 // Note that all nodes in the tree should have same opcode. 2322 bool BOpIsInOrAndTree = BOpc && BOpc == Opc && BOp->hasOneUse(); 2323 if (!BOpIsInOrAndTree || BOp->getParent() != CurBB->getBasicBlock() || 2324 !InBlock(BOpOp0, CurBB->getBasicBlock()) || 2325 !InBlock(BOpOp1, CurBB->getBasicBlock())) { 2326 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB, 2327 TProb, FProb, InvertCond); 2328 return; 2329 } 2330 2331 // Create TmpBB after CurBB. 2332 MachineFunction::iterator BBI(CurBB); 2333 MachineFunction &MF = DAG.getMachineFunction(); 2334 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock()); 2335 CurBB->getParent()->insert(++BBI, TmpBB); 2336 2337 if (Opc == Instruction::Or) { 2338 // Codegen X | Y as: 2339 // BB1: 2340 // jmp_if_X TBB 2341 // jmp TmpBB 2342 // TmpBB: 2343 // jmp_if_Y TBB 2344 // jmp FBB 2345 // 2346 2347 // We have flexibility in setting Prob for BB1 and Prob for TmpBB. 2348 // The requirement is that 2349 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB) 2350 // = TrueProb for original BB. 2351 // Assuming the original probabilities are A and B, one choice is to set 2352 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to 2353 // A/(1+B) and 2B/(1+B). This choice assumes that 2354 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB. 2355 // Another choice is to assume TrueProb for BB1 equals to TrueProb for 2356 // TmpBB, but the math is more complicated. 2357 2358 auto NewTrueProb = TProb / 2; 2359 auto NewFalseProb = TProb / 2 + FProb; 2360 // Emit the LHS condition. 2361 FindMergedConditions(BOpOp0, TBB, TmpBB, CurBB, SwitchBB, Opc, NewTrueProb, 2362 NewFalseProb, InvertCond); 2363 2364 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B). 2365 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb}; 2366 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); 2367 // Emit the RHS condition into TmpBB. 2368 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], 2369 Probs[1], InvertCond); 2370 } else { 2371 assert(Opc == Instruction::And && "Unknown merge op!"); 2372 // Codegen X & Y as: 2373 // BB1: 2374 // jmp_if_X TmpBB 2375 // jmp FBB 2376 // TmpBB: 2377 // jmp_if_Y TBB 2378 // jmp FBB 2379 // 2380 // This requires creation of TmpBB after CurBB. 2381 2382 // We have flexibility in setting Prob for BB1 and Prob for TmpBB. 2383 // The requirement is that 2384 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB) 2385 // = FalseProb for original BB. 2386 // Assuming the original probabilities are A and B, one choice is to set 2387 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to 2388 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 == 2389 // TrueProb for BB1 * FalseProb for TmpBB. 2390 2391 auto NewTrueProb = TProb + FProb / 2; 2392 auto NewFalseProb = FProb / 2; 2393 // Emit the LHS condition. 2394 FindMergedConditions(BOpOp0, TmpBB, FBB, CurBB, SwitchBB, Opc, NewTrueProb, 2395 NewFalseProb, InvertCond); 2396 2397 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A). 2398 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2}; 2399 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end()); 2400 // Emit the RHS condition into TmpBB. 2401 FindMergedConditions(BOpOp1, TBB, FBB, TmpBB, SwitchBB, Opc, Probs[0], 2402 Probs[1], InvertCond); 2403 } 2404 } 2405 2406 /// If the set of cases should be emitted as a series of branches, return true. 2407 /// If we should emit this as a bunch of and/or'd together conditions, return 2408 /// false. 2409 bool 2410 SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) { 2411 if (Cases.size() != 2) return true; 2412 2413 // If this is two comparisons of the same values or'd or and'd together, they 2414 // will get folded into a single comparison, so don't emit two blocks. 2415 if ((Cases[0].CmpLHS == Cases[1].CmpLHS && 2416 Cases[0].CmpRHS == Cases[1].CmpRHS) || 2417 (Cases[0].CmpRHS == Cases[1].CmpLHS && 2418 Cases[0].CmpLHS == Cases[1].CmpRHS)) { 2419 return false; 2420 } 2421 2422 // Handle: (X != null) | (Y != null) --> (X|Y) != 0 2423 // Handle: (X == null) & (Y == null) --> (X|Y) == 0 2424 if (Cases[0].CmpRHS == Cases[1].CmpRHS && 2425 Cases[0].CC == Cases[1].CC && 2426 isa<Constant>(Cases[0].CmpRHS) && 2427 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) { 2428 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB) 2429 return false; 2430 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB) 2431 return false; 2432 } 2433 2434 return true; 2435 } 2436 2437 void SelectionDAGBuilder::visitBr(const BranchInst &I) { 2438 MachineBasicBlock *BrMBB = FuncInfo.MBB; 2439 2440 // Update machine-CFG edges. 2441 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)]; 2442 2443 if (I.isUnconditional()) { 2444 // Update machine-CFG edges. 2445 BrMBB->addSuccessor(Succ0MBB); 2446 2447 // If this is not a fall-through branch or optimizations are switched off, 2448 // emit the branch. 2449 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None) 2450 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 2451 MVT::Other, getControlRoot(), 2452 DAG.getBasicBlock(Succ0MBB))); 2453 2454 return; 2455 } 2456 2457 // If this condition is one of the special cases we handle, do special stuff 2458 // now. 2459 const Value *CondVal = I.getCondition(); 2460 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)]; 2461 2462 // If this is a series of conditions that are or'd or and'd together, emit 2463 // this as a sequence of branches instead of setcc's with and/or operations. 2464 // As long as jumps are not expensive (exceptions for multi-use logic ops, 2465 // unpredictable branches, and vector extracts because those jumps are likely 2466 // expensive for any target), this should improve performance. 2467 // For example, instead of something like: 2468 // cmp A, B 2469 // C = seteq 2470 // cmp D, E 2471 // F = setle 2472 // or C, F 2473 // jnz foo 2474 // Emit: 2475 // cmp A, B 2476 // je foo 2477 // cmp D, E 2478 // jle foo 2479 const Instruction *BOp = dyn_cast<Instruction>(CondVal); 2480 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp && 2481 BOp->hasOneUse() && !I.hasMetadata(LLVMContext::MD_unpredictable)) { 2482 Value *Vec; 2483 const Value *BOp0, *BOp1; 2484 Instruction::BinaryOps Opcode = (Instruction::BinaryOps)0; 2485 if (match(BOp, m_LogicalAnd(m_Value(BOp0), m_Value(BOp1)))) 2486 Opcode = Instruction::And; 2487 else if (match(BOp, m_LogicalOr(m_Value(BOp0), m_Value(BOp1)))) 2488 Opcode = Instruction::Or; 2489 2490 if (Opcode && !(match(BOp0, m_ExtractElt(m_Value(Vec), m_Value())) && 2491 match(BOp1, m_ExtractElt(m_Specific(Vec), m_Value())))) { 2492 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB, Opcode, 2493 getEdgeProbability(BrMBB, Succ0MBB), 2494 getEdgeProbability(BrMBB, Succ1MBB), 2495 /*InvertCond=*/false); 2496 // If the compares in later blocks need to use values not currently 2497 // exported from this block, export them now. This block should always 2498 // be the first entry. 2499 assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); 2500 2501 // Allow some cases to be rejected. 2502 if (ShouldEmitAsBranches(SL->SwitchCases)) { 2503 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) { 2504 ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS); 2505 ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS); 2506 } 2507 2508 // Emit the branch for this block. 2509 visitSwitchCase(SL->SwitchCases[0], BrMBB); 2510 SL->SwitchCases.erase(SL->SwitchCases.begin()); 2511 return; 2512 } 2513 2514 // Okay, we decided not to do this, remove any inserted MBB's and clear 2515 // SwitchCases. 2516 for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) 2517 FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB); 2518 2519 SL->SwitchCases.clear(); 2520 } 2521 } 2522 2523 // Create a CaseBlock record representing this branch. 2524 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()), 2525 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc()); 2526 2527 // Use visitSwitchCase to actually insert the fast branch sequence for this 2528 // cond branch. 2529 visitSwitchCase(CB, BrMBB); 2530 } 2531 2532 /// visitSwitchCase - Emits the necessary code to represent a single node in 2533 /// the binary search tree resulting from lowering a switch instruction. 2534 void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, 2535 MachineBasicBlock *SwitchBB) { 2536 SDValue Cond; 2537 SDValue CondLHS = getValue(CB.CmpLHS); 2538 SDLoc dl = CB.DL; 2539 2540 if (CB.CC == ISD::SETTRUE) { 2541 // Branch or fall through to TrueBB. 2542 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); 2543 SwitchBB->normalizeSuccProbs(); 2544 if (CB.TrueBB != NextBlock(SwitchBB)) { 2545 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(), 2546 DAG.getBasicBlock(CB.TrueBB))); 2547 } 2548 return; 2549 } 2550 2551 auto &TLI = DAG.getTargetLoweringInfo(); 2552 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType()); 2553 2554 // Build the setcc now. 2555 if (!CB.CmpMHS) { 2556 // Fold "(X == true)" to X and "(X == false)" to !X to 2557 // handle common cases produced by branch lowering. 2558 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) && 2559 CB.CC == ISD::SETEQ) 2560 Cond = CondLHS; 2561 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) && 2562 CB.CC == ISD::SETEQ) { 2563 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); 2564 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); 2565 } else { 2566 SDValue CondRHS = getValue(CB.CmpRHS); 2567 2568 // If a pointer's DAG type is larger than its memory type then the DAG 2569 // values are zero-extended. This breaks signed comparisons so truncate 2570 // back to the underlying type before doing the compare. 2571 if (CondLHS.getValueType() != MemVT) { 2572 CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT); 2573 CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT); 2574 } 2575 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC); 2576 } 2577 } else { 2578 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); 2579 2580 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue(); 2581 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue(); 2582 2583 SDValue CmpOp = getValue(CB.CmpMHS); 2584 EVT VT = CmpOp.getValueType(); 2585 2586 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) { 2587 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT), 2588 ISD::SETLE); 2589 } else { 2590 SDValue SUB = DAG.getNode(ISD::SUB, dl, 2591 VT, CmpOp, DAG.getConstant(Low, dl, VT)); 2592 Cond = DAG.getSetCC(dl, MVT::i1, SUB, 2593 DAG.getConstant(High-Low, dl, VT), ISD::SETULE); 2594 } 2595 } 2596 2597 // Update successor info 2598 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); 2599 // TrueBB and FalseBB are always different unless the incoming IR is 2600 // degenerate. This only happens when running llc on weird IR. 2601 if (CB.TrueBB != CB.FalseBB) 2602 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb); 2603 SwitchBB->normalizeSuccProbs(); 2604 2605 // If the lhs block is the next block, invert the condition so that we can 2606 // fall through to the lhs instead of the rhs block. 2607 if (CB.TrueBB == NextBlock(SwitchBB)) { 2608 std::swap(CB.TrueBB, CB.FalseBB); 2609 SDValue True = DAG.getConstant(1, dl, Cond.getValueType()); 2610 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True); 2611 } 2612 2613 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, 2614 MVT::Other, getControlRoot(), Cond, 2615 DAG.getBasicBlock(CB.TrueBB)); 2616 2617 setValue(CurInst, BrCond); 2618 2619 // Insert the false branch. Do this even if it's a fall through branch, 2620 // this makes it easier to do DAG optimizations which require inverting 2621 // the branch condition. 2622 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, 2623 DAG.getBasicBlock(CB.FalseBB)); 2624 2625 DAG.setRoot(BrCond); 2626 } 2627 2628 /// visitJumpTable - Emit JumpTable node in the current MBB 2629 void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) { 2630 // Emit the code for the jump table 2631 assert(JT.Reg != -1U && "Should lower JT Header first!"); 2632 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); 2633 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(), 2634 JT.Reg, PTy); 2635 SDValue Table = DAG.getJumpTable(JT.JTI, PTy); 2636 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(), 2637 MVT::Other, Index.getValue(1), 2638 Table, Index); 2639 DAG.setRoot(BrJumpTable); 2640 } 2641 2642 /// visitJumpTableHeader - This function emits necessary code to produce index 2643 /// in the JumpTable from switch case. 2644 void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT, 2645 JumpTableHeader &JTH, 2646 MachineBasicBlock *SwitchBB) { 2647 SDLoc dl = getCurSDLoc(); 2648 2649 // Subtract the lowest switch case value from the value being switched on. 2650 SDValue SwitchOp = getValue(JTH.SValue); 2651 EVT VT = SwitchOp.getValueType(); 2652 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, 2653 DAG.getConstant(JTH.First, dl, VT)); 2654 2655 // The SDNode we just created, which holds the value being switched on minus 2656 // the smallest case value, needs to be copied to a virtual register so it 2657 // can be used as an index into the jump table in a subsequent basic block. 2658 // This value may be smaller or larger than the target's pointer type, and 2659 // therefore require extension or truncating. 2660 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2661 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout())); 2662 2663 unsigned JumpTableReg = 2664 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout())); 2665 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, 2666 JumpTableReg, SwitchOp); 2667 JT.Reg = JumpTableReg; 2668 2669 if (!JTH.FallthroughUnreachable) { 2670 // Emit the range check for the jump table, and branch to the default block 2671 // for the switch statement if the value being switched on exceeds the 2672 // largest case in the switch. 2673 SDValue CMP = DAG.getSetCC( 2674 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 2675 Sub.getValueType()), 2676 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); 2677 2678 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, 2679 MVT::Other, CopyTo, CMP, 2680 DAG.getBasicBlock(JT.Default)); 2681 2682 // Avoid emitting unnecessary branches to the next block. 2683 if (JT.MBB != NextBlock(SwitchBB)) 2684 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, 2685 DAG.getBasicBlock(JT.MBB)); 2686 2687 DAG.setRoot(BrCond); 2688 } else { 2689 // Avoid emitting unnecessary branches to the next block. 2690 if (JT.MBB != NextBlock(SwitchBB)) 2691 DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo, 2692 DAG.getBasicBlock(JT.MBB))); 2693 else 2694 DAG.setRoot(CopyTo); 2695 } 2696 } 2697 2698 /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global 2699 /// variable if there exists one. 2700 static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, 2701 SDValue &Chain) { 2702 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2703 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); 2704 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); 2705 MachineFunction &MF = DAG.getMachineFunction(); 2706 Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent()); 2707 MachineSDNode *Node = 2708 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); 2709 if (Global) { 2710 MachinePointerInfo MPInfo(Global); 2711 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | 2712 MachineMemOperand::MODereferenceable; 2713 MachineMemOperand *MemRef = MF.getMachineMemOperand( 2714 MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlign(PtrTy)); 2715 DAG.setNodeMemRefs(Node, {MemRef}); 2716 } 2717 if (PtrTy != PtrMemTy) 2718 return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy); 2719 return SDValue(Node, 0); 2720 } 2721 2722 /// Codegen a new tail for a stack protector check ParentMBB which has had its 2723 /// tail spliced into a stack protector check success bb. 2724 /// 2725 /// For a high level explanation of how this fits into the stack protector 2726 /// generation see the comment on the declaration of class 2727 /// StackProtectorDescriptor. 2728 void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, 2729 MachineBasicBlock *ParentBB) { 2730 2731 // First create the loads to the guard/stack slot for the comparison. 2732 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2733 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); 2734 EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); 2735 2736 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo(); 2737 int FI = MFI.getStackProtectorIndex(); 2738 2739 SDValue Guard; 2740 SDLoc dl = getCurSDLoc(); 2741 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy); 2742 const Module &M = *ParentBB->getParent()->getFunction().getParent(); 2743 Align Align = 2744 DAG.getDataLayout().getPrefTypeAlign(Type::getInt8PtrTy(M.getContext())); 2745 2746 // Generate code to load the content of the guard slot. 2747 SDValue GuardVal = DAG.getLoad( 2748 PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr, 2749 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, 2750 MachineMemOperand::MOVolatile); 2751 2752 if (TLI.useStackGuardXorFP()) 2753 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl); 2754 2755 // Retrieve guard check function, nullptr if instrumentation is inlined. 2756 if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) { 2757 // The target provides a guard check function to validate the guard value. 2758 // Generate a call to that function with the content of the guard slot as 2759 // argument. 2760 FunctionType *FnTy = GuardCheckFn->getFunctionType(); 2761 assert(FnTy->getNumParams() == 1 && "Invalid function signature"); 2762 2763 TargetLowering::ArgListTy Args; 2764 TargetLowering::ArgListEntry Entry; 2765 Entry.Node = GuardVal; 2766 Entry.Ty = FnTy->getParamType(0); 2767 if (GuardCheckFn->hasParamAttribute(0, Attribute::AttrKind::InReg)) 2768 Entry.IsInReg = true; 2769 Args.push_back(Entry); 2770 2771 TargetLowering::CallLoweringInfo CLI(DAG); 2772 CLI.setDebugLoc(getCurSDLoc()) 2773 .setChain(DAG.getEntryNode()) 2774 .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(), 2775 getValue(GuardCheckFn), std::move(Args)); 2776 2777 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); 2778 DAG.setRoot(Result.second); 2779 return; 2780 } 2781 2782 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD. 2783 // Otherwise, emit a volatile load to retrieve the stack guard value. 2784 SDValue Chain = DAG.getEntryNode(); 2785 if (TLI.useLoadStackGuardNode()) { 2786 Guard = getLoadStackGuard(DAG, dl, Chain); 2787 } else { 2788 const Value *IRGuard = TLI.getSDagStackGuard(M); 2789 SDValue GuardPtr = getValue(IRGuard); 2790 2791 Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, 2792 MachinePointerInfo(IRGuard, 0), Align, 2793 MachineMemOperand::MOVolatile); 2794 } 2795 2796 // Perform the comparison via a getsetcc. 2797 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(), 2798 *DAG.getContext(), 2799 Guard.getValueType()), 2800 Guard, GuardVal, ISD::SETNE); 2801 2802 // If the guard/stackslot do not equal, branch to failure MBB. 2803 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, 2804 MVT::Other, GuardVal.getOperand(0), 2805 Cmp, DAG.getBasicBlock(SPD.getFailureMBB())); 2806 // Otherwise branch to success MBB. 2807 SDValue Br = DAG.getNode(ISD::BR, dl, 2808 MVT::Other, BrCond, 2809 DAG.getBasicBlock(SPD.getSuccessMBB())); 2810 2811 DAG.setRoot(Br); 2812 } 2813 2814 /// Codegen the failure basic block for a stack protector check. 2815 /// 2816 /// A failure stack protector machine basic block consists simply of a call to 2817 /// __stack_chk_fail(). 2818 /// 2819 /// For a high level explanation of how this fits into the stack protector 2820 /// generation see the comment on the declaration of class 2821 /// StackProtectorDescriptor. 2822 void 2823 SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { 2824 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2825 TargetLowering::MakeLibCallOptions CallOptions; 2826 CallOptions.setDiscardResult(true); 2827 SDValue Chain = 2828 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, 2829 std::nullopt, CallOptions, getCurSDLoc()) 2830 .second; 2831 // On PS4/PS5, the "return address" must still be within the calling 2832 // function, even if it's at the very end, so emit an explicit TRAP here. 2833 // Passing 'true' for doesNotReturn above won't generate the trap for us. 2834 if (TM.getTargetTriple().isPS()) 2835 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); 2836 // WebAssembly needs an unreachable instruction after a non-returning call, 2837 // because the function return type can be different from __stack_chk_fail's 2838 // return type (void). 2839 if (TM.getTargetTriple().isWasm()) 2840 Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); 2841 2842 DAG.setRoot(Chain); 2843 } 2844 2845 /// visitBitTestHeader - This function emits necessary code to produce value 2846 /// suitable for "bit tests" 2847 void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B, 2848 MachineBasicBlock *SwitchBB) { 2849 SDLoc dl = getCurSDLoc(); 2850 2851 // Subtract the minimum value. 2852 SDValue SwitchOp = getValue(B.SValue); 2853 EVT VT = SwitchOp.getValueType(); 2854 SDValue RangeSub = 2855 DAG.getNode(ISD::SUB, dl, VT, SwitchOp, DAG.getConstant(B.First, dl, VT)); 2856 2857 // Determine the type of the test operands. 2858 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2859 bool UsePtrType = false; 2860 if (!TLI.isTypeLegal(VT)) { 2861 UsePtrType = true; 2862 } else { 2863 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i) 2864 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) { 2865 // Switch table case range are encoded into series of masks. 2866 // Just use pointer type, it's guaranteed to fit. 2867 UsePtrType = true; 2868 break; 2869 } 2870 } 2871 SDValue Sub = RangeSub; 2872 if (UsePtrType) { 2873 VT = TLI.getPointerTy(DAG.getDataLayout()); 2874 Sub = DAG.getZExtOrTrunc(Sub, dl, VT); 2875 } 2876 2877 B.RegVT = VT.getSimpleVT(); 2878 B.Reg = FuncInfo.CreateReg(B.RegVT); 2879 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub); 2880 2881 MachineBasicBlock* MBB = B.Cases[0].ThisBB; 2882 2883 if (!B.FallthroughUnreachable) 2884 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb); 2885 addSuccessorWithProb(SwitchBB, MBB, B.Prob); 2886 SwitchBB->normalizeSuccProbs(); 2887 2888 SDValue Root = CopyTo; 2889 if (!B.FallthroughUnreachable) { 2890 // Conditional branch to the default block. 2891 SDValue RangeCmp = DAG.getSetCC(dl, 2892 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), 2893 RangeSub.getValueType()), 2894 RangeSub, DAG.getConstant(B.Range, dl, RangeSub.getValueType()), 2895 ISD::SETUGT); 2896 2897 Root = DAG.getNode(ISD::BRCOND, dl, MVT::Other, Root, RangeCmp, 2898 DAG.getBasicBlock(B.Default)); 2899 } 2900 2901 // Avoid emitting unnecessary branches to the next block. 2902 if (MBB != NextBlock(SwitchBB)) 2903 Root = DAG.getNode(ISD::BR, dl, MVT::Other, Root, DAG.getBasicBlock(MBB)); 2904 2905 DAG.setRoot(Root); 2906 } 2907 2908 /// visitBitTestCase - this function produces one "bit test" 2909 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB, 2910 MachineBasicBlock* NextMBB, 2911 BranchProbability BranchProbToNext, 2912 unsigned Reg, 2913 BitTestCase &B, 2914 MachineBasicBlock *SwitchBB) { 2915 SDLoc dl = getCurSDLoc(); 2916 MVT VT = BB.RegVT; 2917 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT); 2918 SDValue Cmp; 2919 unsigned PopCount = llvm::popcount(B.Mask); 2920 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 2921 if (PopCount == 1) { 2922 // Testing for a single bit; just compare the shift count with what it 2923 // would need to be to shift a 1 bit in that position. 2924 Cmp = DAG.getSetCC( 2925 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), 2926 ShiftOp, DAG.getConstant(llvm::countr_zero(B.Mask), dl, VT), 2927 ISD::SETEQ); 2928 } else if (PopCount == BB.Range) { 2929 // There is only one zero bit in the range, test for it directly. 2930 Cmp = DAG.getSetCC( 2931 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), 2932 ShiftOp, DAG.getConstant(llvm::countr_one(B.Mask), dl, VT), ISD::SETNE); 2933 } else { 2934 // Make desired shift 2935 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT, 2936 DAG.getConstant(1, dl, VT), ShiftOp); 2937 2938 // Emit bit tests and jumps 2939 SDValue AndOp = DAG.getNode(ISD::AND, dl, 2940 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT)); 2941 Cmp = DAG.getSetCC( 2942 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT), 2943 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE); 2944 } 2945 2946 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb. 2947 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb); 2948 // The branch probability from SwitchBB to NextMBB is BranchProbToNext. 2949 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext); 2950 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is 2951 // one as they are relative probabilities (and thus work more like weights), 2952 // and hence we need to normalize them to let the sum of them become one. 2953 SwitchBB->normalizeSuccProbs(); 2954 2955 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl, 2956 MVT::Other, getControlRoot(), 2957 Cmp, DAG.getBasicBlock(B.TargetBB)); 2958 2959 // Avoid emitting unnecessary branches to the next block. 2960 if (NextMBB != NextBlock(SwitchBB)) 2961 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd, 2962 DAG.getBasicBlock(NextMBB)); 2963 2964 DAG.setRoot(BrAnd); 2965 } 2966 2967 void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { 2968 MachineBasicBlock *InvokeMBB = FuncInfo.MBB; 2969 2970 // Retrieve successors. Look through artificial IR level blocks like 2971 // catchswitch for successors. 2972 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)]; 2973 const BasicBlock *EHPadBB = I.getSuccessor(1); 2974 MachineBasicBlock *EHPadMBB = FuncInfo.MBBMap[EHPadBB]; 2975 2976 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't 2977 // have to do anything here to lower funclet bundles. 2978 assert(!I.hasOperandBundlesOtherThan( 2979 {LLVMContext::OB_deopt, LLVMContext::OB_gc_transition, 2980 LLVMContext::OB_gc_live, LLVMContext::OB_funclet, 2981 LLVMContext::OB_cfguardtarget, 2982 LLVMContext::OB_clang_arc_attachedcall}) && 2983 "Cannot lower invokes with arbitrary operand bundles yet!"); 2984 2985 const Value *Callee(I.getCalledOperand()); 2986 const Function *Fn = dyn_cast<Function>(Callee); 2987 if (isa<InlineAsm>(Callee)) 2988 visitInlineAsm(I, EHPadBB); 2989 else if (Fn && Fn->isIntrinsic()) { 2990 switch (Fn->getIntrinsicID()) { 2991 default: 2992 llvm_unreachable("Cannot invoke this intrinsic"); 2993 case Intrinsic::donothing: 2994 // Ignore invokes to @llvm.donothing: jump directly to the next BB. 2995 case Intrinsic::seh_try_begin: 2996 case Intrinsic::seh_scope_begin: 2997 case Intrinsic::seh_try_end: 2998 case Intrinsic::seh_scope_end: 2999 if (EHPadMBB) 3000 // a block referenced by EH table 3001 // so dtor-funclet not removed by opts 3002 EHPadMBB->setMachineBlockAddressTaken(); 3003 break; 3004 case Intrinsic::experimental_patchpoint_void: 3005 case Intrinsic::experimental_patchpoint_i64: 3006 visitPatchpoint(I, EHPadBB); 3007 break; 3008 case Intrinsic::experimental_gc_statepoint: 3009 LowerStatepoint(cast<GCStatepointInst>(I), EHPadBB); 3010 break; 3011 case Intrinsic::wasm_rethrow: { 3012 // This is usually done in visitTargetIntrinsic, but this intrinsic is 3013 // special because it can be invoked, so we manually lower it to a DAG 3014 // node here. 3015 SmallVector<SDValue, 8> Ops; 3016 Ops.push_back(getRoot()); // inchain 3017 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3018 Ops.push_back( 3019 DAG.getTargetConstant(Intrinsic::wasm_rethrow, getCurSDLoc(), 3020 TLI.getPointerTy(DAG.getDataLayout()))); 3021 SDVTList VTs = DAG.getVTList(ArrayRef<EVT>({MVT::Other})); // outchain 3022 DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops)); 3023 break; 3024 } 3025 } 3026 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) { 3027 // Currently we do not lower any intrinsic calls with deopt operand bundles. 3028 // Eventually we will support lowering the @llvm.experimental.deoptimize 3029 // intrinsic, and right now there are no plans to support other intrinsics 3030 // with deopt state. 3031 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB); 3032 } else { 3033 LowerCallTo(I, getValue(Callee), false, false, EHPadBB); 3034 } 3035 3036 // If the value of the invoke is used outside of its defining block, make it 3037 // available as a virtual register. 3038 // We already took care of the exported value for the statepoint instruction 3039 // during call to the LowerStatepoint. 3040 if (!isa<GCStatepointInst>(I)) { 3041 CopyToExportRegsIfNeeded(&I); 3042 } 3043 3044 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests; 3045 BranchProbabilityInfo *BPI = FuncInfo.BPI; 3046 BranchProbability EHPadBBProb = 3047 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB) 3048 : BranchProbability::getZero(); 3049 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests); 3050 3051 // Update successor info. 3052 addSuccessorWithProb(InvokeMBB, Return); 3053 for (auto &UnwindDest : UnwindDests) { 3054 UnwindDest.first->setIsEHPad(); 3055 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second); 3056 } 3057 InvokeMBB->normalizeSuccProbs(); 3058 3059 // Drop into normal successor. 3060 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), 3061 DAG.getBasicBlock(Return))); 3062 } 3063 3064 void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { 3065 MachineBasicBlock *CallBrMBB = FuncInfo.MBB; 3066 3067 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't 3068 // have to do anything here to lower funclet bundles. 3069 assert(!I.hasOperandBundlesOtherThan( 3070 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && 3071 "Cannot lower callbrs with arbitrary operand bundles yet!"); 3072 3073 assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr"); 3074 visitInlineAsm(I); 3075 CopyToExportRegsIfNeeded(&I); 3076 3077 // Retrieve successors. 3078 SmallPtrSet<BasicBlock *, 8> Dests; 3079 Dests.insert(I.getDefaultDest()); 3080 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; 3081 3082 // Update successor info. 3083 addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); 3084 for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { 3085 BasicBlock *Dest = I.getIndirectDest(i); 3086 MachineBasicBlock *Target = FuncInfo.MBBMap[Dest]; 3087 Target->setIsInlineAsmBrIndirectTarget(); 3088 Target->setMachineBlockAddressTaken(); 3089 Target->setLabelMustBeEmitted(); 3090 // Don't add duplicate machine successors. 3091 if (Dests.insert(Dest).second) 3092 addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); 3093 } 3094 CallBrMBB->normalizeSuccProbs(); 3095 3096 // Drop into default successor. 3097 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), 3098 MVT::Other, getControlRoot(), 3099 DAG.getBasicBlock(Return))); 3100 } 3101 3102 void SelectionDAGBuilder::visitResume(const ResumeInst &RI) { 3103 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!"); 3104 } 3105 3106 void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { 3107 assert(FuncInfo.MBB->isEHPad() && 3108 "Call to landingpad not in landing pad!"); 3109 3110 // If there aren't registers to copy the values into (e.g., during SjLj 3111 // exceptions), then don't bother to create these DAG nodes. 3112 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3113 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn(); 3114 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 && 3115 TLI.getExceptionSelectorRegister(PersonalityFn) == 0) 3116 return; 3117 3118 // If landingpad's return type is token type, we don't create DAG nodes 3119 // for its exception pointer and selector value. The extraction of exception 3120 // pointer or selector value from token type landingpads is not currently 3121 // supported. 3122 if (LP.getType()->isTokenTy()) 3123 return; 3124 3125 SmallVector<EVT, 2> ValueVTs; 3126 SDLoc dl = getCurSDLoc(); 3127 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs); 3128 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported"); 3129 3130 // Get the two live-in registers as SDValues. The physregs have already been 3131 // copied into virtual registers. 3132 SDValue Ops[2]; 3133 if (FuncInfo.ExceptionPointerVirtReg) { 3134 Ops[0] = DAG.getZExtOrTrunc( 3135 DAG.getCopyFromReg(DAG.getEntryNode(), dl, 3136 FuncInfo.ExceptionPointerVirtReg, 3137 TLI.getPointerTy(DAG.getDataLayout())), 3138 dl, ValueVTs[0]); 3139 } else { 3140 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())); 3141 } 3142 Ops[1] = DAG.getZExtOrTrunc( 3143 DAG.getCopyFromReg(DAG.getEntryNode(), dl, 3144 FuncInfo.ExceptionSelectorVirtReg, 3145 TLI.getPointerTy(DAG.getDataLayout())), 3146 dl, ValueVTs[1]); 3147 3148 // Merge into one. 3149 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl, 3150 DAG.getVTList(ValueVTs), Ops); 3151 setValue(&LP, Res); 3152 } 3153 3154 void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, 3155 MachineBasicBlock *Last) { 3156 // Update JTCases. 3157 for (JumpTableBlock &JTB : SL->JTCases) 3158 if (JTB.first.HeaderBB == First) 3159 JTB.first.HeaderBB = Last; 3160 3161 // Update BitTestCases. 3162 for (BitTestBlock &BTB : SL->BitTestCases) 3163 if (BTB.Parent == First) 3164 BTB.Parent = Last; 3165 } 3166 3167 void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { 3168 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB; 3169 3170 // Update machine-CFG edges with unique successors. 3171 SmallSet<BasicBlock*, 32> Done; 3172 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) { 3173 BasicBlock *BB = I.getSuccessor(i); 3174 bool Inserted = Done.insert(BB).second; 3175 if (!Inserted) 3176 continue; 3177 3178 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB]; 3179 addSuccessorWithProb(IndirectBrMBB, Succ); 3180 } 3181 IndirectBrMBB->normalizeSuccProbs(); 3182 3183 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(), 3184 MVT::Other, getControlRoot(), 3185 getValue(I.getAddress()))); 3186 } 3187 3188 void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) { 3189 if (!DAG.getTarget().Options.TrapUnreachable) 3190 return; 3191 3192 // We may be able to ignore unreachable behind a noreturn call. 3193 if (DAG.getTarget().Options.NoTrapAfterNoreturn) { 3194 const BasicBlock &BB = *I.getParent(); 3195 if (&I != &BB.front()) { 3196 BasicBlock::const_iterator PredI = 3197 std::prev(BasicBlock::const_iterator(&I)); 3198 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) { 3199 if (Call->doesNotReturn()) 3200 return; 3201 } 3202 } 3203 } 3204 3205 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot())); 3206 } 3207 3208 void SelectionDAGBuilder::visitUnary(const User &I, unsigned Opcode) { 3209 SDNodeFlags Flags; 3210 if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) 3211 Flags.copyFMF(*FPOp); 3212 3213 SDValue Op = getValue(I.getOperand(0)); 3214 SDValue UnNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op.getValueType(), 3215 Op, Flags); 3216 setValue(&I, UnNodeValue); 3217 } 3218 3219 void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) { 3220 SDNodeFlags Flags; 3221 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) { 3222 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap()); 3223 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap()); 3224 } 3225 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) 3226 Flags.setExact(ExactOp->isExact()); 3227 if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) 3228 Flags.copyFMF(*FPOp); 3229 3230 SDValue Op1 = getValue(I.getOperand(0)); 3231 SDValue Op2 = getValue(I.getOperand(1)); 3232 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), 3233 Op1, Op2, Flags); 3234 setValue(&I, BinNodeValue); 3235 } 3236 3237 void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) { 3238 SDValue Op1 = getValue(I.getOperand(0)); 3239 SDValue Op2 = getValue(I.getOperand(1)); 3240 3241 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy( 3242 Op1.getValueType(), DAG.getDataLayout()); 3243 3244 // Coerce the shift amount to the right type if we can. This exposes the 3245 // truncate or zext to optimization early. 3246 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) { 3247 assert(ShiftTy.getSizeInBits() >= Log2_32_Ceil(Op1.getValueSizeInBits()) && 3248 "Unexpected shift type"); 3249 Op2 = DAG.getZExtOrTrunc(Op2, getCurSDLoc(), ShiftTy); 3250 } 3251 3252 bool nuw = false; 3253 bool nsw = false; 3254 bool exact = false; 3255 3256 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) { 3257 3258 if (const OverflowingBinaryOperator *OFBinOp = 3259 dyn_cast<const OverflowingBinaryOperator>(&I)) { 3260 nuw = OFBinOp->hasNoUnsignedWrap(); 3261 nsw = OFBinOp->hasNoSignedWrap(); 3262 } 3263 if (const PossiblyExactOperator *ExactOp = 3264 dyn_cast<const PossiblyExactOperator>(&I)) 3265 exact = ExactOp->isExact(); 3266 } 3267 SDNodeFlags Flags; 3268 Flags.setExact(exact); 3269 Flags.setNoSignedWrap(nsw); 3270 Flags.setNoUnsignedWrap(nuw); 3271 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2, 3272 Flags); 3273 setValue(&I, Res); 3274 } 3275 3276 void SelectionDAGBuilder::visitSDiv(const User &I) { 3277 SDValue Op1 = getValue(I.getOperand(0)); 3278 SDValue Op2 = getValue(I.getOperand(1)); 3279 3280 SDNodeFlags Flags; 3281 Flags.setExact(isa<PossiblyExactOperator>(&I) && 3282 cast<PossiblyExactOperator>(&I)->isExact()); 3283 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1, 3284 Op2, Flags)); 3285 } 3286 3287 void SelectionDAGBuilder::visitICmp(const User &I) { 3288 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE; 3289 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I)) 3290 predicate = IC->getPredicate(); 3291 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I)) 3292 predicate = ICmpInst::Predicate(IC->getPredicate()); 3293 SDValue Op1 = getValue(I.getOperand(0)); 3294 SDValue Op2 = getValue(I.getOperand(1)); 3295 ISD::CondCode Opcode = getICmpCondCode(predicate); 3296 3297 auto &TLI = DAG.getTargetLoweringInfo(); 3298 EVT MemVT = 3299 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); 3300 3301 // If a pointer's DAG type is larger than its memory type then the DAG values 3302 // are zero-extended. This breaks signed comparisons so truncate back to the 3303 // underlying type before doing the compare. 3304 if (Op1.getValueType() != MemVT) { 3305 Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT); 3306 Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT); 3307 } 3308 3309 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3310 I.getType()); 3311 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); 3312 } 3313 3314 void SelectionDAGBuilder::visitFCmp(const User &I) { 3315 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE; 3316 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I)) 3317 predicate = FC->getPredicate(); 3318 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I)) 3319 predicate = FCmpInst::Predicate(FC->getPredicate()); 3320 SDValue Op1 = getValue(I.getOperand(0)); 3321 SDValue Op2 = getValue(I.getOperand(1)); 3322 3323 ISD::CondCode Condition = getFCmpCondCode(predicate); 3324 auto *FPMO = cast<FPMathOperator>(&I); 3325 if (FPMO->hasNoNaNs() || TM.Options.NoNaNsFPMath) 3326 Condition = getFCmpCodeWithoutNaN(Condition); 3327 3328 SDNodeFlags Flags; 3329 Flags.copyFMF(*FPMO); 3330 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); 3331 3332 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3333 I.getType()); 3334 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition)); 3335 } 3336 3337 // Check if the condition of the select has one use or two users that are both 3338 // selects with the same condition. 3339 static bool hasOnlySelectUsers(const Value *Cond) { 3340 return llvm::all_of(Cond->users(), [](const Value *V) { 3341 return isa<SelectInst>(V); 3342 }); 3343 } 3344 3345 void SelectionDAGBuilder::visitSelect(const User &I) { 3346 SmallVector<EVT, 4> ValueVTs; 3347 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), 3348 ValueVTs); 3349 unsigned NumValues = ValueVTs.size(); 3350 if (NumValues == 0) return; 3351 3352 SmallVector<SDValue, 4> Values(NumValues); 3353 SDValue Cond = getValue(I.getOperand(0)); 3354 SDValue LHSVal = getValue(I.getOperand(1)); 3355 SDValue RHSVal = getValue(I.getOperand(2)); 3356 SmallVector<SDValue, 1> BaseOps(1, Cond); 3357 ISD::NodeType OpCode = 3358 Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; 3359 3360 bool IsUnaryAbs = false; 3361 bool Negate = false; 3362 3363 SDNodeFlags Flags; 3364 if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) 3365 Flags.copyFMF(*FPOp); 3366 3367 // Min/max matching is only viable if all output VTs are the same. 3368 if (all_equal(ValueVTs)) { 3369 EVT VT = ValueVTs[0]; 3370 LLVMContext &Ctx = *DAG.getContext(); 3371 auto &TLI = DAG.getTargetLoweringInfo(); 3372 3373 // We care about the legality of the operation after it has been type 3374 // legalized. 3375 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal) 3376 VT = TLI.getTypeToTransformTo(Ctx, VT); 3377 3378 // If the vselect is legal, assume we want to leave this as a vector setcc + 3379 // vselect. Otherwise, if this is going to be scalarized, we want to see if 3380 // min/max is legal on the scalar type. 3381 bool UseScalarMinMax = VT.isVector() && 3382 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT); 3383 3384 // ValueTracking's select pattern matching does not account for -0.0, 3385 // so we can't lower to FMINIMUM/FMAXIMUM because those nodes specify that 3386 // -0.0 is less than +0.0. 3387 Value *LHS, *RHS; 3388 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS); 3389 ISD::NodeType Opc = ISD::DELETED_NODE; 3390 switch (SPR.Flavor) { 3391 case SPF_UMAX: Opc = ISD::UMAX; break; 3392 case SPF_UMIN: Opc = ISD::UMIN; break; 3393 case SPF_SMAX: Opc = ISD::SMAX; break; 3394 case SPF_SMIN: Opc = ISD::SMIN; break; 3395 case SPF_FMINNUM: 3396 switch (SPR.NaNBehavior) { 3397 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?"); 3398 case SPNB_RETURNS_NAN: break; 3399 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break; 3400 case SPNB_RETURNS_ANY: 3401 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT) || 3402 (UseScalarMinMax && 3403 TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()))) 3404 Opc = ISD::FMINNUM; 3405 break; 3406 } 3407 break; 3408 case SPF_FMAXNUM: 3409 switch (SPR.NaNBehavior) { 3410 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?"); 3411 case SPNB_RETURNS_NAN: break; 3412 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break; 3413 case SPNB_RETURNS_ANY: 3414 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT) || 3415 (UseScalarMinMax && 3416 TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()))) 3417 Opc = ISD::FMAXNUM; 3418 break; 3419 } 3420 break; 3421 case SPF_NABS: 3422 Negate = true; 3423 [[fallthrough]]; 3424 case SPF_ABS: 3425 IsUnaryAbs = true; 3426 Opc = ISD::ABS; 3427 break; 3428 default: break; 3429 } 3430 3431 if (!IsUnaryAbs && Opc != ISD::DELETED_NODE && 3432 (TLI.isOperationLegalOrCustom(Opc, VT) || 3433 (UseScalarMinMax && 3434 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) && 3435 // If the underlying comparison instruction is used by any other 3436 // instruction, the consumed instructions won't be destroyed, so it is 3437 // not profitable to convert to a min/max. 3438 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) { 3439 OpCode = Opc; 3440 LHSVal = getValue(LHS); 3441 RHSVal = getValue(RHS); 3442 BaseOps.clear(); 3443 } 3444 3445 if (IsUnaryAbs) { 3446 OpCode = Opc; 3447 LHSVal = getValue(LHS); 3448 BaseOps.clear(); 3449 } 3450 } 3451 3452 if (IsUnaryAbs) { 3453 for (unsigned i = 0; i != NumValues; ++i) { 3454 SDLoc dl = getCurSDLoc(); 3455 EVT VT = LHSVal.getNode()->getValueType(LHSVal.getResNo() + i); 3456 Values[i] = 3457 DAG.getNode(OpCode, dl, VT, LHSVal.getValue(LHSVal.getResNo() + i)); 3458 if (Negate) 3459 Values[i] = DAG.getNegative(Values[i], dl, VT); 3460 } 3461 } else { 3462 for (unsigned i = 0; i != NumValues; ++i) { 3463 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end()); 3464 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); 3465 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i)); 3466 Values[i] = DAG.getNode( 3467 OpCode, getCurSDLoc(), 3468 LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops, Flags); 3469 } 3470 } 3471 3472 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3473 DAG.getVTList(ValueVTs), Values)); 3474 } 3475 3476 void SelectionDAGBuilder::visitTrunc(const User &I) { 3477 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest). 3478 SDValue N = getValue(I.getOperand(0)); 3479 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3480 I.getType()); 3481 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N)); 3482 } 3483 3484 void SelectionDAGBuilder::visitZExt(const User &I) { 3485 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 3486 // ZExt also can't be a cast to bool for same reason. So, nothing much to do 3487 SDValue N = getValue(I.getOperand(0)); 3488 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3489 I.getType()); 3490 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N)); 3491 } 3492 3493 void SelectionDAGBuilder::visitSExt(const User &I) { 3494 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest). 3495 // SExt also can't be a cast to bool for same reason. So, nothing much to do 3496 SDValue N = getValue(I.getOperand(0)); 3497 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3498 I.getType()); 3499 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N)); 3500 } 3501 3502 void SelectionDAGBuilder::visitFPTrunc(const User &I) { 3503 // FPTrunc is never a no-op cast, no need to check 3504 SDValue N = getValue(I.getOperand(0)); 3505 SDLoc dl = getCurSDLoc(); 3506 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3507 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 3508 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N, 3509 DAG.getTargetConstant( 3510 0, dl, TLI.getPointerTy(DAG.getDataLayout())))); 3511 } 3512 3513 void SelectionDAGBuilder::visitFPExt(const User &I) { 3514 // FPExt is never a no-op cast, no need to check 3515 SDValue N = getValue(I.getOperand(0)); 3516 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3517 I.getType()); 3518 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N)); 3519 } 3520 3521 void SelectionDAGBuilder::visitFPToUI(const User &I) { 3522 // FPToUI is never a no-op cast, no need to check 3523 SDValue N = getValue(I.getOperand(0)); 3524 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3525 I.getType()); 3526 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N)); 3527 } 3528 3529 void SelectionDAGBuilder::visitFPToSI(const User &I) { 3530 // FPToSI is never a no-op cast, no need to check 3531 SDValue N = getValue(I.getOperand(0)); 3532 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3533 I.getType()); 3534 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N)); 3535 } 3536 3537 void SelectionDAGBuilder::visitUIToFP(const User &I) { 3538 // UIToFP is never a no-op cast, no need to check 3539 SDValue N = getValue(I.getOperand(0)); 3540 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3541 I.getType()); 3542 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N)); 3543 } 3544 3545 void SelectionDAGBuilder::visitSIToFP(const User &I) { 3546 // SIToFP is never a no-op cast, no need to check 3547 SDValue N = getValue(I.getOperand(0)); 3548 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3549 I.getType()); 3550 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N)); 3551 } 3552 3553 void SelectionDAGBuilder::visitPtrToInt(const User &I) { 3554 // What to do depends on the size of the integer and the size of the pointer. 3555 // We can either truncate, zero extend, or no-op, accordingly. 3556 SDValue N = getValue(I.getOperand(0)); 3557 auto &TLI = DAG.getTargetLoweringInfo(); 3558 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3559 I.getType()); 3560 EVT PtrMemVT = 3561 TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); 3562 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT); 3563 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT); 3564 setValue(&I, N); 3565 } 3566 3567 void SelectionDAGBuilder::visitIntToPtr(const User &I) { 3568 // What to do depends on the size of the integer and the size of the pointer. 3569 // We can either truncate, zero extend, or no-op, accordingly. 3570 SDValue N = getValue(I.getOperand(0)); 3571 auto &TLI = DAG.getTargetLoweringInfo(); 3572 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 3573 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); 3574 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT); 3575 N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT); 3576 setValue(&I, N); 3577 } 3578 3579 void SelectionDAGBuilder::visitBitCast(const User &I) { 3580 SDValue N = getValue(I.getOperand(0)); 3581 SDLoc dl = getCurSDLoc(); 3582 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 3583 I.getType()); 3584 3585 // BitCast assures us that source and destination are the same size so this is 3586 // either a BITCAST or a no-op. 3587 if (DestVT != N.getValueType()) 3588 setValue(&I, DAG.getNode(ISD::BITCAST, dl, 3589 DestVT, N)); // convert types. 3590 // Check if the original LLVM IR Operand was a ConstantInt, because getValue() 3591 // might fold any kind of constant expression to an integer constant and that 3592 // is not what we are looking for. Only recognize a bitcast of a genuine 3593 // constant integer as an opaque constant. 3594 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0))) 3595 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false, 3596 /*isOpaque*/true)); 3597 else 3598 setValue(&I, N); // noop cast. 3599 } 3600 3601 void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) { 3602 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3603 const Value *SV = I.getOperand(0); 3604 SDValue N = getValue(SV); 3605 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 3606 3607 unsigned SrcAS = SV->getType()->getPointerAddressSpace(); 3608 unsigned DestAS = I.getType()->getPointerAddressSpace(); 3609 3610 if (!TM.isNoopAddrSpaceCast(SrcAS, DestAS)) 3611 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS); 3612 3613 setValue(&I, N); 3614 } 3615 3616 void SelectionDAGBuilder::visitInsertElement(const User &I) { 3617 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3618 SDValue InVec = getValue(I.getOperand(0)); 3619 SDValue InVal = getValue(I.getOperand(1)); 3620 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(), 3621 TLI.getVectorIdxTy(DAG.getDataLayout())); 3622 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(), 3623 TLI.getValueType(DAG.getDataLayout(), I.getType()), 3624 InVec, InVal, InIdx)); 3625 } 3626 3627 void SelectionDAGBuilder::visitExtractElement(const User &I) { 3628 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3629 SDValue InVec = getValue(I.getOperand(0)); 3630 SDValue InIdx = DAG.getZExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(), 3631 TLI.getVectorIdxTy(DAG.getDataLayout())); 3632 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(), 3633 TLI.getValueType(DAG.getDataLayout(), I.getType()), 3634 InVec, InIdx)); 3635 } 3636 3637 void SelectionDAGBuilder::visitShuffleVector(const User &I) { 3638 SDValue Src1 = getValue(I.getOperand(0)); 3639 SDValue Src2 = getValue(I.getOperand(1)); 3640 ArrayRef<int> Mask; 3641 if (auto *SVI = dyn_cast<ShuffleVectorInst>(&I)) 3642 Mask = SVI->getShuffleMask(); 3643 else 3644 Mask = cast<ConstantExpr>(I).getShuffleMask(); 3645 SDLoc DL = getCurSDLoc(); 3646 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3647 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 3648 EVT SrcVT = Src1.getValueType(); 3649 3650 if (all_of(Mask, [](int Elem) { return Elem == 0; }) && 3651 VT.isScalableVector()) { 3652 // Canonical splat form of first element of first input vector. 3653 SDValue FirstElt = 3654 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT.getScalarType(), Src1, 3655 DAG.getVectorIdxConstant(0, DL)); 3656 setValue(&I, DAG.getNode(ISD::SPLAT_VECTOR, DL, VT, FirstElt)); 3657 return; 3658 } 3659 3660 // For now, we only handle splats for scalable vectors. 3661 // The DAGCombiner will perform a BUILD_VECTOR -> SPLAT_VECTOR transformation 3662 // for targets that support a SPLAT_VECTOR for non-scalable vector types. 3663 assert(!VT.isScalableVector() && "Unsupported scalable vector shuffle"); 3664 3665 unsigned SrcNumElts = SrcVT.getVectorNumElements(); 3666 unsigned MaskNumElts = Mask.size(); 3667 3668 if (SrcNumElts == MaskNumElts) { 3669 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask)); 3670 return; 3671 } 3672 3673 // Normalize the shuffle vector since mask and vector length don't match. 3674 if (SrcNumElts < MaskNumElts) { 3675 // Mask is longer than the source vectors. We can use concatenate vector to 3676 // make the mask and vectors lengths match. 3677 3678 if (MaskNumElts % SrcNumElts == 0) { 3679 // Mask length is a multiple of the source vector length. 3680 // Check if the shuffle is some kind of concatenation of the input 3681 // vectors. 3682 unsigned NumConcat = MaskNumElts / SrcNumElts; 3683 bool IsConcat = true; 3684 SmallVector<int, 8> ConcatSrcs(NumConcat, -1); 3685 for (unsigned i = 0; i != MaskNumElts; ++i) { 3686 int Idx = Mask[i]; 3687 if (Idx < 0) 3688 continue; 3689 // Ensure the indices in each SrcVT sized piece are sequential and that 3690 // the same source is used for the whole piece. 3691 if ((Idx % SrcNumElts != (i % SrcNumElts)) || 3692 (ConcatSrcs[i / SrcNumElts] >= 0 && 3693 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) { 3694 IsConcat = false; 3695 break; 3696 } 3697 // Remember which source this index came from. 3698 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts; 3699 } 3700 3701 // The shuffle is concatenating multiple vectors together. Just emit 3702 // a CONCAT_VECTORS operation. 3703 if (IsConcat) { 3704 SmallVector<SDValue, 8> ConcatOps; 3705 for (auto Src : ConcatSrcs) { 3706 if (Src < 0) 3707 ConcatOps.push_back(DAG.getUNDEF(SrcVT)); 3708 else if (Src == 0) 3709 ConcatOps.push_back(Src1); 3710 else 3711 ConcatOps.push_back(Src2); 3712 } 3713 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps)); 3714 return; 3715 } 3716 } 3717 3718 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts); 3719 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts; 3720 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), 3721 PaddedMaskNumElts); 3722 3723 // Pad both vectors with undefs to make them the same length as the mask. 3724 SDValue UndefVal = DAG.getUNDEF(SrcVT); 3725 3726 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal); 3727 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal); 3728 MOps1[0] = Src1; 3729 MOps2[0] = Src2; 3730 3731 Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); 3732 Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); 3733 3734 // Readjust mask for new input vector length. 3735 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1); 3736 for (unsigned i = 0; i != MaskNumElts; ++i) { 3737 int Idx = Mask[i]; 3738 if (Idx >= (int)SrcNumElts) 3739 Idx -= SrcNumElts - PaddedMaskNumElts; 3740 MappedOps[i] = Idx; 3741 } 3742 3743 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps); 3744 3745 // If the concatenated vector was padded, extract a subvector with the 3746 // correct number of elements. 3747 if (MaskNumElts != PaddedMaskNumElts) 3748 Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Result, 3749 DAG.getVectorIdxConstant(0, DL)); 3750 3751 setValue(&I, Result); 3752 return; 3753 } 3754 3755 if (SrcNumElts > MaskNumElts) { 3756 // Analyze the access pattern of the vector to see if we can extract 3757 // two subvectors and do the shuffle. 3758 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from 3759 bool CanExtract = true; 3760 for (int Idx : Mask) { 3761 unsigned Input = 0; 3762 if (Idx < 0) 3763 continue; 3764 3765 if (Idx >= (int)SrcNumElts) { 3766 Input = 1; 3767 Idx -= SrcNumElts; 3768 } 3769 3770 // If all the indices come from the same MaskNumElts sized portion of 3771 // the sources we can use extract. Also make sure the extract wouldn't 3772 // extract past the end of the source. 3773 int NewStartIdx = alignDown(Idx, MaskNumElts); 3774 if (NewStartIdx + MaskNumElts > SrcNumElts || 3775 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx)) 3776 CanExtract = false; 3777 // Make sure we always update StartIdx as we use it to track if all 3778 // elements are undef. 3779 StartIdx[Input] = NewStartIdx; 3780 } 3781 3782 if (StartIdx[0] < 0 && StartIdx[1] < 0) { 3783 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used. 3784 return; 3785 } 3786 if (CanExtract) { 3787 // Extract appropriate subvector and generate a vector shuffle 3788 for (unsigned Input = 0; Input < 2; ++Input) { 3789 SDValue &Src = Input == 0 ? Src1 : Src2; 3790 if (StartIdx[Input] < 0) 3791 Src = DAG.getUNDEF(VT); 3792 else { 3793 Src = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src, 3794 DAG.getVectorIdxConstant(StartIdx[Input], DL)); 3795 } 3796 } 3797 3798 // Calculate new mask. 3799 SmallVector<int, 8> MappedOps(Mask); 3800 for (int &Idx : MappedOps) { 3801 if (Idx >= (int)SrcNumElts) 3802 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts; 3803 else if (Idx >= 0) 3804 Idx -= StartIdx[0]; 3805 } 3806 3807 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps)); 3808 return; 3809 } 3810 } 3811 3812 // We can't use either concat vectors or extract subvectors so fall back to 3813 // replacing the shuffle with extract and build vector. 3814 // to insert and build vector. 3815 EVT EltVT = VT.getVectorElementType(); 3816 SmallVector<SDValue,8> Ops; 3817 for (int Idx : Mask) { 3818 SDValue Res; 3819 3820 if (Idx < 0) { 3821 Res = DAG.getUNDEF(EltVT); 3822 } else { 3823 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2; 3824 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts; 3825 3826 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Src, 3827 DAG.getVectorIdxConstant(Idx, DL)); 3828 } 3829 3830 Ops.push_back(Res); 3831 } 3832 3833 setValue(&I, DAG.getBuildVector(VT, DL, Ops)); 3834 } 3835 3836 void SelectionDAGBuilder::visitInsertValue(const InsertValueInst &I) { 3837 ArrayRef<unsigned> Indices = I.getIndices(); 3838 const Value *Op0 = I.getOperand(0); 3839 const Value *Op1 = I.getOperand(1); 3840 Type *AggTy = I.getType(); 3841 Type *ValTy = Op1->getType(); 3842 bool IntoUndef = isa<UndefValue>(Op0); 3843 bool FromUndef = isa<UndefValue>(Op1); 3844 3845 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); 3846 3847 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3848 SmallVector<EVT, 4> AggValueVTs; 3849 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs); 3850 SmallVector<EVT, 4> ValValueVTs; 3851 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); 3852 3853 unsigned NumAggValues = AggValueVTs.size(); 3854 unsigned NumValValues = ValValueVTs.size(); 3855 SmallVector<SDValue, 4> Values(NumAggValues); 3856 3857 // Ignore an insertvalue that produces an empty object 3858 if (!NumAggValues) { 3859 setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); 3860 return; 3861 } 3862 3863 SDValue Agg = getValue(Op0); 3864 unsigned i = 0; 3865 // Copy the beginning value(s) from the original aggregate. 3866 for (; i != LinearIndex; ++i) 3867 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3868 SDValue(Agg.getNode(), Agg.getResNo() + i); 3869 // Copy values from the inserted value(s). 3870 if (NumValValues) { 3871 SDValue Val = getValue(Op1); 3872 for (; i != LinearIndex + NumValValues; ++i) 3873 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3874 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex); 3875 } 3876 // Copy remaining value(s) from the original aggregate. 3877 for (; i != NumAggValues; ++i) 3878 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) : 3879 SDValue(Agg.getNode(), Agg.getResNo() + i); 3880 3881 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3882 DAG.getVTList(AggValueVTs), Values)); 3883 } 3884 3885 void SelectionDAGBuilder::visitExtractValue(const ExtractValueInst &I) { 3886 ArrayRef<unsigned> Indices = I.getIndices(); 3887 const Value *Op0 = I.getOperand(0); 3888 Type *AggTy = Op0->getType(); 3889 Type *ValTy = I.getType(); 3890 bool OutOfUndef = isa<UndefValue>(Op0); 3891 3892 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices); 3893 3894 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 3895 SmallVector<EVT, 4> ValValueVTs; 3896 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs); 3897 3898 unsigned NumValValues = ValValueVTs.size(); 3899 3900 // Ignore a extractvalue that produces an empty object 3901 if (!NumValValues) { 3902 setValue(&I, DAG.getUNDEF(MVT(MVT::Other))); 3903 return; 3904 } 3905 3906 SmallVector<SDValue, 4> Values(NumValValues); 3907 3908 SDValue Agg = getValue(Op0); 3909 // Copy out the selected value(s). 3910 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i) 3911 Values[i - LinearIndex] = 3912 OutOfUndef ? 3913 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) : 3914 SDValue(Agg.getNode(), Agg.getResNo() + i); 3915 3916 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 3917 DAG.getVTList(ValValueVTs), Values)); 3918 } 3919 3920 void SelectionDAGBuilder::visitGetElementPtr(const User &I) { 3921 Value *Op0 = I.getOperand(0); 3922 // Note that the pointer operand may be a vector of pointers. Take the scalar 3923 // element which holds a pointer. 3924 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace(); 3925 SDValue N = getValue(Op0); 3926 SDLoc dl = getCurSDLoc(); 3927 auto &TLI = DAG.getTargetLoweringInfo(); 3928 3929 // Normalize Vector GEP - all scalar operands should be converted to the 3930 // splat vector. 3931 bool IsVectorGEP = I.getType()->isVectorTy(); 3932 ElementCount VectorElementCount = 3933 IsVectorGEP ? cast<VectorType>(I.getType())->getElementCount() 3934 : ElementCount::getFixed(0); 3935 3936 if (IsVectorGEP && !N.getValueType().isVector()) { 3937 LLVMContext &Context = *DAG.getContext(); 3938 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorElementCount); 3939 N = DAG.getSplat(VT, dl, N); 3940 } 3941 3942 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I); 3943 GTI != E; ++GTI) { 3944 const Value *Idx = GTI.getOperand(); 3945 if (StructType *StTy = GTI.getStructTypeOrNull()) { 3946 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue(); 3947 if (Field) { 3948 // N = N + Offset 3949 uint64_t Offset = 3950 DAG.getDataLayout().getStructLayout(StTy)->getElementOffset(Field); 3951 3952 // In an inbounds GEP with an offset that is nonnegative even when 3953 // interpreted as signed, assume there is no unsigned overflow. 3954 SDNodeFlags Flags; 3955 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds()) 3956 Flags.setNoUnsignedWrap(true); 3957 3958 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, 3959 DAG.getConstant(Offset, dl, N.getValueType()), Flags); 3960 } 3961 } else { 3962 // IdxSize is the width of the arithmetic according to IR semantics. 3963 // In SelectionDAG, we may prefer to do arithmetic in a wider bitwidth 3964 // (and fix up the result later). 3965 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS); 3966 MVT IdxTy = MVT::getIntegerVT(IdxSize); 3967 TypeSize ElementSize = 3968 DAG.getDataLayout().getTypeAllocSize(GTI.getIndexedType()); 3969 // We intentionally mask away the high bits here; ElementSize may not 3970 // fit in IdxTy. 3971 APInt ElementMul(IdxSize, ElementSize.getKnownMinValue()); 3972 bool ElementScalable = ElementSize.isScalable(); 3973 3974 // If this is a scalar constant or a splat vector of constants, 3975 // handle it quickly. 3976 const auto *C = dyn_cast<Constant>(Idx); 3977 if (C && isa<VectorType>(C->getType())) 3978 C = C->getSplatValue(); 3979 3980 const auto *CI = dyn_cast_or_null<ConstantInt>(C); 3981 if (CI && CI->isZero()) 3982 continue; 3983 if (CI && !ElementScalable) { 3984 APInt Offs = ElementMul * CI->getValue().sextOrTrunc(IdxSize); 3985 LLVMContext &Context = *DAG.getContext(); 3986 SDValue OffsVal; 3987 if (IsVectorGEP) 3988 OffsVal = DAG.getConstant( 3989 Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorElementCount)); 3990 else 3991 OffsVal = DAG.getConstant(Offs, dl, IdxTy); 3992 3993 // In an inbounds GEP with an offset that is nonnegative even when 3994 // interpreted as signed, assume there is no unsigned overflow. 3995 SDNodeFlags Flags; 3996 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds()) 3997 Flags.setNoUnsignedWrap(true); 3998 3999 OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); 4000 4001 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); 4002 continue; 4003 } 4004 4005 // N = N + Idx * ElementMul; 4006 SDValue IdxN = getValue(Idx); 4007 4008 if (!IdxN.getValueType().isVector() && IsVectorGEP) { 4009 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), 4010 VectorElementCount); 4011 IdxN = DAG.getSplat(VT, dl, IdxN); 4012 } 4013 4014 // If the index is smaller or larger than intptr_t, truncate or extend 4015 // it. 4016 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType()); 4017 4018 if (ElementScalable) { 4019 EVT VScaleTy = N.getValueType().getScalarType(); 4020 SDValue VScale = DAG.getNode( 4021 ISD::VSCALE, dl, VScaleTy, 4022 DAG.getConstant(ElementMul.getZExtValue(), dl, VScaleTy)); 4023 if (IsVectorGEP) 4024 VScale = DAG.getSplatVector(N.getValueType(), dl, VScale); 4025 IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, VScale); 4026 } else { 4027 // If this is a multiply by a power of two, turn it into a shl 4028 // immediately. This is a very common case. 4029 if (ElementMul != 1) { 4030 if (ElementMul.isPowerOf2()) { 4031 unsigned Amt = ElementMul.logBase2(); 4032 IdxN = DAG.getNode(ISD::SHL, dl, 4033 N.getValueType(), IdxN, 4034 DAG.getConstant(Amt, dl, IdxN.getValueType())); 4035 } else { 4036 SDValue Scale = DAG.getConstant(ElementMul.getZExtValue(), dl, 4037 IdxN.getValueType()); 4038 IdxN = DAG.getNode(ISD::MUL, dl, 4039 N.getValueType(), IdxN, Scale); 4040 } 4041 } 4042 } 4043 4044 N = DAG.getNode(ISD::ADD, dl, 4045 N.getValueType(), N, IdxN); 4046 } 4047 } 4048 4049 MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS); 4050 MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS); 4051 if (IsVectorGEP) { 4052 PtrTy = MVT::getVectorVT(PtrTy, VectorElementCount); 4053 PtrMemTy = MVT::getVectorVT(PtrMemTy, VectorElementCount); 4054 } 4055 4056 if (PtrMemTy != PtrTy && !cast<GEPOperator>(I).isInBounds()) 4057 N = DAG.getPtrExtendInReg(N, dl, PtrMemTy); 4058 4059 setValue(&I, N); 4060 } 4061 4062 void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) { 4063 // If this is a fixed sized alloca in the entry block of the function, 4064 // allocate it statically on the stack. 4065 if (FuncInfo.StaticAllocaMap.count(&I)) 4066 return; // getValue will auto-populate this. 4067 4068 SDLoc dl = getCurSDLoc(); 4069 Type *Ty = I.getAllocatedType(); 4070 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4071 auto &DL = DAG.getDataLayout(); 4072 TypeSize TySize = DL.getTypeAllocSize(Ty); 4073 MaybeAlign Alignment = std::max(DL.getPrefTypeAlign(Ty), I.getAlign()); 4074 4075 SDValue AllocSize = getValue(I.getArraySize()); 4076 4077 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), I.getAddressSpace()); 4078 if (AllocSize.getValueType() != IntPtr) 4079 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr); 4080 4081 if (TySize.isScalable()) 4082 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, 4083 DAG.getVScale(dl, IntPtr, 4084 APInt(IntPtr.getScalarSizeInBits(), 4085 TySize.getKnownMinValue()))); 4086 else 4087 AllocSize = 4088 DAG.getNode(ISD::MUL, dl, IntPtr, AllocSize, 4089 DAG.getConstant(TySize.getFixedValue(), dl, IntPtr)); 4090 4091 // Handle alignment. If the requested alignment is less than or equal to 4092 // the stack alignment, ignore it. If the size is greater than or equal to 4093 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node. 4094 Align StackAlign = DAG.getSubtarget().getFrameLowering()->getStackAlign(); 4095 if (*Alignment <= StackAlign) 4096 Alignment = std::nullopt; 4097 4098 const uint64_t StackAlignMask = StackAlign.value() - 1U; 4099 // Round the size of the allocation up to the stack alignment size 4100 // by add SA-1 to the size. This doesn't overflow because we're computing 4101 // an address inside an alloca. 4102 SDNodeFlags Flags; 4103 Flags.setNoUnsignedWrap(true); 4104 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize, 4105 DAG.getConstant(StackAlignMask, dl, IntPtr), Flags); 4106 4107 // Mask out the low bits for alignment purposes. 4108 AllocSize = DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize, 4109 DAG.getConstant(~StackAlignMask, dl, IntPtr)); 4110 4111 SDValue Ops[] = { 4112 getRoot(), AllocSize, 4113 DAG.getConstant(Alignment ? Alignment->value() : 0, dl, IntPtr)}; 4114 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other); 4115 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops); 4116 setValue(&I, DSA); 4117 DAG.setRoot(DSA.getValue(1)); 4118 4119 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects()); 4120 } 4121 4122 void SelectionDAGBuilder::visitLoad(const LoadInst &I) { 4123 if (I.isAtomic()) 4124 return visitAtomicLoad(I); 4125 4126 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4127 const Value *SV = I.getOperand(0); 4128 if (TLI.supportSwiftError()) { 4129 // Swifterror values can come from either a function parameter with 4130 // swifterror attribute or an alloca with swifterror attribute. 4131 if (const Argument *Arg = dyn_cast<Argument>(SV)) { 4132 if (Arg->hasSwiftErrorAttr()) 4133 return visitLoadFromSwiftError(I); 4134 } 4135 4136 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) { 4137 if (Alloca->isSwiftError()) 4138 return visitLoadFromSwiftError(I); 4139 } 4140 } 4141 4142 SDValue Ptr = getValue(SV); 4143 4144 Type *Ty = I.getType(); 4145 SmallVector<EVT, 4> ValueVTs, MemVTs; 4146 SmallVector<uint64_t, 4> Offsets; 4147 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); 4148 unsigned NumValues = ValueVTs.size(); 4149 if (NumValues == 0) 4150 return; 4151 4152 Align Alignment = I.getAlign(); 4153 AAMDNodes AAInfo = I.getAAMetadata(); 4154 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); 4155 bool isVolatile = I.isVolatile(); 4156 MachineMemOperand::Flags MMOFlags = 4157 TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); 4158 4159 SDValue Root; 4160 bool ConstantMemory = false; 4161 if (isVolatile) 4162 // Serialize volatile loads with other side effects. 4163 Root = getRoot(); 4164 else if (NumValues > MaxParallelChains) 4165 Root = getMemoryRoot(); 4166 else if (AA && 4167 AA->pointsToConstantMemory(MemoryLocation( 4168 SV, 4169 LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), 4170 AAInfo))) { 4171 // Do not serialize (non-volatile) loads of constant memory with anything. 4172 Root = DAG.getEntryNode(); 4173 ConstantMemory = true; 4174 MMOFlags |= MachineMemOperand::MOInvariant; 4175 } else { 4176 // Do not serialize non-volatile loads against each other. 4177 Root = DAG.getRoot(); 4178 } 4179 4180 SDLoc dl = getCurSDLoc(); 4181 4182 if (isVolatile) 4183 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG); 4184 4185 // An aggregate load cannot wrap around the address space, so offsets to its 4186 // parts don't wrap either. 4187 SDNodeFlags Flags; 4188 Flags.setNoUnsignedWrap(true); 4189 4190 SmallVector<SDValue, 4> Values(NumValues); 4191 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); 4192 EVT PtrVT = Ptr.getValueType(); 4193 4194 unsigned ChainI = 0; 4195 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 4196 // Serializing loads here may result in excessive register pressure, and 4197 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling 4198 // could recover a bit by hoisting nodes upward in the chain by recognizing 4199 // they are side-effect free or do not alias. The optimizer should really 4200 // avoid this case by converting large object/array copies to llvm.memcpy 4201 // (MaxParallelChains should always remain as failsafe). 4202 if (ChainI == MaxParallelChains) { 4203 assert(PendingLoads.empty() && "PendingLoads must be serialized first"); 4204 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 4205 ArrayRef(Chains.data(), ChainI)); 4206 Root = Chain; 4207 ChainI = 0; 4208 } 4209 SDValue A = DAG.getNode(ISD::ADD, dl, 4210 PtrVT, Ptr, 4211 DAG.getConstant(Offsets[i], dl, PtrVT), 4212 Flags); 4213 4214 SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, 4215 MachinePointerInfo(SV, Offsets[i]), Alignment, 4216 MMOFlags, AAInfo, Ranges); 4217 Chains[ChainI] = L.getValue(1); 4218 4219 if (MemVTs[i] != ValueVTs[i]) 4220 L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]); 4221 4222 Values[i] = L; 4223 } 4224 4225 if (!ConstantMemory) { 4226 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 4227 ArrayRef(Chains.data(), ChainI)); 4228 if (isVolatile) 4229 DAG.setRoot(Chain); 4230 else 4231 PendingLoads.push_back(Chain); 4232 } 4233 4234 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl, 4235 DAG.getVTList(ValueVTs), Values)); 4236 } 4237 4238 void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) { 4239 assert(DAG.getTargetLoweringInfo().supportSwiftError() && 4240 "call visitStoreToSwiftError when backend supports swifterror"); 4241 4242 SmallVector<EVT, 4> ValueVTs; 4243 SmallVector<uint64_t, 4> Offsets; 4244 const Value *SrcV = I.getOperand(0); 4245 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), 4246 SrcV->getType(), ValueVTs, &Offsets); 4247 assert(ValueVTs.size() == 1 && Offsets[0] == 0 && 4248 "expect a single EVT for swifterror"); 4249 4250 SDValue Src = getValue(SrcV); 4251 // Create a virtual register, then update the virtual register. 4252 Register VReg = 4253 SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand()); 4254 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue 4255 // Chain can be getRoot or getControlRoot. 4256 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, 4257 SDValue(Src.getNode(), Src.getResNo())); 4258 DAG.setRoot(CopyNode); 4259 } 4260 4261 void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { 4262 assert(DAG.getTargetLoweringInfo().supportSwiftError() && 4263 "call visitLoadFromSwiftError when backend supports swifterror"); 4264 4265 assert(!I.isVolatile() && 4266 !I.hasMetadata(LLVMContext::MD_nontemporal) && 4267 !I.hasMetadata(LLVMContext::MD_invariant_load) && 4268 "Support volatile, non temporal, invariant for load_from_swift_error"); 4269 4270 const Value *SV = I.getOperand(0); 4271 Type *Ty = I.getType(); 4272 assert( 4273 (!AA || 4274 !AA->pointsToConstantMemory(MemoryLocation( 4275 SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), 4276 I.getAAMetadata()))) && 4277 "load_from_swift_error should not be constant memory"); 4278 4279 SmallVector<EVT, 4> ValueVTs; 4280 SmallVector<uint64_t, 4> Offsets; 4281 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty, 4282 ValueVTs, &Offsets); 4283 assert(ValueVTs.size() == 1 && Offsets[0] == 0 && 4284 "expect a single EVT for swifterror"); 4285 4286 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT 4287 SDValue L = DAG.getCopyFromReg( 4288 getRoot(), getCurSDLoc(), 4289 SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]); 4290 4291 setValue(&I, L); 4292 } 4293 4294 void SelectionDAGBuilder::visitStore(const StoreInst &I) { 4295 if (I.isAtomic()) 4296 return visitAtomicStore(I); 4297 4298 const Value *SrcV = I.getOperand(0); 4299 const Value *PtrV = I.getOperand(1); 4300 4301 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4302 if (TLI.supportSwiftError()) { 4303 // Swifterror values can come from either a function parameter with 4304 // swifterror attribute or an alloca with swifterror attribute. 4305 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) { 4306 if (Arg->hasSwiftErrorAttr()) 4307 return visitStoreToSwiftError(I); 4308 } 4309 4310 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) { 4311 if (Alloca->isSwiftError()) 4312 return visitStoreToSwiftError(I); 4313 } 4314 } 4315 4316 SmallVector<EVT, 4> ValueVTs, MemVTs; 4317 SmallVector<uint64_t, 4> Offsets; 4318 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), 4319 SrcV->getType(), ValueVTs, &MemVTs, &Offsets); 4320 unsigned NumValues = ValueVTs.size(); 4321 if (NumValues == 0) 4322 return; 4323 4324 // Get the lowered operands. Note that we do this after 4325 // checking if NumResults is zero, because with zero results 4326 // the operands won't have values in the map. 4327 SDValue Src = getValue(SrcV); 4328 SDValue Ptr = getValue(PtrV); 4329 4330 SDValue Root = I.isVolatile() ? getRoot() : getMemoryRoot(); 4331 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues)); 4332 SDLoc dl = getCurSDLoc(); 4333 Align Alignment = I.getAlign(); 4334 AAMDNodes AAInfo = I.getAAMetadata(); 4335 4336 auto MMOFlags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); 4337 4338 // An aggregate load cannot wrap around the address space, so offsets to its 4339 // parts don't wrap either. 4340 SDNodeFlags Flags; 4341 Flags.setNoUnsignedWrap(true); 4342 4343 unsigned ChainI = 0; 4344 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) { 4345 // See visitLoad comments. 4346 if (ChainI == MaxParallelChains) { 4347 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 4348 ArrayRef(Chains.data(), ChainI)); 4349 Root = Chain; 4350 ChainI = 0; 4351 } 4352 SDValue Add = 4353 DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(Offsets[i]), dl, Flags); 4354 SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); 4355 if (MemVTs[i] != ValueVTs[i]) 4356 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); 4357 SDValue St = 4358 DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), 4359 Alignment, MMOFlags, AAInfo); 4360 Chains[ChainI] = St; 4361 } 4362 4363 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, 4364 ArrayRef(Chains.data(), ChainI)); 4365 setValue(&I, StoreNode); 4366 DAG.setRoot(StoreNode); 4367 } 4368 4369 void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, 4370 bool IsCompressing) { 4371 SDLoc sdl = getCurSDLoc(); 4372 4373 auto getMaskedStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, 4374 MaybeAlign &Alignment) { 4375 // llvm.masked.store.*(Src0, Ptr, alignment, Mask) 4376 Src0 = I.getArgOperand(0); 4377 Ptr = I.getArgOperand(1); 4378 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getMaybeAlignValue(); 4379 Mask = I.getArgOperand(3); 4380 }; 4381 auto getCompressingStoreOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, 4382 MaybeAlign &Alignment) { 4383 // llvm.masked.compressstore.*(Src0, Ptr, Mask) 4384 Src0 = I.getArgOperand(0); 4385 Ptr = I.getArgOperand(1); 4386 Mask = I.getArgOperand(2); 4387 Alignment = std::nullopt; 4388 }; 4389 4390 Value *PtrOperand, *MaskOperand, *Src0Operand; 4391 MaybeAlign Alignment; 4392 if (IsCompressing) 4393 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); 4394 else 4395 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment); 4396 4397 SDValue Ptr = getValue(PtrOperand); 4398 SDValue Src0 = getValue(Src0Operand); 4399 SDValue Mask = getValue(MaskOperand); 4400 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 4401 4402 EVT VT = Src0.getValueType(); 4403 if (!Alignment) 4404 Alignment = DAG.getEVTAlign(VT); 4405 4406 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 4407 MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, 4408 MemoryLocation::UnknownSize, *Alignment, I.getAAMetadata()); 4409 SDValue StoreNode = 4410 DAG.getMaskedStore(getMemoryRoot(), sdl, Src0, Ptr, Offset, Mask, VT, MMO, 4411 ISD::UNINDEXED, false /* Truncating */, IsCompressing); 4412 DAG.setRoot(StoreNode); 4413 setValue(&I, StoreNode); 4414 } 4415 4416 // Get a uniform base for the Gather/Scatter intrinsic. 4417 // The first argument of the Gather/Scatter intrinsic is a vector of pointers. 4418 // We try to represent it as a base pointer + vector of indices. 4419 // Usually, the vector of pointers comes from a 'getelementptr' instruction. 4420 // The first operand of the GEP may be a single pointer or a vector of pointers 4421 // Example: 4422 // %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind 4423 // or 4424 // %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind 4425 // %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, .. 4426 // 4427 // When the first GEP operand is a single pointer - it is the uniform base we 4428 // are looking for. If first operand of the GEP is a splat vector - we 4429 // extract the splat value and use it as a uniform base. 4430 // In all other cases the function returns 'false'. 4431 static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, 4432 ISD::MemIndexType &IndexType, SDValue &Scale, 4433 SelectionDAGBuilder *SDB, const BasicBlock *CurBB, 4434 uint64_t ElemSize) { 4435 SelectionDAG& DAG = SDB->DAG; 4436 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4437 const DataLayout &DL = DAG.getDataLayout(); 4438 4439 assert(Ptr->getType()->isVectorTy() && "Unexpected pointer type"); 4440 4441 // Handle splat constant pointer. 4442 if (auto *C = dyn_cast<Constant>(Ptr)) { 4443 C = C->getSplatValue(); 4444 if (!C) 4445 return false; 4446 4447 Base = SDB->getValue(C); 4448 4449 ElementCount NumElts = cast<VectorType>(Ptr->getType())->getElementCount(); 4450 EVT VT = EVT::getVectorVT(*DAG.getContext(), TLI.getPointerTy(DL), NumElts); 4451 Index = DAG.getConstant(0, SDB->getCurSDLoc(), VT); 4452 IndexType = ISD::SIGNED_SCALED; 4453 Scale = DAG.getTargetConstant(1, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); 4454 return true; 4455 } 4456 4457 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr); 4458 if (!GEP || GEP->getParent() != CurBB) 4459 return false; 4460 4461 if (GEP->getNumOperands() != 2) 4462 return false; 4463 4464 const Value *BasePtr = GEP->getPointerOperand(); 4465 const Value *IndexVal = GEP->getOperand(GEP->getNumOperands() - 1); 4466 4467 // Make sure the base is scalar and the index is a vector. 4468 if (BasePtr->getType()->isVectorTy() || !IndexVal->getType()->isVectorTy()) 4469 return false; 4470 4471 uint64_t ScaleVal = DL.getTypeAllocSize(GEP->getResultElementType()); 4472 4473 // Target may not support the required addressing mode. 4474 if (ScaleVal != 1 && 4475 !TLI.isLegalScaleForGatherScatter(ScaleVal, ElemSize)) 4476 return false; 4477 4478 Base = SDB->getValue(BasePtr); 4479 Index = SDB->getValue(IndexVal); 4480 IndexType = ISD::SIGNED_SCALED; 4481 4482 Scale = 4483 DAG.getTargetConstant(ScaleVal, SDB->getCurSDLoc(), TLI.getPointerTy(DL)); 4484 return true; 4485 } 4486 4487 void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { 4488 SDLoc sdl = getCurSDLoc(); 4489 4490 // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) 4491 const Value *Ptr = I.getArgOperand(1); 4492 SDValue Src0 = getValue(I.getArgOperand(0)); 4493 SDValue Mask = getValue(I.getArgOperand(3)); 4494 EVT VT = Src0.getValueType(); 4495 Align Alignment = cast<ConstantInt>(I.getArgOperand(2)) 4496 ->getMaybeAlignValue() 4497 .value_or(DAG.getEVTAlign(VT.getScalarType())); 4498 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4499 4500 SDValue Base; 4501 SDValue Index; 4502 ISD::MemIndexType IndexType; 4503 SDValue Scale; 4504 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, 4505 I.getParent(), VT.getScalarStoreSize()); 4506 4507 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); 4508 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 4509 MachinePointerInfo(AS), MachineMemOperand::MOStore, 4510 // TODO: Make MachineMemOperands aware of scalable 4511 // vectors. 4512 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata()); 4513 if (!UniformBase) { 4514 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); 4515 Index = getValue(Ptr); 4516 IndexType = ISD::SIGNED_SCALED; 4517 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); 4518 } 4519 4520 EVT IdxVT = Index.getValueType(); 4521 EVT EltTy = IdxVT.getVectorElementType(); 4522 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { 4523 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); 4524 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); 4525 } 4526 4527 SDValue Ops[] = { getMemoryRoot(), Src0, Mask, Base, Index, Scale }; 4528 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl, 4529 Ops, MMO, IndexType, false); 4530 DAG.setRoot(Scatter); 4531 setValue(&I, Scatter); 4532 } 4533 4534 void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { 4535 SDLoc sdl = getCurSDLoc(); 4536 4537 auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, 4538 MaybeAlign &Alignment) { 4539 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0) 4540 Ptr = I.getArgOperand(0); 4541 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getMaybeAlignValue(); 4542 Mask = I.getArgOperand(2); 4543 Src0 = I.getArgOperand(3); 4544 }; 4545 auto getExpandingLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, 4546 MaybeAlign &Alignment) { 4547 // @llvm.masked.expandload.*(Ptr, Mask, Src0) 4548 Ptr = I.getArgOperand(0); 4549 Alignment = std::nullopt; 4550 Mask = I.getArgOperand(1); 4551 Src0 = I.getArgOperand(2); 4552 }; 4553 4554 Value *PtrOperand, *MaskOperand, *Src0Operand; 4555 MaybeAlign Alignment; 4556 if (IsExpanding) 4557 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); 4558 else 4559 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment); 4560 4561 SDValue Ptr = getValue(PtrOperand); 4562 SDValue Src0 = getValue(Src0Operand); 4563 SDValue Mask = getValue(MaskOperand); 4564 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 4565 4566 EVT VT = Src0.getValueType(); 4567 if (!Alignment) 4568 Alignment = DAG.getEVTAlign(VT); 4569 4570 AAMDNodes AAInfo = I.getAAMetadata(); 4571 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); 4572 4573 // Do not serialize masked loads of constant memory with anything. 4574 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); 4575 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); 4576 4577 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); 4578 4579 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 4580 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, 4581 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); 4582 4583 SDValue Load = 4584 DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Offset, Mask, Src0, VT, MMO, 4585 ISD::UNINDEXED, ISD::NON_EXTLOAD, IsExpanding); 4586 if (AddToChain) 4587 PendingLoads.push_back(Load.getValue(1)); 4588 setValue(&I, Load); 4589 } 4590 4591 void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { 4592 SDLoc sdl = getCurSDLoc(); 4593 4594 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) 4595 const Value *Ptr = I.getArgOperand(0); 4596 SDValue Src0 = getValue(I.getArgOperand(3)); 4597 SDValue Mask = getValue(I.getArgOperand(2)); 4598 4599 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4600 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 4601 Align Alignment = cast<ConstantInt>(I.getArgOperand(1)) 4602 ->getMaybeAlignValue() 4603 .value_or(DAG.getEVTAlign(VT.getScalarType())); 4604 4605 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); 4606 4607 SDValue Root = DAG.getRoot(); 4608 SDValue Base; 4609 SDValue Index; 4610 ISD::MemIndexType IndexType; 4611 SDValue Scale; 4612 bool UniformBase = getUniformBase(Ptr, Base, Index, IndexType, Scale, this, 4613 I.getParent(), VT.getScalarStoreSize()); 4614 unsigned AS = Ptr->getType()->getScalarType()->getPointerAddressSpace(); 4615 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 4616 MachinePointerInfo(AS), MachineMemOperand::MOLoad, 4617 // TODO: Make MachineMemOperands aware of scalable 4618 // vectors. 4619 MemoryLocation::UnknownSize, Alignment, I.getAAMetadata(), Ranges); 4620 4621 if (!UniformBase) { 4622 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout())); 4623 Index = getValue(Ptr); 4624 IndexType = ISD::SIGNED_SCALED; 4625 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout())); 4626 } 4627 4628 EVT IdxVT = Index.getValueType(); 4629 EVT EltTy = IdxVT.getVectorElementType(); 4630 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { 4631 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); 4632 Index = DAG.getNode(ISD::SIGN_EXTEND, sdl, NewIdxVT, Index); 4633 } 4634 4635 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale }; 4636 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl, 4637 Ops, MMO, IndexType, ISD::NON_EXTLOAD); 4638 4639 PendingLoads.push_back(Gather.getValue(1)); 4640 setValue(&I, Gather); 4641 } 4642 4643 void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { 4644 SDLoc dl = getCurSDLoc(); 4645 AtomicOrdering SuccessOrdering = I.getSuccessOrdering(); 4646 AtomicOrdering FailureOrdering = I.getFailureOrdering(); 4647 SyncScope::ID SSID = I.getSyncScopeID(); 4648 4649 SDValue InChain = getRoot(); 4650 4651 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); 4652 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); 4653 4654 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4655 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); 4656 4657 MachineFunction &MF = DAG.getMachineFunction(); 4658 MachineMemOperand *MMO = MF.getMachineMemOperand( 4659 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), 4660 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, SuccessOrdering, 4661 FailureOrdering); 4662 4663 SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, 4664 dl, MemVT, VTs, InChain, 4665 getValue(I.getPointerOperand()), 4666 getValue(I.getCompareOperand()), 4667 getValue(I.getNewValOperand()), MMO); 4668 4669 SDValue OutChain = L.getValue(2); 4670 4671 setValue(&I, L); 4672 DAG.setRoot(OutChain); 4673 } 4674 4675 void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { 4676 SDLoc dl = getCurSDLoc(); 4677 ISD::NodeType NT; 4678 switch (I.getOperation()) { 4679 default: llvm_unreachable("Unknown atomicrmw operation"); 4680 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break; 4681 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break; 4682 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break; 4683 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break; 4684 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break; 4685 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break; 4686 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break; 4687 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break; 4688 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; 4689 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; 4690 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; 4691 case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break; 4692 case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break; 4693 case AtomicRMWInst::FMax: NT = ISD::ATOMIC_LOAD_FMAX; break; 4694 case AtomicRMWInst::FMin: NT = ISD::ATOMIC_LOAD_FMIN; break; 4695 case AtomicRMWInst::UIncWrap: 4696 NT = ISD::ATOMIC_LOAD_UINC_WRAP; 4697 break; 4698 case AtomicRMWInst::UDecWrap: 4699 NT = ISD::ATOMIC_LOAD_UDEC_WRAP; 4700 break; 4701 } 4702 AtomicOrdering Ordering = I.getOrdering(); 4703 SyncScope::ID SSID = I.getSyncScopeID(); 4704 4705 SDValue InChain = getRoot(); 4706 4707 auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); 4708 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4709 auto Flags = TLI.getAtomicMemOperandFlags(I, DAG.getDataLayout()); 4710 4711 MachineFunction &MF = DAG.getMachineFunction(); 4712 MachineMemOperand *MMO = MF.getMachineMemOperand( 4713 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), 4714 DAG.getEVTAlign(MemVT), AAMDNodes(), nullptr, SSID, Ordering); 4715 4716 SDValue L = 4717 DAG.getAtomic(NT, dl, MemVT, InChain, 4718 getValue(I.getPointerOperand()), getValue(I.getValOperand()), 4719 MMO); 4720 4721 SDValue OutChain = L.getValue(1); 4722 4723 setValue(&I, L); 4724 DAG.setRoot(OutChain); 4725 } 4726 4727 void SelectionDAGBuilder::visitFence(const FenceInst &I) { 4728 SDLoc dl = getCurSDLoc(); 4729 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4730 SDValue Ops[3]; 4731 Ops[0] = getRoot(); 4732 Ops[1] = DAG.getTargetConstant((unsigned)I.getOrdering(), dl, 4733 TLI.getFenceOperandTy(DAG.getDataLayout())); 4734 Ops[2] = DAG.getTargetConstant(I.getSyncScopeID(), dl, 4735 TLI.getFenceOperandTy(DAG.getDataLayout())); 4736 SDValue N = DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops); 4737 setValue(&I, N); 4738 DAG.setRoot(N); 4739 } 4740 4741 void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { 4742 SDLoc dl = getCurSDLoc(); 4743 AtomicOrdering Order = I.getOrdering(); 4744 SyncScope::ID SSID = I.getSyncScopeID(); 4745 4746 SDValue InChain = getRoot(); 4747 4748 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4749 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 4750 EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); 4751 4752 if (!TLI.supportsUnalignedAtomics() && 4753 I.getAlign().value() < MemVT.getSizeInBits() / 8) 4754 report_fatal_error("Cannot generate unaligned atomic load"); 4755 4756 auto Flags = TLI.getLoadMemOperandFlags(I, DAG.getDataLayout(), AC, LibInfo); 4757 4758 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 4759 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), 4760 I.getAlign(), AAMDNodes(), nullptr, SSID, Order); 4761 4762 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); 4763 4764 SDValue Ptr = getValue(I.getPointerOperand()); 4765 4766 if (TLI.lowerAtomicLoadAsLoadSDNode(I)) { 4767 // TODO: Once this is better exercised by tests, it should be merged with 4768 // the normal path for loads to prevent future divergence. 4769 SDValue L = DAG.getLoad(MemVT, dl, InChain, Ptr, MMO); 4770 if (MemVT != VT) 4771 L = DAG.getPtrExtOrTrunc(L, dl, VT); 4772 4773 setValue(&I, L); 4774 SDValue OutChain = L.getValue(1); 4775 if (!I.isUnordered()) 4776 DAG.setRoot(OutChain); 4777 else 4778 PendingLoads.push_back(OutChain); 4779 return; 4780 } 4781 4782 SDValue L = DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain, 4783 Ptr, MMO); 4784 4785 SDValue OutChain = L.getValue(1); 4786 if (MemVT != VT) 4787 L = DAG.getPtrExtOrTrunc(L, dl, VT); 4788 4789 setValue(&I, L); 4790 DAG.setRoot(OutChain); 4791 } 4792 4793 void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { 4794 SDLoc dl = getCurSDLoc(); 4795 4796 AtomicOrdering Ordering = I.getOrdering(); 4797 SyncScope::ID SSID = I.getSyncScopeID(); 4798 4799 SDValue InChain = getRoot(); 4800 4801 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4802 EVT MemVT = 4803 TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); 4804 4805 if (!TLI.supportsUnalignedAtomics() && 4806 I.getAlign().value() < MemVT.getSizeInBits() / 8) 4807 report_fatal_error("Cannot generate unaligned atomic store"); 4808 4809 auto Flags = TLI.getStoreMemOperandFlags(I, DAG.getDataLayout()); 4810 4811 MachineFunction &MF = DAG.getMachineFunction(); 4812 MachineMemOperand *MMO = MF.getMachineMemOperand( 4813 MachinePointerInfo(I.getPointerOperand()), Flags, MemVT.getStoreSize(), 4814 I.getAlign(), AAMDNodes(), nullptr, SSID, Ordering); 4815 4816 SDValue Val = getValue(I.getValueOperand()); 4817 if (Val.getValueType() != MemVT) 4818 Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT); 4819 SDValue Ptr = getValue(I.getPointerOperand()); 4820 4821 if (TLI.lowerAtomicStoreAsStoreSDNode(I)) { 4822 // TODO: Once this is better exercised by tests, it should be merged with 4823 // the normal path for stores to prevent future divergence. 4824 SDValue S = DAG.getStore(InChain, dl, Val, Ptr, MMO); 4825 setValue(&I, S); 4826 DAG.setRoot(S); 4827 return; 4828 } 4829 SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, 4830 Ptr, Val, MMO); 4831 4832 setValue(&I, OutChain); 4833 DAG.setRoot(OutChain); 4834 } 4835 4836 /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC 4837 /// node. 4838 void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, 4839 unsigned Intrinsic) { 4840 // Ignore the callsite's attributes. A specific call site may be marked with 4841 // readnone, but the lowering code will expect the chain based on the 4842 // definition. 4843 const Function *F = I.getCalledFunction(); 4844 bool HasChain = !F->doesNotAccessMemory(); 4845 bool OnlyLoad = HasChain && F->onlyReadsMemory(); 4846 4847 // Build the operand list. 4848 SmallVector<SDValue, 8> Ops; 4849 if (HasChain) { // If this intrinsic has side-effects, chainify it. 4850 if (OnlyLoad) { 4851 // We don't need to serialize loads against other loads. 4852 Ops.push_back(DAG.getRoot()); 4853 } else { 4854 Ops.push_back(getRoot()); 4855 } 4856 } 4857 4858 // Info is set by getTgtMemIntrinsic 4859 TargetLowering::IntrinsicInfo Info; 4860 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 4861 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, 4862 DAG.getMachineFunction(), 4863 Intrinsic); 4864 4865 // Add the intrinsic ID as an integer operand if it's not a target intrinsic. 4866 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || 4867 Info.opc == ISD::INTRINSIC_W_CHAIN) 4868 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(), 4869 TLI.getPointerTy(DAG.getDataLayout()))); 4870 4871 // Add all operands of the call to the operand list. 4872 for (unsigned i = 0, e = I.arg_size(); i != e; ++i) { 4873 const Value *Arg = I.getArgOperand(i); 4874 if (!I.paramHasAttr(i, Attribute::ImmArg)) { 4875 Ops.push_back(getValue(Arg)); 4876 continue; 4877 } 4878 4879 // Use TargetConstant instead of a regular constant for immarg. 4880 EVT VT = TLI.getValueType(DAG.getDataLayout(), Arg->getType(), true); 4881 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Arg)) { 4882 assert(CI->getBitWidth() <= 64 && 4883 "large intrinsic immediates not handled"); 4884 Ops.push_back(DAG.getTargetConstant(*CI, SDLoc(), VT)); 4885 } else { 4886 Ops.push_back( 4887 DAG.getTargetConstantFP(*cast<ConstantFP>(Arg), SDLoc(), VT)); 4888 } 4889 } 4890 4891 SmallVector<EVT, 4> ValueVTs; 4892 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); 4893 4894 if (HasChain) 4895 ValueVTs.push_back(MVT::Other); 4896 4897 SDVTList VTs = DAG.getVTList(ValueVTs); 4898 4899 // Propagate fast-math-flags from IR to node(s). 4900 SDNodeFlags Flags; 4901 if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) 4902 Flags.copyFMF(*FPMO); 4903 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); 4904 4905 // Create the node. 4906 SDValue Result; 4907 // In some cases, custom collection of operands from CallInst I may be needed. 4908 TLI.CollectTargetIntrinsicOperands(I, Ops, DAG); 4909 if (IsTgtIntrinsic) { 4910 // This is target intrinsic that touches memory 4911 // 4912 // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic 4913 // didn't yield anything useful. 4914 MachinePointerInfo MPI; 4915 if (Info.ptrVal) 4916 MPI = MachinePointerInfo(Info.ptrVal, Info.offset); 4917 else if (Info.fallbackAddressSpace) 4918 MPI = MachinePointerInfo(*Info.fallbackAddressSpace); 4919 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, 4920 Info.memVT, MPI, Info.align, Info.flags, 4921 Info.size, I.getAAMetadata()); 4922 } else if (!HasChain) { 4923 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); 4924 } else if (!I.getType()->isVoidTy()) { 4925 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); 4926 } else { 4927 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); 4928 } 4929 4930 if (HasChain) { 4931 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); 4932 if (OnlyLoad) 4933 PendingLoads.push_back(Chain); 4934 else 4935 DAG.setRoot(Chain); 4936 } 4937 4938 if (!I.getType()->isVoidTy()) { 4939 if (!isa<VectorType>(I.getType())) 4940 Result = lowerRangeToAssertZExt(DAG, I, Result); 4941 4942 MaybeAlign Alignment = I.getRetAlign(); 4943 4944 // Insert `assertalign` node if there's an alignment. 4945 if (InsertAssertAlign && Alignment) { 4946 Result = 4947 DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); 4948 } 4949 4950 setValue(&I, Result); 4951 } 4952 } 4953 4954 /// GetSignificand - Get the significand and build it into a floating-point 4955 /// number with exponent of 1: 4956 /// 4957 /// Op = (Op & 0x007fffff) | 0x3f800000; 4958 /// 4959 /// where Op is the hexadecimal representation of floating point value. 4960 static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) { 4961 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 4962 DAG.getConstant(0x007fffff, dl, MVT::i32)); 4963 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1, 4964 DAG.getConstant(0x3f800000, dl, MVT::i32)); 4965 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2); 4966 } 4967 4968 /// GetExponent - Get the exponent: 4969 /// 4970 /// (float)(int)(((Op & 0x7f800000) >> 23) - 127); 4971 /// 4972 /// where Op is the hexadecimal representation of floating point value. 4973 static SDValue GetExponent(SelectionDAG &DAG, SDValue Op, 4974 const TargetLowering &TLI, const SDLoc &dl) { 4975 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op, 4976 DAG.getConstant(0x7f800000, dl, MVT::i32)); 4977 SDValue t1 = DAG.getNode( 4978 ISD::SRL, dl, MVT::i32, t0, 4979 DAG.getConstant(23, dl, 4980 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout()))); 4981 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1, 4982 DAG.getConstant(127, dl, MVT::i32)); 4983 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2); 4984 } 4985 4986 /// getF32Constant - Get 32-bit floating point constant. 4987 static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt, 4988 const SDLoc &dl) { 4989 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl, 4990 MVT::f32); 4991 } 4992 4993 static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl, 4994 SelectionDAG &DAG) { 4995 // TODO: What fast-math-flags should be set on the floating-point nodes? 4996 4997 // IntegerPartOfX = ((int32_t)(t0); 4998 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0); 4999 5000 // FractionalPartOfX = t0 - (float)IntegerPartOfX; 5001 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX); 5002 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1); 5003 5004 // IntegerPartOfX <<= 23; 5005 IntegerPartOfX = 5006 DAG.getNode(ISD::SHL, dl, MVT::i32, IntegerPartOfX, 5007 DAG.getConstant(23, dl, 5008 DAG.getTargetLoweringInfo().getShiftAmountTy( 5009 MVT::i32, DAG.getDataLayout()))); 5010 5011 SDValue TwoToFractionalPartOfX; 5012 if (LimitFloatPrecision <= 6) { 5013 // For floating-point precision of 6: 5014 // 5015 // TwoToFractionalPartOfX = 5016 // 0.997535578f + 5017 // (0.735607626f + 0.252464424f * x) * x; 5018 // 5019 // error 0.0144103317, which is 6 bits 5020 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5021 getF32Constant(DAG, 0x3e814304, dl)); 5022 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 5023 getF32Constant(DAG, 0x3f3c50c8, dl)); 5024 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5025 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5026 getF32Constant(DAG, 0x3f7f5e7e, dl)); 5027 } else if (LimitFloatPrecision <= 12) { 5028 // For floating-point precision of 12: 5029 // 5030 // TwoToFractionalPartOfX = 5031 // 0.999892986f + 5032 // (0.696457318f + 5033 // (0.224338339f + 0.792043434e-1f * x) * x) * x; 5034 // 5035 // error 0.000107046256, which is 13 to 14 bits 5036 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5037 getF32Constant(DAG, 0x3da235e3, dl)); 5038 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 5039 getF32Constant(DAG, 0x3e65b8f3, dl)); 5040 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5041 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5042 getF32Constant(DAG, 0x3f324b07, dl)); 5043 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5044 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 5045 getF32Constant(DAG, 0x3f7ff8fd, dl)); 5046 } else { // LimitFloatPrecision <= 18 5047 // For floating-point precision of 18: 5048 // 5049 // TwoToFractionalPartOfX = 5050 // 0.999999982f + 5051 // (0.693148872f + 5052 // (0.240227044f + 5053 // (0.554906021e-1f + 5054 // (0.961591928e-2f + 5055 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x; 5056 // error 2.47208000*10^(-7), which is better than 18 bits 5057 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5058 getF32Constant(DAG, 0x3924b03e, dl)); 5059 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 5060 getF32Constant(DAG, 0x3ab24b87, dl)); 5061 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5062 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5063 getF32Constant(DAG, 0x3c1d8c17, dl)); 5064 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5065 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 5066 getF32Constant(DAG, 0x3d634a1d, dl)); 5067 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 5068 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 5069 getF32Constant(DAG, 0x3e75fe14, dl)); 5070 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 5071 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10, 5072 getF32Constant(DAG, 0x3f317234, dl)); 5073 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X); 5074 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12, 5075 getF32Constant(DAG, 0x3f800000, dl)); 5076 } 5077 5078 // Add the exponent into the result in integer domain. 5079 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX); 5080 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, 5081 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX)); 5082 } 5083 5084 /// expandExp - Lower an exp intrinsic. Handles the special sequences for 5085 /// limited-precision mode. 5086 static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, 5087 const TargetLowering &TLI, SDNodeFlags Flags) { 5088 if (Op.getValueType() == MVT::f32 && 5089 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 5090 5091 // Put the exponent in the right bit position for later addition to the 5092 // final result: 5093 // 5094 // t0 = Op * log2(e) 5095 5096 // TODO: What fast-math-flags should be set here? 5097 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op, 5098 DAG.getConstantFP(numbers::log2ef, dl, MVT::f32)); 5099 return getLimitedPrecisionExp2(t0, dl, DAG); 5100 } 5101 5102 // No special expansion. 5103 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op, Flags); 5104 } 5105 5106 /// expandLog - Lower a log intrinsic. Handles the special sequences for 5107 /// limited-precision mode. 5108 static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, 5109 const TargetLowering &TLI, SDNodeFlags Flags) { 5110 // TODO: What fast-math-flags should be set on the floating-point nodes? 5111 5112 if (Op.getValueType() == MVT::f32 && 5113 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 5114 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 5115 5116 // Scale the exponent by log(2). 5117 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 5118 SDValue LogOfExponent = 5119 DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 5120 DAG.getConstantFP(numbers::ln2f, dl, MVT::f32)); 5121 5122 // Get the significand and build it into a floating-point number with 5123 // exponent of 1. 5124 SDValue X = GetSignificand(DAG, Op1, dl); 5125 5126 SDValue LogOfMantissa; 5127 if (LimitFloatPrecision <= 6) { 5128 // For floating-point precision of 6: 5129 // 5130 // LogofMantissa = 5131 // -1.1609546f + 5132 // (1.4034025f - 0.23903021f * x) * x; 5133 // 5134 // error 0.0034276066, which is better than 8 bits 5135 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5136 getF32Constant(DAG, 0xbe74c456, dl)); 5137 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5138 getF32Constant(DAG, 0x3fb3a2b1, dl)); 5139 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5140 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5141 getF32Constant(DAG, 0x3f949a29, dl)); 5142 } else if (LimitFloatPrecision <= 12) { 5143 // For floating-point precision of 12: 5144 // 5145 // LogOfMantissa = 5146 // -1.7417939f + 5147 // (2.8212026f + 5148 // (-1.4699568f + 5149 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x; 5150 // 5151 // error 0.000061011436, which is 14 bits 5152 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5153 getF32Constant(DAG, 0xbd67b6d6, dl)); 5154 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5155 getF32Constant(DAG, 0x3ee4f4b8, dl)); 5156 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5157 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5158 getF32Constant(DAG, 0x3fbc278b, dl)); 5159 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5160 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5161 getF32Constant(DAG, 0x40348e95, dl)); 5162 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5163 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 5164 getF32Constant(DAG, 0x3fdef31a, dl)); 5165 } else { // LimitFloatPrecision <= 18 5166 // For floating-point precision of 18: 5167 // 5168 // LogOfMantissa = 5169 // -2.1072184f + 5170 // (4.2372794f + 5171 // (-3.7029485f + 5172 // (2.2781945f + 5173 // (-0.87823314f + 5174 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x; 5175 // 5176 // error 0.0000023660568, which is better than 18 bits 5177 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5178 getF32Constant(DAG, 0xbc91e5ac, dl)); 5179 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5180 getF32Constant(DAG, 0x3e4350aa, dl)); 5181 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5182 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5183 getF32Constant(DAG, 0x3f60d3e3, dl)); 5184 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5185 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5186 getF32Constant(DAG, 0x4011cdf0, dl)); 5187 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5188 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 5189 getF32Constant(DAG, 0x406cfd1c, dl)); 5190 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 5191 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 5192 getF32Constant(DAG, 0x408797cb, dl)); 5193 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 5194 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 5195 getF32Constant(DAG, 0x4006dcab, dl)); 5196 } 5197 5198 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa); 5199 } 5200 5201 // No special expansion. 5202 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op, Flags); 5203 } 5204 5205 /// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for 5206 /// limited-precision mode. 5207 static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, 5208 const TargetLowering &TLI, SDNodeFlags Flags) { 5209 // TODO: What fast-math-flags should be set on the floating-point nodes? 5210 5211 if (Op.getValueType() == MVT::f32 && 5212 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 5213 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 5214 5215 // Get the exponent. 5216 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl); 5217 5218 // Get the significand and build it into a floating-point number with 5219 // exponent of 1. 5220 SDValue X = GetSignificand(DAG, Op1, dl); 5221 5222 // Different possible minimax approximations of significand in 5223 // floating-point for various degrees of accuracy over [1,2]. 5224 SDValue Log2ofMantissa; 5225 if (LimitFloatPrecision <= 6) { 5226 // For floating-point precision of 6: 5227 // 5228 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x; 5229 // 5230 // error 0.0049451742, which is more than 7 bits 5231 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5232 getF32Constant(DAG, 0xbeb08fe0, dl)); 5233 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5234 getF32Constant(DAG, 0x40019463, dl)); 5235 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5236 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5237 getF32Constant(DAG, 0x3fd6633d, dl)); 5238 } else if (LimitFloatPrecision <= 12) { 5239 // For floating-point precision of 12: 5240 // 5241 // Log2ofMantissa = 5242 // -2.51285454f + 5243 // (4.07009056f + 5244 // (-2.12067489f + 5245 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x; 5246 // 5247 // error 0.0000876136000, which is better than 13 bits 5248 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5249 getF32Constant(DAG, 0xbda7262e, dl)); 5250 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5251 getF32Constant(DAG, 0x3f25280b, dl)); 5252 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5253 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5254 getF32Constant(DAG, 0x4007b923, dl)); 5255 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5256 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5257 getF32Constant(DAG, 0x40823e2f, dl)); 5258 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5259 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 5260 getF32Constant(DAG, 0x4020d29c, dl)); 5261 } else { // LimitFloatPrecision <= 18 5262 // For floating-point precision of 18: 5263 // 5264 // Log2ofMantissa = 5265 // -3.0400495f + 5266 // (6.1129976f + 5267 // (-5.3420409f + 5268 // (3.2865683f + 5269 // (-1.2669343f + 5270 // (0.27515199f - 5271 // 0.25691327e-1f * x) * x) * x) * x) * x) * x; 5272 // 5273 // error 0.0000018516, which is better than 18 bits 5274 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5275 getF32Constant(DAG, 0xbcd2769e, dl)); 5276 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5277 getF32Constant(DAG, 0x3e8ce0b9, dl)); 5278 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5279 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5280 getF32Constant(DAG, 0x3fa22ae7, dl)); 5281 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5282 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4, 5283 getF32Constant(DAG, 0x40525723, dl)); 5284 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5285 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6, 5286 getF32Constant(DAG, 0x40aaf200, dl)); 5287 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 5288 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8, 5289 getF32Constant(DAG, 0x40c39dad, dl)); 5290 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X); 5291 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10, 5292 getF32Constant(DAG, 0x4042902c, dl)); 5293 } 5294 5295 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa); 5296 } 5297 5298 // No special expansion. 5299 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op, Flags); 5300 } 5301 5302 /// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for 5303 /// limited-precision mode. 5304 static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, 5305 const TargetLowering &TLI, SDNodeFlags Flags) { 5306 // TODO: What fast-math-flags should be set on the floating-point nodes? 5307 5308 if (Op.getValueType() == MVT::f32 && 5309 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 5310 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op); 5311 5312 // Scale the exponent by log10(2) [0.30102999f]. 5313 SDValue Exp = GetExponent(DAG, Op1, TLI, dl); 5314 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp, 5315 getF32Constant(DAG, 0x3e9a209a, dl)); 5316 5317 // Get the significand and build it into a floating-point number with 5318 // exponent of 1. 5319 SDValue X = GetSignificand(DAG, Op1, dl); 5320 5321 SDValue Log10ofMantissa; 5322 if (LimitFloatPrecision <= 6) { 5323 // For floating-point precision of 6: 5324 // 5325 // Log10ofMantissa = 5326 // -0.50419619f + 5327 // (0.60948995f - 0.10380950f * x) * x; 5328 // 5329 // error 0.0014886165, which is 6 bits 5330 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5331 getF32Constant(DAG, 0xbdd49a13, dl)); 5332 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0, 5333 getF32Constant(DAG, 0x3f1c0789, dl)); 5334 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5335 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2, 5336 getF32Constant(DAG, 0x3f011300, dl)); 5337 } else if (LimitFloatPrecision <= 12) { 5338 // For floating-point precision of 12: 5339 // 5340 // Log10ofMantissa = 5341 // -0.64831180f + 5342 // (0.91751397f + 5343 // (-0.31664806f + 0.47637168e-1f * x) * x) * x; 5344 // 5345 // error 0.00019228036, which is better than 12 bits 5346 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5347 getF32Constant(DAG, 0x3d431f31, dl)); 5348 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 5349 getF32Constant(DAG, 0x3ea21fb2, dl)); 5350 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5351 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 5352 getF32Constant(DAG, 0x3f6ae232, dl)); 5353 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5354 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 5355 getF32Constant(DAG, 0x3f25f7c3, dl)); 5356 } else { // LimitFloatPrecision <= 18 5357 // For floating-point precision of 18: 5358 // 5359 // Log10ofMantissa = 5360 // -0.84299375f + 5361 // (1.5327582f + 5362 // (-1.0688956f + 5363 // (0.49102474f + 5364 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x; 5365 // 5366 // error 0.0000037995730, which is better than 18 bits 5367 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X, 5368 getF32Constant(DAG, 0x3c5d51ce, dl)); 5369 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, 5370 getF32Constant(DAG, 0x3e00685a, dl)); 5371 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X); 5372 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2, 5373 getF32Constant(DAG, 0x3efb6798, dl)); 5374 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X); 5375 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4, 5376 getF32Constant(DAG, 0x3f88d192, dl)); 5377 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X); 5378 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6, 5379 getF32Constant(DAG, 0x3fc4316c, dl)); 5380 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X); 5381 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8, 5382 getF32Constant(DAG, 0x3f57ce70, dl)); 5383 } 5384 5385 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa); 5386 } 5387 5388 // No special expansion. 5389 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op, Flags); 5390 } 5391 5392 /// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for 5393 /// limited-precision mode. 5394 static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG, 5395 const TargetLowering &TLI, SDNodeFlags Flags) { 5396 if (Op.getValueType() == MVT::f32 && 5397 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) 5398 return getLimitedPrecisionExp2(Op, dl, DAG); 5399 5400 // No special expansion. 5401 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op, Flags); 5402 } 5403 5404 /// visitPow - Lower a pow intrinsic. Handles the special sequences for 5405 /// limited-precision mode with x == 10.0f. 5406 static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS, 5407 SelectionDAG &DAG, const TargetLowering &TLI, 5408 SDNodeFlags Flags) { 5409 bool IsExp10 = false; 5410 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 && 5411 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) { 5412 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) { 5413 APFloat Ten(10.0f); 5414 IsExp10 = LHSC->isExactlyValue(Ten); 5415 } 5416 } 5417 5418 // TODO: What fast-math-flags should be set on the FMUL node? 5419 if (IsExp10) { 5420 // Put the exponent in the right bit position for later addition to the 5421 // final result: 5422 // 5423 // #define LOG2OF10 3.3219281f 5424 // t0 = Op * LOG2OF10; 5425 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS, 5426 getF32Constant(DAG, 0x40549a78, dl)); 5427 return getLimitedPrecisionExp2(t0, dl, DAG); 5428 } 5429 5430 // No special expansion. 5431 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS, Flags); 5432 } 5433 5434 /// ExpandPowI - Expand a llvm.powi intrinsic. 5435 static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, 5436 SelectionDAG &DAG) { 5437 // If RHS is a constant, we can expand this out to a multiplication tree if 5438 // it's beneficial on the target, otherwise we end up lowering to a call to 5439 // __powidf2 (for example). 5440 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) { 5441 unsigned Val = RHSC->getSExtValue(); 5442 5443 // powi(x, 0) -> 1.0 5444 if (Val == 0) 5445 return DAG.getConstantFP(1.0, DL, LHS.getValueType()); 5446 5447 if (DAG.getTargetLoweringInfo().isBeneficialToExpandPowI( 5448 Val, DAG.shouldOptForSize())) { 5449 // Get the exponent as a positive value. 5450 if ((int)Val < 0) 5451 Val = -Val; 5452 // We use the simple binary decomposition method to generate the multiply 5453 // sequence. There are more optimal ways to do this (for example, 5454 // powi(x,15) generates one more multiply than it should), but this has 5455 // the benefit of being both really simple and much better than a libcall. 5456 SDValue Res; // Logically starts equal to 1.0 5457 SDValue CurSquare = LHS; 5458 // TODO: Intrinsics should have fast-math-flags that propagate to these 5459 // nodes. 5460 while (Val) { 5461 if (Val & 1) { 5462 if (Res.getNode()) 5463 Res = 5464 DAG.getNode(ISD::FMUL, DL, Res.getValueType(), Res, CurSquare); 5465 else 5466 Res = CurSquare; // 1.0*CurSquare. 5467 } 5468 5469 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(), 5470 CurSquare, CurSquare); 5471 Val >>= 1; 5472 } 5473 5474 // If the original was negative, invert the result, producing 1/(x*x*x). 5475 if (RHSC->getSExtValue() < 0) 5476 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(), 5477 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res); 5478 return Res; 5479 } 5480 } 5481 5482 // Otherwise, expand to a libcall. 5483 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS); 5484 } 5485 5486 static SDValue expandDivFix(unsigned Opcode, const SDLoc &DL, 5487 SDValue LHS, SDValue RHS, SDValue Scale, 5488 SelectionDAG &DAG, const TargetLowering &TLI) { 5489 EVT VT = LHS.getValueType(); 5490 bool Signed = Opcode == ISD::SDIVFIX || Opcode == ISD::SDIVFIXSAT; 5491 bool Saturating = Opcode == ISD::SDIVFIXSAT || Opcode == ISD::UDIVFIXSAT; 5492 LLVMContext &Ctx = *DAG.getContext(); 5493 5494 // If the type is legal but the operation isn't, this node might survive all 5495 // the way to operation legalization. If we end up there and we do not have 5496 // the ability to widen the type (if VT*2 is not legal), we cannot expand the 5497 // node. 5498 5499 // Coax the legalizer into expanding the node during type legalization instead 5500 // by bumping the size by one bit. This will force it to Promote, enabling the 5501 // early expansion and avoiding the need to expand later. 5502 5503 // We don't have to do this if Scale is 0; that can always be expanded, unless 5504 // it's a saturating signed operation. Those can experience true integer 5505 // division overflow, a case which we must avoid. 5506 5507 // FIXME: We wouldn't have to do this (or any of the early 5508 // expansion/promotion) if it was possible to expand a libcall of an 5509 // illegal type during operation legalization. But it's not, so things 5510 // get a bit hacky. 5511 unsigned ScaleInt = cast<ConstantSDNode>(Scale)->getZExtValue(); 5512 if ((ScaleInt > 0 || (Saturating && Signed)) && 5513 (TLI.isTypeLegal(VT) || 5514 (VT.isVector() && TLI.isTypeLegal(VT.getVectorElementType())))) { 5515 TargetLowering::LegalizeAction Action = TLI.getFixedPointOperationAction( 5516 Opcode, VT, ScaleInt); 5517 if (Action != TargetLowering::Legal && Action != TargetLowering::Custom) { 5518 EVT PromVT; 5519 if (VT.isScalarInteger()) 5520 PromVT = EVT::getIntegerVT(Ctx, VT.getSizeInBits() + 1); 5521 else if (VT.isVector()) { 5522 PromVT = VT.getVectorElementType(); 5523 PromVT = EVT::getIntegerVT(Ctx, PromVT.getSizeInBits() + 1); 5524 PromVT = EVT::getVectorVT(Ctx, PromVT, VT.getVectorElementCount()); 5525 } else 5526 llvm_unreachable("Wrong VT for DIVFIX?"); 5527 if (Signed) { 5528 LHS = DAG.getSExtOrTrunc(LHS, DL, PromVT); 5529 RHS = DAG.getSExtOrTrunc(RHS, DL, PromVT); 5530 } else { 5531 LHS = DAG.getZExtOrTrunc(LHS, DL, PromVT); 5532 RHS = DAG.getZExtOrTrunc(RHS, DL, PromVT); 5533 } 5534 EVT ShiftTy = TLI.getShiftAmountTy(PromVT, DAG.getDataLayout()); 5535 // For saturating operations, we need to shift up the LHS to get the 5536 // proper saturation width, and then shift down again afterwards. 5537 if (Saturating) 5538 LHS = DAG.getNode(ISD::SHL, DL, PromVT, LHS, 5539 DAG.getConstant(1, DL, ShiftTy)); 5540 SDValue Res = DAG.getNode(Opcode, DL, PromVT, LHS, RHS, Scale); 5541 if (Saturating) 5542 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, DL, PromVT, Res, 5543 DAG.getConstant(1, DL, ShiftTy)); 5544 return DAG.getZExtOrTrunc(Res, DL, VT); 5545 } 5546 } 5547 5548 return DAG.getNode(Opcode, DL, VT, LHS, RHS, Scale); 5549 } 5550 5551 // getUnderlyingArgRegs - Find underlying registers used for a truncated, 5552 // bitcasted, or split argument. Returns a list of <Register, size in bits> 5553 static void 5554 getUnderlyingArgRegs(SmallVectorImpl<std::pair<unsigned, TypeSize>> &Regs, 5555 const SDValue &N) { 5556 switch (N.getOpcode()) { 5557 case ISD::CopyFromReg: { 5558 SDValue Op = N.getOperand(1); 5559 Regs.emplace_back(cast<RegisterSDNode>(Op)->getReg(), 5560 Op.getValueType().getSizeInBits()); 5561 return; 5562 } 5563 case ISD::BITCAST: 5564 case ISD::AssertZext: 5565 case ISD::AssertSext: 5566 case ISD::TRUNCATE: 5567 getUnderlyingArgRegs(Regs, N.getOperand(0)); 5568 return; 5569 case ISD::BUILD_PAIR: 5570 case ISD::BUILD_VECTOR: 5571 case ISD::CONCAT_VECTORS: 5572 for (SDValue Op : N->op_values()) 5573 getUnderlyingArgRegs(Regs, Op); 5574 return; 5575 default: 5576 return; 5577 } 5578 } 5579 5580 /// If the DbgValueInst is a dbg_value of a function argument, create the 5581 /// corresponding DBG_VALUE machine instruction for it now. At the end of 5582 /// instruction selection, they will be inserted to the entry BB. 5583 /// We don't currently support this for variadic dbg_values, as they shouldn't 5584 /// appear for function arguments or in the prologue. 5585 bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( 5586 const Value *V, DILocalVariable *Variable, DIExpression *Expr, 5587 DILocation *DL, FuncArgumentDbgValueKind Kind, const SDValue &N) { 5588 const Argument *Arg = dyn_cast<Argument>(V); 5589 if (!Arg) 5590 return false; 5591 5592 MachineFunction &MF = DAG.getMachineFunction(); 5593 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); 5594 5595 // Helper to create DBG_INSTR_REFs or DBG_VALUEs, depending on what kind 5596 // we've been asked to pursue. 5597 auto MakeVRegDbgValue = [&](Register Reg, DIExpression *FragExpr, 5598 bool Indirect) { 5599 if (Reg.isVirtual() && MF.useDebugInstrRef()) { 5600 // For VRegs, in instruction referencing mode, create a DBG_INSTR_REF 5601 // pointing at the VReg, which will be patched up later. 5602 auto &Inst = TII->get(TargetOpcode::DBG_INSTR_REF); 5603 SmallVector<MachineOperand, 1> MOs({MachineOperand::CreateReg( 5604 /* Reg */ Reg, /* isDef */ false, /* isImp */ false, 5605 /* isKill */ false, /* isDead */ false, 5606 /* isUndef */ false, /* isEarlyClobber */ false, 5607 /* SubReg */ 0, /* isDebug */ true)}); 5608 5609 auto *NewDIExpr = FragExpr; 5610 // We don't have an "Indirect" field in DBG_INSTR_REF, fold that into 5611 // the DIExpression. 5612 if (Indirect) 5613 NewDIExpr = DIExpression::prepend(FragExpr, DIExpression::DerefBefore); 5614 SmallVector<uint64_t, 2> Ops({dwarf::DW_OP_LLVM_arg, 0}); 5615 NewDIExpr = DIExpression::prependOpcodes(NewDIExpr, Ops); 5616 return BuildMI(MF, DL, Inst, false, MOs, Variable, NewDIExpr); 5617 } else { 5618 // Create a completely standard DBG_VALUE. 5619 auto &Inst = TII->get(TargetOpcode::DBG_VALUE); 5620 return BuildMI(MF, DL, Inst, Indirect, Reg, Variable, FragExpr); 5621 } 5622 }; 5623 5624 if (Kind == FuncArgumentDbgValueKind::Value) { 5625 // ArgDbgValues are hoisted to the beginning of the entry block. So we 5626 // should only emit as ArgDbgValue if the dbg.value intrinsic is found in 5627 // the entry block. 5628 bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front(); 5629 if (!IsInEntryBlock) 5630 return false; 5631 5632 // ArgDbgValues are hoisted to the beginning of the entry block. So we 5633 // should only emit as ArgDbgValue if the dbg.value intrinsic describes a 5634 // variable that also is a param. 5635 // 5636 // Although, if we are at the top of the entry block already, we can still 5637 // emit using ArgDbgValue. This might catch some situations when the 5638 // dbg.value refers to an argument that isn't used in the entry block, so 5639 // any CopyToReg node would be optimized out and the only way to express 5640 // this DBG_VALUE is by using the physical reg (or FI) as done in this 5641 // method. ArgDbgValues are hoisted to the beginning of the entry block. So 5642 // we should only emit as ArgDbgValue if the Variable is an argument to the 5643 // current function, and the dbg.value intrinsic is found in the entry 5644 // block. 5645 bool VariableIsFunctionInputArg = Variable->isParameter() && 5646 !DL->getInlinedAt(); 5647 bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder; 5648 if (!IsInPrologue && !VariableIsFunctionInputArg) 5649 return false; 5650 5651 // Here we assume that a function argument on IR level only can be used to 5652 // describe one input parameter on source level. If we for example have 5653 // source code like this 5654 // 5655 // struct A { long x, y; }; 5656 // void foo(struct A a, long b) { 5657 // ... 5658 // b = a.x; 5659 // ... 5660 // } 5661 // 5662 // and IR like this 5663 // 5664 // define void @foo(i32 %a1, i32 %a2, i32 %b) { 5665 // entry: 5666 // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment 5667 // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment 5668 // call void @llvm.dbg.value(metadata i32 %b, "b", 5669 // ... 5670 // call void @llvm.dbg.value(metadata i32 %a1, "b" 5671 // ... 5672 // 5673 // then the last dbg.value is describing a parameter "b" using a value that 5674 // is an argument. But since we already has used %a1 to describe a parameter 5675 // we should not handle that last dbg.value here (that would result in an 5676 // incorrect hoisting of the DBG_VALUE to the function entry). 5677 // Notice that we allow one dbg.value per IR level argument, to accommodate 5678 // for the situation with fragments above. 5679 if (VariableIsFunctionInputArg) { 5680 unsigned ArgNo = Arg->getArgNo(); 5681 if (ArgNo >= FuncInfo.DescribedArgs.size()) 5682 FuncInfo.DescribedArgs.resize(ArgNo + 1, false); 5683 else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo)) 5684 return false; 5685 FuncInfo.DescribedArgs.set(ArgNo); 5686 } 5687 } 5688 5689 bool IsIndirect = false; 5690 std::optional<MachineOperand> Op; 5691 // Some arguments' frame index is recorded during argument lowering. 5692 int FI = FuncInfo.getArgumentFrameIndex(Arg); 5693 if (FI != std::numeric_limits<int>::max()) 5694 Op = MachineOperand::CreateFI(FI); 5695 5696 SmallVector<std::pair<unsigned, TypeSize>, 8> ArgRegsAndSizes; 5697 if (!Op && N.getNode()) { 5698 getUnderlyingArgRegs(ArgRegsAndSizes, N); 5699 Register Reg; 5700 if (ArgRegsAndSizes.size() == 1) 5701 Reg = ArgRegsAndSizes.front().first; 5702 5703 if (Reg && Reg.isVirtual()) { 5704 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 5705 Register PR = RegInfo.getLiveInPhysReg(Reg); 5706 if (PR) 5707 Reg = PR; 5708 } 5709 if (Reg) { 5710 Op = MachineOperand::CreateReg(Reg, false); 5711 IsIndirect = Kind != FuncArgumentDbgValueKind::Value; 5712 } 5713 } 5714 5715 if (!Op && N.getNode()) { 5716 // Check if frame index is available. 5717 SDValue LCandidate = peekThroughBitcasts(N); 5718 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(LCandidate.getNode())) 5719 if (FrameIndexSDNode *FINode = 5720 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 5721 Op = MachineOperand::CreateFI(FINode->getIndex()); 5722 } 5723 5724 if (!Op) { 5725 // Create a DBG_VALUE for each decomposed value in ArgRegs to cover Reg 5726 auto splitMultiRegDbgValue = [&](ArrayRef<std::pair<unsigned, TypeSize>> 5727 SplitRegs) { 5728 unsigned Offset = 0; 5729 for (const auto &RegAndSize : SplitRegs) { 5730 // If the expression is already a fragment, the current register 5731 // offset+size might extend beyond the fragment. In this case, only 5732 // the register bits that are inside the fragment are relevant. 5733 int RegFragmentSizeInBits = RegAndSize.second; 5734 if (auto ExprFragmentInfo = Expr->getFragmentInfo()) { 5735 uint64_t ExprFragmentSizeInBits = ExprFragmentInfo->SizeInBits; 5736 // The register is entirely outside the expression fragment, 5737 // so is irrelevant for debug info. 5738 if (Offset >= ExprFragmentSizeInBits) 5739 break; 5740 // The register is partially outside the expression fragment, only 5741 // the low bits within the fragment are relevant for debug info. 5742 if (Offset + RegFragmentSizeInBits > ExprFragmentSizeInBits) { 5743 RegFragmentSizeInBits = ExprFragmentSizeInBits - Offset; 5744 } 5745 } 5746 5747 auto FragmentExpr = DIExpression::createFragmentExpression( 5748 Expr, Offset, RegFragmentSizeInBits); 5749 Offset += RegAndSize.second; 5750 // If a valid fragment expression cannot be created, the variable's 5751 // correct value cannot be determined and so it is set as Undef. 5752 if (!FragmentExpr) { 5753 SDDbgValue *SDV = DAG.getConstantDbgValue( 5754 Variable, Expr, UndefValue::get(V->getType()), DL, SDNodeOrder); 5755 DAG.AddDbgValue(SDV, false); 5756 continue; 5757 } 5758 MachineInstr *NewMI = 5759 MakeVRegDbgValue(RegAndSize.first, *FragmentExpr, 5760 Kind != FuncArgumentDbgValueKind::Value); 5761 FuncInfo.ArgDbgValues.push_back(NewMI); 5762 } 5763 }; 5764 5765 // Check if ValueMap has reg number. 5766 DenseMap<const Value *, Register>::const_iterator 5767 VMI = FuncInfo.ValueMap.find(V); 5768 if (VMI != FuncInfo.ValueMap.end()) { 5769 const auto &TLI = DAG.getTargetLoweringInfo(); 5770 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second, 5771 V->getType(), std::nullopt); 5772 if (RFV.occupiesMultipleRegs()) { 5773 splitMultiRegDbgValue(RFV.getRegsAndSizes()); 5774 return true; 5775 } 5776 5777 Op = MachineOperand::CreateReg(VMI->second, false); 5778 IsIndirect = Kind != FuncArgumentDbgValueKind::Value; 5779 } else if (ArgRegsAndSizes.size() > 1) { 5780 // This was split due to the calling convention, and no virtual register 5781 // mapping exists for the value. 5782 splitMultiRegDbgValue(ArgRegsAndSizes); 5783 return true; 5784 } 5785 } 5786 5787 if (!Op) 5788 return false; 5789 5790 assert(Variable->isValidLocationForIntrinsic(DL) && 5791 "Expected inlined-at fields to agree"); 5792 MachineInstr *NewMI = nullptr; 5793 5794 if (Op->isReg()) 5795 NewMI = MakeVRegDbgValue(Op->getReg(), Expr, IsIndirect); 5796 else 5797 NewMI = BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), true, *Op, 5798 Variable, Expr); 5799 5800 // Otherwise, use ArgDbgValues. 5801 FuncInfo.ArgDbgValues.push_back(NewMI); 5802 return true; 5803 } 5804 5805 /// Return the appropriate SDDbgValue based on N. 5806 SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N, 5807 DILocalVariable *Variable, 5808 DIExpression *Expr, 5809 const DebugLoc &dl, 5810 unsigned DbgSDNodeOrder) { 5811 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) { 5812 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe 5813 // stack slot locations. 5814 // 5815 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting 5816 // debug values here after optimization: 5817 // 5818 // dbg.value(i32* %px, !"int *px", !DIExpression()), and 5819 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref)) 5820 // 5821 // Both describe the direct values of their associated variables. 5822 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(), 5823 /*IsIndirect*/ false, dl, DbgSDNodeOrder); 5824 } 5825 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(), 5826 /*IsIndirect*/ false, dl, DbgSDNodeOrder); 5827 } 5828 5829 static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) { 5830 switch (Intrinsic) { 5831 case Intrinsic::smul_fix: 5832 return ISD::SMULFIX; 5833 case Intrinsic::umul_fix: 5834 return ISD::UMULFIX; 5835 case Intrinsic::smul_fix_sat: 5836 return ISD::SMULFIXSAT; 5837 case Intrinsic::umul_fix_sat: 5838 return ISD::UMULFIXSAT; 5839 case Intrinsic::sdiv_fix: 5840 return ISD::SDIVFIX; 5841 case Intrinsic::udiv_fix: 5842 return ISD::UDIVFIX; 5843 case Intrinsic::sdiv_fix_sat: 5844 return ISD::SDIVFIXSAT; 5845 case Intrinsic::udiv_fix_sat: 5846 return ISD::UDIVFIXSAT; 5847 default: 5848 llvm_unreachable("Unhandled fixed point intrinsic"); 5849 } 5850 } 5851 5852 void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I, 5853 const char *FunctionName) { 5854 assert(FunctionName && "FunctionName must not be nullptr"); 5855 SDValue Callee = DAG.getExternalSymbol( 5856 FunctionName, 5857 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); 5858 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); 5859 } 5860 5861 /// Given a @llvm.call.preallocated.setup, return the corresponding 5862 /// preallocated call. 5863 static const CallBase *FindPreallocatedCall(const Value *PreallocatedSetup) { 5864 assert(cast<CallBase>(PreallocatedSetup) 5865 ->getCalledFunction() 5866 ->getIntrinsicID() == Intrinsic::call_preallocated_setup && 5867 "expected call_preallocated_setup Value"); 5868 for (const auto *U : PreallocatedSetup->users()) { 5869 auto *UseCall = cast<CallBase>(U); 5870 const Function *Fn = UseCall->getCalledFunction(); 5871 if (!Fn || Fn->getIntrinsicID() != Intrinsic::call_preallocated_arg) { 5872 return UseCall; 5873 } 5874 } 5875 llvm_unreachable("expected corresponding call to preallocated setup/arg"); 5876 } 5877 5878 /// Lower the call to the specified intrinsic function. 5879 void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, 5880 unsigned Intrinsic) { 5881 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 5882 SDLoc sdl = getCurSDLoc(); 5883 DebugLoc dl = getCurDebugLoc(); 5884 SDValue Res; 5885 5886 SDNodeFlags Flags; 5887 if (auto *FPOp = dyn_cast<FPMathOperator>(&I)) 5888 Flags.copyFMF(*FPOp); 5889 5890 switch (Intrinsic) { 5891 default: 5892 // By default, turn this into a target intrinsic node. 5893 visitTargetIntrinsic(I, Intrinsic); 5894 return; 5895 case Intrinsic::vscale: { 5896 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 5897 setValue(&I, DAG.getVScale(sdl, VT, APInt(VT.getSizeInBits(), 1))); 5898 return; 5899 } 5900 case Intrinsic::vastart: visitVAStart(I); return; 5901 case Intrinsic::vaend: visitVAEnd(I); return; 5902 case Intrinsic::vacopy: visitVACopy(I); return; 5903 case Intrinsic::returnaddress: 5904 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, 5905 TLI.getValueType(DAG.getDataLayout(), I.getType()), 5906 getValue(I.getArgOperand(0)))); 5907 return; 5908 case Intrinsic::addressofreturnaddress: 5909 setValue(&I, 5910 DAG.getNode(ISD::ADDROFRETURNADDR, sdl, 5911 TLI.getValueType(DAG.getDataLayout(), I.getType()))); 5912 return; 5913 case Intrinsic::sponentry: 5914 setValue(&I, 5915 DAG.getNode(ISD::SPONENTRY, sdl, 5916 TLI.getValueType(DAG.getDataLayout(), I.getType()))); 5917 return; 5918 case Intrinsic::frameaddress: 5919 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, 5920 TLI.getFrameIndexTy(DAG.getDataLayout()), 5921 getValue(I.getArgOperand(0)))); 5922 return; 5923 case Intrinsic::read_volatile_register: 5924 case Intrinsic::read_register: { 5925 Value *Reg = I.getArgOperand(0); 5926 SDValue Chain = getRoot(); 5927 SDValue RegName = 5928 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); 5929 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 5930 Res = DAG.getNode(ISD::READ_REGISTER, sdl, 5931 DAG.getVTList(VT, MVT::Other), Chain, RegName); 5932 setValue(&I, Res); 5933 DAG.setRoot(Res.getValue(1)); 5934 return; 5935 } 5936 case Intrinsic::write_register: { 5937 Value *Reg = I.getArgOperand(0); 5938 Value *RegValue = I.getArgOperand(1); 5939 SDValue Chain = getRoot(); 5940 SDValue RegName = 5941 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata())); 5942 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, 5943 RegName, getValue(RegValue))); 5944 return; 5945 } 5946 case Intrinsic::memcpy: { 5947 const auto &MCI = cast<MemCpyInst>(I); 5948 SDValue Op1 = getValue(I.getArgOperand(0)); 5949 SDValue Op2 = getValue(I.getArgOperand(1)); 5950 SDValue Op3 = getValue(I.getArgOperand(2)); 5951 // @llvm.memcpy defines 0 and 1 to both mean no alignment. 5952 Align DstAlign = MCI.getDestAlign().valueOrOne(); 5953 Align SrcAlign = MCI.getSourceAlign().valueOrOne(); 5954 Align Alignment = std::min(DstAlign, SrcAlign); 5955 bool isVol = MCI.isVolatile(); 5956 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 5957 // FIXME: Support passing different dest/src alignments to the memcpy DAG 5958 // node. 5959 SDValue Root = isVol ? getRoot() : getMemoryRoot(); 5960 SDValue MC = DAG.getMemcpy( 5961 Root, sdl, Op1, Op2, Op3, Alignment, isVol, 5962 /* AlwaysInline */ false, isTC, MachinePointerInfo(I.getArgOperand(0)), 5963 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA); 5964 updateDAGForMaybeTailCall(MC); 5965 return; 5966 } 5967 case Intrinsic::memcpy_inline: { 5968 const auto &MCI = cast<MemCpyInlineInst>(I); 5969 SDValue Dst = getValue(I.getArgOperand(0)); 5970 SDValue Src = getValue(I.getArgOperand(1)); 5971 SDValue Size = getValue(I.getArgOperand(2)); 5972 assert(isa<ConstantSDNode>(Size) && "memcpy_inline needs constant size"); 5973 // @llvm.memcpy.inline defines 0 and 1 to both mean no alignment. 5974 Align DstAlign = MCI.getDestAlign().valueOrOne(); 5975 Align SrcAlign = MCI.getSourceAlign().valueOrOne(); 5976 Align Alignment = std::min(DstAlign, SrcAlign); 5977 bool isVol = MCI.isVolatile(); 5978 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 5979 // FIXME: Support passing different dest/src alignments to the memcpy DAG 5980 // node. 5981 SDValue MC = DAG.getMemcpy( 5982 getRoot(), sdl, Dst, Src, Size, Alignment, isVol, 5983 /* AlwaysInline */ true, isTC, MachinePointerInfo(I.getArgOperand(0)), 5984 MachinePointerInfo(I.getArgOperand(1)), I.getAAMetadata(), AA); 5985 updateDAGForMaybeTailCall(MC); 5986 return; 5987 } 5988 case Intrinsic::memset: { 5989 const auto &MSI = cast<MemSetInst>(I); 5990 SDValue Op1 = getValue(I.getArgOperand(0)); 5991 SDValue Op2 = getValue(I.getArgOperand(1)); 5992 SDValue Op3 = getValue(I.getArgOperand(2)); 5993 // @llvm.memset defines 0 and 1 to both mean no alignment. 5994 Align Alignment = MSI.getDestAlign().valueOrOne(); 5995 bool isVol = MSI.isVolatile(); 5996 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 5997 SDValue Root = isVol ? getRoot() : getMemoryRoot(); 5998 SDValue MS = DAG.getMemset( 5999 Root, sdl, Op1, Op2, Op3, Alignment, isVol, /* AlwaysInline */ false, 6000 isTC, MachinePointerInfo(I.getArgOperand(0)), I.getAAMetadata()); 6001 updateDAGForMaybeTailCall(MS); 6002 return; 6003 } 6004 case Intrinsic::memset_inline: { 6005 const auto &MSII = cast<MemSetInlineInst>(I); 6006 SDValue Dst = getValue(I.getArgOperand(0)); 6007 SDValue Value = getValue(I.getArgOperand(1)); 6008 SDValue Size = getValue(I.getArgOperand(2)); 6009 assert(isa<ConstantSDNode>(Size) && "memset_inline needs constant size"); 6010 // @llvm.memset defines 0 and 1 to both mean no alignment. 6011 Align DstAlign = MSII.getDestAlign().valueOrOne(); 6012 bool isVol = MSII.isVolatile(); 6013 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 6014 SDValue Root = isVol ? getRoot() : getMemoryRoot(); 6015 SDValue MC = DAG.getMemset(Root, sdl, Dst, Value, Size, DstAlign, isVol, 6016 /* AlwaysInline */ true, isTC, 6017 MachinePointerInfo(I.getArgOperand(0)), 6018 I.getAAMetadata()); 6019 updateDAGForMaybeTailCall(MC); 6020 return; 6021 } 6022 case Intrinsic::memmove: { 6023 const auto &MMI = cast<MemMoveInst>(I); 6024 SDValue Op1 = getValue(I.getArgOperand(0)); 6025 SDValue Op2 = getValue(I.getArgOperand(1)); 6026 SDValue Op3 = getValue(I.getArgOperand(2)); 6027 // @llvm.memmove defines 0 and 1 to both mean no alignment. 6028 Align DstAlign = MMI.getDestAlign().valueOrOne(); 6029 Align SrcAlign = MMI.getSourceAlign().valueOrOne(); 6030 Align Alignment = std::min(DstAlign, SrcAlign); 6031 bool isVol = MMI.isVolatile(); 6032 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 6033 // FIXME: Support passing different dest/src alignments to the memmove DAG 6034 // node. 6035 SDValue Root = isVol ? getRoot() : getMemoryRoot(); 6036 SDValue MM = DAG.getMemmove(Root, sdl, Op1, Op2, Op3, Alignment, isVol, 6037 isTC, MachinePointerInfo(I.getArgOperand(0)), 6038 MachinePointerInfo(I.getArgOperand(1)), 6039 I.getAAMetadata(), AA); 6040 updateDAGForMaybeTailCall(MM); 6041 return; 6042 } 6043 case Intrinsic::memcpy_element_unordered_atomic: { 6044 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I); 6045 SDValue Dst = getValue(MI.getRawDest()); 6046 SDValue Src = getValue(MI.getRawSource()); 6047 SDValue Length = getValue(MI.getLength()); 6048 6049 Type *LengthTy = MI.getLength()->getType(); 6050 unsigned ElemSz = MI.getElementSizeInBytes(); 6051 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 6052 SDValue MC = 6053 DAG.getAtomicMemcpy(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz, 6054 isTC, MachinePointerInfo(MI.getRawDest()), 6055 MachinePointerInfo(MI.getRawSource())); 6056 updateDAGForMaybeTailCall(MC); 6057 return; 6058 } 6059 case Intrinsic::memmove_element_unordered_atomic: { 6060 auto &MI = cast<AtomicMemMoveInst>(I); 6061 SDValue Dst = getValue(MI.getRawDest()); 6062 SDValue Src = getValue(MI.getRawSource()); 6063 SDValue Length = getValue(MI.getLength()); 6064 6065 Type *LengthTy = MI.getLength()->getType(); 6066 unsigned ElemSz = MI.getElementSizeInBytes(); 6067 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 6068 SDValue MC = 6069 DAG.getAtomicMemmove(getRoot(), sdl, Dst, Src, Length, LengthTy, ElemSz, 6070 isTC, MachinePointerInfo(MI.getRawDest()), 6071 MachinePointerInfo(MI.getRawSource())); 6072 updateDAGForMaybeTailCall(MC); 6073 return; 6074 } 6075 case Intrinsic::memset_element_unordered_atomic: { 6076 auto &MI = cast<AtomicMemSetInst>(I); 6077 SDValue Dst = getValue(MI.getRawDest()); 6078 SDValue Val = getValue(MI.getValue()); 6079 SDValue Length = getValue(MI.getLength()); 6080 6081 Type *LengthTy = MI.getLength()->getType(); 6082 unsigned ElemSz = MI.getElementSizeInBytes(); 6083 bool isTC = I.isTailCall() && isInTailCallPosition(I, DAG.getTarget()); 6084 SDValue MC = 6085 DAG.getAtomicMemset(getRoot(), sdl, Dst, Val, Length, LengthTy, ElemSz, 6086 isTC, MachinePointerInfo(MI.getRawDest())); 6087 updateDAGForMaybeTailCall(MC); 6088 return; 6089 } 6090 case Intrinsic::call_preallocated_setup: { 6091 const CallBase *PreallocatedCall = FindPreallocatedCall(&I); 6092 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); 6093 SDValue Res = DAG.getNode(ISD::PREALLOCATED_SETUP, sdl, MVT::Other, 6094 getRoot(), SrcValue); 6095 setValue(&I, Res); 6096 DAG.setRoot(Res); 6097 return; 6098 } 6099 case Intrinsic::call_preallocated_arg: { 6100 const CallBase *PreallocatedCall = FindPreallocatedCall(I.getOperand(0)); 6101 SDValue SrcValue = DAG.getSrcValue(PreallocatedCall); 6102 SDValue Ops[3]; 6103 Ops[0] = getRoot(); 6104 Ops[1] = SrcValue; 6105 Ops[2] = DAG.getTargetConstant(*cast<ConstantInt>(I.getArgOperand(1)), sdl, 6106 MVT::i32); // arg index 6107 SDValue Res = DAG.getNode( 6108 ISD::PREALLOCATED_ARG, sdl, 6109 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Ops); 6110 setValue(&I, Res); 6111 DAG.setRoot(Res.getValue(1)); 6112 return; 6113 } 6114 case Intrinsic::dbg_declare: { 6115 // Debug intrinsics are handled separately in assignment tracking mode. 6116 if (isAssignmentTrackingEnabled(*I.getFunction()->getParent())) 6117 return; 6118 // Assume dbg.declare can not currently use DIArgList, i.e. 6119 // it is non-variadic. 6120 const auto &DI = cast<DbgVariableIntrinsic>(I); 6121 assert(!DI.hasArgList() && "Only dbg.value should currently use DIArgList"); 6122 DILocalVariable *Variable = DI.getVariable(); 6123 DIExpression *Expression = DI.getExpression(); 6124 dropDanglingDebugInfo(Variable, Expression); 6125 assert(Variable && "Missing variable"); 6126 LLVM_DEBUG(dbgs() << "SelectionDAG visiting debug intrinsic: " << DI 6127 << "\n"); 6128 // Check if address has undef value. 6129 const Value *Address = DI.getVariableLocationOp(0); 6130 if (!Address || isa<UndefValue>(Address) || 6131 (Address->use_empty() && !isa<Argument>(Address))) { 6132 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI 6133 << " (bad/undef/unused-arg address)\n"); 6134 return; 6135 } 6136 6137 bool isParameter = Variable->isParameter() || isa<Argument>(Address); 6138 6139 // Check if this variable can be described by a frame index, typically 6140 // either as a static alloca or a byval parameter. 6141 int FI = std::numeric_limits<int>::max(); 6142 if (const auto *AI = 6143 dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) { 6144 if (AI->isStaticAlloca()) { 6145 auto I = FuncInfo.StaticAllocaMap.find(AI); 6146 if (I != FuncInfo.StaticAllocaMap.end()) 6147 FI = I->second; 6148 } 6149 } else if (const auto *Arg = dyn_cast<Argument>( 6150 Address->stripInBoundsConstantOffsets())) { 6151 FI = FuncInfo.getArgumentFrameIndex(Arg); 6152 } 6153 6154 // llvm.dbg.declare is handled as a frame index in the MachineFunction 6155 // variable table. 6156 if (FI != std::numeric_limits<int>::max()) { 6157 LLVM_DEBUG(dbgs() << "Skipping " << DI 6158 << " (variable info stashed in MF side table)\n"); 6159 return; 6160 } 6161 6162 SDValue &N = NodeMap[Address]; 6163 if (!N.getNode() && isa<Argument>(Address)) 6164 // Check unused arguments map. 6165 N = UnusedArgNodeMap[Address]; 6166 SDDbgValue *SDV; 6167 if (N.getNode()) { 6168 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address)) 6169 Address = BCI->getOperand(0); 6170 // Parameters are handled specially. 6171 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode()); 6172 if (isParameter && FINode) { 6173 // Byval parameter. We have a frame index at this point. 6174 SDV = 6175 DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(), 6176 /*IsIndirect*/ true, dl, SDNodeOrder); 6177 } else if (isa<Argument>(Address)) { 6178 // Address is an argument, so try to emit its dbg value using 6179 // virtual register info from the FuncInfo.ValueMap. 6180 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 6181 FuncArgumentDbgValueKind::Declare, N); 6182 return; 6183 } else { 6184 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), 6185 true, dl, SDNodeOrder); 6186 } 6187 DAG.AddDbgValue(SDV, isParameter); 6188 } else { 6189 // If Address is an argument then try to emit its dbg value using 6190 // virtual register info from the FuncInfo.ValueMap. 6191 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, 6192 FuncArgumentDbgValueKind::Declare, N)) { 6193 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI 6194 << " (could not emit func-arg dbg_value)\n"); 6195 } 6196 } 6197 return; 6198 } 6199 case Intrinsic::dbg_label: { 6200 const DbgLabelInst &DI = cast<DbgLabelInst>(I); 6201 DILabel *Label = DI.getLabel(); 6202 assert(Label && "Missing label"); 6203 6204 SDDbgLabel *SDV; 6205 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); 6206 DAG.AddDbgLabel(SDV); 6207 return; 6208 } 6209 case Intrinsic::dbg_assign: { 6210 // Debug intrinsics are handled seperately in assignment tracking mode. 6211 assert(isAssignmentTrackingEnabled(*I.getFunction()->getParent()) && 6212 "expected assignment tracking to be enabled"); 6213 return; 6214 } 6215 case Intrinsic::dbg_value: { 6216 // Debug intrinsics are handled seperately in assignment tracking mode. 6217 if (isAssignmentTrackingEnabled(*I.getFunction()->getParent())) 6218 return; 6219 const DbgValueInst &DI = cast<DbgValueInst>(I); 6220 assert(DI.getVariable() && "Missing variable"); 6221 6222 DILocalVariable *Variable = DI.getVariable(); 6223 DIExpression *Expression = DI.getExpression(); 6224 dropDanglingDebugInfo(Variable, Expression); 6225 SmallVector<Value *, 4> Values(DI.getValues()); 6226 if (Values.empty()) 6227 return; 6228 6229 if (llvm::is_contained(Values, nullptr)) 6230 return; 6231 6232 bool IsVariadic = DI.hasArgList(); 6233 if (!handleDebugValue(Values, Variable, Expression, DI.getDebugLoc(), 6234 SDNodeOrder, IsVariadic)) 6235 addDanglingDebugInfo(&DI, SDNodeOrder); 6236 return; 6237 } 6238 6239 case Intrinsic::eh_typeid_for: { 6240 // Find the type id for the given typeinfo. 6241 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0)); 6242 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); 6243 Res = DAG.getConstant(TypeID, sdl, MVT::i32); 6244 setValue(&I, Res); 6245 return; 6246 } 6247 6248 case Intrinsic::eh_return_i32: 6249 case Intrinsic::eh_return_i64: 6250 DAG.getMachineFunction().setCallsEHReturn(true); 6251 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl, 6252 MVT::Other, 6253 getControlRoot(), 6254 getValue(I.getArgOperand(0)), 6255 getValue(I.getArgOperand(1)))); 6256 return; 6257 case Intrinsic::eh_unwind_init: 6258 DAG.getMachineFunction().setCallsUnwindInit(true); 6259 return; 6260 case Intrinsic::eh_dwarf_cfa: 6261 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, 6262 TLI.getPointerTy(DAG.getDataLayout()), 6263 getValue(I.getArgOperand(0)))); 6264 return; 6265 case Intrinsic::eh_sjlj_callsite: { 6266 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); 6267 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(0)); 6268 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"); 6269 6270 MMI.setCurrentCallSite(CI->getZExtValue()); 6271 return; 6272 } 6273 case Intrinsic::eh_sjlj_functioncontext: { 6274 // Get and store the index of the function context. 6275 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); 6276 AllocaInst *FnCtx = 6277 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts()); 6278 int FI = FuncInfo.StaticAllocaMap[FnCtx]; 6279 MFI.setFunctionContextIndex(FI); 6280 return; 6281 } 6282 case Intrinsic::eh_sjlj_setjmp: { 6283 SDValue Ops[2]; 6284 Ops[0] = getRoot(); 6285 Ops[1] = getValue(I.getArgOperand(0)); 6286 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl, 6287 DAG.getVTList(MVT::i32, MVT::Other), Ops); 6288 setValue(&I, Op.getValue(0)); 6289 DAG.setRoot(Op.getValue(1)); 6290 return; 6291 } 6292 case Intrinsic::eh_sjlj_longjmp: 6293 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, 6294 getRoot(), getValue(I.getArgOperand(0)))); 6295 return; 6296 case Intrinsic::eh_sjlj_setup_dispatch: 6297 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, 6298 getRoot())); 6299 return; 6300 case Intrinsic::masked_gather: 6301 visitMaskedGather(I); 6302 return; 6303 case Intrinsic::masked_load: 6304 visitMaskedLoad(I); 6305 return; 6306 case Intrinsic::masked_scatter: 6307 visitMaskedScatter(I); 6308 return; 6309 case Intrinsic::masked_store: 6310 visitMaskedStore(I); 6311 return; 6312 case Intrinsic::masked_expandload: 6313 visitMaskedLoad(I, true /* IsExpanding */); 6314 return; 6315 case Intrinsic::masked_compressstore: 6316 visitMaskedStore(I, true /* IsCompressing */); 6317 return; 6318 case Intrinsic::powi: 6319 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), 6320 getValue(I.getArgOperand(1)), DAG)); 6321 return; 6322 case Intrinsic::log: 6323 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); 6324 return; 6325 case Intrinsic::log2: 6326 setValue(&I, 6327 expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); 6328 return; 6329 case Intrinsic::log10: 6330 setValue(&I, 6331 expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); 6332 return; 6333 case Intrinsic::exp: 6334 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); 6335 return; 6336 case Intrinsic::exp2: 6337 setValue(&I, 6338 expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI, Flags)); 6339 return; 6340 case Intrinsic::pow: 6341 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), 6342 getValue(I.getArgOperand(1)), DAG, TLI, Flags)); 6343 return; 6344 case Intrinsic::sqrt: 6345 case Intrinsic::fabs: 6346 case Intrinsic::sin: 6347 case Intrinsic::cos: 6348 case Intrinsic::floor: 6349 case Intrinsic::ceil: 6350 case Intrinsic::trunc: 6351 case Intrinsic::rint: 6352 case Intrinsic::nearbyint: 6353 case Intrinsic::round: 6354 case Intrinsic::roundeven: 6355 case Intrinsic::canonicalize: { 6356 unsigned Opcode; 6357 switch (Intrinsic) { 6358 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 6359 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break; 6360 case Intrinsic::fabs: Opcode = ISD::FABS; break; 6361 case Intrinsic::sin: Opcode = ISD::FSIN; break; 6362 case Intrinsic::cos: Opcode = ISD::FCOS; break; 6363 case Intrinsic::floor: Opcode = ISD::FFLOOR; break; 6364 case Intrinsic::ceil: Opcode = ISD::FCEIL; break; 6365 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break; 6366 case Intrinsic::rint: Opcode = ISD::FRINT; break; 6367 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break; 6368 case Intrinsic::round: Opcode = ISD::FROUND; break; 6369 case Intrinsic::roundeven: Opcode = ISD::FROUNDEVEN; break; 6370 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break; 6371 } 6372 6373 setValue(&I, DAG.getNode(Opcode, sdl, 6374 getValue(I.getArgOperand(0)).getValueType(), 6375 getValue(I.getArgOperand(0)), Flags)); 6376 return; 6377 } 6378 case Intrinsic::lround: 6379 case Intrinsic::llround: 6380 case Intrinsic::lrint: 6381 case Intrinsic::llrint: { 6382 unsigned Opcode; 6383 switch (Intrinsic) { 6384 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 6385 case Intrinsic::lround: Opcode = ISD::LROUND; break; 6386 case Intrinsic::llround: Opcode = ISD::LLROUND; break; 6387 case Intrinsic::lrint: Opcode = ISD::LRINT; break; 6388 case Intrinsic::llrint: Opcode = ISD::LLRINT; break; 6389 } 6390 6391 EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6392 setValue(&I, DAG.getNode(Opcode, sdl, RetVT, 6393 getValue(I.getArgOperand(0)))); 6394 return; 6395 } 6396 case Intrinsic::minnum: 6397 setValue(&I, DAG.getNode(ISD::FMINNUM, sdl, 6398 getValue(I.getArgOperand(0)).getValueType(), 6399 getValue(I.getArgOperand(0)), 6400 getValue(I.getArgOperand(1)), Flags)); 6401 return; 6402 case Intrinsic::maxnum: 6403 setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl, 6404 getValue(I.getArgOperand(0)).getValueType(), 6405 getValue(I.getArgOperand(0)), 6406 getValue(I.getArgOperand(1)), Flags)); 6407 return; 6408 case Intrinsic::minimum: 6409 setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl, 6410 getValue(I.getArgOperand(0)).getValueType(), 6411 getValue(I.getArgOperand(0)), 6412 getValue(I.getArgOperand(1)), Flags)); 6413 return; 6414 case Intrinsic::maximum: 6415 setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl, 6416 getValue(I.getArgOperand(0)).getValueType(), 6417 getValue(I.getArgOperand(0)), 6418 getValue(I.getArgOperand(1)), Flags)); 6419 return; 6420 case Intrinsic::copysign: 6421 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, 6422 getValue(I.getArgOperand(0)).getValueType(), 6423 getValue(I.getArgOperand(0)), 6424 getValue(I.getArgOperand(1)), Flags)); 6425 return; 6426 case Intrinsic::arithmetic_fence: { 6427 setValue(&I, DAG.getNode(ISD::ARITH_FENCE, sdl, 6428 getValue(I.getArgOperand(0)).getValueType(), 6429 getValue(I.getArgOperand(0)), Flags)); 6430 return; 6431 } 6432 case Intrinsic::fma: 6433 setValue(&I, DAG.getNode( 6434 ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(), 6435 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), 6436 getValue(I.getArgOperand(2)), Flags)); 6437 return; 6438 #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ 6439 case Intrinsic::INTRINSIC: 6440 #include "llvm/IR/ConstrainedOps.def" 6441 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I)); 6442 return; 6443 #define BEGIN_REGISTER_VP_INTRINSIC(VPID, ...) case Intrinsic::VPID: 6444 #include "llvm/IR/VPIntrinsics.def" 6445 visitVectorPredicationIntrinsic(cast<VPIntrinsic>(I)); 6446 return; 6447 case Intrinsic::fptrunc_round: { 6448 // Get the last argument, the metadata and convert it to an integer in the 6449 // call 6450 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(1))->getMetadata(); 6451 std::optional<RoundingMode> RoundMode = 6452 convertStrToRoundingMode(cast<MDString>(MD)->getString()); 6453 6454 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6455 6456 // Propagate fast-math-flags from IR to node(s). 6457 SDNodeFlags Flags; 6458 Flags.copyFMF(*cast<FPMathOperator>(&I)); 6459 SelectionDAG::FlagInserter FlagsInserter(DAG, Flags); 6460 6461 SDValue Result; 6462 Result = DAG.getNode( 6463 ISD::FPTRUNC_ROUND, sdl, VT, getValue(I.getArgOperand(0)), 6464 DAG.getTargetConstant((int)*RoundMode, sdl, 6465 TLI.getPointerTy(DAG.getDataLayout()))); 6466 setValue(&I, Result); 6467 6468 return; 6469 } 6470 case Intrinsic::fmuladd: { 6471 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6472 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && 6473 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { 6474 setValue(&I, DAG.getNode(ISD::FMA, sdl, 6475 getValue(I.getArgOperand(0)).getValueType(), 6476 getValue(I.getArgOperand(0)), 6477 getValue(I.getArgOperand(1)), 6478 getValue(I.getArgOperand(2)), Flags)); 6479 } else { 6480 // TODO: Intrinsic calls should have fast-math-flags. 6481 SDValue Mul = DAG.getNode( 6482 ISD::FMUL, sdl, getValue(I.getArgOperand(0)).getValueType(), 6483 getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), Flags); 6484 SDValue Add = DAG.getNode(ISD::FADD, sdl, 6485 getValue(I.getArgOperand(0)).getValueType(), 6486 Mul, getValue(I.getArgOperand(2)), Flags); 6487 setValue(&I, Add); 6488 } 6489 return; 6490 } 6491 case Intrinsic::convert_to_fp16: 6492 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16, 6493 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16, 6494 getValue(I.getArgOperand(0)), 6495 DAG.getTargetConstant(0, sdl, 6496 MVT::i32)))); 6497 return; 6498 case Intrinsic::convert_from_fp16: 6499 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl, 6500 TLI.getValueType(DAG.getDataLayout(), I.getType()), 6501 DAG.getNode(ISD::BITCAST, sdl, MVT::f16, 6502 getValue(I.getArgOperand(0))))); 6503 return; 6504 case Intrinsic::fptosi_sat: { 6505 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6506 setValue(&I, DAG.getNode(ISD::FP_TO_SINT_SAT, sdl, VT, 6507 getValue(I.getArgOperand(0)), 6508 DAG.getValueType(VT.getScalarType()))); 6509 return; 6510 } 6511 case Intrinsic::fptoui_sat: { 6512 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6513 setValue(&I, DAG.getNode(ISD::FP_TO_UINT_SAT, sdl, VT, 6514 getValue(I.getArgOperand(0)), 6515 DAG.getValueType(VT.getScalarType()))); 6516 return; 6517 } 6518 case Intrinsic::set_rounding: 6519 Res = DAG.getNode(ISD::SET_ROUNDING, sdl, MVT::Other, 6520 {getRoot(), getValue(I.getArgOperand(0))}); 6521 setValue(&I, Res); 6522 DAG.setRoot(Res.getValue(0)); 6523 return; 6524 case Intrinsic::is_fpclass: { 6525 const DataLayout DLayout = DAG.getDataLayout(); 6526 EVT DestVT = TLI.getValueType(DLayout, I.getType()); 6527 EVT ArgVT = TLI.getValueType(DLayout, I.getArgOperand(0)->getType()); 6528 FPClassTest Test = static_cast<FPClassTest>( 6529 cast<ConstantInt>(I.getArgOperand(1))->getZExtValue()); 6530 MachineFunction &MF = DAG.getMachineFunction(); 6531 const Function &F = MF.getFunction(); 6532 SDValue Op = getValue(I.getArgOperand(0)); 6533 SDNodeFlags Flags; 6534 Flags.setNoFPExcept( 6535 !F.getAttributes().hasFnAttr(llvm::Attribute::StrictFP)); 6536 // If ISD::IS_FPCLASS should be expanded, do it right now, because the 6537 // expansion can use illegal types. Making expansion early allows 6538 // legalizing these types prior to selection. 6539 if (!TLI.isOperationLegalOrCustom(ISD::IS_FPCLASS, ArgVT)) { 6540 SDValue Result = TLI.expandIS_FPCLASS(DestVT, Op, Test, Flags, sdl, DAG); 6541 setValue(&I, Result); 6542 return; 6543 } 6544 6545 SDValue Check = DAG.getTargetConstant(Test, sdl, MVT::i32); 6546 SDValue V = DAG.getNode(ISD::IS_FPCLASS, sdl, DestVT, {Op, Check}, Flags); 6547 setValue(&I, V); 6548 return; 6549 } 6550 case Intrinsic::pcmarker: { 6551 SDValue Tmp = getValue(I.getArgOperand(0)); 6552 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); 6553 return; 6554 } 6555 case Intrinsic::readcyclecounter: { 6556 SDValue Op = getRoot(); 6557 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl, 6558 DAG.getVTList(MVT::i64, MVT::Other), Op); 6559 setValue(&I, Res); 6560 DAG.setRoot(Res.getValue(1)); 6561 return; 6562 } 6563 case Intrinsic::bitreverse: 6564 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, 6565 getValue(I.getArgOperand(0)).getValueType(), 6566 getValue(I.getArgOperand(0)))); 6567 return; 6568 case Intrinsic::bswap: 6569 setValue(&I, DAG.getNode(ISD::BSWAP, sdl, 6570 getValue(I.getArgOperand(0)).getValueType(), 6571 getValue(I.getArgOperand(0)))); 6572 return; 6573 case Intrinsic::cttz: { 6574 SDValue Arg = getValue(I.getArgOperand(0)); 6575 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 6576 EVT Ty = Arg.getValueType(); 6577 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, 6578 sdl, Ty, Arg)); 6579 return; 6580 } 6581 case Intrinsic::ctlz: { 6582 SDValue Arg = getValue(I.getArgOperand(0)); 6583 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1)); 6584 EVT Ty = Arg.getValueType(); 6585 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, 6586 sdl, Ty, Arg)); 6587 return; 6588 } 6589 case Intrinsic::ctpop: { 6590 SDValue Arg = getValue(I.getArgOperand(0)); 6591 EVT Ty = Arg.getValueType(); 6592 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); 6593 return; 6594 } 6595 case Intrinsic::fshl: 6596 case Intrinsic::fshr: { 6597 bool IsFSHL = Intrinsic == Intrinsic::fshl; 6598 SDValue X = getValue(I.getArgOperand(0)); 6599 SDValue Y = getValue(I.getArgOperand(1)); 6600 SDValue Z = getValue(I.getArgOperand(2)); 6601 EVT VT = X.getValueType(); 6602 6603 if (X == Y) { 6604 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR; 6605 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z)); 6606 } else { 6607 auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR; 6608 setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z)); 6609 } 6610 return; 6611 } 6612 case Intrinsic::sadd_sat: { 6613 SDValue Op1 = getValue(I.getArgOperand(0)); 6614 SDValue Op2 = getValue(I.getArgOperand(1)); 6615 setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); 6616 return; 6617 } 6618 case Intrinsic::uadd_sat: { 6619 SDValue Op1 = getValue(I.getArgOperand(0)); 6620 SDValue Op2 = getValue(I.getArgOperand(1)); 6621 setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2)); 6622 return; 6623 } 6624 case Intrinsic::ssub_sat: { 6625 SDValue Op1 = getValue(I.getArgOperand(0)); 6626 SDValue Op2 = getValue(I.getArgOperand(1)); 6627 setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2)); 6628 return; 6629 } 6630 case Intrinsic::usub_sat: { 6631 SDValue Op1 = getValue(I.getArgOperand(0)); 6632 SDValue Op2 = getValue(I.getArgOperand(1)); 6633 setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2)); 6634 return; 6635 } 6636 case Intrinsic::sshl_sat: { 6637 SDValue Op1 = getValue(I.getArgOperand(0)); 6638 SDValue Op2 = getValue(I.getArgOperand(1)); 6639 setValue(&I, DAG.getNode(ISD::SSHLSAT, sdl, Op1.getValueType(), Op1, Op2)); 6640 return; 6641 } 6642 case Intrinsic::ushl_sat: { 6643 SDValue Op1 = getValue(I.getArgOperand(0)); 6644 SDValue Op2 = getValue(I.getArgOperand(1)); 6645 setValue(&I, DAG.getNode(ISD::USHLSAT, sdl, Op1.getValueType(), Op1, Op2)); 6646 return; 6647 } 6648 case Intrinsic::smul_fix: 6649 case Intrinsic::umul_fix: 6650 case Intrinsic::smul_fix_sat: 6651 case Intrinsic::umul_fix_sat: { 6652 SDValue Op1 = getValue(I.getArgOperand(0)); 6653 SDValue Op2 = getValue(I.getArgOperand(1)); 6654 SDValue Op3 = getValue(I.getArgOperand(2)); 6655 setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl, 6656 Op1.getValueType(), Op1, Op2, Op3)); 6657 return; 6658 } 6659 case Intrinsic::sdiv_fix: 6660 case Intrinsic::udiv_fix: 6661 case Intrinsic::sdiv_fix_sat: 6662 case Intrinsic::udiv_fix_sat: { 6663 SDValue Op1 = getValue(I.getArgOperand(0)); 6664 SDValue Op2 = getValue(I.getArgOperand(1)); 6665 SDValue Op3 = getValue(I.getArgOperand(2)); 6666 setValue(&I, expandDivFix(FixedPointIntrinsicToOpcode(Intrinsic), sdl, 6667 Op1, Op2, Op3, DAG, TLI)); 6668 return; 6669 } 6670 case Intrinsic::smax: { 6671 SDValue Op1 = getValue(I.getArgOperand(0)); 6672 SDValue Op2 = getValue(I.getArgOperand(1)); 6673 setValue(&I, DAG.getNode(ISD::SMAX, sdl, Op1.getValueType(), Op1, Op2)); 6674 return; 6675 } 6676 case Intrinsic::smin: { 6677 SDValue Op1 = getValue(I.getArgOperand(0)); 6678 SDValue Op2 = getValue(I.getArgOperand(1)); 6679 setValue(&I, DAG.getNode(ISD::SMIN, sdl, Op1.getValueType(), Op1, Op2)); 6680 return; 6681 } 6682 case Intrinsic::umax: { 6683 SDValue Op1 = getValue(I.getArgOperand(0)); 6684 SDValue Op2 = getValue(I.getArgOperand(1)); 6685 setValue(&I, DAG.getNode(ISD::UMAX, sdl, Op1.getValueType(), Op1, Op2)); 6686 return; 6687 } 6688 case Intrinsic::umin: { 6689 SDValue Op1 = getValue(I.getArgOperand(0)); 6690 SDValue Op2 = getValue(I.getArgOperand(1)); 6691 setValue(&I, DAG.getNode(ISD::UMIN, sdl, Op1.getValueType(), Op1, Op2)); 6692 return; 6693 } 6694 case Intrinsic::abs: { 6695 // TODO: Preserve "int min is poison" arg in SDAG? 6696 SDValue Op1 = getValue(I.getArgOperand(0)); 6697 setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1)); 6698 return; 6699 } 6700 case Intrinsic::stacksave: { 6701 SDValue Op = getRoot(); 6702 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6703 Res = DAG.getNode(ISD::STACKSAVE, sdl, DAG.getVTList(VT, MVT::Other), Op); 6704 setValue(&I, Res); 6705 DAG.setRoot(Res.getValue(1)); 6706 return; 6707 } 6708 case Intrinsic::stackrestore: 6709 Res = getValue(I.getArgOperand(0)); 6710 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); 6711 return; 6712 case Intrinsic::get_dynamic_area_offset: { 6713 SDValue Op = getRoot(); 6714 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); 6715 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6716 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for 6717 // target. 6718 if (PtrTy.getFixedSizeInBits() < ResTy.getFixedSizeInBits()) 6719 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset" 6720 " intrinsic!"); 6721 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy), 6722 Op); 6723 DAG.setRoot(Op); 6724 setValue(&I, Res); 6725 return; 6726 } 6727 case Intrinsic::stackguard: { 6728 MachineFunction &MF = DAG.getMachineFunction(); 6729 const Module &M = *MF.getFunction().getParent(); 6730 SDValue Chain = getRoot(); 6731 if (TLI.useLoadStackGuardNode()) { 6732 Res = getLoadStackGuard(DAG, sdl, Chain); 6733 } else { 6734 EVT PtrTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); 6735 const Value *Global = TLI.getSDagStackGuard(M); 6736 Align Align = DAG.getDataLayout().getPrefTypeAlign(Global->getType()); 6737 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global), 6738 MachinePointerInfo(Global, 0), Align, 6739 MachineMemOperand::MOVolatile); 6740 } 6741 if (TLI.useStackGuardXorFP()) 6742 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl); 6743 DAG.setRoot(Chain); 6744 setValue(&I, Res); 6745 return; 6746 } 6747 case Intrinsic::stackprotector: { 6748 // Emit code into the DAG to store the stack guard onto the stack. 6749 MachineFunction &MF = DAG.getMachineFunction(); 6750 MachineFrameInfo &MFI = MF.getFrameInfo(); 6751 SDValue Src, Chain = getRoot(); 6752 6753 if (TLI.useLoadStackGuardNode()) 6754 Src = getLoadStackGuard(DAG, sdl, Chain); 6755 else 6756 Src = getValue(I.getArgOperand(0)); // The guard's value. 6757 6758 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1)); 6759 6760 int FI = FuncInfo.StaticAllocaMap[Slot]; 6761 MFI.setStackProtectorIndex(FI); 6762 EVT PtrTy = TLI.getFrameIndexTy(DAG.getDataLayout()); 6763 6764 SDValue FIN = DAG.getFrameIndex(FI, PtrTy); 6765 6766 // Store the stack protector onto the stack. 6767 Res = DAG.getStore( 6768 Chain, sdl, Src, FIN, 6769 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), 6770 MaybeAlign(), MachineMemOperand::MOVolatile); 6771 setValue(&I, Res); 6772 DAG.setRoot(Res); 6773 return; 6774 } 6775 case Intrinsic::objectsize: 6776 llvm_unreachable("llvm.objectsize.* should have been lowered already"); 6777 6778 case Intrinsic::is_constant: 6779 llvm_unreachable("llvm.is.constant.* should have been lowered already"); 6780 6781 case Intrinsic::annotation: 6782 case Intrinsic::ptr_annotation: 6783 case Intrinsic::launder_invariant_group: 6784 case Intrinsic::strip_invariant_group: 6785 // Drop the intrinsic, but forward the value 6786 setValue(&I, getValue(I.getOperand(0))); 6787 return; 6788 6789 case Intrinsic::assume: 6790 case Intrinsic::experimental_noalias_scope_decl: 6791 case Intrinsic::var_annotation: 6792 case Intrinsic::sideeffect: 6793 // Discard annotate attributes, noalias scope declarations, assumptions, and 6794 // artificial side-effects. 6795 return; 6796 6797 case Intrinsic::codeview_annotation: { 6798 // Emit a label associated with this metadata. 6799 MachineFunction &MF = DAG.getMachineFunction(); 6800 MCSymbol *Label = 6801 MF.getMMI().getContext().createTempSymbol("annotation", true); 6802 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata(); 6803 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD)); 6804 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label); 6805 DAG.setRoot(Res); 6806 return; 6807 } 6808 6809 case Intrinsic::init_trampoline: { 6810 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts()); 6811 6812 SDValue Ops[6]; 6813 Ops[0] = getRoot(); 6814 Ops[1] = getValue(I.getArgOperand(0)); 6815 Ops[2] = getValue(I.getArgOperand(1)); 6816 Ops[3] = getValue(I.getArgOperand(2)); 6817 Ops[4] = DAG.getSrcValue(I.getArgOperand(0)); 6818 Ops[5] = DAG.getSrcValue(F); 6819 6820 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); 6821 6822 DAG.setRoot(Res); 6823 return; 6824 } 6825 case Intrinsic::adjust_trampoline: 6826 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, 6827 TLI.getPointerTy(DAG.getDataLayout()), 6828 getValue(I.getArgOperand(0)))); 6829 return; 6830 case Intrinsic::gcroot: { 6831 assert(DAG.getMachineFunction().getFunction().hasGC() && 6832 "only valid in functions with gc specified, enforced by Verifier"); 6833 assert(GFI && "implied by previous"); 6834 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts(); 6835 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1)); 6836 6837 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode()); 6838 GFI->addStackRoot(FI->getIndex(), TypeMap); 6839 return; 6840 } 6841 case Intrinsic::gcread: 6842 case Intrinsic::gcwrite: 6843 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); 6844 case Intrinsic::get_rounding: 6845 Res = DAG.getNode(ISD::GET_ROUNDING, sdl, {MVT::i32, MVT::Other}, getRoot()); 6846 setValue(&I, Res); 6847 DAG.setRoot(Res.getValue(1)); 6848 return; 6849 6850 case Intrinsic::expect: 6851 // Just replace __builtin_expect(exp, c) with EXP. 6852 setValue(&I, getValue(I.getArgOperand(0))); 6853 return; 6854 6855 case Intrinsic::ubsantrap: 6856 case Intrinsic::debugtrap: 6857 case Intrinsic::trap: { 6858 StringRef TrapFuncName = 6859 I.getAttributes().getFnAttr("trap-func-name").getValueAsString(); 6860 if (TrapFuncName.empty()) { 6861 switch (Intrinsic) { 6862 case Intrinsic::trap: 6863 DAG.setRoot(DAG.getNode(ISD::TRAP, sdl, MVT::Other, getRoot())); 6864 break; 6865 case Intrinsic::debugtrap: 6866 DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, sdl, MVT::Other, getRoot())); 6867 break; 6868 case Intrinsic::ubsantrap: 6869 DAG.setRoot(DAG.getNode( 6870 ISD::UBSANTRAP, sdl, MVT::Other, getRoot(), 6871 DAG.getTargetConstant( 6872 cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(), sdl, 6873 MVT::i32))); 6874 break; 6875 default: llvm_unreachable("unknown trap intrinsic"); 6876 } 6877 return; 6878 } 6879 TargetLowering::ArgListTy Args; 6880 if (Intrinsic == Intrinsic::ubsantrap) { 6881 Args.push_back(TargetLoweringBase::ArgListEntry()); 6882 Args[0].Val = I.getArgOperand(0); 6883 Args[0].Node = getValue(Args[0].Val); 6884 Args[0].Ty = Args[0].Val->getType(); 6885 } 6886 6887 TargetLowering::CallLoweringInfo CLI(DAG); 6888 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee( 6889 CallingConv::C, I.getType(), 6890 DAG.getExternalSymbol(TrapFuncName.data(), 6891 TLI.getPointerTy(DAG.getDataLayout())), 6892 std::move(Args)); 6893 6894 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); 6895 DAG.setRoot(Result.second); 6896 return; 6897 } 6898 6899 case Intrinsic::uadd_with_overflow: 6900 case Intrinsic::sadd_with_overflow: 6901 case Intrinsic::usub_with_overflow: 6902 case Intrinsic::ssub_with_overflow: 6903 case Intrinsic::umul_with_overflow: 6904 case Intrinsic::smul_with_overflow: { 6905 ISD::NodeType Op; 6906 switch (Intrinsic) { 6907 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 6908 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break; 6909 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break; 6910 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break; 6911 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break; 6912 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break; 6913 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break; 6914 } 6915 SDValue Op1 = getValue(I.getArgOperand(0)); 6916 SDValue Op2 = getValue(I.getArgOperand(1)); 6917 6918 EVT ResultVT = Op1.getValueType(); 6919 EVT OverflowVT = MVT::i1; 6920 if (ResultVT.isVector()) 6921 OverflowVT = EVT::getVectorVT( 6922 *Context, OverflowVT, ResultVT.getVectorElementCount()); 6923 6924 SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT); 6925 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); 6926 return; 6927 } 6928 case Intrinsic::prefetch: { 6929 SDValue Ops[5]; 6930 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 6931 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore; 6932 Ops[0] = DAG.getRoot(); 6933 Ops[1] = getValue(I.getArgOperand(0)); 6934 Ops[2] = getValue(I.getArgOperand(1)); 6935 Ops[3] = getValue(I.getArgOperand(2)); 6936 Ops[4] = getValue(I.getArgOperand(3)); 6937 SDValue Result = DAG.getMemIntrinsicNode( 6938 ISD::PREFETCH, sdl, DAG.getVTList(MVT::Other), Ops, 6939 EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), 6940 /* align */ std::nullopt, Flags); 6941 6942 // Chain the prefetch in parallell with any pending loads, to stay out of 6943 // the way of later optimizations. 6944 PendingLoads.push_back(Result); 6945 Result = getRoot(); 6946 DAG.setRoot(Result); 6947 return; 6948 } 6949 case Intrinsic::lifetime_start: 6950 case Intrinsic::lifetime_end: { 6951 bool IsStart = (Intrinsic == Intrinsic::lifetime_start); 6952 // Stack coloring is not enabled in O0, discard region information. 6953 if (TM.getOptLevel() == CodeGenOpt::None) 6954 return; 6955 6956 const int64_t ObjectSize = 6957 cast<ConstantInt>(I.getArgOperand(0))->getSExtValue(); 6958 Value *const ObjectPtr = I.getArgOperand(1); 6959 SmallVector<const Value *, 4> Allocas; 6960 getUnderlyingObjects(ObjectPtr, Allocas); 6961 6962 for (const Value *Alloca : Allocas) { 6963 const AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(Alloca); 6964 6965 // Could not find an Alloca. 6966 if (!LifetimeObject) 6967 continue; 6968 6969 // First check that the Alloca is static, otherwise it won't have a 6970 // valid frame index. 6971 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); 6972 if (SI == FuncInfo.StaticAllocaMap.end()) 6973 return; 6974 6975 const int FrameIndex = SI->second; 6976 int64_t Offset; 6977 if (GetPointerBaseWithConstantOffset( 6978 ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) 6979 Offset = -1; // Cannot determine offset from alloca to lifetime object. 6980 Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, 6981 Offset); 6982 DAG.setRoot(Res); 6983 } 6984 return; 6985 } 6986 case Intrinsic::pseudoprobe: { 6987 auto Guid = cast<ConstantInt>(I.getArgOperand(0))->getZExtValue(); 6988 auto Index = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue(); 6989 auto Attr = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue(); 6990 Res = DAG.getPseudoProbeNode(sdl, getRoot(), Guid, Index, Attr); 6991 DAG.setRoot(Res); 6992 return; 6993 } 6994 case Intrinsic::invariant_start: 6995 // Discard region information. 6996 setValue(&I, 6997 DAG.getUNDEF(TLI.getValueType(DAG.getDataLayout(), I.getType()))); 6998 return; 6999 case Intrinsic::invariant_end: 7000 // Discard region information. 7001 return; 7002 case Intrinsic::clear_cache: 7003 /// FunctionName may be null. 7004 if (const char *FunctionName = TLI.getClearCacheBuiltinName()) 7005 lowerCallToExternalSymbol(I, FunctionName); 7006 return; 7007 case Intrinsic::donothing: 7008 case Intrinsic::seh_try_begin: 7009 case Intrinsic::seh_scope_begin: 7010 case Intrinsic::seh_try_end: 7011 case Intrinsic::seh_scope_end: 7012 // ignore 7013 return; 7014 case Intrinsic::experimental_stackmap: 7015 visitStackmap(I); 7016 return; 7017 case Intrinsic::experimental_patchpoint_void: 7018 case Intrinsic::experimental_patchpoint_i64: 7019 visitPatchpoint(I); 7020 return; 7021 case Intrinsic::experimental_gc_statepoint: 7022 LowerStatepoint(cast<GCStatepointInst>(I)); 7023 return; 7024 case Intrinsic::experimental_gc_result: 7025 visitGCResult(cast<GCResultInst>(I)); 7026 return; 7027 case Intrinsic::experimental_gc_relocate: 7028 visitGCRelocate(cast<GCRelocateInst>(I)); 7029 return; 7030 case Intrinsic::instrprof_cover: 7031 llvm_unreachable("instrprof failed to lower a cover"); 7032 case Intrinsic::instrprof_increment: 7033 llvm_unreachable("instrprof failed to lower an increment"); 7034 case Intrinsic::instrprof_value_profile: 7035 llvm_unreachable("instrprof failed to lower a value profiling call"); 7036 case Intrinsic::localescape: { 7037 MachineFunction &MF = DAG.getMachineFunction(); 7038 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); 7039 7040 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission 7041 // is the same on all targets. 7042 for (unsigned Idx = 0, E = I.arg_size(); Idx < E; ++Idx) { 7043 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts(); 7044 if (isa<ConstantPointerNull>(Arg)) 7045 continue; // Skip null pointers. They represent a hole in index space. 7046 AllocaInst *Slot = cast<AllocaInst>(Arg); 7047 assert(FuncInfo.StaticAllocaMap.count(Slot) && 7048 "can only escape static allocas"); 7049 int FI = FuncInfo.StaticAllocaMap[Slot]; 7050 MCSymbol *FrameAllocSym = 7051 MF.getMMI().getContext().getOrCreateFrameAllocSymbol( 7052 GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx); 7053 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl, 7054 TII->get(TargetOpcode::LOCAL_ESCAPE)) 7055 .addSym(FrameAllocSym) 7056 .addFrameIndex(FI); 7057 } 7058 7059 return; 7060 } 7061 7062 case Intrinsic::localrecover: { 7063 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx) 7064 MachineFunction &MF = DAG.getMachineFunction(); 7065 7066 // Get the symbol that defines the frame offset. 7067 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts()); 7068 auto *Idx = cast<ConstantInt>(I.getArgOperand(2)); 7069 unsigned IdxVal = 7070 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max())); 7071 MCSymbol *FrameAllocSym = 7072 MF.getMMI().getContext().getOrCreateFrameAllocSymbol( 7073 GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal); 7074 7075 Value *FP = I.getArgOperand(1); 7076 SDValue FPVal = getValue(FP); 7077 EVT PtrVT = FPVal.getValueType(); 7078 7079 // Create a MCSymbol for the label to avoid any target lowering 7080 // that would make this PC relative. 7081 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT); 7082 SDValue OffsetVal = 7083 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym); 7084 7085 // Add the offset to the FP. 7086 SDValue Add = DAG.getMemBasePlusOffset(FPVal, OffsetVal, sdl); 7087 setValue(&I, Add); 7088 7089 return; 7090 } 7091 7092 case Intrinsic::eh_exceptionpointer: 7093 case Intrinsic::eh_exceptioncode: { 7094 // Get the exception pointer vreg, copy from it, and resize it to fit. 7095 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0)); 7096 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); 7097 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT); 7098 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC); 7099 SDValue N = DAG.getCopyFromReg(DAG.getEntryNode(), sdl, VReg, PtrVT); 7100 if (Intrinsic == Intrinsic::eh_exceptioncode) 7101 N = DAG.getZExtOrTrunc(N, sdl, MVT::i32); 7102 setValue(&I, N); 7103 return; 7104 } 7105 case Intrinsic::xray_customevent: { 7106 // Here we want to make sure that the intrinsic behaves as if it has a 7107 // specific calling convention, and only for x86_64. 7108 // FIXME: Support other platforms later. 7109 const auto &Triple = DAG.getTarget().getTargetTriple(); 7110 if (Triple.getArch() != Triple::x86_64) 7111 return; 7112 7113 SmallVector<SDValue, 8> Ops; 7114 7115 // We want to say that we always want the arguments in registers. 7116 SDValue LogEntryVal = getValue(I.getArgOperand(0)); 7117 SDValue StrSizeVal = getValue(I.getArgOperand(1)); 7118 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 7119 SDValue Chain = getRoot(); 7120 Ops.push_back(LogEntryVal); 7121 Ops.push_back(StrSizeVal); 7122 Ops.push_back(Chain); 7123 7124 // We need to enforce the calling convention for the callsite, so that 7125 // argument ordering is enforced correctly, and that register allocation can 7126 // see that some registers may be assumed clobbered and have to preserve 7127 // them across calls to the intrinsic. 7128 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL, 7129 sdl, NodeTys, Ops); 7130 SDValue patchableNode = SDValue(MN, 0); 7131 DAG.setRoot(patchableNode); 7132 setValue(&I, patchableNode); 7133 return; 7134 } 7135 case Intrinsic::xray_typedevent: { 7136 // Here we want to make sure that the intrinsic behaves as if it has a 7137 // specific calling convention, and only for x86_64. 7138 // FIXME: Support other platforms later. 7139 const auto &Triple = DAG.getTarget().getTargetTriple(); 7140 if (Triple.getArch() != Triple::x86_64) 7141 return; 7142 7143 SmallVector<SDValue, 8> Ops; 7144 7145 // We want to say that we always want the arguments in registers. 7146 // It's unclear to me how manipulating the selection DAG here forces callers 7147 // to provide arguments in registers instead of on the stack. 7148 SDValue LogTypeId = getValue(I.getArgOperand(0)); 7149 SDValue LogEntryVal = getValue(I.getArgOperand(1)); 7150 SDValue StrSizeVal = getValue(I.getArgOperand(2)); 7151 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 7152 SDValue Chain = getRoot(); 7153 Ops.push_back(LogTypeId); 7154 Ops.push_back(LogEntryVal); 7155 Ops.push_back(StrSizeVal); 7156 Ops.push_back(Chain); 7157 7158 // We need to enforce the calling convention for the callsite, so that 7159 // argument ordering is enforced correctly, and that register allocation can 7160 // see that some registers may be assumed clobbered and have to preserve 7161 // them across calls to the intrinsic. 7162 MachineSDNode *MN = DAG.getMachineNode( 7163 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, sdl, NodeTys, Ops); 7164 SDValue patchableNode = SDValue(MN, 0); 7165 DAG.setRoot(patchableNode); 7166 setValue(&I, patchableNode); 7167 return; 7168 } 7169 case Intrinsic::experimental_deoptimize: 7170 LowerDeoptimizeCall(&I); 7171 return; 7172 case Intrinsic::experimental_stepvector: 7173 visitStepVector(I); 7174 return; 7175 case Intrinsic::vector_reduce_fadd: 7176 case Intrinsic::vector_reduce_fmul: 7177 case Intrinsic::vector_reduce_add: 7178 case Intrinsic::vector_reduce_mul: 7179 case Intrinsic::vector_reduce_and: 7180 case Intrinsic::vector_reduce_or: 7181 case Intrinsic::vector_reduce_xor: 7182 case Intrinsic::vector_reduce_smax: 7183 case Intrinsic::vector_reduce_smin: 7184 case Intrinsic::vector_reduce_umax: 7185 case Intrinsic::vector_reduce_umin: 7186 case Intrinsic::vector_reduce_fmax: 7187 case Intrinsic::vector_reduce_fmin: 7188 visitVectorReduce(I, Intrinsic); 7189 return; 7190 7191 case Intrinsic::icall_branch_funnel: { 7192 SmallVector<SDValue, 16> Ops; 7193 Ops.push_back(getValue(I.getArgOperand(0))); 7194 7195 int64_t Offset; 7196 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset( 7197 I.getArgOperand(1), Offset, DAG.getDataLayout())); 7198 if (!Base) 7199 report_fatal_error( 7200 "llvm.icall.branch.funnel operand must be a GlobalValue"); 7201 Ops.push_back(DAG.getTargetGlobalAddress(Base, sdl, MVT::i64, 0)); 7202 7203 struct BranchFunnelTarget { 7204 int64_t Offset; 7205 SDValue Target; 7206 }; 7207 SmallVector<BranchFunnelTarget, 8> Targets; 7208 7209 for (unsigned Op = 1, N = I.arg_size(); Op != N; Op += 2) { 7210 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset( 7211 I.getArgOperand(Op), Offset, DAG.getDataLayout())); 7212 if (ElemBase != Base) 7213 report_fatal_error("all llvm.icall.branch.funnel operands must refer " 7214 "to the same GlobalValue"); 7215 7216 SDValue Val = getValue(I.getArgOperand(Op + 1)); 7217 auto *GA = dyn_cast<GlobalAddressSDNode>(Val); 7218 if (!GA) 7219 report_fatal_error( 7220 "llvm.icall.branch.funnel operand must be a GlobalValue"); 7221 Targets.push_back({Offset, DAG.getTargetGlobalAddress( 7222 GA->getGlobal(), sdl, Val.getValueType(), 7223 GA->getOffset())}); 7224 } 7225 llvm::sort(Targets, 7226 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) { 7227 return T1.Offset < T2.Offset; 7228 }); 7229 7230 for (auto &T : Targets) { 7231 Ops.push_back(DAG.getTargetConstant(T.Offset, sdl, MVT::i32)); 7232 Ops.push_back(T.Target); 7233 } 7234 7235 Ops.push_back(DAG.getRoot()); // Chain 7236 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, sdl, 7237 MVT::Other, Ops), 7238 0); 7239 DAG.setRoot(N); 7240 setValue(&I, N); 7241 HasTailCall = true; 7242 return; 7243 } 7244 7245 case Intrinsic::wasm_landingpad_index: 7246 // Information this intrinsic contained has been transferred to 7247 // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely 7248 // delete it now. 7249 return; 7250 7251 case Intrinsic::aarch64_settag: 7252 case Intrinsic::aarch64_settag_zero: { 7253 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 7254 bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero; 7255 SDValue Val = TSI.EmitTargetCodeForSetTag( 7256 DAG, sdl, getRoot(), getValue(I.getArgOperand(0)), 7257 getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)), 7258 ZeroMemory); 7259 DAG.setRoot(Val); 7260 setValue(&I, Val); 7261 return; 7262 } 7263 case Intrinsic::ptrmask: { 7264 SDValue Ptr = getValue(I.getOperand(0)); 7265 SDValue Const = getValue(I.getOperand(1)); 7266 7267 EVT PtrVT = Ptr.getValueType(); 7268 setValue(&I, DAG.getNode(ISD::AND, sdl, PtrVT, Ptr, 7269 DAG.getZExtOrTrunc(Const, sdl, PtrVT))); 7270 return; 7271 } 7272 case Intrinsic::threadlocal_address: { 7273 setValue(&I, getValue(I.getOperand(0))); 7274 return; 7275 } 7276 case Intrinsic::get_active_lane_mask: { 7277 EVT CCVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 7278 SDValue Index = getValue(I.getOperand(0)); 7279 EVT ElementVT = Index.getValueType(); 7280 7281 if (!TLI.shouldExpandGetActiveLaneMask(CCVT, ElementVT)) { 7282 visitTargetIntrinsic(I, Intrinsic); 7283 return; 7284 } 7285 7286 SDValue TripCount = getValue(I.getOperand(1)); 7287 auto VecTy = CCVT.changeVectorElementType(ElementVT); 7288 7289 SDValue VectorIndex = DAG.getSplat(VecTy, sdl, Index); 7290 SDValue VectorTripCount = DAG.getSplat(VecTy, sdl, TripCount); 7291 SDValue VectorStep = DAG.getStepVector(sdl, VecTy); 7292 SDValue VectorInduction = DAG.getNode( 7293 ISD::UADDSAT, sdl, VecTy, VectorIndex, VectorStep); 7294 SDValue SetCC = DAG.getSetCC(sdl, CCVT, VectorInduction, 7295 VectorTripCount, ISD::CondCode::SETULT); 7296 setValue(&I, SetCC); 7297 return; 7298 } 7299 case Intrinsic::vector_insert: { 7300 SDValue Vec = getValue(I.getOperand(0)); 7301 SDValue SubVec = getValue(I.getOperand(1)); 7302 SDValue Index = getValue(I.getOperand(2)); 7303 7304 // The intrinsic's index type is i64, but the SDNode requires an index type 7305 // suitable for the target. Convert the index as required. 7306 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 7307 if (Index.getValueType() != VectorIdxTy) 7308 Index = DAG.getVectorIdxConstant( 7309 cast<ConstantSDNode>(Index)->getZExtValue(), sdl); 7310 7311 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 7312 setValue(&I, DAG.getNode(ISD::INSERT_SUBVECTOR, sdl, ResultVT, Vec, SubVec, 7313 Index)); 7314 return; 7315 } 7316 case Intrinsic::vector_extract: { 7317 SDValue Vec = getValue(I.getOperand(0)); 7318 SDValue Index = getValue(I.getOperand(1)); 7319 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 7320 7321 // The intrinsic's index type is i64, but the SDNode requires an index type 7322 // suitable for the target. Convert the index as required. 7323 MVT VectorIdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); 7324 if (Index.getValueType() != VectorIdxTy) 7325 Index = DAG.getVectorIdxConstant( 7326 cast<ConstantSDNode>(Index)->getZExtValue(), sdl); 7327 7328 setValue(&I, 7329 DAG.getNode(ISD::EXTRACT_SUBVECTOR, sdl, ResultVT, Vec, Index)); 7330 return; 7331 } 7332 case Intrinsic::experimental_vector_reverse: 7333 visitVectorReverse(I); 7334 return; 7335 case Intrinsic::experimental_vector_splice: 7336 visitVectorSplice(I); 7337 return; 7338 case Intrinsic::callbr_landingpad: 7339 visitCallBrLandingPad(I); 7340 return; 7341 case Intrinsic::experimental_vector_interleave2: 7342 visitVectorInterleave(I); 7343 return; 7344 case Intrinsic::experimental_vector_deinterleave2: 7345 visitVectorDeinterleave(I); 7346 return; 7347 } 7348 } 7349 7350 void SelectionDAGBuilder::visitConstrainedFPIntrinsic( 7351 const ConstrainedFPIntrinsic &FPI) { 7352 SDLoc sdl = getCurSDLoc(); 7353 7354 // We do not need to serialize constrained FP intrinsics against 7355 // each other or against (nonvolatile) loads, so they can be 7356 // chained like loads. 7357 SDValue Chain = DAG.getRoot(); 7358 SmallVector<SDValue, 4> Opers; 7359 Opers.push_back(Chain); 7360 if (FPI.isUnaryOp()) { 7361 Opers.push_back(getValue(FPI.getArgOperand(0))); 7362 } else if (FPI.isTernaryOp()) { 7363 Opers.push_back(getValue(FPI.getArgOperand(0))); 7364 Opers.push_back(getValue(FPI.getArgOperand(1))); 7365 Opers.push_back(getValue(FPI.getArgOperand(2))); 7366 } else { 7367 Opers.push_back(getValue(FPI.getArgOperand(0))); 7368 Opers.push_back(getValue(FPI.getArgOperand(1))); 7369 } 7370 7371 auto pushOutChain = [this](SDValue Result, fp::ExceptionBehavior EB) { 7372 assert(Result.getNode()->getNumValues() == 2); 7373 7374 // Push node to the appropriate list so that future instructions can be 7375 // chained up correctly. 7376 SDValue OutChain = Result.getValue(1); 7377 switch (EB) { 7378 case fp::ExceptionBehavior::ebIgnore: 7379 // The only reason why ebIgnore nodes still need to be chained is that 7380 // they might depend on the current rounding mode, and therefore must 7381 // not be moved across instruction that may change that mode. 7382 [[fallthrough]]; 7383 case fp::ExceptionBehavior::ebMayTrap: 7384 // These must not be moved across calls or instructions that may change 7385 // floating-point exception masks. 7386 PendingConstrainedFP.push_back(OutChain); 7387 break; 7388 case fp::ExceptionBehavior::ebStrict: 7389 // These must not be moved across calls or instructions that may change 7390 // floating-point exception masks or read floating-point exception flags. 7391 // In addition, they cannot be optimized out even if unused. 7392 PendingConstrainedFPStrict.push_back(OutChain); 7393 break; 7394 } 7395 }; 7396 7397 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7398 EVT VT = TLI.getValueType(DAG.getDataLayout(), FPI.getType()); 7399 SDVTList VTs = DAG.getVTList(VT, MVT::Other); 7400 fp::ExceptionBehavior EB = *FPI.getExceptionBehavior(); 7401 7402 SDNodeFlags Flags; 7403 if (EB == fp::ExceptionBehavior::ebIgnore) 7404 Flags.setNoFPExcept(true); 7405 7406 if (auto *FPOp = dyn_cast<FPMathOperator>(&FPI)) 7407 Flags.copyFMF(*FPOp); 7408 7409 unsigned Opcode; 7410 switch (FPI.getIntrinsicID()) { 7411 default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. 7412 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \ 7413 case Intrinsic::INTRINSIC: \ 7414 Opcode = ISD::STRICT_##DAGN; \ 7415 break; 7416 #include "llvm/IR/ConstrainedOps.def" 7417 case Intrinsic::experimental_constrained_fmuladd: { 7418 Opcode = ISD::STRICT_FMA; 7419 // Break fmuladd into fmul and fadd. 7420 if (TM.Options.AllowFPOpFusion == FPOpFusion::Strict || 7421 !TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { 7422 Opers.pop_back(); 7423 SDValue Mul = DAG.getNode(ISD::STRICT_FMUL, sdl, VTs, Opers, Flags); 7424 pushOutChain(Mul, EB); 7425 Opcode = ISD::STRICT_FADD; 7426 Opers.clear(); 7427 Opers.push_back(Mul.getValue(1)); 7428 Opers.push_back(Mul.getValue(0)); 7429 Opers.push_back(getValue(FPI.getArgOperand(2))); 7430 } 7431 break; 7432 } 7433 } 7434 7435 // A few strict DAG nodes carry additional operands that are not 7436 // set up by the default code above. 7437 switch (Opcode) { 7438 default: break; 7439 case ISD::STRICT_FP_ROUND: 7440 Opers.push_back( 7441 DAG.getTargetConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()))); 7442 break; 7443 case ISD::STRICT_FSETCC: 7444 case ISD::STRICT_FSETCCS: { 7445 auto *FPCmp = dyn_cast<ConstrainedFPCmpIntrinsic>(&FPI); 7446 ISD::CondCode Condition = getFCmpCondCode(FPCmp->getPredicate()); 7447 if (TM.Options.NoNaNsFPMath) 7448 Condition = getFCmpCodeWithoutNaN(Condition); 7449 Opers.push_back(DAG.getCondCode(Condition)); 7450 break; 7451 } 7452 } 7453 7454 SDValue Result = DAG.getNode(Opcode, sdl, VTs, Opers, Flags); 7455 pushOutChain(Result, EB); 7456 7457 SDValue FPResult = Result.getValue(0); 7458 setValue(&FPI, FPResult); 7459 } 7460 7461 static unsigned getISDForVPIntrinsic(const VPIntrinsic &VPIntrin) { 7462 std::optional<unsigned> ResOPC; 7463 switch (VPIntrin.getIntrinsicID()) { 7464 case Intrinsic::vp_ctlz: { 7465 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne(); 7466 ResOPC = IsZeroUndef ? ISD::VP_CTLZ_ZERO_UNDEF : ISD::VP_CTLZ; 7467 break; 7468 } 7469 case Intrinsic::vp_cttz: { 7470 bool IsZeroUndef = cast<ConstantInt>(VPIntrin.getArgOperand(1))->isOne(); 7471 ResOPC = IsZeroUndef ? ISD::VP_CTTZ_ZERO_UNDEF : ISD::VP_CTTZ; 7472 break; 7473 } 7474 #define HELPER_MAP_VPID_TO_VPSD(VPID, VPSD) \ 7475 case Intrinsic::VPID: \ 7476 ResOPC = ISD::VPSD; \ 7477 break; 7478 #include "llvm/IR/VPIntrinsics.def" 7479 } 7480 7481 if (!ResOPC) 7482 llvm_unreachable( 7483 "Inconsistency: no SDNode available for this VPIntrinsic!"); 7484 7485 if (*ResOPC == ISD::VP_REDUCE_SEQ_FADD || 7486 *ResOPC == ISD::VP_REDUCE_SEQ_FMUL) { 7487 if (VPIntrin.getFastMathFlags().allowReassoc()) 7488 return *ResOPC == ISD::VP_REDUCE_SEQ_FADD ? ISD::VP_REDUCE_FADD 7489 : ISD::VP_REDUCE_FMUL; 7490 } 7491 7492 return *ResOPC; 7493 } 7494 7495 void SelectionDAGBuilder::visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT, 7496 SmallVector<SDValue, 7> &OpValues) { 7497 SDLoc DL = getCurSDLoc(); 7498 Value *PtrOperand = VPIntrin.getArgOperand(0); 7499 MaybeAlign Alignment = VPIntrin.getPointerAlignment(); 7500 AAMDNodes AAInfo = VPIntrin.getAAMetadata(); 7501 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); 7502 SDValue LD; 7503 bool AddToChain = true; 7504 // Do not serialize variable-length loads of constant memory with 7505 // anything. 7506 if (!Alignment) 7507 Alignment = DAG.getEVTAlign(VT); 7508 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); 7509 AddToChain = !AA || !AA->pointsToConstantMemory(ML); 7510 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); 7511 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 7512 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, 7513 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); 7514 LD = DAG.getLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], OpValues[2], 7515 MMO, false /*IsExpanding */); 7516 if (AddToChain) 7517 PendingLoads.push_back(LD.getValue(1)); 7518 setValue(&VPIntrin, LD); 7519 } 7520 7521 void SelectionDAGBuilder::visitVPGather(const VPIntrinsic &VPIntrin, EVT VT, 7522 SmallVector<SDValue, 7> &OpValues) { 7523 SDLoc DL = getCurSDLoc(); 7524 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7525 Value *PtrOperand = VPIntrin.getArgOperand(0); 7526 MaybeAlign Alignment = VPIntrin.getPointerAlignment(); 7527 AAMDNodes AAInfo = VPIntrin.getAAMetadata(); 7528 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); 7529 SDValue LD; 7530 if (!Alignment) 7531 Alignment = DAG.getEVTAlign(VT.getScalarType()); 7532 unsigned AS = 7533 PtrOperand->getType()->getScalarType()->getPointerAddressSpace(); 7534 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 7535 MachinePointerInfo(AS), MachineMemOperand::MOLoad, 7536 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); 7537 SDValue Base, Index, Scale; 7538 ISD::MemIndexType IndexType; 7539 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale, 7540 this, VPIntrin.getParent(), 7541 VT.getScalarStoreSize()); 7542 if (!UniformBase) { 7543 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout())); 7544 Index = getValue(PtrOperand); 7545 IndexType = ISD::SIGNED_SCALED; 7546 Scale = DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); 7547 } 7548 EVT IdxVT = Index.getValueType(); 7549 EVT EltTy = IdxVT.getVectorElementType(); 7550 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { 7551 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); 7552 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index); 7553 } 7554 LD = DAG.getGatherVP( 7555 DAG.getVTList(VT, MVT::Other), VT, DL, 7556 {DAG.getRoot(), Base, Index, Scale, OpValues[1], OpValues[2]}, MMO, 7557 IndexType); 7558 PendingLoads.push_back(LD.getValue(1)); 7559 setValue(&VPIntrin, LD); 7560 } 7561 7562 void SelectionDAGBuilder::visitVPStore(const VPIntrinsic &VPIntrin, 7563 SmallVector<SDValue, 7> &OpValues) { 7564 SDLoc DL = getCurSDLoc(); 7565 Value *PtrOperand = VPIntrin.getArgOperand(1); 7566 EVT VT = OpValues[0].getValueType(); 7567 MaybeAlign Alignment = VPIntrin.getPointerAlignment(); 7568 AAMDNodes AAInfo = VPIntrin.getAAMetadata(); 7569 SDValue ST; 7570 if (!Alignment) 7571 Alignment = DAG.getEVTAlign(VT); 7572 SDValue Ptr = OpValues[1]; 7573 SDValue Offset = DAG.getUNDEF(Ptr.getValueType()); 7574 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 7575 MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, 7576 MemoryLocation::UnknownSize, *Alignment, AAInfo); 7577 ST = DAG.getStoreVP(getMemoryRoot(), DL, OpValues[0], Ptr, Offset, 7578 OpValues[2], OpValues[3], VT, MMO, ISD::UNINDEXED, 7579 /* IsTruncating */ false, /*IsCompressing*/ false); 7580 DAG.setRoot(ST); 7581 setValue(&VPIntrin, ST); 7582 } 7583 7584 void SelectionDAGBuilder::visitVPScatter(const VPIntrinsic &VPIntrin, 7585 SmallVector<SDValue, 7> &OpValues) { 7586 SDLoc DL = getCurSDLoc(); 7587 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7588 Value *PtrOperand = VPIntrin.getArgOperand(1); 7589 EVT VT = OpValues[0].getValueType(); 7590 MaybeAlign Alignment = VPIntrin.getPointerAlignment(); 7591 AAMDNodes AAInfo = VPIntrin.getAAMetadata(); 7592 SDValue ST; 7593 if (!Alignment) 7594 Alignment = DAG.getEVTAlign(VT.getScalarType()); 7595 unsigned AS = 7596 PtrOperand->getType()->getScalarType()->getPointerAddressSpace(); 7597 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 7598 MachinePointerInfo(AS), MachineMemOperand::MOStore, 7599 MemoryLocation::UnknownSize, *Alignment, AAInfo); 7600 SDValue Base, Index, Scale; 7601 ISD::MemIndexType IndexType; 7602 bool UniformBase = getUniformBase(PtrOperand, Base, Index, IndexType, Scale, 7603 this, VPIntrin.getParent(), 7604 VT.getScalarStoreSize()); 7605 if (!UniformBase) { 7606 Base = DAG.getConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout())); 7607 Index = getValue(PtrOperand); 7608 IndexType = ISD::SIGNED_SCALED; 7609 Scale = 7610 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())); 7611 } 7612 EVT IdxVT = Index.getValueType(); 7613 EVT EltTy = IdxVT.getVectorElementType(); 7614 if (TLI.shouldExtendGSIndex(IdxVT, EltTy)) { 7615 EVT NewIdxVT = IdxVT.changeVectorElementType(EltTy); 7616 Index = DAG.getNode(ISD::SIGN_EXTEND, DL, NewIdxVT, Index); 7617 } 7618 ST = DAG.getScatterVP(DAG.getVTList(MVT::Other), VT, DL, 7619 {getMemoryRoot(), OpValues[0], Base, Index, Scale, 7620 OpValues[2], OpValues[3]}, 7621 MMO, IndexType); 7622 DAG.setRoot(ST); 7623 setValue(&VPIntrin, ST); 7624 } 7625 7626 void SelectionDAGBuilder::visitVPStridedLoad( 7627 const VPIntrinsic &VPIntrin, EVT VT, SmallVectorImpl<SDValue> &OpValues) { 7628 SDLoc DL = getCurSDLoc(); 7629 Value *PtrOperand = VPIntrin.getArgOperand(0); 7630 MaybeAlign Alignment = VPIntrin.getPointerAlignment(); 7631 if (!Alignment) 7632 Alignment = DAG.getEVTAlign(VT.getScalarType()); 7633 AAMDNodes AAInfo = VPIntrin.getAAMetadata(); 7634 const MDNode *Ranges = VPIntrin.getMetadata(LLVMContext::MD_range); 7635 MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); 7636 bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); 7637 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); 7638 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 7639 MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, 7640 MemoryLocation::UnknownSize, *Alignment, AAInfo, Ranges); 7641 7642 SDValue LD = DAG.getStridedLoadVP(VT, DL, InChain, OpValues[0], OpValues[1], 7643 OpValues[2], OpValues[3], MMO, 7644 false /*IsExpanding*/); 7645 7646 if (AddToChain) 7647 PendingLoads.push_back(LD.getValue(1)); 7648 setValue(&VPIntrin, LD); 7649 } 7650 7651 void SelectionDAGBuilder::visitVPStridedStore( 7652 const VPIntrinsic &VPIntrin, SmallVectorImpl<SDValue> &OpValues) { 7653 SDLoc DL = getCurSDLoc(); 7654 Value *PtrOperand = VPIntrin.getArgOperand(1); 7655 EVT VT = OpValues[0].getValueType(); 7656 MaybeAlign Alignment = VPIntrin.getPointerAlignment(); 7657 if (!Alignment) 7658 Alignment = DAG.getEVTAlign(VT.getScalarType()); 7659 AAMDNodes AAInfo = VPIntrin.getAAMetadata(); 7660 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( 7661 MachinePointerInfo(PtrOperand), MachineMemOperand::MOStore, 7662 MemoryLocation::UnknownSize, *Alignment, AAInfo); 7663 7664 SDValue ST = DAG.getStridedStoreVP( 7665 getMemoryRoot(), DL, OpValues[0], OpValues[1], 7666 DAG.getUNDEF(OpValues[1].getValueType()), OpValues[2], OpValues[3], 7667 OpValues[4], VT, MMO, ISD::UNINDEXED, /*IsTruncating*/ false, 7668 /*IsCompressing*/ false); 7669 7670 DAG.setRoot(ST); 7671 setValue(&VPIntrin, ST); 7672 } 7673 7674 void SelectionDAGBuilder::visitVPCmp(const VPCmpIntrinsic &VPIntrin) { 7675 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7676 SDLoc DL = getCurSDLoc(); 7677 7678 ISD::CondCode Condition; 7679 CmpInst::Predicate CondCode = VPIntrin.getPredicate(); 7680 bool IsFP = VPIntrin.getOperand(0)->getType()->isFPOrFPVectorTy(); 7681 if (IsFP) { 7682 // FIXME: Regular fcmps are FPMathOperators which may have fast-math (nnan) 7683 // flags, but calls that don't return floating-point types can't be 7684 // FPMathOperators, like vp.fcmp. This affects constrained fcmp too. 7685 Condition = getFCmpCondCode(CondCode); 7686 if (TM.Options.NoNaNsFPMath) 7687 Condition = getFCmpCodeWithoutNaN(Condition); 7688 } else { 7689 Condition = getICmpCondCode(CondCode); 7690 } 7691 7692 SDValue Op1 = getValue(VPIntrin.getOperand(0)); 7693 SDValue Op2 = getValue(VPIntrin.getOperand(1)); 7694 // #2 is the condition code 7695 SDValue MaskOp = getValue(VPIntrin.getOperand(3)); 7696 SDValue EVL = getValue(VPIntrin.getOperand(4)); 7697 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy(); 7698 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && 7699 "Unexpected target EVL type"); 7700 EVL = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, EVL); 7701 7702 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 7703 VPIntrin.getType()); 7704 setValue(&VPIntrin, 7705 DAG.getSetCCVP(DL, DestVT, Op1, Op2, Condition, MaskOp, EVL)); 7706 } 7707 7708 void SelectionDAGBuilder::visitVectorPredicationIntrinsic( 7709 const VPIntrinsic &VPIntrin) { 7710 SDLoc DL = getCurSDLoc(); 7711 unsigned Opcode = getISDForVPIntrinsic(VPIntrin); 7712 7713 auto IID = VPIntrin.getIntrinsicID(); 7714 7715 if (const auto *CmpI = dyn_cast<VPCmpIntrinsic>(&VPIntrin)) 7716 return visitVPCmp(*CmpI); 7717 7718 SmallVector<EVT, 4> ValueVTs; 7719 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7720 ComputeValueVTs(TLI, DAG.getDataLayout(), VPIntrin.getType(), ValueVTs); 7721 SDVTList VTs = DAG.getVTList(ValueVTs); 7722 7723 auto EVLParamPos = VPIntrinsic::getVectorLengthParamPos(IID); 7724 7725 MVT EVLParamVT = TLI.getVPExplicitVectorLengthTy(); 7726 assert(EVLParamVT.isScalarInteger() && EVLParamVT.bitsGE(MVT::i32) && 7727 "Unexpected target EVL type"); 7728 7729 // Request operands. 7730 SmallVector<SDValue, 7> OpValues; 7731 for (unsigned I = 0; I < VPIntrin.arg_size(); ++I) { 7732 auto Op = getValue(VPIntrin.getArgOperand(I)); 7733 if (I == EVLParamPos) 7734 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, EVLParamVT, Op); 7735 OpValues.push_back(Op); 7736 } 7737 7738 switch (Opcode) { 7739 default: { 7740 SDNodeFlags SDFlags; 7741 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin)) 7742 SDFlags.copyFMF(*FPMO); 7743 SDValue Result = DAG.getNode(Opcode, DL, VTs, OpValues, SDFlags); 7744 setValue(&VPIntrin, Result); 7745 break; 7746 } 7747 case ISD::VP_LOAD: 7748 visitVPLoad(VPIntrin, ValueVTs[0], OpValues); 7749 break; 7750 case ISD::VP_GATHER: 7751 visitVPGather(VPIntrin, ValueVTs[0], OpValues); 7752 break; 7753 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: 7754 visitVPStridedLoad(VPIntrin, ValueVTs[0], OpValues); 7755 break; 7756 case ISD::VP_STORE: 7757 visitVPStore(VPIntrin, OpValues); 7758 break; 7759 case ISD::VP_SCATTER: 7760 visitVPScatter(VPIntrin, OpValues); 7761 break; 7762 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: 7763 visitVPStridedStore(VPIntrin, OpValues); 7764 break; 7765 case ISD::VP_FMULADD: { 7766 assert(OpValues.size() == 5 && "Unexpected number of operands"); 7767 SDNodeFlags SDFlags; 7768 if (auto *FPMO = dyn_cast<FPMathOperator>(&VPIntrin)) 7769 SDFlags.copyFMF(*FPMO); 7770 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && 7771 TLI.isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), ValueVTs[0])) { 7772 setValue(&VPIntrin, DAG.getNode(ISD::VP_FMA, DL, VTs, OpValues, SDFlags)); 7773 } else { 7774 SDValue Mul = DAG.getNode( 7775 ISD::VP_FMUL, DL, VTs, 7776 {OpValues[0], OpValues[1], OpValues[3], OpValues[4]}, SDFlags); 7777 SDValue Add = 7778 DAG.getNode(ISD::VP_FADD, DL, VTs, 7779 {Mul, OpValues[2], OpValues[3], OpValues[4]}, SDFlags); 7780 setValue(&VPIntrin, Add); 7781 } 7782 break; 7783 } 7784 case ISD::VP_INTTOPTR: { 7785 SDValue N = OpValues[0]; 7786 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), VPIntrin.getType()); 7787 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), VPIntrin.getType()); 7788 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1], 7789 OpValues[2]); 7790 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1], 7791 OpValues[2]); 7792 setValue(&VPIntrin, N); 7793 break; 7794 } 7795 case ISD::VP_PTRTOINT: { 7796 SDValue N = OpValues[0]; 7797 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 7798 VPIntrin.getType()); 7799 EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), 7800 VPIntrin.getOperand(0)->getType()); 7801 N = DAG.getVPPtrExtOrTrunc(getCurSDLoc(), PtrMemVT, N, OpValues[1], 7802 OpValues[2]); 7803 N = DAG.getVPZExtOrTrunc(getCurSDLoc(), DestVT, N, OpValues[1], 7804 OpValues[2]); 7805 setValue(&VPIntrin, N); 7806 break; 7807 } 7808 case ISD::VP_ABS: 7809 case ISD::VP_CTLZ: 7810 case ISD::VP_CTLZ_ZERO_UNDEF: 7811 case ISD::VP_CTTZ: 7812 case ISD::VP_CTTZ_ZERO_UNDEF: { 7813 SDValue Result = 7814 DAG.getNode(Opcode, DL, VTs, {OpValues[0], OpValues[2], OpValues[3]}); 7815 setValue(&VPIntrin, Result); 7816 break; 7817 } 7818 } 7819 } 7820 7821 SDValue SelectionDAGBuilder::lowerStartEH(SDValue Chain, 7822 const BasicBlock *EHPadBB, 7823 MCSymbol *&BeginLabel) { 7824 MachineFunction &MF = DAG.getMachineFunction(); 7825 MachineModuleInfo &MMI = MF.getMMI(); 7826 7827 // Insert a label before the invoke call to mark the try range. This can be 7828 // used to detect deletion of the invoke via the MachineModuleInfo. 7829 BeginLabel = MMI.getContext().createTempSymbol(); 7830 7831 // For SjLj, keep track of which landing pads go with which invokes 7832 // so as to maintain the ordering of pads in the LSDA. 7833 unsigned CallSiteIndex = MMI.getCurrentCallSite(); 7834 if (CallSiteIndex) { 7835 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex); 7836 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex); 7837 7838 // Now that the call site is handled, stop tracking it. 7839 MMI.setCurrentCallSite(0); 7840 } 7841 7842 return DAG.getEHLabel(getCurSDLoc(), Chain, BeginLabel); 7843 } 7844 7845 SDValue SelectionDAGBuilder::lowerEndEH(SDValue Chain, const InvokeInst *II, 7846 const BasicBlock *EHPadBB, 7847 MCSymbol *BeginLabel) { 7848 assert(BeginLabel && "BeginLabel should've been set"); 7849 7850 MachineFunction &MF = DAG.getMachineFunction(); 7851 MachineModuleInfo &MMI = MF.getMMI(); 7852 7853 // Insert a label at the end of the invoke call to mark the try range. This 7854 // can be used to detect deletion of the invoke via the MachineModuleInfo. 7855 MCSymbol *EndLabel = MMI.getContext().createTempSymbol(); 7856 Chain = DAG.getEHLabel(getCurSDLoc(), Chain, EndLabel); 7857 7858 // Inform MachineModuleInfo of range. 7859 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn()); 7860 // There is a platform (e.g. wasm) that uses funclet style IR but does not 7861 // actually use outlined funclets and their LSDA info style. 7862 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) { 7863 assert(II && "II should've been set"); 7864 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo(); 7865 EHInfo->addIPToStateRange(II, BeginLabel, EndLabel); 7866 } else if (!isScopedEHPersonality(Pers)) { 7867 assert(EHPadBB); 7868 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel); 7869 } 7870 7871 return Chain; 7872 } 7873 7874 std::pair<SDValue, SDValue> 7875 SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI, 7876 const BasicBlock *EHPadBB) { 7877 MCSymbol *BeginLabel = nullptr; 7878 7879 if (EHPadBB) { 7880 // Both PendingLoads and PendingExports must be flushed here; 7881 // this call might not return. 7882 (void)getRoot(); 7883 DAG.setRoot(lowerStartEH(getControlRoot(), EHPadBB, BeginLabel)); 7884 CLI.setChain(getRoot()); 7885 } 7886 7887 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7888 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI); 7889 7890 assert((CLI.IsTailCall || Result.second.getNode()) && 7891 "Non-null chain expected with non-tail call!"); 7892 assert((Result.second.getNode() || !Result.first.getNode()) && 7893 "Null value expected with tail call!"); 7894 7895 if (!Result.second.getNode()) { 7896 // As a special case, a null chain means that a tail call has been emitted 7897 // and the DAG root is already updated. 7898 HasTailCall = true; 7899 7900 // Since there's no actual continuation from this block, nothing can be 7901 // relying on us setting vregs for them. 7902 PendingExports.clear(); 7903 } else { 7904 DAG.setRoot(Result.second); 7905 } 7906 7907 if (EHPadBB) { 7908 DAG.setRoot(lowerEndEH(getRoot(), cast_or_null<InvokeInst>(CLI.CB), EHPadBB, 7909 BeginLabel)); 7910 } 7911 7912 return Result; 7913 } 7914 7915 void SelectionDAGBuilder::LowerCallTo(const CallBase &CB, SDValue Callee, 7916 bool isTailCall, 7917 bool isMustTailCall, 7918 const BasicBlock *EHPadBB) { 7919 auto &DL = DAG.getDataLayout(); 7920 FunctionType *FTy = CB.getFunctionType(); 7921 Type *RetTy = CB.getType(); 7922 7923 TargetLowering::ArgListTy Args; 7924 Args.reserve(CB.arg_size()); 7925 7926 const Value *SwiftErrorVal = nullptr; 7927 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 7928 7929 if (isTailCall) { 7930 // Avoid emitting tail calls in functions with the disable-tail-calls 7931 // attribute. 7932 auto *Caller = CB.getParent()->getParent(); 7933 if (Caller->getFnAttribute("disable-tail-calls").getValueAsString() == 7934 "true" && !isMustTailCall) 7935 isTailCall = false; 7936 7937 // We can't tail call inside a function with a swifterror argument. Lowering 7938 // does not support this yet. It would have to move into the swifterror 7939 // register before the call. 7940 if (TLI.supportSwiftError() && 7941 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) 7942 isTailCall = false; 7943 } 7944 7945 for (auto I = CB.arg_begin(), E = CB.arg_end(); I != E; ++I) { 7946 TargetLowering::ArgListEntry Entry; 7947 const Value *V = *I; 7948 7949 // Skip empty types 7950 if (V->getType()->isEmptyTy()) 7951 continue; 7952 7953 SDValue ArgNode = getValue(V); 7954 Entry.Node = ArgNode; Entry.Ty = V->getType(); 7955 7956 Entry.setAttributes(&CB, I - CB.arg_begin()); 7957 7958 // Use swifterror virtual register as input to the call. 7959 if (Entry.IsSwiftError && TLI.supportSwiftError()) { 7960 SwiftErrorVal = V; 7961 // We find the virtual register for the actual swifterror argument. 7962 // Instead of using the Value, we use the virtual register instead. 7963 Entry.Node = 7964 DAG.getRegister(SwiftError.getOrCreateVRegUseAt(&CB, FuncInfo.MBB, V), 7965 EVT(TLI.getPointerTy(DL))); 7966 } 7967 7968 Args.push_back(Entry); 7969 7970 // If we have an explicit sret argument that is an Instruction, (i.e., it 7971 // might point to function-local memory), we can't meaningfully tail-call. 7972 if (Entry.IsSRet && isa<Instruction>(V)) 7973 isTailCall = false; 7974 } 7975 7976 // If call site has a cfguardtarget operand bundle, create and add an 7977 // additional ArgListEntry. 7978 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_cfguardtarget)) { 7979 TargetLowering::ArgListEntry Entry; 7980 Value *V = Bundle->Inputs[0]; 7981 SDValue ArgNode = getValue(V); 7982 Entry.Node = ArgNode; 7983 Entry.Ty = V->getType(); 7984 Entry.IsCFGuardTarget = true; 7985 Args.push_back(Entry); 7986 } 7987 7988 // Check if target-independent constraints permit a tail call here. 7989 // Target-dependent constraints are checked within TLI->LowerCallTo. 7990 if (isTailCall && !isInTailCallPosition(CB, DAG.getTarget())) 7991 isTailCall = false; 7992 7993 // Disable tail calls if there is an swifterror argument. Targets have not 7994 // been updated to support tail calls. 7995 if (TLI.supportSwiftError() && SwiftErrorVal) 7996 isTailCall = false; 7997 7998 ConstantInt *CFIType = nullptr; 7999 if (CB.isIndirectCall()) { 8000 if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_kcfi)) { 8001 if (!TLI.supportKCFIBundles()) 8002 report_fatal_error( 8003 "Target doesn't support calls with kcfi operand bundles."); 8004 CFIType = cast<ConstantInt>(Bundle->Inputs[0]); 8005 assert(CFIType->getType()->isIntegerTy(32) && "Invalid CFI type"); 8006 } 8007 } 8008 8009 TargetLowering::CallLoweringInfo CLI(DAG); 8010 CLI.setDebugLoc(getCurSDLoc()) 8011 .setChain(getRoot()) 8012 .setCallee(RetTy, FTy, Callee, std::move(Args), CB) 8013 .setTailCall(isTailCall) 8014 .setConvergent(CB.isConvergent()) 8015 .setIsPreallocated( 8016 CB.countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0) 8017 .setCFIType(CFIType); 8018 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB); 8019 8020 if (Result.first.getNode()) { 8021 Result.first = lowerRangeToAssertZExt(DAG, CB, Result.first); 8022 setValue(&CB, Result.first); 8023 } 8024 8025 // The last element of CLI.InVals has the SDValue for swifterror return. 8026 // Here we copy it to a virtual register and update SwiftErrorMap for 8027 // book-keeping. 8028 if (SwiftErrorVal && TLI.supportSwiftError()) { 8029 // Get the last element of InVals. 8030 SDValue Src = CLI.InVals.back(); 8031 Register VReg = 8032 SwiftError.getOrCreateVRegDefAt(&CB, FuncInfo.MBB, SwiftErrorVal); 8033 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src); 8034 DAG.setRoot(CopyNode); 8035 } 8036 } 8037 8038 static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, 8039 SelectionDAGBuilder &Builder) { 8040 // Check to see if this load can be trivially constant folded, e.g. if the 8041 // input is from a string literal. 8042 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) { 8043 // Cast pointer to the type we really want to load. 8044 Type *LoadTy = 8045 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits()); 8046 if (LoadVT.isVector()) 8047 LoadTy = FixedVectorType::get(LoadTy, LoadVT.getVectorNumElements()); 8048 8049 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput), 8050 PointerType::getUnqual(LoadTy)); 8051 8052 if (const Constant *LoadCst = 8053 ConstantFoldLoadFromConstPtr(const_cast<Constant *>(LoadInput), 8054 LoadTy, Builder.DAG.getDataLayout())) 8055 return Builder.getValue(LoadCst); 8056 } 8057 8058 // Otherwise, we have to emit the load. If the pointer is to unfoldable but 8059 // still constant memory, the input chain can be the entry node. 8060 SDValue Root; 8061 bool ConstantMemory = false; 8062 8063 // Do not serialize (non-volatile) loads of constant memory with anything. 8064 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) { 8065 Root = Builder.DAG.getEntryNode(); 8066 ConstantMemory = true; 8067 } else { 8068 // Do not serialize non-volatile loads against each other. 8069 Root = Builder.DAG.getRoot(); 8070 } 8071 8072 SDValue Ptr = Builder.getValue(PtrVal); 8073 SDValue LoadVal = 8074 Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root, Ptr, 8075 MachinePointerInfo(PtrVal), Align(1)); 8076 8077 if (!ConstantMemory) 8078 Builder.PendingLoads.push_back(LoadVal.getValue(1)); 8079 return LoadVal; 8080 } 8081 8082 /// Record the value for an instruction that produces an integer result, 8083 /// converting the type where necessary. 8084 void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, 8085 SDValue Value, 8086 bool IsSigned) { 8087 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 8088 I.getType(), true); 8089 if (IsSigned) 8090 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT); 8091 else 8092 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT); 8093 setValue(&I, Value); 8094 } 8095 8096 /// See if we can lower a memcmp/bcmp call into an optimized form. If so, return 8097 /// true and lower it. Otherwise return false, and it will be lowered like a 8098 /// normal call. 8099 /// The caller already checked that \p I calls the appropriate LibFunc with a 8100 /// correct prototype. 8101 bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) { 8102 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); 8103 const Value *Size = I.getArgOperand(2); 8104 const ConstantSDNode *CSize = dyn_cast<ConstantSDNode>(getValue(Size)); 8105 if (CSize && CSize->getZExtValue() == 0) { 8106 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), 8107 I.getType(), true); 8108 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT)); 8109 return true; 8110 } 8111 8112 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 8113 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp( 8114 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS), 8115 getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS)); 8116 if (Res.first.getNode()) { 8117 processIntegerCallValue(I, Res.first, true); 8118 PendingLoads.push_back(Res.second); 8119 return true; 8120 } 8121 8122 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0 8123 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0 8124 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I)) 8125 return false; 8126 8127 // If the target has a fast compare for the given size, it will return a 8128 // preferred load type for that size. Require that the load VT is legal and 8129 // that the target supports unaligned loads of that type. Otherwise, return 8130 // INVALID. 8131 auto hasFastLoadsAndCompare = [&](unsigned NumBits) { 8132 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 8133 MVT LVT = TLI.hasFastEqualityCompare(NumBits); 8134 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) { 8135 // TODO: Handle 5 byte compare as 4-byte + 1 byte. 8136 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads. 8137 // TODO: Check alignment of src and dest ptrs. 8138 unsigned DstAS = LHS->getType()->getPointerAddressSpace(); 8139 unsigned SrcAS = RHS->getType()->getPointerAddressSpace(); 8140 if (!TLI.isTypeLegal(LVT) || 8141 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) || 8142 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS)) 8143 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE; 8144 } 8145 8146 return LVT; 8147 }; 8148 8149 // This turns into unaligned loads. We only do this if the target natively 8150 // supports the MVT we'll be loading or if it is small enough (<= 4) that 8151 // we'll only produce a small number of byte loads. 8152 MVT LoadVT; 8153 unsigned NumBitsToCompare = CSize->getZExtValue() * 8; 8154 switch (NumBitsToCompare) { 8155 default: 8156 return false; 8157 case 16: 8158 LoadVT = MVT::i16; 8159 break; 8160 case 32: 8161 LoadVT = MVT::i32; 8162 break; 8163 case 64: 8164 case 128: 8165 case 256: 8166 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare); 8167 break; 8168 } 8169 8170 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE) 8171 return false; 8172 8173 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this); 8174 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this); 8175 8176 // Bitcast to a wide integer type if the loads are vectors. 8177 if (LoadVT.isVector()) { 8178 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits()); 8179 LoadL = DAG.getBitcast(CmpVT, LoadL); 8180 LoadR = DAG.getBitcast(CmpVT, LoadR); 8181 } 8182 8183 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE); 8184 processIntegerCallValue(I, Cmp, false); 8185 return true; 8186 } 8187 8188 /// See if we can lower a memchr call into an optimized form. If so, return 8189 /// true and lower it. Otherwise return false, and it will be lowered like a 8190 /// normal call. 8191 /// The caller already checked that \p I calls the appropriate LibFunc with a 8192 /// correct prototype. 8193 bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { 8194 const Value *Src = I.getArgOperand(0); 8195 const Value *Char = I.getArgOperand(1); 8196 const Value *Length = I.getArgOperand(2); 8197 8198 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 8199 std::pair<SDValue, SDValue> Res = 8200 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(), 8201 getValue(Src), getValue(Char), getValue(Length), 8202 MachinePointerInfo(Src)); 8203 if (Res.first.getNode()) { 8204 setValue(&I, Res.first); 8205 PendingLoads.push_back(Res.second); 8206 return true; 8207 } 8208 8209 return false; 8210 } 8211 8212 /// See if we can lower a mempcpy call into an optimized form. If so, return 8213 /// true and lower it. Otherwise return false, and it will be lowered like a 8214 /// normal call. 8215 /// The caller already checked that \p I calls the appropriate LibFunc with a 8216 /// correct prototype. 8217 bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { 8218 SDValue Dst = getValue(I.getArgOperand(0)); 8219 SDValue Src = getValue(I.getArgOperand(1)); 8220 SDValue Size = getValue(I.getArgOperand(2)); 8221 8222 Align DstAlign = DAG.InferPtrAlign(Dst).valueOrOne(); 8223 Align SrcAlign = DAG.InferPtrAlign(Src).valueOrOne(); 8224 // DAG::getMemcpy needs Alignment to be defined. 8225 Align Alignment = std::min(DstAlign, SrcAlign); 8226 8227 bool isVol = false; 8228 SDLoc sdl = getCurSDLoc(); 8229 8230 // In the mempcpy context we need to pass in a false value for isTailCall 8231 // because the return pointer needs to be adjusted by the size of 8232 // the copied memory. 8233 SDValue Root = isVol ? getRoot() : getMemoryRoot(); 8234 SDValue MC = DAG.getMemcpy(Root, sdl, Dst, Src, Size, Alignment, isVol, false, 8235 /*isTailCall=*/false, 8236 MachinePointerInfo(I.getArgOperand(0)), 8237 MachinePointerInfo(I.getArgOperand(1)), 8238 I.getAAMetadata()); 8239 assert(MC.getNode() != nullptr && 8240 "** memcpy should not be lowered as TailCall in mempcpy context **"); 8241 DAG.setRoot(MC); 8242 8243 // Check if Size needs to be truncated or extended. 8244 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType()); 8245 8246 // Adjust return pointer to point just past the last dst byte. 8247 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(), 8248 Dst, Size); 8249 setValue(&I, DstPlusSize); 8250 return true; 8251 } 8252 8253 /// See if we can lower a strcpy call into an optimized form. If so, return 8254 /// true and lower it, otherwise return false and it will be lowered like a 8255 /// normal call. 8256 /// The caller already checked that \p I calls the appropriate LibFunc with a 8257 /// correct prototype. 8258 bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { 8259 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 8260 8261 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 8262 std::pair<SDValue, SDValue> Res = 8263 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(), 8264 getValue(Arg0), getValue(Arg1), 8265 MachinePointerInfo(Arg0), 8266 MachinePointerInfo(Arg1), isStpcpy); 8267 if (Res.first.getNode()) { 8268 setValue(&I, Res.first); 8269 DAG.setRoot(Res.second); 8270 return true; 8271 } 8272 8273 return false; 8274 } 8275 8276 /// See if we can lower a strcmp call into an optimized form. If so, return 8277 /// true and lower it, otherwise return false and it will be lowered like a 8278 /// normal call. 8279 /// The caller already checked that \p I calls the appropriate LibFunc with a 8280 /// correct prototype. 8281 bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { 8282 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 8283 8284 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 8285 std::pair<SDValue, SDValue> Res = 8286 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(), 8287 getValue(Arg0), getValue(Arg1), 8288 MachinePointerInfo(Arg0), 8289 MachinePointerInfo(Arg1)); 8290 if (Res.first.getNode()) { 8291 processIntegerCallValue(I, Res.first, true); 8292 PendingLoads.push_back(Res.second); 8293 return true; 8294 } 8295 8296 return false; 8297 } 8298 8299 /// See if we can lower a strlen call into an optimized form. If so, return 8300 /// true and lower it, otherwise return false and it will be lowered like a 8301 /// normal call. 8302 /// The caller already checked that \p I calls the appropriate LibFunc with a 8303 /// correct prototype. 8304 bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { 8305 const Value *Arg0 = I.getArgOperand(0); 8306 8307 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 8308 std::pair<SDValue, SDValue> Res = 8309 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(), 8310 getValue(Arg0), MachinePointerInfo(Arg0)); 8311 if (Res.first.getNode()) { 8312 processIntegerCallValue(I, Res.first, false); 8313 PendingLoads.push_back(Res.second); 8314 return true; 8315 } 8316 8317 return false; 8318 } 8319 8320 /// See if we can lower a strnlen call into an optimized form. If so, return 8321 /// true and lower it, otherwise return false and it will be lowered like a 8322 /// normal call. 8323 /// The caller already checked that \p I calls the appropriate LibFunc with a 8324 /// correct prototype. 8325 bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { 8326 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); 8327 8328 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); 8329 std::pair<SDValue, SDValue> Res = 8330 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(), 8331 getValue(Arg0), getValue(Arg1), 8332 MachinePointerInfo(Arg0)); 8333 if (Res.first.getNode()) { 8334 processIntegerCallValue(I, Res.first, false); 8335 PendingLoads.push_back(Res.second); 8336 return true; 8337 } 8338 8339 return false; 8340 } 8341 8342 /// See if we can lower a unary floating-point operation into an SDNode with 8343 /// the specified Opcode. If so, return true and lower it, otherwise return 8344 /// false and it will be lowered like a normal call. 8345 /// The caller already checked that \p I calls the appropriate LibFunc with a 8346 /// correct prototype. 8347 bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, 8348 unsigned Opcode) { 8349 // We already checked this call's prototype; verify it doesn't modify errno. 8350 if (!I.onlyReadsMemory()) 8351 return false; 8352 8353 SDNodeFlags Flags; 8354 Flags.copyFMF(cast<FPMathOperator>(I)); 8355 8356 SDValue Tmp = getValue(I.getArgOperand(0)); 8357 setValue(&I, 8358 DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp, Flags)); 8359 return true; 8360 } 8361 8362 /// See if we can lower a binary floating-point operation into an SDNode with 8363 /// the specified Opcode. If so, return true and lower it. Otherwise return 8364 /// false, and it will be lowered like a normal call. 8365 /// The caller already checked that \p I calls the appropriate LibFunc with a 8366 /// correct prototype. 8367 bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I, 8368 unsigned Opcode) { 8369 // We already checked this call's prototype; verify it doesn't modify errno. 8370 if (!I.onlyReadsMemory()) 8371 return false; 8372 8373 SDNodeFlags Flags; 8374 Flags.copyFMF(cast<FPMathOperator>(I)); 8375 8376 SDValue Tmp0 = getValue(I.getArgOperand(0)); 8377 SDValue Tmp1 = getValue(I.getArgOperand(1)); 8378 EVT VT = Tmp0.getValueType(); 8379 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1, Flags)); 8380 return true; 8381 } 8382 8383 void SelectionDAGBuilder::visitCall(const CallInst &I) { 8384 // Handle inline assembly differently. 8385 if (I.isInlineAsm()) { 8386 visitInlineAsm(I); 8387 return; 8388 } 8389 8390 diagnoseDontCall(I); 8391 8392 if (Function *F = I.getCalledFunction()) { 8393 if (F->isDeclaration()) { 8394 // Is this an LLVM intrinsic or a target-specific intrinsic? 8395 unsigned IID = F->getIntrinsicID(); 8396 if (!IID) 8397 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo()) 8398 IID = II->getIntrinsicID(F); 8399 8400 if (IID) { 8401 visitIntrinsicCall(I, IID); 8402 return; 8403 } 8404 } 8405 8406 // Check for well-known libc/libm calls. If the function is internal, it 8407 // can't be a library call. Don't do the check if marked as nobuiltin for 8408 // some reason or the call site requires strict floating point semantics. 8409 LibFunc Func; 8410 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() && 8411 F->hasName() && LibInfo->getLibFunc(*F, Func) && 8412 LibInfo->hasOptimizedCodeGen(Func)) { 8413 switch (Func) { 8414 default: break; 8415 case LibFunc_bcmp: 8416 if (visitMemCmpBCmpCall(I)) 8417 return; 8418 break; 8419 case LibFunc_copysign: 8420 case LibFunc_copysignf: 8421 case LibFunc_copysignl: 8422 // We already checked this call's prototype; verify it doesn't modify 8423 // errno. 8424 if (I.onlyReadsMemory()) { 8425 SDValue LHS = getValue(I.getArgOperand(0)); 8426 SDValue RHS = getValue(I.getArgOperand(1)); 8427 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(), 8428 LHS.getValueType(), LHS, RHS)); 8429 return; 8430 } 8431 break; 8432 case LibFunc_fabs: 8433 case LibFunc_fabsf: 8434 case LibFunc_fabsl: 8435 if (visitUnaryFloatCall(I, ISD::FABS)) 8436 return; 8437 break; 8438 case LibFunc_fmin: 8439 case LibFunc_fminf: 8440 case LibFunc_fminl: 8441 if (visitBinaryFloatCall(I, ISD::FMINNUM)) 8442 return; 8443 break; 8444 case LibFunc_fmax: 8445 case LibFunc_fmaxf: 8446 case LibFunc_fmaxl: 8447 if (visitBinaryFloatCall(I, ISD::FMAXNUM)) 8448 return; 8449 break; 8450 case LibFunc_sin: 8451 case LibFunc_sinf: 8452 case LibFunc_sinl: 8453 if (visitUnaryFloatCall(I, ISD::FSIN)) 8454 return; 8455 break; 8456 case LibFunc_cos: 8457 case LibFunc_cosf: 8458 case LibFunc_cosl: 8459 if (visitUnaryFloatCall(I, ISD::FCOS)) 8460 return; 8461 break; 8462 case LibFunc_sqrt: 8463 case LibFunc_sqrtf: 8464 case LibFunc_sqrtl: 8465 case LibFunc_sqrt_finite: 8466 case LibFunc_sqrtf_finite: 8467 case LibFunc_sqrtl_finite: 8468 if (visitUnaryFloatCall(I, ISD::FSQRT)) 8469 return; 8470 break; 8471 case LibFunc_floor: 8472 case LibFunc_floorf: 8473 case LibFunc_floorl: 8474 if (visitUnaryFloatCall(I, ISD::FFLOOR)) 8475 return; 8476 break; 8477 case LibFunc_nearbyint: 8478 case LibFunc_nearbyintf: 8479 case LibFunc_nearbyintl: 8480 if (visitUnaryFloatCall(I, ISD::FNEARBYINT)) 8481 return; 8482 break; 8483 case LibFunc_ceil: 8484 case LibFunc_ceilf: 8485 case LibFunc_ceill: 8486 if (visitUnaryFloatCall(I, ISD::FCEIL)) 8487 return; 8488 break; 8489 case LibFunc_rint: 8490 case LibFunc_rintf: 8491 case LibFunc_rintl: 8492 if (visitUnaryFloatCall(I, ISD::FRINT)) 8493 return; 8494 break; 8495 case LibFunc_round: 8496 case LibFunc_roundf: 8497 case LibFunc_roundl: 8498 if (visitUnaryFloatCall(I, ISD::FROUND)) 8499 return; 8500 break; 8501 case LibFunc_trunc: 8502 case LibFunc_truncf: 8503 case LibFunc_truncl: 8504 if (visitUnaryFloatCall(I, ISD::FTRUNC)) 8505 return; 8506 break; 8507 case LibFunc_log2: 8508 case LibFunc_log2f: 8509 case LibFunc_log2l: 8510 if (visitUnaryFloatCall(I, ISD::FLOG2)) 8511 return; 8512 break; 8513 case LibFunc_exp2: 8514 case LibFunc_exp2f: 8515 case LibFunc_exp2l: 8516 if (visitUnaryFloatCall(I, ISD::FEXP2)) 8517 return; 8518 break; 8519 case LibFunc_memcmp: 8520 if (visitMemCmpBCmpCall(I)) 8521 return; 8522 break; 8523 case LibFunc_mempcpy: 8524 if (visitMemPCpyCall(I)) 8525 return; 8526 break; 8527 case LibFunc_memchr: 8528 if (visitMemChrCall(I)) 8529 return; 8530 break; 8531 case LibFunc_strcpy: 8532 if (visitStrCpyCall(I, false)) 8533 return; 8534 break; 8535 case LibFunc_stpcpy: 8536 if (visitStrCpyCall(I, true)) 8537 return; 8538 break; 8539 case LibFunc_strcmp: 8540 if (visitStrCmpCall(I)) 8541 return; 8542 break; 8543 case LibFunc_strlen: 8544 if (visitStrLenCall(I)) 8545 return; 8546 break; 8547 case LibFunc_strnlen: 8548 if (visitStrNLenCall(I)) 8549 return; 8550 break; 8551 } 8552 } 8553 } 8554 8555 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't 8556 // have to do anything here to lower funclet bundles. 8557 // CFGuardTarget bundles are lowered in LowerCallTo. 8558 assert(!I.hasOperandBundlesOtherThan( 8559 {LLVMContext::OB_deopt, LLVMContext::OB_funclet, 8560 LLVMContext::OB_cfguardtarget, LLVMContext::OB_preallocated, 8561 LLVMContext::OB_clang_arc_attachedcall, LLVMContext::OB_kcfi}) && 8562 "Cannot lower calls with arbitrary operand bundles!"); 8563 8564 SDValue Callee = getValue(I.getCalledOperand()); 8565 8566 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) 8567 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr); 8568 else 8569 // Check if we can potentially perform a tail call. More detailed checking 8570 // is be done within LowerCallTo, after more information about the call is 8571 // known. 8572 LowerCallTo(I, Callee, I.isTailCall(), I.isMustTailCall()); 8573 } 8574 8575 namespace { 8576 8577 /// AsmOperandInfo - This contains information for each constraint that we are 8578 /// lowering. 8579 class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo { 8580 public: 8581 /// CallOperand - If this is the result output operand or a clobber 8582 /// this is null, otherwise it is the incoming operand to the CallInst. 8583 /// This gets modified as the asm is processed. 8584 SDValue CallOperand; 8585 8586 /// AssignedRegs - If this is a register or register class operand, this 8587 /// contains the set of register corresponding to the operand. 8588 RegsForValue AssignedRegs; 8589 8590 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info) 8591 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) { 8592 } 8593 8594 /// Whether or not this operand accesses memory 8595 bool hasMemory(const TargetLowering &TLI) const { 8596 // Indirect operand accesses access memory. 8597 if (isIndirect) 8598 return true; 8599 8600 for (const auto &Code : Codes) 8601 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory) 8602 return true; 8603 8604 return false; 8605 } 8606 }; 8607 8608 8609 } // end anonymous namespace 8610 8611 /// Make sure that the output operand \p OpInfo and its corresponding input 8612 /// operand \p MatchingOpInfo have compatible constraint types (otherwise error 8613 /// out). 8614 static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo, 8615 SDISelAsmOperandInfo &MatchingOpInfo, 8616 SelectionDAG &DAG) { 8617 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT) 8618 return; 8619 8620 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); 8621 const auto &TLI = DAG.getTargetLoweringInfo(); 8622 8623 std::pair<unsigned, const TargetRegisterClass *> MatchRC = 8624 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode, 8625 OpInfo.ConstraintVT); 8626 std::pair<unsigned, const TargetRegisterClass *> InputRC = 8627 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode, 8628 MatchingOpInfo.ConstraintVT); 8629 if ((OpInfo.ConstraintVT.isInteger() != 8630 MatchingOpInfo.ConstraintVT.isInteger()) || 8631 (MatchRC.second != InputRC.second)) { 8632 // FIXME: error out in a more elegant fashion 8633 report_fatal_error("Unsupported asm: input constraint" 8634 " with a matching output constraint of" 8635 " incompatible type!"); 8636 } 8637 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT; 8638 } 8639 8640 /// Get a direct memory input to behave well as an indirect operand. 8641 /// This may introduce stores, hence the need for a \p Chain. 8642 /// \return The (possibly updated) chain. 8643 static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, 8644 SDISelAsmOperandInfo &OpInfo, 8645 SelectionDAG &DAG) { 8646 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 8647 8648 // If we don't have an indirect input, put it in the constpool if we can, 8649 // otherwise spill it to a stack slot. 8650 // TODO: This isn't quite right. We need to handle these according to 8651 // the addressing mode that the constraint wants. Also, this may take 8652 // an additional register for the computation and we don't want that 8653 // either. 8654 8655 // If the operand is a float, integer, or vector constant, spill to a 8656 // constant pool entry to get its address. 8657 const Value *OpVal = OpInfo.CallOperandVal; 8658 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) || 8659 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) { 8660 OpInfo.CallOperand = DAG.getConstantPool( 8661 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout())); 8662 return Chain; 8663 } 8664 8665 // Otherwise, create a stack slot and emit a store to it before the asm. 8666 Type *Ty = OpVal->getType(); 8667 auto &DL = DAG.getDataLayout(); 8668 uint64_t TySize = DL.getTypeAllocSize(Ty); 8669 MachineFunction &MF = DAG.getMachineFunction(); 8670 int SSFI = MF.getFrameInfo().CreateStackObject( 8671 TySize, DL.getPrefTypeAlign(Ty), false); 8672 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL)); 8673 Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot, 8674 MachinePointerInfo::getFixedStack(MF, SSFI), 8675 TLI.getMemValueType(DL, Ty)); 8676 OpInfo.CallOperand = StackSlot; 8677 8678 return Chain; 8679 } 8680 8681 /// GetRegistersForValue - Assign registers (virtual or physical) for the 8682 /// specified operand. We prefer to assign virtual registers, to allow the 8683 /// register allocator to handle the assignment process. However, if the asm 8684 /// uses features that we can't model on machineinstrs, we have SDISel do the 8685 /// allocation. This produces generally horrible, but correct, code. 8686 /// 8687 /// OpInfo describes the operand 8688 /// RefOpInfo describes the matching operand if any, the operand otherwise 8689 static std::optional<unsigned> 8690 getRegistersForValue(SelectionDAG &DAG, const SDLoc &DL, 8691 SDISelAsmOperandInfo &OpInfo, 8692 SDISelAsmOperandInfo &RefOpInfo) { 8693 LLVMContext &Context = *DAG.getContext(); 8694 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 8695 8696 MachineFunction &MF = DAG.getMachineFunction(); 8697 SmallVector<unsigned, 4> Regs; 8698 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 8699 8700 // No work to do for memory/address operands. 8701 if (OpInfo.ConstraintType == TargetLowering::C_Memory || 8702 OpInfo.ConstraintType == TargetLowering::C_Address) 8703 return std::nullopt; 8704 8705 // If this is a constraint for a single physreg, or a constraint for a 8706 // register class, find it. 8707 unsigned AssignedReg; 8708 const TargetRegisterClass *RC; 8709 std::tie(AssignedReg, RC) = TLI.getRegForInlineAsmConstraint( 8710 &TRI, RefOpInfo.ConstraintCode, RefOpInfo.ConstraintVT); 8711 // RC is unset only on failure. Return immediately. 8712 if (!RC) 8713 return std::nullopt; 8714 8715 // Get the actual register value type. This is important, because the user 8716 // may have asked for (e.g.) the AX register in i32 type. We need to 8717 // remember that AX is actually i16 to get the right extension. 8718 const MVT RegVT = *TRI.legalclasstypes_begin(*RC); 8719 8720 if (OpInfo.ConstraintVT != MVT::Other && RegVT != MVT::Untyped) { 8721 // If this is an FP operand in an integer register (or visa versa), or more 8722 // generally if the operand value disagrees with the register class we plan 8723 // to stick it in, fix the operand type. 8724 // 8725 // If this is an input value, the bitcast to the new type is done now. 8726 // Bitcast for output value is done at the end of visitInlineAsm(). 8727 if ((OpInfo.Type == InlineAsm::isOutput || 8728 OpInfo.Type == InlineAsm::isInput) && 8729 !TRI.isTypeLegalForClass(*RC, OpInfo.ConstraintVT)) { 8730 // Try to convert to the first EVT that the reg class contains. If the 8731 // types are identical size, use a bitcast to convert (e.g. two differing 8732 // vector types). Note: output bitcast is done at the end of 8733 // visitInlineAsm(). 8734 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) { 8735 // Exclude indirect inputs while they are unsupported because the code 8736 // to perform the load is missing and thus OpInfo.CallOperand still 8737 // refers to the input address rather than the pointed-to value. 8738 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect) 8739 OpInfo.CallOperand = 8740 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand); 8741 OpInfo.ConstraintVT = RegVT; 8742 // If the operand is an FP value and we want it in integer registers, 8743 // use the corresponding integer type. This turns an f64 value into 8744 // i64, which can be passed with two i32 values on a 32-bit machine. 8745 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) { 8746 MVT VT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits()); 8747 if (OpInfo.Type == InlineAsm::isInput) 8748 OpInfo.CallOperand = 8749 DAG.getNode(ISD::BITCAST, DL, VT, OpInfo.CallOperand); 8750 OpInfo.ConstraintVT = VT; 8751 } 8752 } 8753 } 8754 8755 // No need to allocate a matching input constraint since the constraint it's 8756 // matching to has already been allocated. 8757 if (OpInfo.isMatchingInputConstraint()) 8758 return std::nullopt; 8759 8760 EVT ValueVT = OpInfo.ConstraintVT; 8761 if (OpInfo.ConstraintVT == MVT::Other) 8762 ValueVT = RegVT; 8763 8764 // Initialize NumRegs. 8765 unsigned NumRegs = 1; 8766 if (OpInfo.ConstraintVT != MVT::Other) 8767 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT, RegVT); 8768 8769 // If this is a constraint for a specific physical register, like {r17}, 8770 // assign it now. 8771 8772 // If this associated to a specific register, initialize iterator to correct 8773 // place. If virtual, make sure we have enough registers 8774 8775 // Initialize iterator if necessary 8776 TargetRegisterClass::iterator I = RC->begin(); 8777 MachineRegisterInfo &RegInfo = MF.getRegInfo(); 8778 8779 // Do not check for single registers. 8780 if (AssignedReg) { 8781 I = std::find(I, RC->end(), AssignedReg); 8782 if (I == RC->end()) { 8783 // RC does not contain the selected register, which indicates a 8784 // mismatch between the register and the required type/bitwidth. 8785 return {AssignedReg}; 8786 } 8787 } 8788 8789 for (; NumRegs; --NumRegs, ++I) { 8790 assert(I != RC->end() && "Ran out of registers to allocate!"); 8791 Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC); 8792 Regs.push_back(R); 8793 } 8794 8795 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT); 8796 return std::nullopt; 8797 } 8798 8799 static unsigned 8800 findMatchingInlineAsmOperand(unsigned OperandNo, 8801 const std::vector<SDValue> &AsmNodeOperands) { 8802 // Scan until we find the definition we already emitted of this operand. 8803 unsigned CurOp = InlineAsm::Op_FirstOperand; 8804 for (; OperandNo; --OperandNo) { 8805 // Advance to the next operand. 8806 unsigned OpFlag = 8807 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 8808 assert((InlineAsm::isRegDefKind(OpFlag) || 8809 InlineAsm::isRegDefEarlyClobberKind(OpFlag) || 8810 InlineAsm::isMemKind(OpFlag)) && 8811 "Skipped past definitions?"); 8812 CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1; 8813 } 8814 return CurOp; 8815 } 8816 8817 namespace { 8818 8819 class ExtraFlags { 8820 unsigned Flags = 0; 8821 8822 public: 8823 explicit ExtraFlags(const CallBase &Call) { 8824 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); 8825 if (IA->hasSideEffects()) 8826 Flags |= InlineAsm::Extra_HasSideEffects; 8827 if (IA->isAlignStack()) 8828 Flags |= InlineAsm::Extra_IsAlignStack; 8829 if (Call.isConvergent()) 8830 Flags |= InlineAsm::Extra_IsConvergent; 8831 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect; 8832 } 8833 8834 void update(const TargetLowering::AsmOperandInfo &OpInfo) { 8835 // Ideally, we would only check against memory constraints. However, the 8836 // meaning of an Other constraint can be target-specific and we can't easily 8837 // reason about it. Therefore, be conservative and set MayLoad/MayStore 8838 // for Other constraints as well. 8839 if (OpInfo.ConstraintType == TargetLowering::C_Memory || 8840 OpInfo.ConstraintType == TargetLowering::C_Other) { 8841 if (OpInfo.Type == InlineAsm::isInput) 8842 Flags |= InlineAsm::Extra_MayLoad; 8843 else if (OpInfo.Type == InlineAsm::isOutput) 8844 Flags |= InlineAsm::Extra_MayStore; 8845 else if (OpInfo.Type == InlineAsm::isClobber) 8846 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore); 8847 } 8848 } 8849 8850 unsigned get() const { return Flags; } 8851 }; 8852 8853 } // end anonymous namespace 8854 8855 static bool isFunction(SDValue Op) { 8856 if (Op && Op.getOpcode() == ISD::GlobalAddress) { 8857 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op)) { 8858 auto Fn = dyn_cast_or_null<Function>(GA->getGlobal()); 8859 8860 // In normal "call dllimport func" instruction (non-inlineasm) it force 8861 // indirect access by specifing call opcode. And usually specially print 8862 // asm with indirect symbol (i.g: "*") according to opcode. Inline asm can 8863 // not do in this way now. (In fact, this is similar with "Data Access" 8864 // action). So here we ignore dllimport function. 8865 if (Fn && !Fn->hasDLLImportStorageClass()) 8866 return true; 8867 } 8868 } 8869 return false; 8870 } 8871 8872 /// visitInlineAsm - Handle a call to an InlineAsm object. 8873 void SelectionDAGBuilder::visitInlineAsm(const CallBase &Call, 8874 const BasicBlock *EHPadBB) { 8875 const InlineAsm *IA = cast<InlineAsm>(Call.getCalledOperand()); 8876 8877 /// ConstraintOperands - Information about all of the constraints. 8878 SmallVector<SDISelAsmOperandInfo, 16> ConstraintOperands; 8879 8880 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 8881 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints( 8882 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), Call); 8883 8884 // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack, 8885 // AsmDialect, MayLoad, MayStore). 8886 bool HasSideEffect = IA->hasSideEffects(); 8887 ExtraFlags ExtraInfo(Call); 8888 8889 for (auto &T : TargetConstraints) { 8890 ConstraintOperands.push_back(SDISelAsmOperandInfo(T)); 8891 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back(); 8892 8893 if (OpInfo.CallOperandVal) 8894 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); 8895 8896 if (!HasSideEffect) 8897 HasSideEffect = OpInfo.hasMemory(TLI); 8898 8899 // Determine if this InlineAsm MayLoad or MayStore based on the constraints. 8900 // FIXME: Could we compute this on OpInfo rather than T? 8901 8902 // Compute the constraint code and ConstraintType to use. 8903 TLI.ComputeConstraintToUse(T, SDValue()); 8904 8905 if (T.ConstraintType == TargetLowering::C_Immediate && 8906 OpInfo.CallOperand && !isa<ConstantSDNode>(OpInfo.CallOperand)) 8907 // We've delayed emitting a diagnostic like the "n" constraint because 8908 // inlining could cause an integer showing up. 8909 return emitInlineAsmError(Call, "constraint '" + Twine(T.ConstraintCode) + 8910 "' expects an integer constant " 8911 "expression"); 8912 8913 ExtraInfo.update(T); 8914 } 8915 8916 // We won't need to flush pending loads if this asm doesn't touch 8917 // memory and is nonvolatile. 8918 SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot(); 8919 8920 bool EmitEHLabels = isa<InvokeInst>(Call); 8921 if (EmitEHLabels) { 8922 assert(EHPadBB && "InvokeInst must have an EHPadBB"); 8923 } 8924 bool IsCallBr = isa<CallBrInst>(Call); 8925 8926 if (IsCallBr || EmitEHLabels) { 8927 // If this is a callbr or invoke we need to flush pending exports since 8928 // inlineasm_br and invoke are terminators. 8929 // We need to do this before nodes are glued to the inlineasm_br node. 8930 Chain = getControlRoot(); 8931 } 8932 8933 MCSymbol *BeginLabel = nullptr; 8934 if (EmitEHLabels) { 8935 Chain = lowerStartEH(Chain, EHPadBB, BeginLabel); 8936 } 8937 8938 int OpNo = -1; 8939 SmallVector<StringRef> AsmStrs; 8940 IA->collectAsmStrs(AsmStrs); 8941 8942 // Second pass over the constraints: compute which constraint option to use. 8943 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { 8944 if (OpInfo.hasArg() || OpInfo.Type == InlineAsm::isOutput) 8945 OpNo++; 8946 8947 // If this is an output operand with a matching input operand, look up the 8948 // matching input. If their types mismatch, e.g. one is an integer, the 8949 // other is floating point, or their sizes are different, flag it as an 8950 // error. 8951 if (OpInfo.hasMatchingInput()) { 8952 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput]; 8953 patchMatchingInput(OpInfo, Input, DAG); 8954 } 8955 8956 // Compute the constraint code and ConstraintType to use. 8957 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); 8958 8959 if ((OpInfo.ConstraintType == TargetLowering::C_Memory && 8960 OpInfo.Type == InlineAsm::isClobber) || 8961 OpInfo.ConstraintType == TargetLowering::C_Address) 8962 continue; 8963 8964 // In Linux PIC model, there are 4 cases about value/label addressing: 8965 // 8966 // 1: Function call or Label jmp inside the module. 8967 // 2: Data access (such as global variable, static variable) inside module. 8968 // 3: Function call or Label jmp outside the module. 8969 // 4: Data access (such as global variable) outside the module. 8970 // 8971 // Due to current llvm inline asm architecture designed to not "recognize" 8972 // the asm code, there are quite troubles for us to treat mem addressing 8973 // differently for same value/adress used in different instuctions. 8974 // For example, in pic model, call a func may in plt way or direclty 8975 // pc-related, but lea/mov a function adress may use got. 8976 // 8977 // Here we try to "recognize" function call for the case 1 and case 3 in 8978 // inline asm. And try to adjust the constraint for them. 8979 // 8980 // TODO: Due to current inline asm didn't encourage to jmp to the outsider 8981 // label, so here we don't handle jmp function label now, but we need to 8982 // enhance it (especilly in PIC model) if we meet meaningful requirements. 8983 if (OpInfo.isIndirect && isFunction(OpInfo.CallOperand) && 8984 TLI.isInlineAsmTargetBranch(AsmStrs, OpNo) && 8985 TM.getCodeModel() != CodeModel::Large) { 8986 OpInfo.isIndirect = false; 8987 OpInfo.ConstraintType = TargetLowering::C_Address; 8988 } 8989 8990 // If this is a memory input, and if the operand is not indirect, do what we 8991 // need to provide an address for the memory input. 8992 if (OpInfo.ConstraintType == TargetLowering::C_Memory && 8993 !OpInfo.isIndirect) { 8994 assert((OpInfo.isMultipleAlternative || 8995 (OpInfo.Type == InlineAsm::isInput)) && 8996 "Can only indirectify direct input operands!"); 8997 8998 // Memory operands really want the address of the value. 8999 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG); 9000 9001 // There is no longer a Value* corresponding to this operand. 9002 OpInfo.CallOperandVal = nullptr; 9003 9004 // It is now an indirect operand. 9005 OpInfo.isIndirect = true; 9006 } 9007 9008 } 9009 9010 // AsmNodeOperands - The operands for the ISD::INLINEASM node. 9011 std::vector<SDValue> AsmNodeOperands; 9012 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain 9013 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol( 9014 IA->getAsmString().c_str(), TLI.getProgramPointerTy(DAG.getDataLayout()))); 9015 9016 // If we have a !srcloc metadata node associated with it, we want to attach 9017 // this to the ultimately generated inline asm machineinstr. To do this, we 9018 // pass in the third operand as this (potentially null) inline asm MDNode. 9019 const MDNode *SrcLoc = Call.getMetadata("srcloc"); 9020 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc)); 9021 9022 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore 9023 // bits as operand 3. 9024 AsmNodeOperands.push_back(DAG.getTargetConstant( 9025 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); 9026 9027 // Third pass: Loop over operands to prepare DAG-level operands.. As part of 9028 // this, assign virtual and physical registers for inputs and otput. 9029 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { 9030 // Assign Registers. 9031 SDISelAsmOperandInfo &RefOpInfo = 9032 OpInfo.isMatchingInputConstraint() 9033 ? ConstraintOperands[OpInfo.getMatchedOperand()] 9034 : OpInfo; 9035 const auto RegError = 9036 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); 9037 if (RegError) { 9038 const MachineFunction &MF = DAG.getMachineFunction(); 9039 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 9040 const char *RegName = TRI.getName(*RegError); 9041 emitInlineAsmError(Call, "register '" + Twine(RegName) + 9042 "' allocated for constraint '" + 9043 Twine(OpInfo.ConstraintCode) + 9044 "' does not match required type"); 9045 return; 9046 } 9047 9048 auto DetectWriteToReservedRegister = [&]() { 9049 const MachineFunction &MF = DAG.getMachineFunction(); 9050 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 9051 for (unsigned Reg : OpInfo.AssignedRegs.Regs) { 9052 if (Register::isPhysicalRegister(Reg) && 9053 TRI.isInlineAsmReadOnlyReg(MF, Reg)) { 9054 const char *RegName = TRI.getName(Reg); 9055 emitInlineAsmError(Call, "write to reserved register '" + 9056 Twine(RegName) + "'"); 9057 return true; 9058 } 9059 } 9060 return false; 9061 }; 9062 assert((OpInfo.ConstraintType != TargetLowering::C_Address || 9063 (OpInfo.Type == InlineAsm::isInput && 9064 !OpInfo.isMatchingInputConstraint())) && 9065 "Only address as input operand is allowed."); 9066 9067 switch (OpInfo.Type) { 9068 case InlineAsm::isOutput: 9069 if (OpInfo.ConstraintType == TargetLowering::C_Memory) { 9070 unsigned ConstraintID = 9071 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); 9072 assert(ConstraintID != InlineAsm::Constraint_Unknown && 9073 "Failed to convert memory constraint code to constraint id."); 9074 9075 // Add information to the INLINEASM node to know about this output. 9076 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 9077 OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID); 9078 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(), 9079 MVT::i32)); 9080 AsmNodeOperands.push_back(OpInfo.CallOperand); 9081 } else { 9082 // Otherwise, this outputs to a register (directly for C_Register / 9083 // C_RegisterClass, and a target-defined fashion for 9084 // C_Immediate/C_Other). Find a register that we can use. 9085 if (OpInfo.AssignedRegs.Regs.empty()) { 9086 emitInlineAsmError( 9087 Call, "couldn't allocate output register for constraint '" + 9088 Twine(OpInfo.ConstraintCode) + "'"); 9089 return; 9090 } 9091 9092 if (DetectWriteToReservedRegister()) 9093 return; 9094 9095 // Add information to the INLINEASM node to know that this register is 9096 // set. 9097 OpInfo.AssignedRegs.AddInlineAsmOperands( 9098 OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber 9099 : InlineAsm::Kind_RegDef, 9100 false, 0, getCurSDLoc(), DAG, AsmNodeOperands); 9101 } 9102 break; 9103 9104 case InlineAsm::isInput: 9105 case InlineAsm::isLabel: { 9106 SDValue InOperandVal = OpInfo.CallOperand; 9107 9108 if (OpInfo.isMatchingInputConstraint()) { 9109 // If this is required to match an output register we have already set, 9110 // just use its register. 9111 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(), 9112 AsmNodeOperands); 9113 unsigned OpFlag = 9114 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue(); 9115 if (InlineAsm::isRegDefKind(OpFlag) || 9116 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) { 9117 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs. 9118 if (OpInfo.isIndirect) { 9119 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c 9120 emitInlineAsmError(Call, "inline asm not supported yet: " 9121 "don't know how to handle tied " 9122 "indirect register inputs"); 9123 return; 9124 } 9125 9126 SmallVector<unsigned, 4> Regs; 9127 MachineFunction &MF = DAG.getMachineFunction(); 9128 MachineRegisterInfo &MRI = MF.getRegInfo(); 9129 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); 9130 auto *R = cast<RegisterSDNode>(AsmNodeOperands[CurOp+1]); 9131 Register TiedReg = R->getReg(); 9132 MVT RegVT = R->getSimpleValueType(0); 9133 const TargetRegisterClass *RC = 9134 TiedReg.isVirtual() ? MRI.getRegClass(TiedReg) 9135 : RegVT != MVT::Untyped ? TLI.getRegClassFor(RegVT) 9136 : TRI.getMinimalPhysRegClass(TiedReg); 9137 unsigned NumRegs = InlineAsm::getNumOperandRegisters(OpFlag); 9138 for (unsigned i = 0; i != NumRegs; ++i) 9139 Regs.push_back(MRI.createVirtualRegister(RC)); 9140 9141 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType()); 9142 9143 SDLoc dl = getCurSDLoc(); 9144 // Use the produced MatchedRegs object to 9145 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, &Call); 9146 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, 9147 true, OpInfo.getMatchedOperand(), dl, 9148 DAG, AsmNodeOperands); 9149 break; 9150 } 9151 9152 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!"); 9153 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 && 9154 "Unexpected number of operands"); 9155 // Add information to the INLINEASM node to know about this input. 9156 // See InlineAsm.h isUseOperandTiedToDef. 9157 OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag); 9158 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag, 9159 OpInfo.getMatchedOperand()); 9160 AsmNodeOperands.push_back(DAG.getTargetConstant( 9161 OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); 9162 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]); 9163 break; 9164 } 9165 9166 // Treat indirect 'X' constraint as memory. 9167 if (OpInfo.ConstraintType == TargetLowering::C_Other && 9168 OpInfo.isIndirect) 9169 OpInfo.ConstraintType = TargetLowering::C_Memory; 9170 9171 if (OpInfo.ConstraintType == TargetLowering::C_Immediate || 9172 OpInfo.ConstraintType == TargetLowering::C_Other) { 9173 std::vector<SDValue> Ops; 9174 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode, 9175 Ops, DAG); 9176 if (Ops.empty()) { 9177 if (OpInfo.ConstraintType == TargetLowering::C_Immediate) 9178 if (isa<ConstantSDNode>(InOperandVal)) { 9179 emitInlineAsmError(Call, "value out of range for constraint '" + 9180 Twine(OpInfo.ConstraintCode) + "'"); 9181 return; 9182 } 9183 9184 emitInlineAsmError(Call, 9185 "invalid operand for inline asm constraint '" + 9186 Twine(OpInfo.ConstraintCode) + "'"); 9187 return; 9188 } 9189 9190 // Add information to the INLINEASM node to know about this input. 9191 unsigned ResOpType = 9192 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size()); 9193 AsmNodeOperands.push_back(DAG.getTargetConstant( 9194 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); 9195 llvm::append_range(AsmNodeOperands, Ops); 9196 break; 9197 } 9198 9199 if (OpInfo.ConstraintType == TargetLowering::C_Memory) { 9200 assert((OpInfo.isIndirect || 9201 OpInfo.ConstraintType != TargetLowering::C_Memory) && 9202 "Operand must be indirect to be a mem!"); 9203 assert(InOperandVal.getValueType() == 9204 TLI.getPointerTy(DAG.getDataLayout()) && 9205 "Memory operands expect pointer values"); 9206 9207 unsigned ConstraintID = 9208 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); 9209 assert(ConstraintID != InlineAsm::Constraint_Unknown && 9210 "Failed to convert memory constraint code to constraint id."); 9211 9212 // Add information to the INLINEASM node to know about this input. 9213 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 9214 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID); 9215 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType, 9216 getCurSDLoc(), 9217 MVT::i32)); 9218 AsmNodeOperands.push_back(InOperandVal); 9219 break; 9220 } 9221 9222 if (OpInfo.ConstraintType == TargetLowering::C_Address) { 9223 assert(InOperandVal.getValueType() == 9224 TLI.getPointerTy(DAG.getDataLayout()) && 9225 "Address operands expect pointer values"); 9226 9227 unsigned ConstraintID = 9228 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); 9229 assert(ConstraintID != InlineAsm::Constraint_Unknown && 9230 "Failed to convert memory constraint code to constraint id."); 9231 9232 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1); 9233 9234 SDValue AsmOp = InOperandVal; 9235 if (isFunction(InOperandVal)) { 9236 auto *GA = cast<GlobalAddressSDNode>(InOperandVal); 9237 ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Func, 1); 9238 AsmOp = DAG.getTargetGlobalAddress(GA->getGlobal(), getCurSDLoc(), 9239 InOperandVal.getValueType(), 9240 GA->getOffset()); 9241 } 9242 9243 // Add information to the INLINEASM node to know about this input. 9244 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID); 9245 9246 AsmNodeOperands.push_back( 9247 DAG.getTargetConstant(ResOpType, getCurSDLoc(), MVT::i32)); 9248 9249 AsmNodeOperands.push_back(AsmOp); 9250 break; 9251 } 9252 9253 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass || 9254 OpInfo.ConstraintType == TargetLowering::C_Register) && 9255 "Unknown constraint type!"); 9256 9257 // TODO: Support this. 9258 if (OpInfo.isIndirect) { 9259 emitInlineAsmError( 9260 Call, "Don't know how to handle indirect register inputs yet " 9261 "for constraint '" + 9262 Twine(OpInfo.ConstraintCode) + "'"); 9263 return; 9264 } 9265 9266 // Copy the input into the appropriate registers. 9267 if (OpInfo.AssignedRegs.Regs.empty()) { 9268 emitInlineAsmError(Call, 9269 "couldn't allocate input reg for constraint '" + 9270 Twine(OpInfo.ConstraintCode) + "'"); 9271 return; 9272 } 9273 9274 if (DetectWriteToReservedRegister()) 9275 return; 9276 9277 SDLoc dl = getCurSDLoc(); 9278 9279 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag, 9280 &Call); 9281 9282 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0, 9283 dl, DAG, AsmNodeOperands); 9284 break; 9285 } 9286 case InlineAsm::isClobber: 9287 // Add the clobbered value to the operand list, so that the register 9288 // allocator is aware that the physreg got clobbered. 9289 if (!OpInfo.AssignedRegs.Regs.empty()) 9290 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber, 9291 false, 0, getCurSDLoc(), DAG, 9292 AsmNodeOperands); 9293 break; 9294 } 9295 } 9296 9297 // Finish up input operands. Set the input chain and add the flag last. 9298 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; 9299 if (Flag.getNode()) AsmNodeOperands.push_back(Flag); 9300 9301 unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM; 9302 Chain = DAG.getNode(ISDOpc, getCurSDLoc(), 9303 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); 9304 Flag = Chain.getValue(1); 9305 9306 // Do additional work to generate outputs. 9307 9308 SmallVector<EVT, 1> ResultVTs; 9309 SmallVector<SDValue, 1> ResultValues; 9310 SmallVector<SDValue, 8> OutChains; 9311 9312 llvm::Type *CallResultType = Call.getType(); 9313 ArrayRef<Type *> ResultTypes; 9314 if (StructType *StructResult = dyn_cast<StructType>(CallResultType)) 9315 ResultTypes = StructResult->elements(); 9316 else if (!CallResultType->isVoidTy()) 9317 ResultTypes = ArrayRef(CallResultType); 9318 9319 auto CurResultType = ResultTypes.begin(); 9320 auto handleRegAssign = [&](SDValue V) { 9321 assert(CurResultType != ResultTypes.end() && "Unexpected value"); 9322 assert((*CurResultType)->isSized() && "Unexpected unsized type"); 9323 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType); 9324 ++CurResultType; 9325 // If the type of the inline asm call site return value is different but has 9326 // same size as the type of the asm output bitcast it. One example of this 9327 // is for vectors with different width / number of elements. This can 9328 // happen for register classes that can contain multiple different value 9329 // types. The preg or vreg allocated may not have the same VT as was 9330 // expected. 9331 // 9332 // This can also happen for a return value that disagrees with the register 9333 // class it is put in, eg. a double in a general-purpose register on a 9334 // 32-bit machine. 9335 if (ResultVT != V.getValueType() && 9336 ResultVT.getSizeInBits() == V.getValueSizeInBits()) 9337 V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V); 9338 else if (ResultVT != V.getValueType() && ResultVT.isInteger() && 9339 V.getValueType().isInteger()) { 9340 // If a result value was tied to an input value, the computed result 9341 // may have a wider width than the expected result. Extract the 9342 // relevant portion. 9343 V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V); 9344 } 9345 assert(ResultVT == V.getValueType() && "Asm result value mismatch!"); 9346 ResultVTs.push_back(ResultVT); 9347 ResultValues.push_back(V); 9348 }; 9349 9350 // Deal with output operands. 9351 for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { 9352 if (OpInfo.Type == InlineAsm::isOutput) { 9353 SDValue Val; 9354 // Skip trivial output operands. 9355 if (OpInfo.AssignedRegs.Regs.empty()) 9356 continue; 9357 9358 switch (OpInfo.ConstraintType) { 9359 case TargetLowering::C_Register: 9360 case TargetLowering::C_RegisterClass: 9361 Val = OpInfo.AssignedRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), 9362 Chain, &Flag, &Call); 9363 break; 9364 case TargetLowering::C_Immediate: 9365 case TargetLowering::C_Other: 9366 Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(), 9367 OpInfo, DAG); 9368 break; 9369 case TargetLowering::C_Memory: 9370 break; // Already handled. 9371 case TargetLowering::C_Address: 9372 break; // Silence warning. 9373 case TargetLowering::C_Unknown: 9374 assert(false && "Unexpected unknown constraint"); 9375 } 9376 9377 // Indirect output manifest as stores. Record output chains. 9378 if (OpInfo.isIndirect) { 9379 const Value *Ptr = OpInfo.CallOperandVal; 9380 assert(Ptr && "Expected value CallOperandVal for indirect asm operand"); 9381 SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr), 9382 MachinePointerInfo(Ptr)); 9383 OutChains.push_back(Store); 9384 } else { 9385 // generate CopyFromRegs to associated registers. 9386 assert(!Call.getType()->isVoidTy() && "Bad inline asm!"); 9387 if (Val.getOpcode() == ISD::MERGE_VALUES) { 9388 for (const SDValue &V : Val->op_values()) 9389 handleRegAssign(V); 9390 } else 9391 handleRegAssign(Val); 9392 } 9393 } 9394 } 9395 9396 // Set results. 9397 if (!ResultValues.empty()) { 9398 assert(CurResultType == ResultTypes.end() && 9399 "Mismatch in number of ResultTypes"); 9400 assert(ResultValues.size() == ResultTypes.size() && 9401 "Mismatch in number of output operands in asm result"); 9402 9403 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 9404 DAG.getVTList(ResultVTs), ResultValues); 9405 setValue(&Call, V); 9406 } 9407 9408 // Collect store chains. 9409 if (!OutChains.empty()) 9410 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains); 9411 9412 if (EmitEHLabels) { 9413 Chain = lowerEndEH(Chain, cast<InvokeInst>(&Call), EHPadBB, BeginLabel); 9414 } 9415 9416 // Only Update Root if inline assembly has a memory effect. 9417 if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr || 9418 EmitEHLabels) 9419 DAG.setRoot(Chain); 9420 } 9421 9422 void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call, 9423 const Twine &Message) { 9424 LLVMContext &Ctx = *DAG.getContext(); 9425 Ctx.emitError(&Call, Message); 9426 9427 // Make sure we leave the DAG in a valid state 9428 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9429 SmallVector<EVT, 1> ValueVTs; 9430 ComputeValueVTs(TLI, DAG.getDataLayout(), Call.getType(), ValueVTs); 9431 9432 if (ValueVTs.empty()) 9433 return; 9434 9435 SmallVector<SDValue, 1> Ops; 9436 for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i) 9437 Ops.push_back(DAG.getUNDEF(ValueVTs[i])); 9438 9439 setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc())); 9440 } 9441 9442 void SelectionDAGBuilder::visitVAStart(const CallInst &I) { 9443 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), 9444 MVT::Other, getRoot(), 9445 getValue(I.getArgOperand(0)), 9446 DAG.getSrcValue(I.getArgOperand(0)))); 9447 } 9448 9449 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { 9450 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9451 const DataLayout &DL = DAG.getDataLayout(); 9452 SDValue V = DAG.getVAArg( 9453 TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(), 9454 getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)), 9455 DL.getABITypeAlign(I.getType()).value()); 9456 DAG.setRoot(V.getValue(1)); 9457 9458 if (I.getType()->isPointerTy()) 9459 V = DAG.getPtrExtOrTrunc( 9460 V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType())); 9461 setValue(&I, V); 9462 } 9463 9464 void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { 9465 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), 9466 MVT::Other, getRoot(), 9467 getValue(I.getArgOperand(0)), 9468 DAG.getSrcValue(I.getArgOperand(0)))); 9469 } 9470 9471 void SelectionDAGBuilder::visitVACopy(const CallInst &I) { 9472 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), 9473 MVT::Other, getRoot(), 9474 getValue(I.getArgOperand(0)), 9475 getValue(I.getArgOperand(1)), 9476 DAG.getSrcValue(I.getArgOperand(0)), 9477 DAG.getSrcValue(I.getArgOperand(1)))); 9478 } 9479 9480 SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, 9481 const Instruction &I, 9482 SDValue Op) { 9483 const MDNode *Range = I.getMetadata(LLVMContext::MD_range); 9484 if (!Range) 9485 return Op; 9486 9487 ConstantRange CR = getConstantRangeFromMetadata(*Range); 9488 if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped()) 9489 return Op; 9490 9491 APInt Lo = CR.getUnsignedMin(); 9492 if (!Lo.isMinValue()) 9493 return Op; 9494 9495 APInt Hi = CR.getUnsignedMax(); 9496 unsigned Bits = std::max(Hi.getActiveBits(), 9497 static_cast<unsigned>(IntegerType::MIN_INT_BITS)); 9498 9499 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits); 9500 9501 SDLoc SL = getCurSDLoc(); 9502 9503 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op, 9504 DAG.getValueType(SmallVT)); 9505 unsigned NumVals = Op.getNode()->getNumValues(); 9506 if (NumVals == 1) 9507 return ZExt; 9508 9509 SmallVector<SDValue, 4> Ops; 9510 9511 Ops.push_back(ZExt); 9512 for (unsigned I = 1; I != NumVals; ++I) 9513 Ops.push_back(Op.getValue(I)); 9514 9515 return DAG.getMergeValues(Ops, SL); 9516 } 9517 9518 /// Populate a CallLowerinInfo (into \p CLI) based on the properties of 9519 /// the call being lowered. 9520 /// 9521 /// This is a helper for lowering intrinsics that follow a target calling 9522 /// convention or require stack pointer adjustment. Only a subset of the 9523 /// intrinsic's operands need to participate in the calling convention. 9524 void SelectionDAGBuilder::populateCallLoweringInfo( 9525 TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, 9526 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, 9527 bool IsPatchPoint) { 9528 TargetLowering::ArgListTy Args; 9529 Args.reserve(NumArgs); 9530 9531 // Populate the argument list. 9532 // Attributes for args start at offset 1, after the return attribute. 9533 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; 9534 ArgI != ArgE; ++ArgI) { 9535 const Value *V = Call->getOperand(ArgI); 9536 9537 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); 9538 9539 TargetLowering::ArgListEntry Entry; 9540 Entry.Node = getValue(V); 9541 Entry.Ty = V->getType(); 9542 Entry.setAttributes(Call, ArgI); 9543 Args.push_back(Entry); 9544 } 9545 9546 CLI.setDebugLoc(getCurSDLoc()) 9547 .setChain(getRoot()) 9548 .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args)) 9549 .setDiscardResult(Call->use_empty()) 9550 .setIsPatchPoint(IsPatchPoint) 9551 .setIsPreallocated( 9552 Call->countOperandBundlesOfType(LLVMContext::OB_preallocated) != 0); 9553 } 9554 9555 /// Add a stack map intrinsic call's live variable operands to a stackmap 9556 /// or patchpoint target node's operand list. 9557 /// 9558 /// Constants are converted to TargetConstants purely as an optimization to 9559 /// avoid constant materialization and register allocation. 9560 /// 9561 /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not 9562 /// generate addess computation nodes, and so FinalizeISel can convert the 9563 /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids 9564 /// address materialization and register allocation, but may also be required 9565 /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an 9566 /// alloca in the entry block, then the runtime may assume that the alloca's 9567 /// StackMap location can be read immediately after compilation and that the 9568 /// location is valid at any point during execution (this is similar to the 9569 /// assumption made by the llvm.gcroot intrinsic). If the alloca's location were 9570 /// only available in a register, then the runtime would need to trap when 9571 /// execution reaches the StackMap in order to read the alloca's location. 9572 static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, 9573 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops, 9574 SelectionDAGBuilder &Builder) { 9575 SelectionDAG &DAG = Builder.DAG; 9576 for (unsigned I = StartIdx; I < Call.arg_size(); I++) { 9577 SDValue Op = Builder.getValue(Call.getArgOperand(I)); 9578 9579 // Things on the stack are pointer-typed, meaning that they are already 9580 // legal and can be emitted directly to target nodes. 9581 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op)) { 9582 Ops.push_back(DAG.getTargetFrameIndex(FI->getIndex(), Op.getValueType())); 9583 } else { 9584 // Otherwise emit a target independent node to be legalised. 9585 Ops.push_back(Builder.getValue(Call.getArgOperand(I))); 9586 } 9587 } 9588 } 9589 9590 /// Lower llvm.experimental.stackmap. 9591 void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { 9592 // void @llvm.experimental.stackmap(i64 <id>, i32 <numShadowBytes>, 9593 // [live variables...]) 9594 9595 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value."); 9596 9597 SDValue Chain, InFlag, Callee; 9598 SmallVector<SDValue, 32> Ops; 9599 9600 SDLoc DL = getCurSDLoc(); 9601 Callee = getValue(CI.getCalledOperand()); 9602 9603 // The stackmap intrinsic only records the live variables (the arguments 9604 // passed to it) and emits NOPS (if requested). Unlike the patchpoint 9605 // intrinsic, this won't be lowered to a function call. This means we don't 9606 // have to worry about calling conventions and target specific lowering code. 9607 // Instead we perform the call lowering right here. 9608 // 9609 // chain, flag = CALLSEQ_START(chain, 0, 0) 9610 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag) 9611 // chain, flag = CALLSEQ_END(chain, 0, 0, flag) 9612 // 9613 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL); 9614 InFlag = Chain.getValue(1); 9615 9616 // Add the STACKMAP operands, starting with DAG house-keeping. 9617 Ops.push_back(Chain); 9618 Ops.push_back(InFlag); 9619 9620 // Add the <id>, <numShadowBytes> operands. 9621 // 9622 // These do not require legalisation, and can be emitted directly to target 9623 // constant nodes. 9624 SDValue ID = getValue(CI.getArgOperand(0)); 9625 assert(ID.getValueType() == MVT::i64); 9626 SDValue IDConst = DAG.getTargetConstant( 9627 cast<ConstantSDNode>(ID)->getZExtValue(), DL, ID.getValueType()); 9628 Ops.push_back(IDConst); 9629 9630 SDValue Shad = getValue(CI.getArgOperand(1)); 9631 assert(Shad.getValueType() == MVT::i32); 9632 SDValue ShadConst = DAG.getTargetConstant( 9633 cast<ConstantSDNode>(Shad)->getZExtValue(), DL, Shad.getValueType()); 9634 Ops.push_back(ShadConst); 9635 9636 // Add the live variables. 9637 addStackMapLiveVars(CI, 2, DL, Ops, *this); 9638 9639 // Create the STACKMAP node. 9640 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 9641 Chain = DAG.getNode(ISD::STACKMAP, DL, NodeTys, Ops); 9642 InFlag = Chain.getValue(1); 9643 9644 Chain = DAG.getCALLSEQ_END(Chain, 0, 0, InFlag, DL); 9645 9646 // Stackmaps don't generate values, so nothing goes into the NodeMap. 9647 9648 // Set the root to the target-lowered call chain. 9649 DAG.setRoot(Chain); 9650 9651 // Inform the Frame Information that we have a stackmap in this function. 9652 FuncInfo.MF->getFrameInfo().setHasStackMap(); 9653 } 9654 9655 /// Lower llvm.experimental.patchpoint directly to its target opcode. 9656 void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, 9657 const BasicBlock *EHPadBB) { 9658 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>, 9659 // i32 <numBytes>, 9660 // i8* <target>, 9661 // i32 <numArgs>, 9662 // [Args...], 9663 // [live variables...]) 9664 9665 CallingConv::ID CC = CB.getCallingConv(); 9666 bool IsAnyRegCC = CC == CallingConv::AnyReg; 9667 bool HasDef = !CB.getType()->isVoidTy(); 9668 SDLoc dl = getCurSDLoc(); 9669 SDValue Callee = getValue(CB.getArgOperand(PatchPointOpers::TargetPos)); 9670 9671 // Handle immediate and symbolic callees. 9672 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee)) 9673 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl, 9674 /*isTarget=*/true); 9675 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee)) 9676 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(), 9677 SDLoc(SymbolicCallee), 9678 SymbolicCallee->getValueType(0)); 9679 9680 // Get the real number of arguments participating in the call <numArgs> 9681 SDValue NArgVal = getValue(CB.getArgOperand(PatchPointOpers::NArgPos)); 9682 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue(); 9683 9684 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs> 9685 // Intrinsics include all meta-operands up to but not including CC. 9686 unsigned NumMetaOpers = PatchPointOpers::CCPos; 9687 assert(CB.arg_size() >= NumMetaOpers + NumArgs && 9688 "Not enough arguments provided to the patchpoint intrinsic"); 9689 9690 // For AnyRegCC the arguments are lowered later on manually. 9691 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs; 9692 Type *ReturnTy = 9693 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CB.getType(); 9694 9695 TargetLowering::CallLoweringInfo CLI(DAG); 9696 populateCallLoweringInfo(CLI, &CB, NumMetaOpers, NumCallArgs, Callee, 9697 ReturnTy, true); 9698 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB); 9699 9700 SDNode *CallEnd = Result.second.getNode(); 9701 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg)) 9702 CallEnd = CallEnd->getOperand(0).getNode(); 9703 9704 /// Get a call instruction from the call sequence chain. 9705 /// Tail calls are not allowed. 9706 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && 9707 "Expected a callseq node."); 9708 SDNode *Call = CallEnd->getOperand(0).getNode(); 9709 bool HasGlue = Call->getGluedNode(); 9710 9711 // Replace the target specific call node with the patchable intrinsic. 9712 SmallVector<SDValue, 8> Ops; 9713 9714 // Push the chain. 9715 Ops.push_back(*(Call->op_begin())); 9716 9717 // Optionally, push the glue (if any). 9718 if (HasGlue) 9719 Ops.push_back(*(Call->op_end() - 1)); 9720 9721 // Push the register mask info. 9722 if (HasGlue) 9723 Ops.push_back(*(Call->op_end() - 2)); 9724 else 9725 Ops.push_back(*(Call->op_end() - 1)); 9726 9727 // Add the <id> and <numBytes> constants. 9728 SDValue IDVal = getValue(CB.getArgOperand(PatchPointOpers::IDPos)); 9729 Ops.push_back(DAG.getTargetConstant( 9730 cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64)); 9731 SDValue NBytesVal = getValue(CB.getArgOperand(PatchPointOpers::NBytesPos)); 9732 Ops.push_back(DAG.getTargetConstant( 9733 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl, 9734 MVT::i32)); 9735 9736 // Add the callee. 9737 Ops.push_back(Callee); 9738 9739 // Adjust <numArgs> to account for any arguments that have been passed on the 9740 // stack instead. 9741 // Call Node: Chain, Target, {Args}, RegMask, [Glue] 9742 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3); 9743 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs; 9744 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32)); 9745 9746 // Add the calling convention 9747 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32)); 9748 9749 // Add the arguments we omitted previously. The register allocator should 9750 // place these in any free register. 9751 if (IsAnyRegCC) 9752 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i) 9753 Ops.push_back(getValue(CB.getArgOperand(i))); 9754 9755 // Push the arguments from the call instruction. 9756 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1; 9757 Ops.append(Call->op_begin() + 2, e); 9758 9759 // Push live variables for the stack map. 9760 addStackMapLiveVars(CB, NumMetaOpers + NumArgs, dl, Ops, *this); 9761 9762 SDVTList NodeTys; 9763 if (IsAnyRegCC && HasDef) { 9764 // Create the return types based on the intrinsic definition 9765 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9766 SmallVector<EVT, 3> ValueVTs; 9767 ComputeValueVTs(TLI, DAG.getDataLayout(), CB.getType(), ValueVTs); 9768 assert(ValueVTs.size() == 1 && "Expected only one return value type."); 9769 9770 // There is always a chain and a glue type at the end 9771 ValueVTs.push_back(MVT::Other); 9772 ValueVTs.push_back(MVT::Glue); 9773 NodeTys = DAG.getVTList(ValueVTs); 9774 } else 9775 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); 9776 9777 // Replace the target specific call node with a PATCHPOINT node. 9778 SDValue PPV = DAG.getNode(ISD::PATCHPOINT, dl, NodeTys, Ops); 9779 9780 // Update the NodeMap. 9781 if (HasDef) { 9782 if (IsAnyRegCC) 9783 setValue(&CB, SDValue(PPV.getNode(), 0)); 9784 else 9785 setValue(&CB, Result.first); 9786 } 9787 9788 // Fixup the consumers of the intrinsic. The chain and glue may be used in the 9789 // call sequence. Furthermore the location of the chain and glue can change 9790 // when the AnyReg calling convention is used and the intrinsic returns a 9791 // value. 9792 if (IsAnyRegCC && HasDef) { 9793 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)}; 9794 SDValue To[] = {PPV.getValue(1), PPV.getValue(2)}; 9795 DAG.ReplaceAllUsesOfValuesWith(From, To, 2); 9796 } else 9797 DAG.ReplaceAllUsesWith(Call, PPV.getNode()); 9798 DAG.DeleteNode(Call); 9799 9800 // Inform the Frame Information that we have a patchpoint in this function. 9801 FuncInfo.MF->getFrameInfo().setHasPatchPoint(); 9802 } 9803 9804 void SelectionDAGBuilder::visitVectorReduce(const CallInst &I, 9805 unsigned Intrinsic) { 9806 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 9807 SDValue Op1 = getValue(I.getArgOperand(0)); 9808 SDValue Op2; 9809 if (I.arg_size() > 1) 9810 Op2 = getValue(I.getArgOperand(1)); 9811 SDLoc dl = getCurSDLoc(); 9812 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 9813 SDValue Res; 9814 SDNodeFlags SDFlags; 9815 if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) 9816 SDFlags.copyFMF(*FPMO); 9817 9818 switch (Intrinsic) { 9819 case Intrinsic::vector_reduce_fadd: 9820 if (SDFlags.hasAllowReassociation()) 9821 Res = DAG.getNode(ISD::FADD, dl, VT, Op1, 9822 DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2, SDFlags), 9823 SDFlags); 9824 else 9825 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FADD, dl, VT, Op1, Op2, SDFlags); 9826 break; 9827 case Intrinsic::vector_reduce_fmul: 9828 if (SDFlags.hasAllowReassociation()) 9829 Res = DAG.getNode(ISD::FMUL, dl, VT, Op1, 9830 DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2, SDFlags), 9831 SDFlags); 9832 else 9833 Res = DAG.getNode(ISD::VECREDUCE_SEQ_FMUL, dl, VT, Op1, Op2, SDFlags); 9834 break; 9835 case Intrinsic::vector_reduce_add: 9836 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1); 9837 break; 9838 case Intrinsic::vector_reduce_mul: 9839 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1); 9840 break; 9841 case Intrinsic::vector_reduce_and: 9842 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1); 9843 break; 9844 case Intrinsic::vector_reduce_or: 9845 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1); 9846 break; 9847 case Intrinsic::vector_reduce_xor: 9848 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1); 9849 break; 9850 case Intrinsic::vector_reduce_smax: 9851 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1); 9852 break; 9853 case Intrinsic::vector_reduce_smin: 9854 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1); 9855 break; 9856 case Intrinsic::vector_reduce_umax: 9857 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1); 9858 break; 9859 case Intrinsic::vector_reduce_umin: 9860 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1); 9861 break; 9862 case Intrinsic::vector_reduce_fmax: 9863 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1, SDFlags); 9864 break; 9865 case Intrinsic::vector_reduce_fmin: 9866 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1, SDFlags); 9867 break; 9868 default: 9869 llvm_unreachable("Unhandled vector reduce intrinsic"); 9870 } 9871 setValue(&I, Res); 9872 } 9873 9874 /// Returns an AttributeList representing the attributes applied to the return 9875 /// value of the given call. 9876 static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) { 9877 SmallVector<Attribute::AttrKind, 2> Attrs; 9878 if (CLI.RetSExt) 9879 Attrs.push_back(Attribute::SExt); 9880 if (CLI.RetZExt) 9881 Attrs.push_back(Attribute::ZExt); 9882 if (CLI.IsInReg) 9883 Attrs.push_back(Attribute::InReg); 9884 9885 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex, 9886 Attrs); 9887 } 9888 9889 /// TargetLowering::LowerCallTo - This is the default LowerCallTo 9890 /// implementation, which just calls LowerCall. 9891 /// FIXME: When all targets are 9892 /// migrated to using LowerCall, this hook should be integrated into SDISel. 9893 std::pair<SDValue, SDValue> 9894 TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { 9895 // Handle the incoming return values from the call. 9896 CLI.Ins.clear(); 9897 Type *OrigRetTy = CLI.RetTy; 9898 SmallVector<EVT, 4> RetTys; 9899 SmallVector<uint64_t, 4> Offsets; 9900 auto &DL = CLI.DAG.getDataLayout(); 9901 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets); 9902 9903 if (CLI.IsPostTypeLegalization) { 9904 // If we are lowering a libcall after legalization, split the return type. 9905 SmallVector<EVT, 4> OldRetTys; 9906 SmallVector<uint64_t, 4> OldOffsets; 9907 RetTys.swap(OldRetTys); 9908 Offsets.swap(OldOffsets); 9909 9910 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) { 9911 EVT RetVT = OldRetTys[i]; 9912 uint64_t Offset = OldOffsets[i]; 9913 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT); 9914 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT); 9915 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8; 9916 RetTys.append(NumRegs, RegisterVT); 9917 for (unsigned j = 0; j != NumRegs; ++j) 9918 Offsets.push_back(Offset + j * RegisterVTByteSZ); 9919 } 9920 } 9921 9922 SmallVector<ISD::OutputArg, 4> Outs; 9923 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL); 9924 9925 bool CanLowerReturn = 9926 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(), 9927 CLI.IsVarArg, Outs, CLI.RetTy->getContext()); 9928 9929 SDValue DemoteStackSlot; 9930 int DemoteStackIdx = -100; 9931 if (!CanLowerReturn) { 9932 // FIXME: equivalent assert? 9933 // assert(!CS.hasInAllocaArgument() && 9934 // "sret demotion is incompatible with inalloca"); 9935 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy); 9936 Align Alignment = DL.getPrefTypeAlign(CLI.RetTy); 9937 MachineFunction &MF = CLI.DAG.getMachineFunction(); 9938 DemoteStackIdx = 9939 MF.getFrameInfo().CreateStackObject(TySize, Alignment, false); 9940 Type *StackSlotPtrType = PointerType::get(CLI.RetTy, 9941 DL.getAllocaAddrSpace()); 9942 9943 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL)); 9944 ArgListEntry Entry; 9945 Entry.Node = DemoteStackSlot; 9946 Entry.Ty = StackSlotPtrType; 9947 Entry.IsSExt = false; 9948 Entry.IsZExt = false; 9949 Entry.IsInReg = false; 9950 Entry.IsSRet = true; 9951 Entry.IsNest = false; 9952 Entry.IsByVal = false; 9953 Entry.IsByRef = false; 9954 Entry.IsReturned = false; 9955 Entry.IsSwiftSelf = false; 9956 Entry.IsSwiftAsync = false; 9957 Entry.IsSwiftError = false; 9958 Entry.IsCFGuardTarget = false; 9959 Entry.Alignment = Alignment; 9960 CLI.getArgs().insert(CLI.getArgs().begin(), Entry); 9961 CLI.NumFixedArgs += 1; 9962 CLI.getArgs()[0].IndirectType = CLI.RetTy; 9963 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext()); 9964 9965 // sret demotion isn't compatible with tail-calls, since the sret argument 9966 // points into the callers stack frame. 9967 CLI.IsTailCall = false; 9968 } else { 9969 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( 9970 CLI.RetTy, CLI.CallConv, CLI.IsVarArg, DL); 9971 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 9972 ISD::ArgFlagsTy Flags; 9973 if (NeedsRegBlock) { 9974 Flags.setInConsecutiveRegs(); 9975 if (I == RetTys.size() - 1) 9976 Flags.setInConsecutiveRegsLast(); 9977 } 9978 EVT VT = RetTys[I]; 9979 MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), 9980 CLI.CallConv, VT); 9981 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(), 9982 CLI.CallConv, VT); 9983 for (unsigned i = 0; i != NumRegs; ++i) { 9984 ISD::InputArg MyFlags; 9985 MyFlags.Flags = Flags; 9986 MyFlags.VT = RegisterVT; 9987 MyFlags.ArgVT = VT; 9988 MyFlags.Used = CLI.IsReturnValueUsed; 9989 if (CLI.RetTy->isPointerTy()) { 9990 MyFlags.Flags.setPointer(); 9991 MyFlags.Flags.setPointerAddrSpace( 9992 cast<PointerType>(CLI.RetTy)->getAddressSpace()); 9993 } 9994 if (CLI.RetSExt) 9995 MyFlags.Flags.setSExt(); 9996 if (CLI.RetZExt) 9997 MyFlags.Flags.setZExt(); 9998 if (CLI.IsInReg) 9999 MyFlags.Flags.setInReg(); 10000 CLI.Ins.push_back(MyFlags); 10001 } 10002 } 10003 } 10004 10005 // We push in swifterror return as the last element of CLI.Ins. 10006 ArgListTy &Args = CLI.getArgs(); 10007 if (supportSwiftError()) { 10008 for (const ArgListEntry &Arg : Args) { 10009 if (Arg.IsSwiftError) { 10010 ISD::InputArg MyFlags; 10011 MyFlags.VT = getPointerTy(DL); 10012 MyFlags.ArgVT = EVT(getPointerTy(DL)); 10013 MyFlags.Flags.setSwiftError(); 10014 CLI.Ins.push_back(MyFlags); 10015 } 10016 } 10017 } 10018 10019 // Handle all of the outgoing arguments. 10020 CLI.Outs.clear(); 10021 CLI.OutVals.clear(); 10022 for (unsigned i = 0, e = Args.size(); i != e; ++i) { 10023 SmallVector<EVT, 4> ValueVTs; 10024 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs); 10025 // FIXME: Split arguments if CLI.IsPostTypeLegalization 10026 Type *FinalType = Args[i].Ty; 10027 if (Args[i].IsByVal) 10028 FinalType = Args[i].IndirectType; 10029 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( 10030 FinalType, CLI.CallConv, CLI.IsVarArg, DL); 10031 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues; 10032 ++Value) { 10033 EVT VT = ValueVTs[Value]; 10034 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext()); 10035 SDValue Op = SDValue(Args[i].Node.getNode(), 10036 Args[i].Node.getResNo() + Value); 10037 ISD::ArgFlagsTy Flags; 10038 10039 // Certain targets (such as MIPS), may have a different ABI alignment 10040 // for a type depending on the context. Give the target a chance to 10041 // specify the alignment it wants. 10042 const Align OriginalAlignment(getABIAlignmentForCallingConv(ArgTy, DL)); 10043 Flags.setOrigAlign(OriginalAlignment); 10044 10045 if (Args[i].Ty->isPointerTy()) { 10046 Flags.setPointer(); 10047 Flags.setPointerAddrSpace( 10048 cast<PointerType>(Args[i].Ty)->getAddressSpace()); 10049 } 10050 if (Args[i].IsZExt) 10051 Flags.setZExt(); 10052 if (Args[i].IsSExt) 10053 Flags.setSExt(); 10054 if (Args[i].IsInReg) { 10055 // If we are using vectorcall calling convention, a structure that is 10056 // passed InReg - is surely an HVA 10057 if (CLI.CallConv == CallingConv::X86_VectorCall && 10058 isa<StructType>(FinalType)) { 10059 // The first value of a structure is marked 10060 if (0 == Value) 10061 Flags.setHvaStart(); 10062 Flags.setHva(); 10063 } 10064 // Set InReg Flag 10065 Flags.setInReg(); 10066 } 10067 if (Args[i].IsSRet) 10068 Flags.setSRet(); 10069 if (Args[i].IsSwiftSelf) 10070 Flags.setSwiftSelf(); 10071 if (Args[i].IsSwiftAsync) 10072 Flags.setSwiftAsync(); 10073 if (Args[i].IsSwiftError) 10074 Flags.setSwiftError(); 10075 if (Args[i].IsCFGuardTarget) 10076 Flags.setCFGuardTarget(); 10077 if (Args[i].IsByVal) 10078 Flags.setByVal(); 10079 if (Args[i].IsByRef) 10080 Flags.setByRef(); 10081 if (Args[i].IsPreallocated) { 10082 Flags.setPreallocated(); 10083 // Set the byval flag for CCAssignFn callbacks that don't know about 10084 // preallocated. This way we can know how many bytes we should've 10085 // allocated and how many bytes a callee cleanup function will pop. If 10086 // we port preallocated to more targets, we'll have to add custom 10087 // preallocated handling in the various CC lowering callbacks. 10088 Flags.setByVal(); 10089 } 10090 if (Args[i].IsInAlloca) { 10091 Flags.setInAlloca(); 10092 // Set the byval flag for CCAssignFn callbacks that don't know about 10093 // inalloca. This way we can know how many bytes we should've allocated 10094 // and how many bytes a callee cleanup function will pop. If we port 10095 // inalloca to more targets, we'll have to add custom inalloca handling 10096 // in the various CC lowering callbacks. 10097 Flags.setByVal(); 10098 } 10099 Align MemAlign; 10100 if (Args[i].IsByVal || Args[i].IsInAlloca || Args[i].IsPreallocated) { 10101 unsigned FrameSize = DL.getTypeAllocSize(Args[i].IndirectType); 10102 Flags.setByValSize(FrameSize); 10103 10104 // info is not there but there are cases it cannot get right. 10105 if (auto MA = Args[i].Alignment) 10106 MemAlign = *MA; 10107 else 10108 MemAlign = Align(getByValTypeAlignment(Args[i].IndirectType, DL)); 10109 } else if (auto MA = Args[i].Alignment) { 10110 MemAlign = *MA; 10111 } else { 10112 MemAlign = OriginalAlignment; 10113 } 10114 Flags.setMemAlign(MemAlign); 10115 if (Args[i].IsNest) 10116 Flags.setNest(); 10117 if (NeedsRegBlock) 10118 Flags.setInConsecutiveRegs(); 10119 10120 MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), 10121 CLI.CallConv, VT); 10122 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(), 10123 CLI.CallConv, VT); 10124 SmallVector<SDValue, 4> Parts(NumParts); 10125 ISD::NodeType ExtendKind = ISD::ANY_EXTEND; 10126 10127 if (Args[i].IsSExt) 10128 ExtendKind = ISD::SIGN_EXTEND; 10129 else if (Args[i].IsZExt) 10130 ExtendKind = ISD::ZERO_EXTEND; 10131 10132 // Conservatively only handle 'returned' on non-vectors that can be lowered, 10133 // for now. 10134 if (Args[i].IsReturned && !Op.getValueType().isVector() && 10135 CanLowerReturn) { 10136 assert((CLI.RetTy == Args[i].Ty || 10137 (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && 10138 CLI.RetTy->getPointerAddressSpace() == 10139 Args[i].Ty->getPointerAddressSpace())) && 10140 RetTys.size() == NumValues && "unexpected use of 'returned'"); 10141 // Before passing 'returned' to the target lowering code, ensure that 10142 // either the register MVT and the actual EVT are the same size or that 10143 // the return value and argument are extended in the same way; in these 10144 // cases it's safe to pass the argument register value unchanged as the 10145 // return register value (although it's at the target's option whether 10146 // to do so) 10147 // TODO: allow code generation to take advantage of partially preserved 10148 // registers rather than clobbering the entire register when the 10149 // parameter extension method is not compatible with the return 10150 // extension method 10151 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) || 10152 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt && 10153 CLI.RetZExt == Args[i].IsZExt)) 10154 Flags.setReturned(); 10155 } 10156 10157 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT, CLI.CB, 10158 CLI.CallConv, ExtendKind); 10159 10160 for (unsigned j = 0; j != NumParts; ++j) { 10161 // if it isn't first piece, alignment must be 1 10162 // For scalable vectors the scalable part is currently handled 10163 // by individual targets, so we just use the known minimum size here. 10164 ISD::OutputArg MyFlags( 10165 Flags, Parts[j].getValueType().getSimpleVT(), VT, 10166 i < CLI.NumFixedArgs, i, 10167 j * Parts[j].getValueType().getStoreSize().getKnownMinValue()); 10168 if (NumParts > 1 && j == 0) 10169 MyFlags.Flags.setSplit(); 10170 else if (j != 0) { 10171 MyFlags.Flags.setOrigAlign(Align(1)); 10172 if (j == NumParts - 1) 10173 MyFlags.Flags.setSplitEnd(); 10174 } 10175 10176 CLI.Outs.push_back(MyFlags); 10177 CLI.OutVals.push_back(Parts[j]); 10178 } 10179 10180 if (NeedsRegBlock && Value == NumValues - 1) 10181 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast(); 10182 } 10183 } 10184 10185 SmallVector<SDValue, 4> InVals; 10186 CLI.Chain = LowerCall(CLI, InVals); 10187 10188 // Update CLI.InVals to use outside of this function. 10189 CLI.InVals = InVals; 10190 10191 // Verify that the target's LowerCall behaved as expected. 10192 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && 10193 "LowerCall didn't return a valid chain!"); 10194 assert((!CLI.IsTailCall || InVals.empty()) && 10195 "LowerCall emitted a return value for a tail call!"); 10196 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && 10197 "LowerCall didn't emit the correct number of values!"); 10198 10199 // For a tail call, the return value is merely live-out and there aren't 10200 // any nodes in the DAG representing it. Return a special value to 10201 // indicate that a tail call has been emitted and no more Instructions 10202 // should be processed in the current block. 10203 if (CLI.IsTailCall) { 10204 CLI.DAG.setRoot(CLI.Chain); 10205 return std::make_pair(SDValue(), SDValue()); 10206 } 10207 10208 #ifndef NDEBUG 10209 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) { 10210 assert(InVals[i].getNode() && "LowerCall emitted a null value!"); 10211 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && 10212 "LowerCall emitted a value with the wrong type!"); 10213 } 10214 #endif 10215 10216 SmallVector<SDValue, 4> ReturnValues; 10217 if (!CanLowerReturn) { 10218 // The instruction result is the result of loading from the 10219 // hidden sret parameter. 10220 SmallVector<EVT, 1> PVTs; 10221 Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace()); 10222 10223 ComputeValueVTs(*this, DL, PtrRetTy, PVTs); 10224 assert(PVTs.size() == 1 && "Pointers should fit in one register"); 10225 EVT PtrVT = PVTs[0]; 10226 10227 unsigned NumValues = RetTys.size(); 10228 ReturnValues.resize(NumValues); 10229 SmallVector<SDValue, 4> Chains(NumValues); 10230 10231 // An aggregate return value cannot wrap around the address space, so 10232 // offsets to its parts don't wrap either. 10233 SDNodeFlags Flags; 10234 Flags.setNoUnsignedWrap(true); 10235 10236 MachineFunction &MF = CLI.DAG.getMachineFunction(); 10237 Align HiddenSRetAlign = MF.getFrameInfo().getObjectAlign(DemoteStackIdx); 10238 for (unsigned i = 0; i < NumValues; ++i) { 10239 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot, 10240 CLI.DAG.getConstant(Offsets[i], CLI.DL, 10241 PtrVT), Flags); 10242 SDValue L = CLI.DAG.getLoad( 10243 RetTys[i], CLI.DL, CLI.Chain, Add, 10244 MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(), 10245 DemoteStackIdx, Offsets[i]), 10246 HiddenSRetAlign); 10247 ReturnValues[i] = L; 10248 Chains[i] = L.getValue(1); 10249 } 10250 10251 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains); 10252 } else { 10253 // Collect the legal value parts into potentially illegal values 10254 // that correspond to the original function's return values. 10255 std::optional<ISD::NodeType> AssertOp; 10256 if (CLI.RetSExt) 10257 AssertOp = ISD::AssertSext; 10258 else if (CLI.RetZExt) 10259 AssertOp = ISD::AssertZext; 10260 unsigned CurReg = 0; 10261 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { 10262 EVT VT = RetTys[I]; 10263 MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), 10264 CLI.CallConv, VT); 10265 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(), 10266 CLI.CallConv, VT); 10267 10268 ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg], 10269 NumRegs, RegisterVT, VT, nullptr, 10270 CLI.CallConv, AssertOp)); 10271 CurReg += NumRegs; 10272 } 10273 10274 // For a function returning void, there is no return value. We can't create 10275 // such a node, so we just return a null return value in that case. In 10276 // that case, nothing will actually look at the value. 10277 if (ReturnValues.empty()) 10278 return std::make_pair(SDValue(), CLI.Chain); 10279 } 10280 10281 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL, 10282 CLI.DAG.getVTList(RetTys), ReturnValues); 10283 return std::make_pair(Res, CLI.Chain); 10284 } 10285 10286 /// Places new result values for the node in Results (their number 10287 /// and types must exactly match those of the original return values of 10288 /// the node), or leaves Results empty, which indicates that the node is not 10289 /// to be custom lowered after all. 10290 void TargetLowering::LowerOperationWrapper(SDNode *N, 10291 SmallVectorImpl<SDValue> &Results, 10292 SelectionDAG &DAG) const { 10293 SDValue Res = LowerOperation(SDValue(N, 0), DAG); 10294 10295 if (!Res.getNode()) 10296 return; 10297 10298 // If the original node has one result, take the return value from 10299 // LowerOperation as is. It might not be result number 0. 10300 if (N->getNumValues() == 1) { 10301 Results.push_back(Res); 10302 return; 10303 } 10304 10305 // If the original node has multiple results, then the return node should 10306 // have the same number of results. 10307 assert((N->getNumValues() == Res->getNumValues()) && 10308 "Lowering returned the wrong number of results!"); 10309 10310 // Places new result values base on N result number. 10311 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I) 10312 Results.push_back(Res.getValue(I)); 10313 } 10314 10315 SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { 10316 llvm_unreachable("LowerOperation not implemented for this target!"); 10317 } 10318 10319 void SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, 10320 unsigned Reg, 10321 ISD::NodeType ExtendType) { 10322 SDValue Op = getNonRegisterValue(V); 10323 assert((Op.getOpcode() != ISD::CopyFromReg || 10324 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && 10325 "Copy from a reg to the same reg!"); 10326 assert(!Register::isPhysicalRegister(Reg) && "Is a physreg"); 10327 10328 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 10329 // If this is an InlineAsm we have to match the registers required, not the 10330 // notional registers required by the type. 10331 10332 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(), 10333 std::nullopt); // This is not an ABI copy. 10334 SDValue Chain = DAG.getEntryNode(); 10335 10336 if (ExtendType == ISD::ANY_EXTEND) { 10337 auto PreferredExtendIt = FuncInfo.PreferredExtendType.find(V); 10338 if (PreferredExtendIt != FuncInfo.PreferredExtendType.end()) 10339 ExtendType = PreferredExtendIt->second; 10340 } 10341 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType); 10342 PendingExports.push_back(Chain); 10343 } 10344 10345 #include "llvm/CodeGen/SelectionDAGISel.h" 10346 10347 /// isOnlyUsedInEntryBlock - If the specified argument is only used in the 10348 /// entry block, return true. This includes arguments used by switches, since 10349 /// the switch may expand into multiple basic blocks. 10350 static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) { 10351 // With FastISel active, we may be splitting blocks, so force creation 10352 // of virtual registers for all non-dead arguments. 10353 if (FastISel) 10354 return A->use_empty(); 10355 10356 const BasicBlock &Entry = A->getParent()->front(); 10357 for (const User *U : A->users()) 10358 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U)) 10359 return false; // Use not in entry block. 10360 10361 return true; 10362 } 10363 10364 using ArgCopyElisionMapTy = 10365 DenseMap<const Argument *, 10366 std::pair<const AllocaInst *, const StoreInst *>>; 10367 10368 /// Scan the entry block of the function in FuncInfo for arguments that look 10369 /// like copies into a local alloca. Record any copied arguments in 10370 /// ArgCopyElisionCandidates. 10371 static void 10372 findArgumentCopyElisionCandidates(const DataLayout &DL, 10373 FunctionLoweringInfo *FuncInfo, 10374 ArgCopyElisionMapTy &ArgCopyElisionCandidates) { 10375 // Record the state of every static alloca used in the entry block. Argument 10376 // allocas are all used in the entry block, so we need approximately as many 10377 // entries as we have arguments. 10378 enum StaticAllocaInfo { Unknown, Clobbered, Elidable }; 10379 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas; 10380 unsigned NumArgs = FuncInfo->Fn->arg_size(); 10381 StaticAllocas.reserve(NumArgs * 2); 10382 10383 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * { 10384 if (!V) 10385 return nullptr; 10386 V = V->stripPointerCasts(); 10387 const auto *AI = dyn_cast<AllocaInst>(V); 10388 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI)) 10389 return nullptr; 10390 auto Iter = StaticAllocas.insert({AI, Unknown}); 10391 return &Iter.first->second; 10392 }; 10393 10394 // Look for stores of arguments to static allocas. Look through bitcasts and 10395 // GEPs to handle type coercions, as long as the alloca is fully initialized 10396 // by the store. Any non-store use of an alloca escapes it and any subsequent 10397 // unanalyzed store might write it. 10398 // FIXME: Handle structs initialized with multiple stores. 10399 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) { 10400 // Look for stores, and handle non-store uses conservatively. 10401 const auto *SI = dyn_cast<StoreInst>(&I); 10402 if (!SI) { 10403 // We will look through cast uses, so ignore them completely. 10404 if (I.isCast()) 10405 continue; 10406 // Ignore debug info and pseudo op intrinsics, they don't escape or store 10407 // to allocas. 10408 if (I.isDebugOrPseudoInst()) 10409 continue; 10410 // This is an unknown instruction. Assume it escapes or writes to all 10411 // static alloca operands. 10412 for (const Use &U : I.operands()) { 10413 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U)) 10414 *Info = StaticAllocaInfo::Clobbered; 10415 } 10416 continue; 10417 } 10418 10419 // If the stored value is a static alloca, mark it as escaped. 10420 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand())) 10421 *Info = StaticAllocaInfo::Clobbered; 10422 10423 // Check if the destination is a static alloca. 10424 const Value *Dst = SI->getPointerOperand()->stripPointerCasts(); 10425 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst); 10426 if (!Info) 10427 continue; 10428 const AllocaInst *AI = cast<AllocaInst>(Dst); 10429 10430 // Skip allocas that have been initialized or clobbered. 10431 if (*Info != StaticAllocaInfo::Unknown) 10432 continue; 10433 10434 // Check if the stored value is an argument, and that this store fully 10435 // initializes the alloca. 10436 // If the argument type has padding bits we can't directly forward a pointer 10437 // as the upper bits may contain garbage. 10438 // Don't elide copies from the same argument twice. 10439 const Value *Val = SI->getValueOperand()->stripPointerCasts(); 10440 const auto *Arg = dyn_cast<Argument>(Val); 10441 if (!Arg || Arg->hasPassPointeeByValueCopyAttr() || 10442 Arg->getType()->isEmptyTy() || 10443 DL.getTypeStoreSize(Arg->getType()) != 10444 DL.getTypeAllocSize(AI->getAllocatedType()) || 10445 !DL.typeSizeEqualsStoreSize(Arg->getType()) || 10446 ArgCopyElisionCandidates.count(Arg)) { 10447 *Info = StaticAllocaInfo::Clobbered; 10448 continue; 10449 } 10450 10451 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AI 10452 << '\n'); 10453 10454 // Mark this alloca and store for argument copy elision. 10455 *Info = StaticAllocaInfo::Elidable; 10456 ArgCopyElisionCandidates.insert({Arg, {AI, SI}}); 10457 10458 // Stop scanning if we've seen all arguments. This will happen early in -O0 10459 // builds, which is useful, because -O0 builds have large entry blocks and 10460 // many allocas. 10461 if (ArgCopyElisionCandidates.size() == NumArgs) 10462 break; 10463 } 10464 } 10465 10466 /// Try to elide argument copies from memory into a local alloca. Succeeds if 10467 /// ArgVal is a load from a suitable fixed stack object. 10468 static void tryToElideArgumentCopy( 10469 FunctionLoweringInfo &FuncInfo, SmallVectorImpl<SDValue> &Chains, 10470 DenseMap<int, int> &ArgCopyElisionFrameIndexMap, 10471 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs, 10472 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg, 10473 SDValue ArgVal, bool &ArgHasUses) { 10474 // Check if this is a load from a fixed stack object. 10475 auto *LNode = dyn_cast<LoadSDNode>(ArgVal); 10476 if (!LNode) 10477 return; 10478 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()); 10479 if (!FINode) 10480 return; 10481 10482 // Check that the fixed stack object is the right size and alignment. 10483 // Look at the alignment that the user wrote on the alloca instead of looking 10484 // at the stack object. 10485 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg); 10486 assert(ArgCopyIter != ArgCopyElisionCandidates.end()); 10487 const AllocaInst *AI = ArgCopyIter->second.first; 10488 int FixedIndex = FINode->getIndex(); 10489 int &AllocaIndex = FuncInfo.StaticAllocaMap[AI]; 10490 int OldIndex = AllocaIndex; 10491 MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo(); 10492 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) { 10493 LLVM_DEBUG( 10494 dbgs() << " argument copy elision failed due to bad fixed stack " 10495 "object size\n"); 10496 return; 10497 } 10498 Align RequiredAlignment = AI->getAlign(); 10499 if (MFI.getObjectAlign(FixedIndex) < RequiredAlignment) { 10500 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca " 10501 "greater than stack argument alignment (" 10502 << DebugStr(RequiredAlignment) << " vs " 10503 << DebugStr(MFI.getObjectAlign(FixedIndex)) << ")\n"); 10504 return; 10505 } 10506 10507 // Perform the elision. Delete the old stack object and replace its only use 10508 // in the variable info map. Mark the stack object as mutable. 10509 LLVM_DEBUG({ 10510 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n' 10511 << " Replacing frame index " << OldIndex << " with " << FixedIndex 10512 << '\n'; 10513 }); 10514 MFI.RemoveStackObject(OldIndex); 10515 MFI.setIsImmutableObjectIndex(FixedIndex, false); 10516 AllocaIndex = FixedIndex; 10517 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex}); 10518 Chains.push_back(ArgVal.getValue(1)); 10519 10520 // Avoid emitting code for the store implementing the copy. 10521 const StoreInst *SI = ArgCopyIter->second.second; 10522 ElidedArgCopyInstrs.insert(SI); 10523 10524 // Check for uses of the argument again so that we can avoid exporting ArgVal 10525 // if it is't used by anything other than the store. 10526 for (const Value *U : Arg.users()) { 10527 if (U != SI) { 10528 ArgHasUses = true; 10529 break; 10530 } 10531 } 10532 } 10533 10534 void SelectionDAGISel::LowerArguments(const Function &F) { 10535 SelectionDAG &DAG = SDB->DAG; 10536 SDLoc dl = SDB->getCurSDLoc(); 10537 const DataLayout &DL = DAG.getDataLayout(); 10538 SmallVector<ISD::InputArg, 16> Ins; 10539 10540 // In Naked functions we aren't going to save any registers. 10541 if (F.hasFnAttribute(Attribute::Naked)) 10542 return; 10543 10544 if (!FuncInfo->CanLowerReturn) { 10545 // Put in an sret pointer parameter before all the other parameters. 10546 SmallVector<EVT, 1> ValueVTs; 10547 ComputeValueVTs(*TLI, DAG.getDataLayout(), 10548 F.getReturnType()->getPointerTo( 10549 DAG.getDataLayout().getAllocaAddrSpace()), 10550 ValueVTs); 10551 10552 // NOTE: Assuming that a pointer will never break down to more than one VT 10553 // or one register. 10554 ISD::ArgFlagsTy Flags; 10555 Flags.setSRet(); 10556 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]); 10557 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true, 10558 ISD::InputArg::NoArgIndex, 0); 10559 Ins.push_back(RetArg); 10560 } 10561 10562 // Look for stores of arguments to static allocas. Mark such arguments with a 10563 // flag to ask the target to give us the memory location of that argument if 10564 // available. 10565 ArgCopyElisionMapTy ArgCopyElisionCandidates; 10566 findArgumentCopyElisionCandidates(DL, FuncInfo.get(), 10567 ArgCopyElisionCandidates); 10568 10569 // Set up the incoming argument description vector. 10570 for (const Argument &Arg : F.args()) { 10571 unsigned ArgNo = Arg.getArgNo(); 10572 SmallVector<EVT, 4> ValueVTs; 10573 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); 10574 bool isArgValueUsed = !Arg.use_empty(); 10575 unsigned PartBase = 0; 10576 Type *FinalType = Arg.getType(); 10577 if (Arg.hasAttribute(Attribute::ByVal)) 10578 FinalType = Arg.getParamByValType(); 10579 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( 10580 FinalType, F.getCallingConv(), F.isVarArg(), DL); 10581 for (unsigned Value = 0, NumValues = ValueVTs.size(); 10582 Value != NumValues; ++Value) { 10583 EVT VT = ValueVTs[Value]; 10584 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext()); 10585 ISD::ArgFlagsTy Flags; 10586 10587 10588 if (Arg.getType()->isPointerTy()) { 10589 Flags.setPointer(); 10590 Flags.setPointerAddrSpace( 10591 cast<PointerType>(Arg.getType())->getAddressSpace()); 10592 } 10593 if (Arg.hasAttribute(Attribute::ZExt)) 10594 Flags.setZExt(); 10595 if (Arg.hasAttribute(Attribute::SExt)) 10596 Flags.setSExt(); 10597 if (Arg.hasAttribute(Attribute::InReg)) { 10598 // If we are using vectorcall calling convention, a structure that is 10599 // passed InReg - is surely an HVA 10600 if (F.getCallingConv() == CallingConv::X86_VectorCall && 10601 isa<StructType>(Arg.getType())) { 10602 // The first value of a structure is marked 10603 if (0 == Value) 10604 Flags.setHvaStart(); 10605 Flags.setHva(); 10606 } 10607 // Set InReg Flag 10608 Flags.setInReg(); 10609 } 10610 if (Arg.hasAttribute(Attribute::StructRet)) 10611 Flags.setSRet(); 10612 if (Arg.hasAttribute(Attribute::SwiftSelf)) 10613 Flags.setSwiftSelf(); 10614 if (Arg.hasAttribute(Attribute::SwiftAsync)) 10615 Flags.setSwiftAsync(); 10616 if (Arg.hasAttribute(Attribute::SwiftError)) 10617 Flags.setSwiftError(); 10618 if (Arg.hasAttribute(Attribute::ByVal)) 10619 Flags.setByVal(); 10620 if (Arg.hasAttribute(Attribute::ByRef)) 10621 Flags.setByRef(); 10622 if (Arg.hasAttribute(Attribute::InAlloca)) { 10623 Flags.setInAlloca(); 10624 // Set the byval flag for CCAssignFn callbacks that don't know about 10625 // inalloca. This way we can know how many bytes we should've allocated 10626 // and how many bytes a callee cleanup function will pop. If we port 10627 // inalloca to more targets, we'll have to add custom inalloca handling 10628 // in the various CC lowering callbacks. 10629 Flags.setByVal(); 10630 } 10631 if (Arg.hasAttribute(Attribute::Preallocated)) { 10632 Flags.setPreallocated(); 10633 // Set the byval flag for CCAssignFn callbacks that don't know about 10634 // preallocated. This way we can know how many bytes we should've 10635 // allocated and how many bytes a callee cleanup function will pop. If 10636 // we port preallocated to more targets, we'll have to add custom 10637 // preallocated handling in the various CC lowering callbacks. 10638 Flags.setByVal(); 10639 } 10640 10641 // Certain targets (such as MIPS), may have a different ABI alignment 10642 // for a type depending on the context. Give the target a chance to 10643 // specify the alignment it wants. 10644 const Align OriginalAlignment( 10645 TLI->getABIAlignmentForCallingConv(ArgTy, DL)); 10646 Flags.setOrigAlign(OriginalAlignment); 10647 10648 Align MemAlign; 10649 Type *ArgMemTy = nullptr; 10650 if (Flags.isByVal() || Flags.isInAlloca() || Flags.isPreallocated() || 10651 Flags.isByRef()) { 10652 if (!ArgMemTy) 10653 ArgMemTy = Arg.getPointeeInMemoryValueType(); 10654 10655 uint64_t MemSize = DL.getTypeAllocSize(ArgMemTy); 10656 10657 // For in-memory arguments, size and alignment should be passed from FE. 10658 // BE will guess if this info is not there but there are cases it cannot 10659 // get right. 10660 if (auto ParamAlign = Arg.getParamStackAlign()) 10661 MemAlign = *ParamAlign; 10662 else if ((ParamAlign = Arg.getParamAlign())) 10663 MemAlign = *ParamAlign; 10664 else 10665 MemAlign = Align(TLI->getByValTypeAlignment(ArgMemTy, DL)); 10666 if (Flags.isByRef()) 10667 Flags.setByRefSize(MemSize); 10668 else 10669 Flags.setByValSize(MemSize); 10670 } else if (auto ParamAlign = Arg.getParamStackAlign()) { 10671 MemAlign = *ParamAlign; 10672 } else { 10673 MemAlign = OriginalAlignment; 10674 } 10675 Flags.setMemAlign(MemAlign); 10676 10677 if (Arg.hasAttribute(Attribute::Nest)) 10678 Flags.setNest(); 10679 if (NeedsRegBlock) 10680 Flags.setInConsecutiveRegs(); 10681 if (ArgCopyElisionCandidates.count(&Arg)) 10682 Flags.setCopyElisionCandidate(); 10683 if (Arg.hasAttribute(Attribute::Returned)) 10684 Flags.setReturned(); 10685 10686 MVT RegisterVT = TLI->getRegisterTypeForCallingConv( 10687 *CurDAG->getContext(), F.getCallingConv(), VT); 10688 unsigned NumRegs = TLI->getNumRegistersForCallingConv( 10689 *CurDAG->getContext(), F.getCallingConv(), VT); 10690 for (unsigned i = 0; i != NumRegs; ++i) { 10691 // For scalable vectors, use the minimum size; individual targets 10692 // are responsible for handling scalable vector arguments and 10693 // return values. 10694 ISD::InputArg MyFlags( 10695 Flags, RegisterVT, VT, isArgValueUsed, ArgNo, 10696 PartBase + i * RegisterVT.getStoreSize().getKnownMinValue()); 10697 if (NumRegs > 1 && i == 0) 10698 MyFlags.Flags.setSplit(); 10699 // if it isn't first piece, alignment must be 1 10700 else if (i > 0) { 10701 MyFlags.Flags.setOrigAlign(Align(1)); 10702 if (i == NumRegs - 1) 10703 MyFlags.Flags.setSplitEnd(); 10704 } 10705 Ins.push_back(MyFlags); 10706 } 10707 if (NeedsRegBlock && Value == NumValues - 1) 10708 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast(); 10709 PartBase += VT.getStoreSize().getKnownMinValue(); 10710 } 10711 } 10712 10713 // Call the target to set up the argument values. 10714 SmallVector<SDValue, 8> InVals; 10715 SDValue NewRoot = TLI->LowerFormalArguments( 10716 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals); 10717 10718 // Verify that the target's LowerFormalArguments behaved as expected. 10719 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && 10720 "LowerFormalArguments didn't return a valid chain!"); 10721 assert(InVals.size() == Ins.size() && 10722 "LowerFormalArguments didn't emit the correct number of values!"); 10723 LLVM_DEBUG({ 10724 for (unsigned i = 0, e = Ins.size(); i != e; ++i) { 10725 assert(InVals[i].getNode() && 10726 "LowerFormalArguments emitted a null value!"); 10727 assert(EVT(Ins[i].VT) == InVals[i].getValueType() && 10728 "LowerFormalArguments emitted a value with the wrong type!"); 10729 } 10730 }); 10731 10732 // Update the DAG with the new chain value resulting from argument lowering. 10733 DAG.setRoot(NewRoot); 10734 10735 // Set up the argument values. 10736 unsigned i = 0; 10737 if (!FuncInfo->CanLowerReturn) { 10738 // Create a virtual register for the sret pointer, and put in a copy 10739 // from the sret argument into it. 10740 SmallVector<EVT, 1> ValueVTs; 10741 ComputeValueVTs(*TLI, DAG.getDataLayout(), 10742 F.getReturnType()->getPointerTo( 10743 DAG.getDataLayout().getAllocaAddrSpace()), 10744 ValueVTs); 10745 MVT VT = ValueVTs[0].getSimpleVT(); 10746 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT); 10747 std::optional<ISD::NodeType> AssertOp; 10748 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT, 10749 nullptr, F.getCallingConv(), AssertOp); 10750 10751 MachineFunction& MF = SDB->DAG.getMachineFunction(); 10752 MachineRegisterInfo& RegInfo = MF.getRegInfo(); 10753 Register SRetReg = 10754 RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT)); 10755 FuncInfo->DemoteRegister = SRetReg; 10756 NewRoot = 10757 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue); 10758 DAG.setRoot(NewRoot); 10759 10760 // i indexes lowered arguments. Bump it past the hidden sret argument. 10761 ++i; 10762 } 10763 10764 SmallVector<SDValue, 4> Chains; 10765 DenseMap<int, int> ArgCopyElisionFrameIndexMap; 10766 for (const Argument &Arg : F.args()) { 10767 SmallVector<SDValue, 4> ArgValues; 10768 SmallVector<EVT, 4> ValueVTs; 10769 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs); 10770 unsigned NumValues = ValueVTs.size(); 10771 if (NumValues == 0) 10772 continue; 10773 10774 bool ArgHasUses = !Arg.use_empty(); 10775 10776 // Elide the copying store if the target loaded this argument from a 10777 // suitable fixed stack object. 10778 if (Ins[i].Flags.isCopyElisionCandidate()) { 10779 tryToElideArgumentCopy(*FuncInfo, Chains, ArgCopyElisionFrameIndexMap, 10780 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg, 10781 InVals[i], ArgHasUses); 10782 } 10783 10784 // If this argument is unused then remember its value. It is used to generate 10785 // debugging information. 10786 bool isSwiftErrorArg = 10787 TLI->supportSwiftError() && 10788 Arg.hasAttribute(Attribute::SwiftError); 10789 if (!ArgHasUses && !isSwiftErrorArg) { 10790 SDB->setUnusedArgValue(&Arg, InVals[i]); 10791 10792 // Also remember any frame index for use in FastISel. 10793 if (FrameIndexSDNode *FI = 10794 dyn_cast<FrameIndexSDNode>(InVals[i].getNode())) 10795 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); 10796 } 10797 10798 for (unsigned Val = 0; Val != NumValues; ++Val) { 10799 EVT VT = ValueVTs[Val]; 10800 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(), 10801 F.getCallingConv(), VT); 10802 unsigned NumParts = TLI->getNumRegistersForCallingConv( 10803 *CurDAG->getContext(), F.getCallingConv(), VT); 10804 10805 // Even an apparent 'unused' swifterror argument needs to be returned. So 10806 // we do generate a copy for it that can be used on return from the 10807 // function. 10808 if (ArgHasUses || isSwiftErrorArg) { 10809 std::optional<ISD::NodeType> AssertOp; 10810 if (Arg.hasAttribute(Attribute::SExt)) 10811 AssertOp = ISD::AssertSext; 10812 else if (Arg.hasAttribute(Attribute::ZExt)) 10813 AssertOp = ISD::AssertZext; 10814 10815 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts, 10816 PartVT, VT, nullptr, 10817 F.getCallingConv(), AssertOp)); 10818 } 10819 10820 i += NumParts; 10821 } 10822 10823 // We don't need to do anything else for unused arguments. 10824 if (ArgValues.empty()) 10825 continue; 10826 10827 // Note down frame index. 10828 if (FrameIndexSDNode *FI = 10829 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode())) 10830 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); 10831 10832 SDValue Res = DAG.getMergeValues(ArrayRef(ArgValues.data(), NumValues), 10833 SDB->getCurSDLoc()); 10834 10835 SDB->setValue(&Arg, Res); 10836 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) { 10837 // We want to associate the argument with the frame index, among 10838 // involved operands, that correspond to the lowest address. The 10839 // getCopyFromParts function, called earlier, is swapping the order of 10840 // the operands to BUILD_PAIR depending on endianness. The result of 10841 // that swapping is that the least significant bits of the argument will 10842 // be in the first operand of the BUILD_PAIR node, and the most 10843 // significant bits will be in the second operand. 10844 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0; 10845 if (LoadSDNode *LNode = 10846 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode())) 10847 if (FrameIndexSDNode *FI = 10848 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode())) 10849 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex()); 10850 } 10851 10852 // Analyses past this point are naive and don't expect an assertion. 10853 if (Res.getOpcode() == ISD::AssertZext) 10854 Res = Res.getOperand(0); 10855 10856 // Update the SwiftErrorVRegDefMap. 10857 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) { 10858 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); 10859 if (Register::isVirtualRegister(Reg)) 10860 SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(), 10861 Reg); 10862 } 10863 10864 // If this argument is live outside of the entry block, insert a copy from 10865 // wherever we got it to the vreg that other BB's will reference it as. 10866 if (Res.getOpcode() == ISD::CopyFromReg) { 10867 // If we can, though, try to skip creating an unnecessary vreg. 10868 // FIXME: This isn't very clean... it would be nice to make this more 10869 // general. 10870 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg(); 10871 if (Register::isVirtualRegister(Reg)) { 10872 FuncInfo->ValueMap[&Arg] = Reg; 10873 continue; 10874 } 10875 } 10876 if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) { 10877 FuncInfo->InitializeRegForValue(&Arg); 10878 SDB->CopyToExportRegsIfNeeded(&Arg); 10879 } 10880 } 10881 10882 if (!Chains.empty()) { 10883 Chains.push_back(NewRoot); 10884 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); 10885 } 10886 10887 DAG.setRoot(NewRoot); 10888 10889 assert(i == InVals.size() && "Argument register count mismatch!"); 10890 10891 // If any argument copy elisions occurred and we have debug info, update the 10892 // stale frame indices used in the dbg.declare variable info table. 10893 MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo(); 10894 if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) { 10895 for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) { 10896 auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot); 10897 if (I != ArgCopyElisionFrameIndexMap.end()) 10898 VI.Slot = I->second; 10899 } 10900 } 10901 10902 // Finally, if the target has anything special to do, allow it to do so. 10903 emitFunctionEntryCode(); 10904 } 10905 10906 /// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to 10907 /// ensure constants are generated when needed. Remember the virtual registers 10908 /// that need to be added to the Machine PHI nodes as input. We cannot just 10909 /// directly add them, because expansion might result in multiple MBB's for one 10910 /// BB. As such, the start of the BB might correspond to a different MBB than 10911 /// the end. 10912 void 10913 SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { 10914 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 10915 10916 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled; 10917 10918 // Check PHI nodes in successors that expect a value to be available from this 10919 // block. 10920 for (const BasicBlock *SuccBB : successors(LLVMBB->getTerminator())) { 10921 if (!isa<PHINode>(SuccBB->begin())) continue; 10922 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB]; 10923 10924 // If this terminator has multiple identical successors (common for 10925 // switches), only handle each succ once. 10926 if (!SuccsHandled.insert(SuccMBB).second) 10927 continue; 10928 10929 MachineBasicBlock::iterator MBBI = SuccMBB->begin(); 10930 10931 // At this point we know that there is a 1-1 correspondence between LLVM PHI 10932 // nodes and Machine PHI nodes, but the incoming operands have not been 10933 // emitted yet. 10934 for (const PHINode &PN : SuccBB->phis()) { 10935 // Ignore dead phi's. 10936 if (PN.use_empty()) 10937 continue; 10938 10939 // Skip empty types 10940 if (PN.getType()->isEmptyTy()) 10941 continue; 10942 10943 unsigned Reg; 10944 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB); 10945 10946 if (const auto *C = dyn_cast<Constant>(PHIOp)) { 10947 unsigned &RegOut = ConstantsOut[C]; 10948 if (RegOut == 0) { 10949 RegOut = FuncInfo.CreateRegs(C); 10950 // We need to zero/sign extend ConstantInt phi operands to match 10951 // assumptions in FunctionLoweringInfo::ComputePHILiveOutRegInfo. 10952 ISD::NodeType ExtendType = ISD::ANY_EXTEND; 10953 if (auto *CI = dyn_cast<ConstantInt>(C)) 10954 ExtendType = TLI.signExtendConstant(CI) ? ISD::SIGN_EXTEND 10955 : ISD::ZERO_EXTEND; 10956 CopyValueToVirtualRegister(C, RegOut, ExtendType); 10957 } 10958 Reg = RegOut; 10959 } else { 10960 DenseMap<const Value *, Register>::iterator I = 10961 FuncInfo.ValueMap.find(PHIOp); 10962 if (I != FuncInfo.ValueMap.end()) 10963 Reg = I->second; 10964 else { 10965 assert(isa<AllocaInst>(PHIOp) && 10966 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && 10967 "Didn't codegen value into a register!??"); 10968 Reg = FuncInfo.CreateRegs(PHIOp); 10969 CopyValueToVirtualRegister(PHIOp, Reg); 10970 } 10971 } 10972 10973 // Remember that this register needs to added to the machine PHI node as 10974 // the input for this MBB. 10975 SmallVector<EVT, 4> ValueVTs; 10976 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs); 10977 for (EVT VT : ValueVTs) { 10978 const unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT); 10979 for (unsigned i = 0; i != NumRegisters; ++i) 10980 FuncInfo.PHINodesToUpdate.push_back( 10981 std::make_pair(&*MBBI++, Reg + i)); 10982 Reg += NumRegisters; 10983 } 10984 } 10985 } 10986 10987 ConstantsOut.clear(); 10988 } 10989 10990 MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) { 10991 MachineFunction::iterator I(MBB); 10992 if (++I == FuncInfo.MF->end()) 10993 return nullptr; 10994 return &*I; 10995 } 10996 10997 /// During lowering new call nodes can be created (such as memset, etc.). 10998 /// Those will become new roots of the current DAG, but complications arise 10999 /// when they are tail calls. In such cases, the call lowering will update 11000 /// the root, but the builder still needs to know that a tail call has been 11001 /// lowered in order to avoid generating an additional return. 11002 void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) { 11003 // If the node is null, we do have a tail call. 11004 if (MaybeTC.getNode() != nullptr) 11005 DAG.setRoot(MaybeTC); 11006 else 11007 HasTailCall = true; 11008 } 11009 11010 void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, 11011 MachineBasicBlock *SwitchMBB, 11012 MachineBasicBlock *DefaultMBB) { 11013 MachineFunction *CurMF = FuncInfo.MF; 11014 MachineBasicBlock *NextMBB = nullptr; 11015 MachineFunction::iterator BBI(W.MBB); 11016 if (++BBI != FuncInfo.MF->end()) 11017 NextMBB = &*BBI; 11018 11019 unsigned Size = W.LastCluster - W.FirstCluster + 1; 11020 11021 BranchProbabilityInfo *BPI = FuncInfo.BPI; 11022 11023 if (Size == 2 && W.MBB == SwitchMBB) { 11024 // If any two of the cases has the same destination, and if one value 11025 // is the same as the other, but has one bit unset that the other has set, 11026 // use bit manipulation to do two compares at once. For example: 11027 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)" 11028 // TODO: This could be extended to merge any 2 cases in switches with 3 11029 // cases. 11030 // TODO: Handle cases where W.CaseBB != SwitchBB. 11031 CaseCluster &Small = *W.FirstCluster; 11032 CaseCluster &Big = *W.LastCluster; 11033 11034 if (Small.Low == Small.High && Big.Low == Big.High && 11035 Small.MBB == Big.MBB) { 11036 const APInt &SmallValue = Small.Low->getValue(); 11037 const APInt &BigValue = Big.Low->getValue(); 11038 11039 // Check that there is only one bit different. 11040 APInt CommonBit = BigValue ^ SmallValue; 11041 if (CommonBit.isPowerOf2()) { 11042 SDValue CondLHS = getValue(Cond); 11043 EVT VT = CondLHS.getValueType(); 11044 SDLoc DL = getCurSDLoc(); 11045 11046 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS, 11047 DAG.getConstant(CommonBit, DL, VT)); 11048 SDValue Cond = DAG.getSetCC( 11049 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT), 11050 ISD::SETEQ); 11051 11052 // Update successor info. 11053 // Both Small and Big will jump to Small.BB, so we sum up the 11054 // probabilities. 11055 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob); 11056 if (BPI) 11057 addSuccessorWithProb( 11058 SwitchMBB, DefaultMBB, 11059 // The default destination is the first successor in IR. 11060 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0)); 11061 else 11062 addSuccessorWithProb(SwitchMBB, DefaultMBB); 11063 11064 // Insert the true branch. 11065 SDValue BrCond = 11066 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond, 11067 DAG.getBasicBlock(Small.MBB)); 11068 // Insert the false branch. 11069 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond, 11070 DAG.getBasicBlock(DefaultMBB)); 11071 11072 DAG.setRoot(BrCond); 11073 return; 11074 } 11075 } 11076 } 11077 11078 if (TM.getOptLevel() != CodeGenOpt::None) { 11079 // Here, we order cases by probability so the most likely case will be 11080 // checked first. However, two clusters can have the same probability in 11081 // which case their relative ordering is non-deterministic. So we use Low 11082 // as a tie-breaker as clusters are guaranteed to never overlap. 11083 llvm::sort(W.FirstCluster, W.LastCluster + 1, 11084 [](const CaseCluster &a, const CaseCluster &b) { 11085 return a.Prob != b.Prob ? 11086 a.Prob > b.Prob : 11087 a.Low->getValue().slt(b.Low->getValue()); 11088 }); 11089 11090 // Rearrange the case blocks so that the last one falls through if possible 11091 // without changing the order of probabilities. 11092 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) { 11093 --I; 11094 if (I->Prob > W.LastCluster->Prob) 11095 break; 11096 if (I->Kind == CC_Range && I->MBB == NextMBB) { 11097 std::swap(*I, *W.LastCluster); 11098 break; 11099 } 11100 } 11101 } 11102 11103 // Compute total probability. 11104 BranchProbability DefaultProb = W.DefaultProb; 11105 BranchProbability UnhandledProbs = DefaultProb; 11106 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) 11107 UnhandledProbs += I->Prob; 11108 11109 MachineBasicBlock *CurMBB = W.MBB; 11110 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) { 11111 bool FallthroughUnreachable = false; 11112 MachineBasicBlock *Fallthrough; 11113 if (I == W.LastCluster) { 11114 // For the last cluster, fall through to the default destination. 11115 Fallthrough = DefaultMBB; 11116 FallthroughUnreachable = isa<UnreachableInst>( 11117 DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg()); 11118 } else { 11119 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); 11120 CurMF->insert(BBI, Fallthrough); 11121 // Put Cond in a virtual register to make it available from the new blocks. 11122 ExportFromCurrentBlock(Cond); 11123 } 11124 UnhandledProbs -= I->Prob; 11125 11126 switch (I->Kind) { 11127 case CC_JumpTable: { 11128 // FIXME: Optimize away range check based on pivot comparisons. 11129 JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; 11130 SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; 11131 11132 // The jump block hasn't been inserted yet; insert it here. 11133 MachineBasicBlock *JumpMBB = JT->MBB; 11134 CurMF->insert(BBI, JumpMBB); 11135 11136 auto JumpProb = I->Prob; 11137 auto FallthroughProb = UnhandledProbs; 11138 11139 // If the default statement is a target of the jump table, we evenly 11140 // distribute the default probability to successors of CurMBB. Also 11141 // update the probability on the edge from JumpMBB to Fallthrough. 11142 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), 11143 SE = JumpMBB->succ_end(); 11144 SI != SE; ++SI) { 11145 if (*SI == DefaultMBB) { 11146 JumpProb += DefaultProb / 2; 11147 FallthroughProb -= DefaultProb / 2; 11148 JumpMBB->setSuccProbability(SI, DefaultProb / 2); 11149 JumpMBB->normalizeSuccProbs(); 11150 break; 11151 } 11152 } 11153 11154 if (FallthroughUnreachable) 11155 JTH->FallthroughUnreachable = true; 11156 11157 if (!JTH->FallthroughUnreachable) 11158 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); 11159 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); 11160 CurMBB->normalizeSuccProbs(); 11161 11162 // The jump table header will be inserted in our current block, do the 11163 // range check, and fall through to our fallthrough block. 11164 JTH->HeaderBB = CurMBB; 11165 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader. 11166 11167 // If we're in the right place, emit the jump table header right now. 11168 if (CurMBB == SwitchMBB) { 11169 visitJumpTableHeader(*JT, *JTH, SwitchMBB); 11170 JTH->Emitted = true; 11171 } 11172 break; 11173 } 11174 case CC_BitTests: { 11175 // FIXME: Optimize away range check based on pivot comparisons. 11176 BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex]; 11177 11178 // The bit test blocks haven't been inserted yet; insert them here. 11179 for (BitTestCase &BTC : BTB->Cases) 11180 CurMF->insert(BBI, BTC.ThisBB); 11181 11182 // Fill in fields of the BitTestBlock. 11183 BTB->Parent = CurMBB; 11184 BTB->Default = Fallthrough; 11185 11186 BTB->DefaultProb = UnhandledProbs; 11187 // If the cases in bit test don't form a contiguous range, we evenly 11188 // distribute the probability on the edge to Fallthrough to two 11189 // successors of CurMBB. 11190 if (!BTB->ContiguousRange) { 11191 BTB->Prob += DefaultProb / 2; 11192 BTB->DefaultProb -= DefaultProb / 2; 11193 } 11194 11195 if (FallthroughUnreachable) 11196 BTB->FallthroughUnreachable = true; 11197 11198 // If we're in the right place, emit the bit test header right now. 11199 if (CurMBB == SwitchMBB) { 11200 visitBitTestHeader(*BTB, SwitchMBB); 11201 BTB->Emitted = true; 11202 } 11203 break; 11204 } 11205 case CC_Range: { 11206 const Value *RHS, *LHS, *MHS; 11207 ISD::CondCode CC; 11208 if (I->Low == I->High) { 11209 // Check Cond == I->Low. 11210 CC = ISD::SETEQ; 11211 LHS = Cond; 11212 RHS=I->Low; 11213 MHS = nullptr; 11214 } else { 11215 // Check I->Low <= Cond <= I->High. 11216 CC = ISD::SETLE; 11217 LHS = I->Low; 11218 MHS = Cond; 11219 RHS = I->High; 11220 } 11221 11222 // If Fallthrough is unreachable, fold away the comparison. 11223 if (FallthroughUnreachable) 11224 CC = ISD::SETTRUE; 11225 11226 // The false probability is the sum of all unhandled cases. 11227 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, 11228 getCurSDLoc(), I->Prob, UnhandledProbs); 11229 11230 if (CurMBB == SwitchMBB) 11231 visitSwitchCase(CB, SwitchMBB); 11232 else 11233 SL->SwitchCases.push_back(CB); 11234 11235 break; 11236 } 11237 } 11238 CurMBB = Fallthrough; 11239 } 11240 } 11241 11242 unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC, 11243 CaseClusterIt First, 11244 CaseClusterIt Last) { 11245 return std::count_if(First, Last + 1, [&](const CaseCluster &X) { 11246 if (X.Prob != CC.Prob) 11247 return X.Prob > CC.Prob; 11248 11249 // Ties are broken by comparing the case value. 11250 return X.Low->getValue().slt(CC.Low->getValue()); 11251 }); 11252 } 11253 11254 void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList, 11255 const SwitchWorkListItem &W, 11256 Value *Cond, 11257 MachineBasicBlock *SwitchMBB) { 11258 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && 11259 "Clusters not sorted?"); 11260 11261 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!"); 11262 11263 // Balance the tree based on branch probabilities to create a near-optimal (in 11264 // terms of search time given key frequency) binary search tree. See e.g. Kurt 11265 // Mehlhorn "Nearly Optimal Binary Search Trees" (1975). 11266 CaseClusterIt LastLeft = W.FirstCluster; 11267 CaseClusterIt FirstRight = W.LastCluster; 11268 auto LeftProb = LastLeft->Prob + W.DefaultProb / 2; 11269 auto RightProb = FirstRight->Prob + W.DefaultProb / 2; 11270 11271 // Move LastLeft and FirstRight towards each other from opposite directions to 11272 // find a partitioning of the clusters which balances the probability on both 11273 // sides. If LeftProb and RightProb are equal, alternate which side is 11274 // taken to ensure 0-probability nodes are distributed evenly. 11275 unsigned I = 0; 11276 while (LastLeft + 1 < FirstRight) { 11277 if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1))) 11278 LeftProb += (++LastLeft)->Prob; 11279 else 11280 RightProb += (--FirstRight)->Prob; 11281 I++; 11282 } 11283 11284 while (true) { 11285 // Our binary search tree differs from a typical BST in that ours can have up 11286 // to three values in each leaf. The pivot selection above doesn't take that 11287 // into account, which means the tree might require more nodes and be less 11288 // efficient. We compensate for this here. 11289 11290 unsigned NumLeft = LastLeft - W.FirstCluster + 1; 11291 unsigned NumRight = W.LastCluster - FirstRight + 1; 11292 11293 if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) { 11294 // If one side has less than 3 clusters, and the other has more than 3, 11295 // consider taking a cluster from the other side. 11296 11297 if (NumLeft < NumRight) { 11298 // Consider moving the first cluster on the right to the left side. 11299 CaseCluster &CC = *FirstRight; 11300 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); 11301 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); 11302 if (LeftSideRank <= RightSideRank) { 11303 // Moving the cluster to the left does not demote it. 11304 ++LastLeft; 11305 ++FirstRight; 11306 continue; 11307 } 11308 } else { 11309 assert(NumRight < NumLeft); 11310 // Consider moving the last element on the left to the right side. 11311 CaseCluster &CC = *LastLeft; 11312 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft); 11313 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster); 11314 if (RightSideRank <= LeftSideRank) { 11315 // Moving the cluster to the right does not demot it. 11316 --LastLeft; 11317 --FirstRight; 11318 continue; 11319 } 11320 } 11321 } 11322 break; 11323 } 11324 11325 assert(LastLeft + 1 == FirstRight); 11326 assert(LastLeft >= W.FirstCluster); 11327 assert(FirstRight <= W.LastCluster); 11328 11329 // Use the first element on the right as pivot since we will make less-than 11330 // comparisons against it. 11331 CaseClusterIt PivotCluster = FirstRight; 11332 assert(PivotCluster > W.FirstCluster); 11333 assert(PivotCluster <= W.LastCluster); 11334 11335 CaseClusterIt FirstLeft = W.FirstCluster; 11336 CaseClusterIt LastRight = W.LastCluster; 11337 11338 const ConstantInt *Pivot = PivotCluster->Low; 11339 11340 // New blocks will be inserted immediately after the current one. 11341 MachineFunction::iterator BBI(W.MBB); 11342 ++BBI; 11343 11344 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster, 11345 // we can branch to its destination directly if it's squeezed exactly in 11346 // between the known lower bound and Pivot - 1. 11347 MachineBasicBlock *LeftMBB; 11348 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range && 11349 FirstLeft->Low == W.GE && 11350 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) { 11351 LeftMBB = FirstLeft->MBB; 11352 } else { 11353 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); 11354 FuncInfo.MF->insert(BBI, LeftMBB); 11355 WorkList.push_back( 11356 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2}); 11357 // Put Cond in a virtual register to make it available from the new blocks. 11358 ExportFromCurrentBlock(Cond); 11359 } 11360 11361 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a 11362 // single cluster, RHS.Low == Pivot, and we can branch to its destination 11363 // directly if RHS.High equals the current upper bound. 11364 MachineBasicBlock *RightMBB; 11365 if (FirstRight == LastRight && FirstRight->Kind == CC_Range && 11366 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) { 11367 RightMBB = FirstRight->MBB; 11368 } else { 11369 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock()); 11370 FuncInfo.MF->insert(BBI, RightMBB); 11371 WorkList.push_back( 11372 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2}); 11373 // Put Cond in a virtual register to make it available from the new blocks. 11374 ExportFromCurrentBlock(Cond); 11375 } 11376 11377 // Create the CaseBlock record that will be used to lower the branch. 11378 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB, 11379 getCurSDLoc(), LeftProb, RightProb); 11380 11381 if (W.MBB == SwitchMBB) 11382 visitSwitchCase(CB, SwitchMBB); 11383 else 11384 SL->SwitchCases.push_back(CB); 11385 } 11386 11387 // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb 11388 // from the swith statement. 11389 static BranchProbability scaleCaseProbality(BranchProbability CaseProb, 11390 BranchProbability PeeledCaseProb) { 11391 if (PeeledCaseProb == BranchProbability::getOne()) 11392 return BranchProbability::getZero(); 11393 BranchProbability SwitchProb = PeeledCaseProb.getCompl(); 11394 11395 uint32_t Numerator = CaseProb.getNumerator(); 11396 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator()); 11397 return BranchProbability(Numerator, std::max(Numerator, Denominator)); 11398 } 11399 11400 // Try to peel the top probability case if it exceeds the threshold. 11401 // Return current MachineBasicBlock for the switch statement if the peeling 11402 // does not occur. 11403 // If the peeling is performed, return the newly created MachineBasicBlock 11404 // for the peeled switch statement. Also update Clusters to remove the peeled 11405 // case. PeeledCaseProb is the BranchProbability for the peeled case. 11406 MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster( 11407 const SwitchInst &SI, CaseClusterVector &Clusters, 11408 BranchProbability &PeeledCaseProb) { 11409 MachineBasicBlock *SwitchMBB = FuncInfo.MBB; 11410 // Don't perform if there is only one cluster or optimizing for size. 11411 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 || 11412 TM.getOptLevel() == CodeGenOpt::None || 11413 SwitchMBB->getParent()->getFunction().hasMinSize()) 11414 return SwitchMBB; 11415 11416 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100); 11417 unsigned PeeledCaseIndex = 0; 11418 bool SwitchPeeled = false; 11419 for (unsigned Index = 0; Index < Clusters.size(); ++Index) { 11420 CaseCluster &CC = Clusters[Index]; 11421 if (CC.Prob < TopCaseProb) 11422 continue; 11423 TopCaseProb = CC.Prob; 11424 PeeledCaseIndex = Index; 11425 SwitchPeeled = true; 11426 } 11427 if (!SwitchPeeled) 11428 return SwitchMBB; 11429 11430 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: " 11431 << TopCaseProb << "\n"); 11432 11433 // Record the MBB for the peeled switch statement. 11434 MachineFunction::iterator BBI(SwitchMBB); 11435 ++BBI; 11436 MachineBasicBlock *PeeledSwitchMBB = 11437 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock()); 11438 FuncInfo.MF->insert(BBI, PeeledSwitchMBB); 11439 11440 ExportFromCurrentBlock(SI.getCondition()); 11441 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex; 11442 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt, 11443 nullptr, nullptr, TopCaseProb.getCompl()}; 11444 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB); 11445 11446 Clusters.erase(PeeledCaseIt); 11447 for (CaseCluster &CC : Clusters) { 11448 LLVM_DEBUG( 11449 dbgs() << "Scale the probablity for one cluster, before scaling: " 11450 << CC.Prob << "\n"); 11451 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb); 11452 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n"); 11453 } 11454 PeeledCaseProb = TopCaseProb; 11455 return PeeledSwitchMBB; 11456 } 11457 11458 void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { 11459 // Extract cases from the switch. 11460 BranchProbabilityInfo *BPI = FuncInfo.BPI; 11461 CaseClusterVector Clusters; 11462 Clusters.reserve(SI.getNumCases()); 11463 for (auto I : SI.cases()) { 11464 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()]; 11465 const ConstantInt *CaseVal = I.getCaseValue(); 11466 BranchProbability Prob = 11467 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex()) 11468 : BranchProbability(1, SI.getNumCases() + 1); 11469 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob)); 11470 } 11471 11472 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()]; 11473 11474 // Cluster adjacent cases with the same destination. We do this at all 11475 // optimization levels because it's cheap to do and will make codegen faster 11476 // if there are many clusters. 11477 sortAndRangeify(Clusters); 11478 11479 // The branch probablity of the peeled case. 11480 BranchProbability PeeledCaseProb = BranchProbability::getZero(); 11481 MachineBasicBlock *PeeledSwitchMBB = 11482 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb); 11483 11484 // If there is only the default destination, jump there directly. 11485 MachineBasicBlock *SwitchMBB = FuncInfo.MBB; 11486 if (Clusters.empty()) { 11487 assert(PeeledSwitchMBB == SwitchMBB); 11488 SwitchMBB->addSuccessor(DefaultMBB); 11489 if (DefaultMBB != NextBlock(SwitchMBB)) { 11490 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, 11491 getControlRoot(), DAG.getBasicBlock(DefaultMBB))); 11492 } 11493 return; 11494 } 11495 11496 SL->findJumpTables(Clusters, &SI, DefaultMBB, DAG.getPSI(), DAG.getBFI()); 11497 SL->findBitTestClusters(Clusters, &SI); 11498 11499 LLVM_DEBUG({ 11500 dbgs() << "Case clusters: "; 11501 for (const CaseCluster &C : Clusters) { 11502 if (C.Kind == CC_JumpTable) 11503 dbgs() << "JT:"; 11504 if (C.Kind == CC_BitTests) 11505 dbgs() << "BT:"; 11506 11507 C.Low->getValue().print(dbgs(), true); 11508 if (C.Low != C.High) { 11509 dbgs() << '-'; 11510 C.High->getValue().print(dbgs(), true); 11511 } 11512 dbgs() << ' '; 11513 } 11514 dbgs() << '\n'; 11515 }); 11516 11517 assert(!Clusters.empty()); 11518 SwitchWorkList WorkList; 11519 CaseClusterIt First = Clusters.begin(); 11520 CaseClusterIt Last = Clusters.end() - 1; 11521 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB); 11522 // Scale the branchprobability for DefaultMBB if the peel occurs and 11523 // DefaultMBB is not replaced. 11524 if (PeeledCaseProb != BranchProbability::getZero() && 11525 DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()]) 11526 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb); 11527 WorkList.push_back( 11528 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb}); 11529 11530 while (!WorkList.empty()) { 11531 SwitchWorkListItem W = WorkList.pop_back_val(); 11532 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1; 11533 11534 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None && 11535 !DefaultMBB->getParent()->getFunction().hasMinSize()) { 11536 // For optimized builds, lower large range as a balanced binary tree. 11537 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB); 11538 continue; 11539 } 11540 11541 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB); 11542 } 11543 } 11544 11545 void SelectionDAGBuilder::visitStepVector(const CallInst &I) { 11546 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11547 auto DL = getCurSDLoc(); 11548 EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 11549 setValue(&I, DAG.getStepVector(DL, ResultVT)); 11550 } 11551 11552 void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { 11553 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11554 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 11555 11556 SDLoc DL = getCurSDLoc(); 11557 SDValue V = getValue(I.getOperand(0)); 11558 assert(VT == V.getValueType() && "Malformed vector.reverse!"); 11559 11560 if (VT.isScalableVector()) { 11561 setValue(&I, DAG.getNode(ISD::VECTOR_REVERSE, DL, VT, V)); 11562 return; 11563 } 11564 11565 // Use VECTOR_SHUFFLE for the fixed-length vector 11566 // to maintain existing behavior. 11567 SmallVector<int, 8> Mask; 11568 unsigned NumElts = VT.getVectorMinNumElements(); 11569 for (unsigned i = 0; i != NumElts; ++i) 11570 Mask.push_back(NumElts - 1 - i); 11571 11572 setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask)); 11573 } 11574 11575 void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I) { 11576 auto DL = getCurSDLoc(); 11577 SDValue InVec = getValue(I.getOperand(0)); 11578 EVT OutVT = 11579 InVec.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); 11580 11581 unsigned OutNumElts = OutVT.getVectorMinNumElements(); 11582 11583 // ISD Node needs the input vectors split into two equal parts 11584 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, 11585 DAG.getVectorIdxConstant(0, DL)); 11586 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OutVT, InVec, 11587 DAG.getVectorIdxConstant(OutNumElts, DL)); 11588 11589 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing 11590 // legalisation and combines. 11591 if (OutVT.isFixedLengthVector()) { 11592 SDValue Even = DAG.getVectorShuffle(OutVT, DL, Lo, Hi, 11593 createStrideMask(0, 2, OutNumElts)); 11594 SDValue Odd = DAG.getVectorShuffle(OutVT, DL, Lo, Hi, 11595 createStrideMask(1, 2, OutNumElts)); 11596 SDValue Res = DAG.getMergeValues({Even, Odd}, getCurSDLoc()); 11597 setValue(&I, Res); 11598 return; 11599 } 11600 11601 SDValue Res = DAG.getNode(ISD::VECTOR_DEINTERLEAVE, DL, 11602 DAG.getVTList(OutVT, OutVT), Lo, Hi); 11603 setValue(&I, Res); 11604 return; 11605 } 11606 11607 void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I) { 11608 auto DL = getCurSDLoc(); 11609 EVT InVT = getValue(I.getOperand(0)).getValueType(); 11610 SDValue InVec0 = getValue(I.getOperand(0)); 11611 SDValue InVec1 = getValue(I.getOperand(1)); 11612 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11613 EVT OutVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 11614 11615 // Use VECTOR_SHUFFLE for fixed-length vectors to benefit from existing 11616 // legalisation and combines. 11617 if (OutVT.isFixedLengthVector()) { 11618 unsigned NumElts = InVT.getVectorMinNumElements(); 11619 SDValue V = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, InVec0, InVec1); 11620 setValue(&I, DAG.getVectorShuffle(OutVT, DL, V, DAG.getUNDEF(OutVT), 11621 createInterleaveMask(NumElts, 2))); 11622 return; 11623 } 11624 11625 SDValue Res = DAG.getNode(ISD::VECTOR_INTERLEAVE, DL, 11626 DAG.getVTList(InVT, InVT), InVec0, InVec1); 11627 Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, OutVT, Res.getValue(0), 11628 Res.getValue(1)); 11629 setValue(&I, Res); 11630 return; 11631 } 11632 11633 void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) { 11634 SmallVector<EVT, 4> ValueVTs; 11635 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(), 11636 ValueVTs); 11637 unsigned NumValues = ValueVTs.size(); 11638 if (NumValues == 0) return; 11639 11640 SmallVector<SDValue, 4> Values(NumValues); 11641 SDValue Op = getValue(I.getOperand(0)); 11642 11643 for (unsigned i = 0; i != NumValues; ++i) 11644 Values[i] = DAG.getNode(ISD::FREEZE, getCurSDLoc(), ValueVTs[i], 11645 SDValue(Op.getNode(), Op.getResNo() + i)); 11646 11647 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 11648 DAG.getVTList(ValueVTs), Values)); 11649 } 11650 11651 void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) { 11652 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11653 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); 11654 11655 SDLoc DL = getCurSDLoc(); 11656 SDValue V1 = getValue(I.getOperand(0)); 11657 SDValue V2 = getValue(I.getOperand(1)); 11658 int64_t Imm = cast<ConstantInt>(I.getOperand(2))->getSExtValue(); 11659 11660 // VECTOR_SHUFFLE doesn't support a scalable mask so use a dedicated node. 11661 if (VT.isScalableVector()) { 11662 MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout()); 11663 setValue(&I, DAG.getNode(ISD::VECTOR_SPLICE, DL, VT, V1, V2, 11664 DAG.getConstant(Imm, DL, IdxVT))); 11665 return; 11666 } 11667 11668 unsigned NumElts = VT.getVectorNumElements(); 11669 11670 uint64_t Idx = (NumElts + Imm) % NumElts; 11671 11672 // Use VECTOR_SHUFFLE to maintain original behaviour for fixed-length vectors. 11673 SmallVector<int, 8> Mask; 11674 for (unsigned i = 0; i < NumElts; ++i) 11675 Mask.push_back(Idx + i); 11676 setValue(&I, DAG.getVectorShuffle(VT, DL, V1, V2, Mask)); 11677 } 11678 11679 // Consider the following MIR after SelectionDAG, which produces output in 11680 // phyregs in the first case or virtregs in the second case. 11681 // 11682 // INLINEASM_BR ..., implicit-def $ebx, ..., implicit-def $edx 11683 // %5:gr32 = COPY $ebx 11684 // %6:gr32 = COPY $edx 11685 // %1:gr32 = COPY %6:gr32 11686 // %0:gr32 = COPY %5:gr32 11687 // 11688 // INLINEASM_BR ..., def %5:gr32, ..., def %6:gr32 11689 // %1:gr32 = COPY %6:gr32 11690 // %0:gr32 = COPY %5:gr32 11691 // 11692 // Given %0, we'd like to return $ebx in the first case and %5 in the second. 11693 // Given %1, we'd like to return $edx in the first case and %6 in the second. 11694 // 11695 // If a callbr has outputs, it will have a single mapping in FuncInfo.ValueMap 11696 // to a single virtreg (such as %0). The remaining outputs monotonically 11697 // increase in virtreg number from there. If a callbr has no outputs, then it 11698 // should not have a corresponding callbr landingpad; in fact, the callbr 11699 // landingpad would not even be able to refer to such a callbr. 11700 static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) { 11701 MachineInstr *MI = MRI.def_begin(Reg)->getParent(); 11702 // There is definitely at least one copy. 11703 assert(MI->getOpcode() == TargetOpcode::COPY && 11704 "start of copy chain MUST be COPY"); 11705 Reg = MI->getOperand(1).getReg(); 11706 MI = MRI.def_begin(Reg)->getParent(); 11707 // There may be an optional second copy. 11708 if (MI->getOpcode() == TargetOpcode::COPY) { 11709 assert(Reg.isVirtual() && "expected COPY of virtual register"); 11710 Reg = MI->getOperand(1).getReg(); 11711 assert(Reg.isPhysical() && "expected COPY of physical register"); 11712 MI = MRI.def_begin(Reg)->getParent(); 11713 } 11714 // The start of the chain must be an INLINEASM_BR. 11715 assert(MI->getOpcode() == TargetOpcode::INLINEASM_BR && 11716 "end of copy chain MUST be INLINEASM_BR"); 11717 return Reg; 11718 } 11719 11720 // We must do this walk rather than the simpler 11721 // setValue(&I, getCopyFromRegs(CBR, CBR->getType())); 11722 // otherwise we will end up with copies of virtregs only valid along direct 11723 // edges. 11724 void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) { 11725 SmallVector<EVT, 8> ResultVTs; 11726 SmallVector<SDValue, 8> ResultValues; 11727 const auto *CBR = 11728 cast<CallBrInst>(I.getParent()->getUniquePredecessor()->getTerminator()); 11729 11730 const TargetLowering &TLI = DAG.getTargetLoweringInfo(); 11731 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo(); 11732 MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo(); 11733 11734 unsigned InitialDef = FuncInfo.ValueMap[CBR]; 11735 SDValue Chain = DAG.getRoot(); 11736 11737 // Re-parse the asm constraints string. 11738 TargetLowering::AsmOperandInfoVector TargetConstraints = 11739 TLI.ParseConstraints(DAG.getDataLayout(), TRI, *CBR); 11740 for (auto &T : TargetConstraints) { 11741 SDISelAsmOperandInfo OpInfo(T); 11742 if (OpInfo.Type != InlineAsm::isOutput) 11743 continue; 11744 11745 // Pencil in OpInfo.ConstraintType and OpInfo.ConstraintVT based on the 11746 // individual constraint. 11747 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG); 11748 11749 switch (OpInfo.ConstraintType) { 11750 case TargetLowering::C_Register: 11751 case TargetLowering::C_RegisterClass: { 11752 // Fill in OpInfo.AssignedRegs.Regs. 11753 getRegistersForValue(DAG, getCurSDLoc(), OpInfo, OpInfo); 11754 11755 // getRegistersForValue may produce 1 to many registers based on whether 11756 // the OpInfo.ConstraintVT is legal on the target or not. 11757 for (size_t i = 0, e = OpInfo.AssignedRegs.Regs.size(); i != e; ++i) { 11758 Register OriginalDef = FollowCopyChain(MRI, InitialDef++); 11759 if (Register::isPhysicalRegister(OriginalDef)) 11760 FuncInfo.MBB->addLiveIn(OriginalDef); 11761 // Update the assigned registers to use the original defs. 11762 OpInfo.AssignedRegs.Regs[i] = OriginalDef; 11763 } 11764 11765 SDValue V = OpInfo.AssignedRegs.getCopyFromRegs( 11766 DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, CBR); 11767 ResultValues.push_back(V); 11768 ResultVTs.push_back(OpInfo.ConstraintVT); 11769 break; 11770 } 11771 case TargetLowering::C_Other: { 11772 SDValue Flag; 11773 SDValue V = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(), 11774 OpInfo, DAG); 11775 ++InitialDef; 11776 ResultValues.push_back(V); 11777 ResultVTs.push_back(OpInfo.ConstraintVT); 11778 break; 11779 } 11780 default: 11781 break; 11782 } 11783 } 11784 SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), 11785 DAG.getVTList(ResultVTs), ResultValues); 11786 setValue(&I, V); 11787 } 11788