1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG)); 53 SDValue R = SDValue(); 54 55 switch (N->getOpcode()) { 56 // clang-format off 57 default: 58 #ifndef NDEBUG 59 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 60 N->dump(&DAG); dbgs() << "\n"; 61 #endif 62 report_fatal_error("Do not know how to soften the result of this " 63 "operator!"); 64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break; 65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; 66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 70 case ISD::EXTRACT_VECTOR_ELT: 71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 73 case ISD::STRICT_FMINNUM: 74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 75 case ISD::STRICT_FMAXNUM: 76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 77 case ISD::STRICT_FADD: 78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 79 case ISD::STRICT_FACOS: 80 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break; 81 case ISD::STRICT_FASIN: 82 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break; 83 case ISD::STRICT_FATAN: 84 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break; 85 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 86 case ISD::STRICT_FCEIL: 87 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 88 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 89 case ISD::STRICT_FCOS: 90 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 91 case ISD::STRICT_FCOSH: 92 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break; 93 case ISD::STRICT_FDIV: 94 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 95 case ISD::STRICT_FEXP: 96 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 97 case ISD::STRICT_FEXP2: 98 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 99 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break; 100 case ISD::STRICT_FFLOOR: 101 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 102 case ISD::STRICT_FLOG: 103 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 104 case ISD::STRICT_FLOG2: 105 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 106 case ISD::STRICT_FLOG10: 107 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 108 case ISD::STRICT_FMA: 109 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 110 case ISD::STRICT_FMUL: 111 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 112 case ISD::STRICT_FNEARBYINT: 113 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 114 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 115 case ISD::STRICT_FP_EXTEND: 116 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 117 case ISD::STRICT_FP_ROUND: 118 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 119 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 120 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break; 121 case ISD::STRICT_FPOW: 122 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 123 case ISD::STRICT_FPOWI: 124 case ISD::FPOWI: 125 case ISD::FLDEXP: 126 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break; 127 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break; 128 case ISD::STRICT_FREM: 129 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 130 case ISD::STRICT_FRINT: 131 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 132 case ISD::STRICT_FROUND: 133 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 134 case ISD::STRICT_FROUNDEVEN: 135 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 136 case ISD::STRICT_FSIN: 137 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 138 case ISD::STRICT_FSINH: 139 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break; 140 case ISD::STRICT_FSQRT: 141 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 142 case ISD::STRICT_FSUB: 143 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 144 case ISD::STRICT_FTAN: 145 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break; 146 case ISD::STRICT_FTANH: 147 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break; 148 case ISD::STRICT_FTRUNC: 149 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 150 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 151 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break; 152 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 153 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 154 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 155 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 156 case ISD::STRICT_SINT_TO_FP: 157 case ISD::STRICT_UINT_TO_FP: 158 case ISD::SINT_TO_FP: 159 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 160 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 161 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 162 case ISD::VECREDUCE_FADD: 163 case ISD::VECREDUCE_FMUL: 164 case ISD::VECREDUCE_FMIN: 165 case ISD::VECREDUCE_FMAX: 166 case ISD::VECREDUCE_FMAXIMUM: 167 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break; 168 case ISD::VECREDUCE_SEQ_FADD: 169 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break; 170 // clang-format on 171 } 172 173 // If R is null, the sub-method took care of registering the result. 174 if (R.getNode()) { 175 assert(R.getNode() != N); 176 SetSoftenedFloat(SDValue(N, ResNo), R); 177 } 178 } 179 180 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 181 bool IsStrict = N->isStrictFPOpcode(); 182 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 183 unsigned Offset = IsStrict ? 1 : 0; 184 assert(N->getNumOperands() == (1 + Offset) && 185 "Unexpected number of operands!"); 186 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 187 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 188 TargetLowering::MakeLibCallOptions CallOptions; 189 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 190 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 191 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 192 CallOptions, SDLoc(N), 193 Chain); 194 if (IsStrict) 195 ReplaceValueWith(SDValue(N, 1), Tmp.second); 196 return Tmp.first; 197 } 198 199 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 200 bool IsStrict = N->isStrictFPOpcode(); 201 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 202 unsigned Offset = IsStrict ? 1 : 0; 203 assert(N->getNumOperands() == (2 + Offset) && 204 "Unexpected number of operands!"); 205 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 206 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 207 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 208 TargetLowering::MakeLibCallOptions CallOptions; 209 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 210 N->getOperand(1 + Offset).getValueType() }; 211 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 212 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 213 CallOptions, SDLoc(N), 214 Chain); 215 if (IsStrict) 216 ReplaceValueWith(SDValue(N, 1), Tmp.second); 217 return Tmp.first; 218 } 219 220 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 221 return BitConvertToInteger(N->getOperand(0)); 222 } 223 224 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 225 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 226 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 227 GetSoftenedFloat(N->getOperand(0))); 228 } 229 230 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { 231 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 232 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty, 233 GetSoftenedFloat(N->getOperand(0))); 234 return NewFence; 235 } 236 237 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 238 unsigned ResNo) { 239 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 240 return BitConvertToInteger(Op); 241 } 242 243 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 244 // Convert the inputs to integers, and build a new pair out of them. 245 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 246 TLI.getTypeToTransformTo(*DAG.getContext(), 247 N->getValueType(0)), 248 BitConvertToInteger(N->getOperand(0)), 249 BitConvertToInteger(N->getOperand(1))); 250 } 251 252 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 253 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 254 // In ppcf128, the high 64 bits are always first in memory regardless 255 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 256 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 257 // way. However, APInt's are serialized in an Endian-sensitive fashion, 258 // so on big-Endian targets, the two doubles are output in the wrong 259 // order. Fix this by manually flipping the order of the high 64 bits 260 // and the low 64 bits here. 261 if (DAG.getDataLayout().isBigEndian() && 262 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 263 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 264 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 265 APInt Val(128, words); 266 return DAG.getConstant(Val, SDLoc(CN), 267 TLI.getTypeToTransformTo(*DAG.getContext(), 268 CN->getValueType(0))); 269 } else { 270 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 271 TLI.getTypeToTransformTo(*DAG.getContext(), 272 CN->getValueType(0))); 273 } 274 } 275 276 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) { 277 SDValue Src = N->getOperand(0); 278 assert(Src.getValueType() == MVT::ppcf128 && 279 "In floats only ppcf128 can be extracted by element!"); 280 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N), 281 N->getValueType(0).changeTypeToInteger(), 282 DAG.getBitcast(MVT::i128, Src), N->getOperand(1)); 283 } 284 285 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 286 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 287 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 288 NewOp.getValueType().getVectorElementType(), 289 NewOp, N->getOperand(1)); 290 } 291 292 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 293 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 294 unsigned Size = NVT.getSizeInBits(); 295 296 // Mask = ~(1 << (Size-1)) 297 APInt API = APInt::getAllOnes(Size); 298 API.clearBit(Size - 1); 299 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 300 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 301 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 302 } 303 304 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 305 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 306 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 308 RTLIB::FMIN_F32, 309 RTLIB::FMIN_F64, 310 RTLIB::FMIN_F80, 311 RTLIB::FMIN_F128, 312 RTLIB::FMIN_PPCF128)); 313 } 314 315 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 316 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 317 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 318 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 319 RTLIB::FMAX_F32, 320 RTLIB::FMAX_F64, 321 RTLIB::FMAX_F80, 322 RTLIB::FMAX_F128, 323 RTLIB::FMAX_PPCF128)); 324 } 325 326 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 327 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 328 RTLIB::ADD_F32, 329 RTLIB::ADD_F64, 330 RTLIB::ADD_F80, 331 RTLIB::ADD_F128, 332 RTLIB::ADD_PPCF128)); 333 } 334 335 SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) { 336 return SoftenFloatRes_Unary( 337 N, GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64, 338 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128)); 339 } 340 341 SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) { 342 return SoftenFloatRes_Unary( 343 N, GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64, 344 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128)); 345 } 346 347 SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) { 348 return SoftenFloatRes_Unary( 349 N, GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64, 350 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128)); 351 } 352 353 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 354 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 355 RTLIB::CBRT_F32, 356 RTLIB::CBRT_F64, 357 RTLIB::CBRT_F80, 358 RTLIB::CBRT_F128, 359 RTLIB::CBRT_PPCF128)); 360 } 361 362 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 363 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 364 RTLIB::CEIL_F32, 365 RTLIB::CEIL_F64, 366 RTLIB::CEIL_F80, 367 RTLIB::CEIL_F128, 368 RTLIB::CEIL_PPCF128)); 369 } 370 371 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 372 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 373 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 374 SDLoc dl(N); 375 376 EVT LVT = LHS.getValueType(); 377 EVT RVT = RHS.getValueType(); 378 379 unsigned LSize = LVT.getSizeInBits(); 380 unsigned RSize = RVT.getSizeInBits(); 381 382 // First get the sign bit of second operand. 383 SDValue SignBit = DAG.getNode( 384 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 385 DAG.getConstant(RSize - 1, dl, 386 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 387 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 388 389 // Shift right or sign-extend it if the two operands have different types. 390 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 391 if (SizeDiff > 0) { 392 SignBit = 393 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 394 DAG.getConstant(SizeDiff, dl, 395 TLI.getShiftAmountTy(SignBit.getValueType(), 396 DAG.getDataLayout()))); 397 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 398 } else if (SizeDiff < 0) { 399 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 400 SignBit = 401 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 402 DAG.getConstant(-SizeDiff, dl, 403 TLI.getShiftAmountTy(SignBit.getValueType(), 404 DAG.getDataLayout()))); 405 } 406 407 // Clear the sign bit of the first operand. 408 SDValue Mask = DAG.getNode( 409 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 410 DAG.getConstant(LSize - 1, dl, 411 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 412 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 413 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 414 415 // Or the value with the sign bit. 416 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 417 } 418 419 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 420 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 421 RTLIB::COS_F32, 422 RTLIB::COS_F64, 423 RTLIB::COS_F80, 424 RTLIB::COS_F128, 425 RTLIB::COS_PPCF128)); 426 } 427 428 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) { 429 return SoftenFloatRes_Unary( 430 N, GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64, 431 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128)); 432 } 433 434 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 435 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 436 RTLIB::DIV_F32, 437 RTLIB::DIV_F64, 438 RTLIB::DIV_F80, 439 RTLIB::DIV_F128, 440 RTLIB::DIV_PPCF128)); 441 } 442 443 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 444 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 445 RTLIB::EXP_F32, 446 RTLIB::EXP_F64, 447 RTLIB::EXP_F80, 448 RTLIB::EXP_F128, 449 RTLIB::EXP_PPCF128)); 450 } 451 452 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 453 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 454 RTLIB::EXP2_F32, 455 RTLIB::EXP2_F64, 456 RTLIB::EXP2_F80, 457 RTLIB::EXP2_F128, 458 RTLIB::EXP2_PPCF128)); 459 } 460 461 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) { 462 return SoftenFloatRes_Unary( 463 N, 464 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64, 465 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128)); 466 } 467 468 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 469 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 470 RTLIB::FLOOR_F32, 471 RTLIB::FLOOR_F64, 472 RTLIB::FLOOR_F80, 473 RTLIB::FLOOR_F128, 474 RTLIB::FLOOR_PPCF128)); 475 } 476 477 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 478 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 479 RTLIB::LOG_F32, 480 RTLIB::LOG_F64, 481 RTLIB::LOG_F80, 482 RTLIB::LOG_F128, 483 RTLIB::LOG_PPCF128)); 484 } 485 486 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 487 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 488 RTLIB::LOG2_F32, 489 RTLIB::LOG2_F64, 490 RTLIB::LOG2_F80, 491 RTLIB::LOG2_F128, 492 RTLIB::LOG2_PPCF128)); 493 } 494 495 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 497 RTLIB::LOG10_F32, 498 RTLIB::LOG10_F64, 499 RTLIB::LOG10_F80, 500 RTLIB::LOG10_F128, 501 RTLIB::LOG10_PPCF128)); 502 } 503 504 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 505 bool IsStrict = N->isStrictFPOpcode(); 506 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 507 unsigned Offset = IsStrict ? 1 : 0; 508 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 509 GetSoftenedFloat(N->getOperand(1 + Offset)), 510 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 511 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 512 TargetLowering::MakeLibCallOptions CallOptions; 513 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 514 N->getOperand(1 + Offset).getValueType(), 515 N->getOperand(2 + Offset).getValueType() }; 516 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 517 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 518 GetFPLibCall(N->getValueType(0), 519 RTLIB::FMA_F32, 520 RTLIB::FMA_F64, 521 RTLIB::FMA_F80, 522 RTLIB::FMA_F128, 523 RTLIB::FMA_PPCF128), 524 NVT, Ops, CallOptions, SDLoc(N), Chain); 525 if (IsStrict) 526 ReplaceValueWith(SDValue(N, 1), Tmp.second); 527 return Tmp.first; 528 } 529 530 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 531 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 532 RTLIB::MUL_F32, 533 RTLIB::MUL_F64, 534 RTLIB::MUL_F80, 535 RTLIB::MUL_F128, 536 RTLIB::MUL_PPCF128)); 537 } 538 539 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 540 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 541 RTLIB::NEARBYINT_F32, 542 RTLIB::NEARBYINT_F64, 543 RTLIB::NEARBYINT_F80, 544 RTLIB::NEARBYINT_F128, 545 RTLIB::NEARBYINT_PPCF128)); 546 } 547 548 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 549 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 550 SDLoc dl(N); 551 552 // Expand Y = FNEG(X) -> Y = X ^ sign mask 553 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 554 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 555 DAG.getConstant(SignMask, dl, NVT)); 556 } 557 558 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 559 bool IsStrict = N->isStrictFPOpcode(); 560 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 561 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 562 563 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 564 565 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 566 Op = GetPromotedFloat(Op); 567 // If the promotion did the FP_EXTEND to the destination type for us, 568 // there's nothing left to do here. 569 if (Op.getValueType() == N->getValueType(0)) { 570 if (IsStrict) 571 ReplaceValueWith(SDValue(N, 1), Chain); 572 return BitConvertToInteger(Op); 573 } 574 } 575 576 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16 577 // -> f32, so proceed in two stages. Also, it's entirely possible for both 578 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather 579 // than FP16_TO_FP. 580 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) && 581 N->getValueType(0) != MVT::f32) { 582 if (IsStrict) { 583 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 584 { MVT::f32, MVT::Other }, { Chain, Op }); 585 Chain = Op.getValue(1); 586 } else { 587 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 588 } 589 } 590 591 if (Op.getValueType() == MVT::bf16) { 592 // FIXME: Need ReplaceValueWith on chain in strict case 593 return SoftenFloatRes_BF16_TO_FP(N); 594 } 595 596 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 597 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 598 TargetLowering::MakeLibCallOptions CallOptions; 599 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 600 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 601 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 602 CallOptions, SDLoc(N), 603 Chain); 604 if (IsStrict) 605 ReplaceValueWith(SDValue(N, 1), Tmp.second); 606 return Tmp.first; 607 } 608 609 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 610 // nodes? 611 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 612 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 613 SDValue Op = N->getOperand(0); 614 TargetLowering::MakeLibCallOptions CallOptions; 615 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 616 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 617 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 618 CallOptions, SDLoc(N)).first; 619 if (N->getValueType(0) == MVT::f32) 620 return Res32; 621 622 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 623 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 624 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 625 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 626 } 627 628 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 629 // nodes? 630 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) { 631 assert(N->getValueType(0) == MVT::f32 && 632 "Can only soften BF16_TO_FP with f32 result"); 633 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 634 SDValue Op = N->getOperand(0); 635 SDLoc DL(N); 636 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT, 637 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op)); 638 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op, 639 DAG.getShiftAmountConstant(16, NVT, DL)); 640 return Res; 641 } 642 643 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 644 bool IsStrict = N->isStrictFPOpcode(); 645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 646 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 647 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 648 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 649 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 650 TargetLowering::MakeLibCallOptions CallOptions; 651 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 652 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 653 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 654 CallOptions, SDLoc(N), 655 Chain); 656 if (IsStrict) 657 ReplaceValueWith(SDValue(N, 1), Tmp.second); 658 return Tmp.first; 659 } 660 661 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 662 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 663 RTLIB::POW_F32, 664 RTLIB::POW_F64, 665 RTLIB::POW_F80, 666 RTLIB::POW_F128, 667 RTLIB::POW_PPCF128)); 668 } 669 670 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) { 671 bool IsStrict = N->isStrictFPOpcode(); 672 unsigned Offset = IsStrict ? 1 : 0; 673 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 674 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 675 "Unsupported power type!"); 676 bool IsPowI = 677 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; 678 679 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) 680 : RTLIB::getLDEXP(N->getValueType(0)); 681 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 682 if (!TLI.getLibcallName(LC)) { 683 // Some targets don't have a powi libcall; use pow instead. 684 // FIXME: Implement this if some target needs it. 685 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 686 return DAG.getUNDEF(N->getValueType(0)); 687 } 688 689 if (DAG.getLibInfo().getIntSize() != 690 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 691 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 692 // would use the wrong type for the argument. 693 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 694 return DAG.getUNDEF(N->getValueType(0)); 695 } 696 697 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 698 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 699 N->getOperand(1 + Offset) }; 700 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 701 TargetLowering::MakeLibCallOptions CallOptions; 702 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 703 N->getOperand(1 + Offset).getValueType() }; 704 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 705 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 706 CallOptions, SDLoc(N), 707 Chain); 708 if (IsStrict) 709 ReplaceValueWith(SDValue(N, 1), Tmp.second); 710 return Tmp.first; 711 } 712 713 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) { 714 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp"); 715 EVT VT0 = N->getValueType(0); 716 EVT VT1 = N->getValueType(1); 717 RTLIB::Libcall LC = RTLIB::getFREXP(VT0); 718 719 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) { 720 // If the exponent does not match with sizeof(int) a libcall would use the 721 // wrong type for the argument. 722 // TODO: Should be able to handle mismatches. 723 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)"); 724 return DAG.getUNDEF(N->getValueType(0)); 725 } 726 727 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0); 728 SDValue StackSlot = DAG.CreateStackTemporary(VT1); 729 730 SDLoc DL(N); 731 732 TargetLowering::MakeLibCallOptions CallOptions; 733 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot}; 734 EVT OpsVT[2] = {VT0, StackSlot.getValueType()}; 735 736 // TODO: setTypeListBeforeSoften can't properly express multiple return types, 737 // but we only really need to handle the 0th one for softening anyway. 738 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true); 739 740 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL, 741 /*Chain=*/SDValue()); 742 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 743 auto PtrInfo = 744 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 745 746 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo); 747 748 ReplaceValueWith(SDValue(N, 1), LoadExp); 749 return ReturnVal; 750 } 751 752 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 753 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 754 RTLIB::REM_F32, 755 RTLIB::REM_F64, 756 RTLIB::REM_F80, 757 RTLIB::REM_F128, 758 RTLIB::REM_PPCF128)); 759 } 760 761 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 762 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 763 RTLIB::RINT_F32, 764 RTLIB::RINT_F64, 765 RTLIB::RINT_F80, 766 RTLIB::RINT_F128, 767 RTLIB::RINT_PPCF128)); 768 } 769 770 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 771 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 772 RTLIB::ROUND_F32, 773 RTLIB::ROUND_F64, 774 RTLIB::ROUND_F80, 775 RTLIB::ROUND_F128, 776 RTLIB::ROUND_PPCF128)); 777 } 778 779 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 780 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 781 RTLIB::ROUNDEVEN_F32, 782 RTLIB::ROUNDEVEN_F64, 783 RTLIB::ROUNDEVEN_F80, 784 RTLIB::ROUNDEVEN_F128, 785 RTLIB::ROUNDEVEN_PPCF128)); 786 } 787 788 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 789 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 790 RTLIB::SIN_F32, 791 RTLIB::SIN_F64, 792 RTLIB::SIN_F80, 793 RTLIB::SIN_F128, 794 RTLIB::SIN_PPCF128)); 795 } 796 797 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) { 798 return SoftenFloatRes_Unary( 799 N, GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64, 800 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128)); 801 } 802 803 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 804 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 805 RTLIB::SQRT_F32, 806 RTLIB::SQRT_F64, 807 RTLIB::SQRT_F80, 808 RTLIB::SQRT_F128, 809 RTLIB::SQRT_PPCF128)); 810 } 811 812 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 813 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 814 RTLIB::SUB_F32, 815 RTLIB::SUB_F64, 816 RTLIB::SUB_F80, 817 RTLIB::SUB_F128, 818 RTLIB::SUB_PPCF128)); 819 } 820 821 SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) { 822 return SoftenFloatRes_Unary( 823 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64, 824 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128)); 825 } 826 827 SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) { 828 return SoftenFloatRes_Unary( 829 N, GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64, 830 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128)); 831 } 832 833 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 834 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 835 RTLIB::TRUNC_F32, 836 RTLIB::TRUNC_F64, 837 RTLIB::TRUNC_F80, 838 RTLIB::TRUNC_F128, 839 RTLIB::TRUNC_PPCF128)); 840 } 841 842 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 843 LoadSDNode *L = cast<LoadSDNode>(N); 844 EVT VT = N->getValueType(0); 845 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 846 SDLoc dl(N); 847 848 auto MMOFlags = 849 L->getMemOperand()->getFlags() & 850 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 851 SDValue NewL; 852 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 853 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 854 L->getChain(), L->getBasePtr(), L->getOffset(), 855 L->getPointerInfo(), NVT, L->getOriginalAlign(), 856 MMOFlags, L->getAAInfo()); 857 // Legalized the chain result - switch anything that used the old chain to 858 // use the new one. 859 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 860 return NewL; 861 } 862 863 // Do a non-extending load followed by FP_EXTEND. 864 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 865 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 866 L->getPointerInfo(), L->getMemoryVT(), 867 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 868 // Legalized the chain result - switch anything that used the old chain to 869 // use the new one. 870 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 871 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 872 return BitConvertToInteger(ExtendNode); 873 } 874 875 SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) { 876 AtomicSDNode *L = cast<AtomicSDNode>(N); 877 EVT VT = N->getValueType(0); 878 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 879 SDLoc dl(N); 880 881 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 882 SDValue NewL = 883 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other), 884 {L->getChain(), L->getBasePtr()}, L->getMemOperand()); 885 886 // Legalized the chain result - switch anything that used the old chain to 887 // use the new one. 888 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 889 return NewL; 890 } 891 892 report_fatal_error("softening fp extending atomic load not handled"); 893 } 894 895 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 896 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 897 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 898 return DAG.getSelect(SDLoc(N), 899 LHS.getValueType(), N->getOperand(0), LHS, RHS); 900 } 901 902 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 903 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 904 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 905 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 906 LHS.getValueType(), N->getOperand(0), 907 N->getOperand(1), LHS, RHS, N->getOperand(4)); 908 } 909 910 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 911 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 912 N->getValueType(0))); 913 } 914 915 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 916 SDValue Chain = N->getOperand(0); // Get the chain. 917 SDValue Ptr = N->getOperand(1); // Get the pointer. 918 EVT VT = N->getValueType(0); 919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 920 SDLoc dl(N); 921 922 SDValue NewVAARG; 923 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 924 N->getConstantOperandVal(3)); 925 926 // Legalized the chain result - switch anything that used the old chain to 927 // use the new one. 928 if (N != NewVAARG.getValue(1).getNode()) 929 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 930 return NewVAARG; 931 } 932 933 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 934 bool IsStrict = N->isStrictFPOpcode(); 935 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 936 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 937 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 938 EVT RVT = N->getValueType(0); 939 EVT NVT = EVT(); 940 SDLoc dl(N); 941 942 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 943 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 944 // match. Look for an appropriate libcall. 945 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 946 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 947 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 948 NVT = (MVT::SimpleValueType)t; 949 // The source needs to big enough to hold the operand. 950 if (NVT.bitsGE(SVT)) 951 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 952 } 953 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 954 955 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 956 // Sign/zero extend the argument if the libcall takes a larger type. 957 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 958 NVT, N->getOperand(IsStrict ? 1 : 0)); 959 TargetLowering::MakeLibCallOptions CallOptions; 960 CallOptions.setSExt(Signed); 961 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 962 std::pair<SDValue, SDValue> Tmp = 963 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 964 Op, CallOptions, dl, Chain); 965 966 if (IsStrict) 967 ReplaceValueWith(SDValue(N, 1), Tmp.second); 968 return Tmp.first; 969 } 970 971 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 972 // Expand and soften recursively. 973 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 974 return SDValue(); 975 } 976 977 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 978 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 979 return SDValue(); 980 } 981 982 //===----------------------------------------------------------------------===// 983 // Convert Float Operand to Integer 984 //===----------------------------------------------------------------------===// 985 986 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 987 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG)); 988 SDValue Res = SDValue(); 989 990 switch (N->getOpcode()) { 991 default: 992 #ifndef NDEBUG 993 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 994 N->dump(&DAG); dbgs() << "\n"; 995 #endif 996 report_fatal_error("Do not know how to soften this operator's operand!"); 997 998 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 999 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 1000 case ISD::STRICT_FP_TO_FP16: 1001 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 1002 case ISD::FP_TO_BF16: 1003 case ISD::STRICT_FP_TO_BF16: 1004 case ISD::STRICT_FP_ROUND: 1005 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 1006 case ISD::STRICT_FP_TO_SINT: 1007 case ISD::STRICT_FP_TO_UINT: 1008 case ISD::FP_TO_SINT: 1009 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 1010 case ISD::FP_TO_SINT_SAT: 1011 case ISD::FP_TO_UINT_SAT: 1012 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 1013 case ISD::STRICT_LROUND: 1014 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 1015 case ISD::STRICT_LLROUND: 1016 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 1017 case ISD::STRICT_LRINT: 1018 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 1019 case ISD::STRICT_LLRINT: 1020 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 1021 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 1022 case ISD::STRICT_FSETCC: 1023 case ISD::STRICT_FSETCCS: 1024 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 1025 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 1026 case ISD::ATOMIC_STORE: 1027 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo); 1028 break; 1029 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 1030 } 1031 1032 // If the result is null, the sub-method took care of registering results etc. 1033 if (!Res.getNode()) return false; 1034 1035 // If the result is N, the sub-method updated N in place. Tell the legalizer 1036 // core about this to re-analyze. 1037 if (Res.getNode() == N) 1038 return true; 1039 1040 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1041 "Invalid operand softening"); 1042 1043 ReplaceValueWith(SDValue(N, 0), Res); 1044 return false; 1045 } 1046 1047 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 1048 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 1049 1050 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 1051 } 1052 1053 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 1054 // We actually deal with the partially-softened FP_TO_FP16 node too, which 1055 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 1056 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 1057 N->getOpcode() == ISD::STRICT_FP_TO_FP16 || 1058 N->getOpcode() == ISD::FP_TO_BF16 || 1059 N->getOpcode() == ISD::STRICT_FP_TO_BF16 || 1060 N->getOpcode() == ISD::STRICT_FP_ROUND); 1061 1062 bool IsStrict = N->isStrictFPOpcode(); 1063 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1064 EVT SVT = Op.getValueType(); 1065 EVT RVT = N->getValueType(0); 1066 EVT FloatRVT = RVT; 1067 if (N->getOpcode() == ISD::FP_TO_FP16 || 1068 N->getOpcode() == ISD::STRICT_FP_TO_FP16) 1069 FloatRVT = MVT::f16; 1070 else if (N->getOpcode() == ISD::FP_TO_BF16 || 1071 N->getOpcode() == ISD::STRICT_FP_TO_BF16) 1072 FloatRVT = MVT::bf16; 1073 1074 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 1075 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 1076 1077 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1078 Op = GetSoftenedFloat(Op); 1079 TargetLowering::MakeLibCallOptions CallOptions; 1080 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1081 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 1082 CallOptions, SDLoc(N), 1083 Chain); 1084 if (IsStrict) { 1085 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1086 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1087 return SDValue(); 1088 } 1089 return Tmp.first; 1090 } 1091 1092 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 1093 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1094 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1095 1096 EVT VT = NewLHS.getValueType(); 1097 NewLHS = GetSoftenedFloat(NewLHS); 1098 NewRHS = GetSoftenedFloat(NewRHS); 1099 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1100 N->getOperand(2), N->getOperand(3)); 1101 1102 // If softenSetCCOperands returned a scalar, we need to compare the result 1103 // against zero to select between true and false values. 1104 if (!NewRHS.getNode()) { 1105 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1106 CCCode = ISD::SETNE; 1107 } 1108 1109 // Update N to have the operands specified. 1110 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1111 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1112 N->getOperand(4)), 1113 0); 1114 } 1115 1116 // Even if the result type is legal, no libcall may exactly match. (e.g. We 1117 // don't have FP-i8 conversions) This helper method looks for an appropriate 1118 // promoted libcall. 1119 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 1120 bool Signed) { 1121 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1122 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 1123 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 1124 ++IntVT) { 1125 Promoted = (MVT::SimpleValueType)IntVT; 1126 // The type needs to big enough to hold the result. 1127 if (Promoted.bitsGE(RetVT)) 1128 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 1129 : RTLIB::getFPTOUINT(SrcVT, Promoted); 1130 } 1131 return LC; 1132 } 1133 1134 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 1135 bool IsStrict = N->isStrictFPOpcode(); 1136 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1137 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1138 1139 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1140 EVT SVT = Op.getValueType(); 1141 EVT RVT = N->getValueType(0); 1142 EVT NVT = EVT(); 1143 SDLoc dl(N); 1144 1145 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 1146 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 1147 // match, eg. we don't have fp -> i8 conversions. 1148 // Look for an appropriate libcall. 1149 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 1150 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1151 "Unsupported FP_TO_XINT!"); 1152 1153 Op = GetSoftenedFloat(Op); 1154 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1155 TargetLowering::MakeLibCallOptions CallOptions; 1156 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1157 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1158 CallOptions, dl, Chain); 1159 1160 // Truncate the result if the libcall returns a larger type. 1161 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 1162 1163 if (!IsStrict) 1164 return Res; 1165 1166 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1167 ReplaceValueWith(SDValue(N, 0), Res); 1168 return SDValue(); 1169 } 1170 1171 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 1172 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 1173 return Res; 1174 } 1175 1176 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 1177 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1178 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1179 1180 EVT VT = NewLHS.getValueType(); 1181 NewLHS = GetSoftenedFloat(NewLHS); 1182 NewRHS = GetSoftenedFloat(NewRHS); 1183 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1184 N->getOperand(0), N->getOperand(1)); 1185 1186 // If softenSetCCOperands returned a scalar, we need to compare the result 1187 // against zero to select between true and false values. 1188 if (!NewRHS.getNode()) { 1189 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1190 CCCode = ISD::SETNE; 1191 } 1192 1193 // Update N to have the operands specified. 1194 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1195 N->getOperand(2), N->getOperand(3), 1196 DAG.getCondCode(CCCode)), 1197 0); 1198 } 1199 1200 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1201 bool IsStrict = N->isStrictFPOpcode(); 1202 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1203 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1204 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1205 ISD::CondCode CCCode = 1206 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1207 1208 EVT VT = Op0.getValueType(); 1209 SDValue NewLHS = GetSoftenedFloat(Op0); 1210 SDValue NewRHS = GetSoftenedFloat(Op1); 1211 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1212 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1213 1214 // Update N to have the operands specified. 1215 if (NewRHS.getNode()) { 1216 if (IsStrict) 1217 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1218 NewRHS, DAG.getCondCode(CCCode)); 1219 else 1220 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1221 DAG.getCondCode(CCCode)), 0); 1222 } 1223 1224 // Otherwise, softenSetCCOperands returned a scalar, use it. 1225 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1226 "Unexpected setcc expansion!"); 1227 1228 if (IsStrict) { 1229 ReplaceValueWith(SDValue(N, 0), NewLHS); 1230 ReplaceValueWith(SDValue(N, 1), Chain); 1231 return SDValue(); 1232 } 1233 return NewLHS; 1234 } 1235 1236 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1237 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1238 assert(OpNo == 1 && "Can only soften the stored value!"); 1239 StoreSDNode *ST = cast<StoreSDNode>(N); 1240 SDValue Val = ST->getValue(); 1241 SDLoc dl(N); 1242 1243 if (ST->isTruncatingStore()) 1244 // Do an FP_ROUND followed by a non-truncating store. 1245 Val = BitConvertToInteger( 1246 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val, 1247 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); 1248 else 1249 Val = GetSoftenedFloat(Val); 1250 1251 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1252 ST->getMemOperand()); 1253 } 1254 1255 SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) { 1256 assert(OpNo == 1 && "Can only soften the stored value!"); 1257 AtomicSDNode *ST = cast<AtomicSDNode>(N); 1258 SDValue Val = ST->getVal(); 1259 EVT VT = Val.getValueType(); 1260 SDLoc dl(N); 1261 1262 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled"); 1263 1264 SDValue NewVal = GetSoftenedFloat(Val); 1265 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal, 1266 ST->getBasePtr(), ST->getMemOperand()); 1267 } 1268 1269 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1270 SDValue LHS = N->getOperand(0); 1271 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1272 SDLoc dl(N); 1273 1274 EVT LVT = LHS.getValueType(); 1275 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1276 EVT RVT = RHS.getValueType(); 1277 1278 unsigned LSize = LVT.getSizeInBits(); 1279 unsigned RSize = RVT.getSizeInBits(); 1280 1281 // Shift right or sign-extend it if the two operands have different types. 1282 int SizeDiff = RSize - LSize; 1283 if (SizeDiff > 0) { 1284 RHS = 1285 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1286 DAG.getConstant(SizeDiff, dl, 1287 TLI.getShiftAmountTy(RHS.getValueType(), 1288 DAG.getDataLayout()))); 1289 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1290 } else if (SizeDiff < 0) { 1291 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1292 RHS = 1293 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1294 DAG.getConstant(-SizeDiff, dl, 1295 TLI.getShiftAmountTy(RHS.getValueType(), 1296 DAG.getDataLayout()))); 1297 } 1298 1299 RHS = DAG.getBitcast(LVT, RHS); 1300 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1301 } 1302 1303 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1304 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1305 bool IsStrict = N->isStrictFPOpcode(); 1306 unsigned Offset = IsStrict ? 1 : 0; 1307 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1308 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1309 TargetLowering::MakeLibCallOptions CallOptions; 1310 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1311 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1312 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1313 CallOptions, SDLoc(N), 1314 Chain); 1315 if (IsStrict) { 1316 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1317 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1318 return SDValue(); 1319 } 1320 1321 return Tmp.first; 1322 } 1323 1324 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1325 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1326 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1327 RTLIB::LROUND_F32, 1328 RTLIB::LROUND_F64, 1329 RTLIB::LROUND_F80, 1330 RTLIB::LROUND_F128, 1331 RTLIB::LROUND_PPCF128)); 1332 } 1333 1334 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1335 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1336 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1337 RTLIB::LLROUND_F32, 1338 RTLIB::LLROUND_F64, 1339 RTLIB::LLROUND_F80, 1340 RTLIB::LLROUND_F128, 1341 RTLIB::LLROUND_PPCF128)); 1342 } 1343 1344 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1345 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1346 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1347 RTLIB::LRINT_F32, 1348 RTLIB::LRINT_F64, 1349 RTLIB::LRINT_F80, 1350 RTLIB::LRINT_F128, 1351 RTLIB::LRINT_PPCF128)); 1352 } 1353 1354 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1355 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1356 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1357 RTLIB::LLRINT_F32, 1358 RTLIB::LLRINT_F64, 1359 RTLIB::LLRINT_F80, 1360 RTLIB::LLRINT_F128, 1361 RTLIB::LLRINT_PPCF128)); 1362 } 1363 1364 //===----------------------------------------------------------------------===// 1365 // Float Result Expansion 1366 //===----------------------------------------------------------------------===// 1367 1368 /// ExpandFloatResult - This method is called when the specified result of the 1369 /// specified node is found to need expansion. At this point, the node may also 1370 /// have invalid operands or may have other results that need promotion, we just 1371 /// know that (at least) one result needs expansion. 1372 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1373 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG)); 1374 SDValue Lo, Hi; 1375 Lo = Hi = SDValue(); 1376 1377 // See if the target wants to custom expand this node. 1378 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1379 return; 1380 1381 switch (N->getOpcode()) { 1382 default: 1383 #ifndef NDEBUG 1384 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1385 N->dump(&DAG); dbgs() << "\n"; 1386 #endif 1387 report_fatal_error("Do not know how to expand the result of this " 1388 "operator!"); 1389 // clang-format off 1390 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1391 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 1392 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1393 1394 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1395 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1396 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1397 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1398 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1399 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1400 1401 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1402 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1403 case ISD::STRICT_FMINNUM: 1404 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1405 case ISD::STRICT_FMAXNUM: 1406 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1407 case ISD::STRICT_FADD: 1408 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1409 case ISD::STRICT_FACOS: 1410 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break; 1411 case ISD::STRICT_FASIN: 1412 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break; 1413 case ISD::STRICT_FATAN: 1414 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break; 1415 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1416 case ISD::STRICT_FCEIL: 1417 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1418 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1419 case ISD::STRICT_FCOS: 1420 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1421 case ISD::STRICT_FCOSH: 1422 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break; 1423 case ISD::STRICT_FDIV: 1424 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1425 case ISD::STRICT_FEXP: 1426 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1427 case ISD::STRICT_FEXP2: 1428 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1429 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break; 1430 case ISD::STRICT_FFLOOR: 1431 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1432 case ISD::STRICT_FLOG: 1433 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1434 case ISD::STRICT_FLOG2: 1435 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1436 case ISD::STRICT_FLOG10: 1437 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1438 case ISD::STRICT_FMA: 1439 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1440 case ISD::STRICT_FMUL: 1441 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1442 case ISD::STRICT_FNEARBYINT: 1443 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1444 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1445 case ISD::STRICT_FP_EXTEND: 1446 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1447 case ISD::STRICT_FPOW: 1448 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1449 case ISD::STRICT_FPOWI: 1450 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1451 case ISD::FLDEXP: 1452 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break; 1453 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1454 case ISD::STRICT_FRINT: 1455 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1456 case ISD::STRICT_FROUND: 1457 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1458 case ISD::STRICT_FROUNDEVEN: 1459 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1460 case ISD::STRICT_FSIN: 1461 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1462 case ISD::STRICT_FSINH: 1463 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break; 1464 case ISD::STRICT_FSQRT: 1465 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1466 case ISD::STRICT_FSUB: 1467 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1468 case ISD::STRICT_FTAN: 1469 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break; 1470 case ISD::STRICT_FTANH: 1471 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break; 1472 case ISD::STRICT_FTRUNC: 1473 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1474 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1475 case ISD::STRICT_SINT_TO_FP: 1476 case ISD::STRICT_UINT_TO_FP: 1477 case ISD::SINT_TO_FP: 1478 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1479 case ISD::STRICT_FREM: 1480 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1481 // clang-format on 1482 } 1483 1484 // If Lo/Hi is null, the sub-method took care of registering results etc. 1485 if (Lo.getNode()) 1486 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1487 } 1488 1489 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1490 SDValue &Hi) { 1491 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1492 assert(NVT.getSizeInBits() == 64 && 1493 "Do not know how to expand this float constant!"); 1494 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1495 SDLoc dl(N); 1496 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1497 APInt(64, C.getRawData()[1])), 1498 dl, NVT); 1499 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1500 APInt(64, C.getRawData()[0])), 1501 dl, NVT); 1502 } 1503 1504 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1505 SDValue &Lo, SDValue &Hi) { 1506 bool IsStrict = N->isStrictFPOpcode(); 1507 unsigned Offset = IsStrict ? 1 : 0; 1508 SDValue Op = N->getOperand(0 + Offset); 1509 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1510 TargetLowering::MakeLibCallOptions CallOptions; 1511 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1512 Op, CallOptions, SDLoc(N), 1513 Chain); 1514 if (IsStrict) 1515 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1516 GetPairElements(Tmp.first, Lo, Hi); 1517 } 1518 1519 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1520 SDValue &Lo, SDValue &Hi) { 1521 bool IsStrict = N->isStrictFPOpcode(); 1522 unsigned Offset = IsStrict ? 1 : 0; 1523 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1524 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1525 TargetLowering::MakeLibCallOptions CallOptions; 1526 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1527 Ops, CallOptions, SDLoc(N), 1528 Chain); 1529 if (IsStrict) 1530 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1531 GetPairElements(Tmp.first, Lo, Hi); 1532 } 1533 1534 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1535 SDValue &Hi) { 1536 assert(N->getValueType(0) == MVT::ppcf128 && 1537 "Logic only correct for ppcf128!"); 1538 SDLoc dl(N); 1539 SDValue Tmp; 1540 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1541 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1542 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1543 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1544 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1545 ISD::SETEQ); 1546 } 1547 1548 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1549 SDValue &Hi) { 1550 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1551 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1552 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1553 RTLIB::FMIN_PPCF128), Lo, Hi); 1554 } 1555 1556 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1557 SDValue &Hi) { 1558 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1559 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1560 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1561 RTLIB::FMAX_PPCF128), Lo, Hi); 1562 } 1563 1564 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1565 SDValue &Hi) { 1566 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1567 RTLIB::ADD_F32, RTLIB::ADD_F64, 1568 RTLIB::ADD_F80, RTLIB::ADD_F128, 1569 RTLIB::ADD_PPCF128), Lo, Hi); 1570 } 1571 1572 void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo, 1573 SDValue &Hi) { 1574 ExpandFloatRes_Unary(N, 1575 GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, 1576 RTLIB::ACOS_F64, RTLIB::ACOS_F80, 1577 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128), 1578 Lo, Hi); 1579 } 1580 1581 void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo, 1582 SDValue &Hi) { 1583 ExpandFloatRes_Unary(N, 1584 GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, 1585 RTLIB::ASIN_F64, RTLIB::ASIN_F80, 1586 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128), 1587 Lo, Hi); 1588 } 1589 1590 void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo, 1591 SDValue &Hi) { 1592 ExpandFloatRes_Unary(N, 1593 GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, 1594 RTLIB::ATAN_F64, RTLIB::ATAN_F80, 1595 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128), 1596 Lo, Hi); 1597 } 1598 1599 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1600 SDValue &Hi) { 1601 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1602 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1603 RTLIB::CBRT_F128, 1604 RTLIB::CBRT_PPCF128), Lo, Hi); 1605 } 1606 1607 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1608 SDValue &Lo, SDValue &Hi) { 1609 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1610 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1611 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1612 RTLIB::CEIL_PPCF128), Lo, Hi); 1613 } 1614 1615 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1616 SDValue &Lo, SDValue &Hi) { 1617 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1618 RTLIB::COPYSIGN_F32, 1619 RTLIB::COPYSIGN_F64, 1620 RTLIB::COPYSIGN_F80, 1621 RTLIB::COPYSIGN_F128, 1622 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1623 } 1624 1625 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1626 SDValue &Lo, SDValue &Hi) { 1627 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1628 RTLIB::COS_F32, RTLIB::COS_F64, 1629 RTLIB::COS_F80, RTLIB::COS_F128, 1630 RTLIB::COS_PPCF128), Lo, Hi); 1631 } 1632 1633 void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo, 1634 SDValue &Hi) { 1635 ExpandFloatRes_Unary(N, 1636 GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, 1637 RTLIB::COSH_F64, RTLIB::COSH_F80, 1638 RTLIB::COSH_F128, RTLIB::COSH_PPCF128), 1639 Lo, Hi); 1640 } 1641 1642 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1643 SDValue &Hi) { 1644 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1645 RTLIB::DIV_F32, 1646 RTLIB::DIV_F64, 1647 RTLIB::DIV_F80, 1648 RTLIB::DIV_F128, 1649 RTLIB::DIV_PPCF128), Lo, Hi); 1650 } 1651 1652 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1653 SDValue &Lo, SDValue &Hi) { 1654 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1655 RTLIB::EXP_F32, RTLIB::EXP_F64, 1656 RTLIB::EXP_F80, RTLIB::EXP_F128, 1657 RTLIB::EXP_PPCF128), Lo, Hi); 1658 } 1659 1660 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1661 SDValue &Lo, SDValue &Hi) { 1662 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1663 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1664 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1665 RTLIB::EXP2_PPCF128), Lo, Hi); 1666 } 1667 1668 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo, 1669 SDValue &Hi) { 1670 ExpandFloatRes_Unary(N, 1671 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, 1672 RTLIB::EXP10_F64, RTLIB::EXP10_F80, 1673 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128), 1674 Lo, Hi); 1675 } 1676 1677 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1678 SDValue &Lo, SDValue &Hi) { 1679 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1680 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1681 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1682 RTLIB::FLOOR_PPCF128), Lo, Hi); 1683 } 1684 1685 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1686 SDValue &Lo, SDValue &Hi) { 1687 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1688 RTLIB::LOG_F32, RTLIB::LOG_F64, 1689 RTLIB::LOG_F80, RTLIB::LOG_F128, 1690 RTLIB::LOG_PPCF128), Lo, Hi); 1691 } 1692 1693 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1694 SDValue &Lo, SDValue &Hi) { 1695 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1696 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1697 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1698 RTLIB::LOG2_PPCF128), Lo, Hi); 1699 } 1700 1701 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1702 SDValue &Lo, SDValue &Hi) { 1703 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1704 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1705 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1706 RTLIB::LOG10_PPCF128), Lo, Hi); 1707 } 1708 1709 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1710 SDValue &Hi) { 1711 bool IsStrict = N->isStrictFPOpcode(); 1712 unsigned Offset = IsStrict ? 1 : 0; 1713 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1714 N->getOperand(2 + Offset) }; 1715 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1716 TargetLowering::MakeLibCallOptions CallOptions; 1717 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1718 RTLIB::FMA_F32, 1719 RTLIB::FMA_F64, 1720 RTLIB::FMA_F80, 1721 RTLIB::FMA_F128, 1722 RTLIB::FMA_PPCF128), 1723 N->getValueType(0), Ops, CallOptions, 1724 SDLoc(N), Chain); 1725 if (IsStrict) 1726 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1727 GetPairElements(Tmp.first, Lo, Hi); 1728 } 1729 1730 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1731 SDValue &Hi) { 1732 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1733 RTLIB::MUL_F32, 1734 RTLIB::MUL_F64, 1735 RTLIB::MUL_F80, 1736 RTLIB::MUL_F128, 1737 RTLIB::MUL_PPCF128), Lo, Hi); 1738 } 1739 1740 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1741 SDValue &Lo, SDValue &Hi) { 1742 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1743 RTLIB::NEARBYINT_F32, 1744 RTLIB::NEARBYINT_F64, 1745 RTLIB::NEARBYINT_F80, 1746 RTLIB::NEARBYINT_F128, 1747 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1748 } 1749 1750 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1751 SDValue &Hi) { 1752 SDLoc dl(N); 1753 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1754 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1755 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1756 } 1757 1758 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1759 SDValue &Hi) { 1760 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1761 SDLoc dl(N); 1762 bool IsStrict = N->isStrictFPOpcode(); 1763 1764 SDValue Chain; 1765 if (IsStrict) { 1766 // If the expanded type is the same as the input type, just bypass the node. 1767 if (NVT == N->getOperand(1).getValueType()) { 1768 Hi = N->getOperand(1); 1769 Chain = N->getOperand(0); 1770 } else { 1771 // Other we need to extend. 1772 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1773 { N->getOperand(0), N->getOperand(1) }); 1774 Chain = Hi.getValue(1); 1775 } 1776 } else { 1777 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1778 } 1779 1780 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1781 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1782 1783 if (IsStrict) 1784 ReplaceValueWith(SDValue(N, 1), Chain); 1785 } 1786 1787 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1788 SDValue &Lo, SDValue &Hi) { 1789 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1790 RTLIB::POW_F32, RTLIB::POW_F64, 1791 RTLIB::POW_F80, RTLIB::POW_F128, 1792 RTLIB::POW_PPCF128), Lo, Hi); 1793 } 1794 1795 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1796 SDValue &Lo, SDValue &Hi) { 1797 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1798 } 1799 1800 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo, 1801 SDValue &Hi) { 1802 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi); 1803 } 1804 1805 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1806 SDValue &Lo, SDValue &Hi) { 1807 assert(N->getValueType(0) == MVT::ppcf128 && 1808 "Logic only correct for ppcf128!"); 1809 1810 SDLoc dl(N); 1811 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1812 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1813 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1814 } 1815 1816 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1817 SDValue &Lo, SDValue &Hi) { 1818 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1819 RTLIB::REM_F32, RTLIB::REM_F64, 1820 RTLIB::REM_F80, RTLIB::REM_F128, 1821 RTLIB::REM_PPCF128), Lo, Hi); 1822 } 1823 1824 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1825 SDValue &Lo, SDValue &Hi) { 1826 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1827 RTLIB::RINT_F32, RTLIB::RINT_F64, 1828 RTLIB::RINT_F80, RTLIB::RINT_F128, 1829 RTLIB::RINT_PPCF128), Lo, Hi); 1830 } 1831 1832 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1833 SDValue &Lo, SDValue &Hi) { 1834 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1835 RTLIB::ROUND_F32, 1836 RTLIB::ROUND_F64, 1837 RTLIB::ROUND_F80, 1838 RTLIB::ROUND_F128, 1839 RTLIB::ROUND_PPCF128), Lo, Hi); 1840 } 1841 1842 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1843 SDValue &Lo, SDValue &Hi) { 1844 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1845 RTLIB::ROUNDEVEN_F32, 1846 RTLIB::ROUNDEVEN_F64, 1847 RTLIB::ROUNDEVEN_F80, 1848 RTLIB::ROUNDEVEN_F128, 1849 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1850 } 1851 1852 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1853 SDValue &Lo, SDValue &Hi) { 1854 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1855 RTLIB::SIN_F32, RTLIB::SIN_F64, 1856 RTLIB::SIN_F80, RTLIB::SIN_F128, 1857 RTLIB::SIN_PPCF128), Lo, Hi); 1858 } 1859 1860 void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo, 1861 SDValue &Hi) { 1862 ExpandFloatRes_Unary(N, 1863 GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, 1864 RTLIB::SINH_F64, RTLIB::SINH_F80, 1865 RTLIB::SINH_F128, RTLIB::SINH_PPCF128), 1866 Lo, Hi); 1867 } 1868 1869 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1870 SDValue &Lo, SDValue &Hi) { 1871 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1872 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1873 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1874 RTLIB::SQRT_PPCF128), Lo, Hi); 1875 } 1876 1877 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1878 SDValue &Hi) { 1879 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1880 RTLIB::SUB_F32, 1881 RTLIB::SUB_F64, 1882 RTLIB::SUB_F80, 1883 RTLIB::SUB_F128, 1884 RTLIB::SUB_PPCF128), Lo, Hi); 1885 } 1886 1887 void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo, 1888 SDValue &Hi) { 1889 ExpandFloatRes_Unary(N, 1890 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, 1891 RTLIB::TAN_F64, RTLIB::TAN_F80, 1892 RTLIB::TAN_F128, RTLIB::TAN_PPCF128), 1893 Lo, Hi); 1894 } 1895 1896 void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo, 1897 SDValue &Hi) { 1898 ExpandFloatRes_Unary(N, 1899 GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, 1900 RTLIB::TANH_F64, RTLIB::TANH_F80, 1901 RTLIB::TANH_F128, RTLIB::TANH_PPCF128), 1902 Lo, Hi); 1903 } 1904 1905 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 1906 SDValue &Lo, SDValue &Hi) { 1907 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1908 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 1909 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 1910 RTLIB::TRUNC_PPCF128), Lo, Hi); 1911 } 1912 1913 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 1914 SDValue &Hi) { 1915 if (ISD::isNormalLoad(N)) { 1916 ExpandRes_NormalLoad(N, Lo, Hi); 1917 return; 1918 } 1919 1920 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 1921 LoadSDNode *LD = cast<LoadSDNode>(N); 1922 SDValue Chain = LD->getChain(); 1923 SDValue Ptr = LD->getBasePtr(); 1924 SDLoc dl(N); 1925 1926 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 1927 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 1928 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 1929 1930 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 1931 LD->getMemoryVT(), LD->getMemOperand()); 1932 1933 // Remember the chain. 1934 Chain = Hi.getValue(1); 1935 1936 // The low part is zero. 1937 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1938 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1939 1940 // Modified the chain - switch anything that used the old chain to use the 1941 // new one. 1942 ReplaceValueWith(SDValue(LD, 1), Chain); 1943 } 1944 1945 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 1946 SDValue &Hi) { 1947 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 1948 EVT VT = N->getValueType(0); 1949 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1950 bool Strict = N->isStrictFPOpcode(); 1951 SDValue Src = N->getOperand(Strict ? 1 : 0); 1952 EVT SrcVT = Src.getValueType(); 1953 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 1954 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1955 SDLoc dl(N); 1956 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 1957 1958 // TODO: Any other flags to propagate? 1959 SDNodeFlags Flags; 1960 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 1961 1962 // First do an SINT_TO_FP, whether the original was signed or unsigned. 1963 // When promoting partial word types to i32 we must honor the signedness, 1964 // though. 1965 if (SrcVT.bitsLE(MVT::i32)) { 1966 // The integer can be represented exactly in an f64. 1967 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT), 1968 APInt(NVT.getSizeInBits(), 0)), dl, NVT); 1969 if (Strict) { 1970 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 1971 {Chain, Src}, Flags); 1972 Chain = Hi.getValue(1); 1973 } else 1974 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 1975 } else { 1976 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1977 if (SrcVT.bitsLE(MVT::i64)) { 1978 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1979 MVT::i64, Src); 1980 LC = RTLIB::SINTTOFP_I64_PPCF128; 1981 } else if (SrcVT.bitsLE(MVT::i128)) { 1982 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 1983 LC = RTLIB::SINTTOFP_I128_PPCF128; 1984 } 1985 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1986 1987 TargetLowering::MakeLibCallOptions CallOptions; 1988 CallOptions.setSExt(true); 1989 std::pair<SDValue, SDValue> Tmp = 1990 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 1991 if (Strict) 1992 Chain = Tmp.second; 1993 GetPairElements(Tmp.first, Lo, Hi); 1994 } 1995 1996 // No need to complement for unsigned 32-bit integers 1997 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 1998 if (Strict) 1999 ReplaceValueWith(SDValue(N, 1), Chain); 2000 2001 return; 2002 } 2003 2004 // Unsigned - fix up the SINT_TO_FP value just calculated. 2005 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 2006 // keep semantics correctness if the integer is not exactly representable 2007 // here. See ExpandLegalINT_TO_FP. 2008 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 2009 SrcVT = Src.getValueType(); 2010 2011 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 2012 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 2013 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 2014 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 2015 ArrayRef<uint64_t> Parts; 2016 2017 switch (SrcVT.getSimpleVT().SimpleTy) { 2018 default: 2019 llvm_unreachable("Unsupported UINT_TO_FP!"); 2020 case MVT::i32: 2021 Parts = TwoE32; 2022 break; 2023 case MVT::i64: 2024 Parts = TwoE64; 2025 break; 2026 case MVT::i128: 2027 Parts = TwoE128; 2028 break; 2029 } 2030 2031 // TODO: Are there other fast-math-flags to propagate to this FADD? 2032 SDValue NewLo = DAG.getConstantFP( 2033 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 2034 if (Strict) { 2035 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 2036 {Chain, Hi, NewLo}, Flags); 2037 Chain = Lo.getValue(1); 2038 ReplaceValueWith(SDValue(N, 1), Chain); 2039 } else 2040 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 2041 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 2042 Lo, Hi, ISD::SETLT); 2043 GetPairElements(Lo, Lo, Hi); 2044 } 2045 2046 2047 //===----------------------------------------------------------------------===// 2048 // Float Operand Expansion 2049 //===----------------------------------------------------------------------===// 2050 2051 /// ExpandFloatOperand - This method is called when the specified operand of the 2052 /// specified node is found to need expansion. At this point, all of the result 2053 /// types of the node are known to be legal, but other operands of the node may 2054 /// need promotion or expansion as well as the specified one. 2055 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 2056 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG)); 2057 SDValue Res = SDValue(); 2058 2059 // See if the target wants to custom expand this node. 2060 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2061 return false; 2062 2063 switch (N->getOpcode()) { 2064 default: 2065 #ifndef NDEBUG 2066 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 2067 N->dump(&DAG); dbgs() << "\n"; 2068 #endif 2069 report_fatal_error("Do not know how to expand this operator's operand!"); 2070 2071 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2072 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2073 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2074 2075 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 2076 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 2077 case ISD::STRICT_FP_ROUND: 2078 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 2079 case ISD::STRICT_FP_TO_SINT: 2080 case ISD::STRICT_FP_TO_UINT: 2081 case ISD::FP_TO_SINT: 2082 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 2083 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 2084 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 2085 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 2086 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 2087 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 2088 case ISD::STRICT_FSETCC: 2089 case ISD::STRICT_FSETCCS: 2090 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 2091 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 2092 OpNo); break; 2093 } 2094 2095 // If the result is null, the sub-method took care of registering results etc. 2096 if (!Res.getNode()) return false; 2097 2098 // If the result is N, the sub-method updated N in place. Tell the legalizer 2099 // core about this. 2100 if (Res.getNode() == N) 2101 return true; 2102 2103 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2104 "Invalid operand expansion"); 2105 2106 ReplaceValueWith(SDValue(N, 0), Res); 2107 return false; 2108 } 2109 2110 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 2111 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2112 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 2113 SDValue &NewRHS, 2114 ISD::CondCode &CCCode, 2115 const SDLoc &dl, SDValue &Chain, 2116 bool IsSignaling) { 2117 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2118 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 2119 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 2120 2121 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 2122 2123 // FIXME: This generated code sucks. We want to generate 2124 // FCMPU crN, hi1, hi2 2125 // BNE crN, L: 2126 // FCMPU crN, lo1, lo2 2127 // The following can be improved, but not that much. 2128 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 2129 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 2130 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 2131 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 2132 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 2133 RHSLo, CCCode, OutputChain, IsSignaling); 2134 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 2135 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 2136 Tmp1 = 2137 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 2138 ISD::SETUNE, OutputChain, IsSignaling); 2139 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 2140 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 2141 RHSHi, CCCode, OutputChain, IsSignaling); 2142 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 2143 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 2144 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 2145 NewRHS = SDValue(); // LHS is the result, not a compare. 2146 Chain = OutputChain; 2147 } 2148 2149 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 2150 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2151 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2152 SDValue Chain; 2153 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 2154 2155 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2156 // against zero to select between true and false values. 2157 if (!NewRHS.getNode()) { 2158 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2159 CCCode = ISD::SETNE; 2160 } 2161 2162 // Update N to have the operands specified. 2163 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2164 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2165 N->getOperand(4)), 0); 2166 } 2167 2168 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 2169 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 2170 "Logic only correct for ppcf128!"); 2171 SDValue Lo, Hi; 2172 GetExpandedFloat(N->getOperand(1), Lo, Hi); 2173 // The ppcf128 value is providing only the sign; take it from the 2174 // higher-order double (which must have the larger magnitude). 2175 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 2176 N->getValueType(0), N->getOperand(0), Hi); 2177 } 2178 2179 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 2180 bool IsStrict = N->isStrictFPOpcode(); 2181 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 2182 "Logic only correct for ppcf128!"); 2183 SDValue Lo, Hi; 2184 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 2185 2186 if (!IsStrict) 2187 // Round it the rest of the way (e.g. to f32) if needed. 2188 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 2189 N->getValueType(0), Hi, N->getOperand(1)); 2190 2191 // Eliminate the node if the input float type is the same as the output float 2192 // type. 2193 if (Hi.getValueType() == N->getValueType(0)) { 2194 // Connect the output chain to the input chain, unlinking the node. 2195 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 2196 ReplaceValueWith(SDValue(N, 0), Hi); 2197 return SDValue(); 2198 } 2199 2200 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 2201 {N->getValueType(0), MVT::Other}, 2202 {N->getOperand(0), Hi, N->getOperand(2)}); 2203 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 2204 ReplaceValueWith(SDValue(N, 0), Expansion); 2205 return SDValue(); 2206 } 2207 2208 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 2209 EVT RVT = N->getValueType(0); 2210 SDLoc dl(N); 2211 2212 bool IsStrict = N->isStrictFPOpcode(); 2213 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 2214 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 2215 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2216 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2217 2218 EVT NVT; 2219 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 2220 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 2221 "Unsupported FP_TO_XINT!"); 2222 TargetLowering::MakeLibCallOptions CallOptions; 2223 std::pair<SDValue, SDValue> Tmp = 2224 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 2225 if (!IsStrict) 2226 return Tmp.first; 2227 2228 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2229 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2230 return SDValue(); 2231 } 2232 2233 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 2234 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2235 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2236 SDValue Chain; 2237 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 2238 2239 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2240 // against zero to select between true and false values. 2241 if (!NewRHS.getNode()) { 2242 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2243 CCCode = ISD::SETNE; 2244 } 2245 2246 // Update N to have the operands specified. 2247 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2248 N->getOperand(2), N->getOperand(3), 2249 DAG.getCondCode(CCCode)), 0); 2250 } 2251 2252 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 2253 bool IsStrict = N->isStrictFPOpcode(); 2254 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 2255 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 2256 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2257 ISD::CondCode CCCode = 2258 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 2259 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 2260 N->getOpcode() == ISD::STRICT_FSETCCS); 2261 2262 // FloatExpandSetCCOperands always returned a scalar. 2263 assert(!NewRHS.getNode() && "Expect to return scalar"); 2264 assert(NewLHS.getValueType() == N->getValueType(0) && 2265 "Unexpected setcc expansion!"); 2266 if (Chain) { 2267 ReplaceValueWith(SDValue(N, 0), NewLHS); 2268 ReplaceValueWith(SDValue(N, 1), Chain); 2269 return SDValue(); 2270 } 2271 return NewLHS; 2272 } 2273 2274 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 2275 if (ISD::isNormalStore(N)) 2276 return ExpandOp_NormalStore(N, OpNo); 2277 2278 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2279 assert(OpNo == 1 && "Can only expand the stored value so far"); 2280 StoreSDNode *ST = cast<StoreSDNode>(N); 2281 2282 SDValue Chain = ST->getChain(); 2283 SDValue Ptr = ST->getBasePtr(); 2284 2285 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 2286 ST->getValue().getValueType()); 2287 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2288 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2289 (void)NVT; 2290 2291 SDValue Lo, Hi; 2292 GetExpandedOp(ST->getValue(), Lo, Hi); 2293 2294 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 2295 ST->getMemoryVT(), ST->getMemOperand()); 2296 } 2297 2298 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 2299 EVT RVT = N->getValueType(0); 2300 EVT RetVT = N->getOperand(0).getValueType(); 2301 TargetLowering::MakeLibCallOptions CallOptions; 2302 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2303 RTLIB::LROUND_F32, 2304 RTLIB::LROUND_F64, 2305 RTLIB::LROUND_F80, 2306 RTLIB::LROUND_F128, 2307 RTLIB::LROUND_PPCF128), 2308 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2309 } 2310 2311 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2312 EVT RVT = N->getValueType(0); 2313 EVT RetVT = N->getOperand(0).getValueType(); 2314 TargetLowering::MakeLibCallOptions CallOptions; 2315 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2316 RTLIB::LLROUND_F32, 2317 RTLIB::LLROUND_F64, 2318 RTLIB::LLROUND_F80, 2319 RTLIB::LLROUND_F128, 2320 RTLIB::LLROUND_PPCF128), 2321 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2322 } 2323 2324 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2325 EVT RVT = N->getValueType(0); 2326 EVT RetVT = N->getOperand(0).getValueType(); 2327 TargetLowering::MakeLibCallOptions CallOptions; 2328 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2329 RTLIB::LRINT_F32, 2330 RTLIB::LRINT_F64, 2331 RTLIB::LRINT_F80, 2332 RTLIB::LRINT_F128, 2333 RTLIB::LRINT_PPCF128), 2334 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2335 } 2336 2337 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2338 EVT RVT = N->getValueType(0); 2339 EVT RetVT = N->getOperand(0).getValueType(); 2340 TargetLowering::MakeLibCallOptions CallOptions; 2341 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2342 RTLIB::LLRINT_F32, 2343 RTLIB::LLRINT_F64, 2344 RTLIB::LLRINT_F80, 2345 RTLIB::LLRINT_F128, 2346 RTLIB::LLRINT_PPCF128), 2347 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2348 } 2349 2350 //===----------------------------------------------------------------------===// 2351 // Float Operand Promotion 2352 //===----------------------------------------------------------------------===// 2353 // 2354 2355 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2356 if (OpVT == MVT::f16) { 2357 return ISD::FP16_TO_FP; 2358 } else if (RetVT == MVT::f16) { 2359 return ISD::FP_TO_FP16; 2360 } else if (OpVT == MVT::bf16) { 2361 return ISD::BF16_TO_FP; 2362 } else if (RetVT == MVT::bf16) { 2363 return ISD::FP_TO_BF16; 2364 } 2365 2366 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2367 } 2368 2369 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { 2370 if (OpVT == MVT::f16) 2371 return ISD::STRICT_FP16_TO_FP; 2372 2373 if (RetVT == MVT::f16) 2374 return ISD::STRICT_FP_TO_FP16; 2375 2376 if (OpVT == MVT::bf16) 2377 return ISD::STRICT_BF16_TO_FP; 2378 2379 if (RetVT == MVT::bf16) 2380 return ISD::STRICT_FP_TO_BF16; 2381 2382 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2383 } 2384 2385 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2386 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG)); 2387 SDValue R = SDValue(); 2388 2389 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2390 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2391 return false; 2392 } 2393 2394 // Nodes that use a promotion-requiring floating point operand, but doesn't 2395 // produce a promotion-requiring floating point result, need to be legalized 2396 // to use the promoted float operand. Nodes that produce at least one 2397 // promotion-requiring floating point result have their operands legalized as 2398 // a part of PromoteFloatResult. 2399 // clang-format off 2400 switch (N->getOpcode()) { 2401 default: 2402 #ifndef NDEBUG 2403 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2404 N->dump(&DAG); dbgs() << "\n"; 2405 #endif 2406 report_fatal_error("Do not know how to promote this operator's operand!"); 2407 2408 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2409 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2410 case ISD::FP_TO_SINT: 2411 case ISD::FP_TO_UINT: 2412 case ISD::LRINT: 2413 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break; 2414 case ISD::FP_TO_SINT_SAT: 2415 case ISD::FP_TO_UINT_SAT: 2416 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2417 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2418 case ISD::STRICT_FP_EXTEND: 2419 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo); 2420 break; 2421 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2422 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2423 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2424 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break; 2425 } 2426 // clang-format on 2427 2428 if (R.getNode()) 2429 ReplaceValueWith(SDValue(N, 0), R); 2430 return false; 2431 } 2432 2433 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2434 SDValue Op = N->getOperand(0); 2435 EVT OpVT = Op->getValueType(0); 2436 2437 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2438 EVT PromotedVT = Promoted->getValueType(0); 2439 2440 // Convert the promoted float value to the desired IVT. 2441 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2442 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2443 IVT, Promoted); 2444 // The final result type might not be an scalar so we need a bitcast. The 2445 // bitcast will be further legalized if needed. 2446 return DAG.getBitcast(N->getValueType(0), Convert); 2447 } 2448 2449 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2450 // PromoteFloatRes_FCOPYSIGN. 2451 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2452 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2453 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2454 2455 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2456 N->getOperand(0), Op1); 2457 } 2458 2459 // Convert the promoted float value to the desired integer type 2460 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) { 2461 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2462 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2463 } 2464 2465 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2466 unsigned OpNo) { 2467 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2468 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2469 N->getOperand(1)); 2470 } 2471 2472 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2473 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2474 EVT VT = N->getValueType(0); 2475 2476 // Desired VT is same as promoted type. Use promoted float directly. 2477 if (VT == Op->getValueType(0)) 2478 return Op; 2479 2480 // Else, extend the promoted float value to the desired VT. 2481 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2482 } 2483 2484 SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, 2485 unsigned OpNo) { 2486 assert(OpNo == 1 && "Promoting unpromotable operand"); 2487 2488 SDValue Op = GetPromotedFloat(N->getOperand(1)); 2489 EVT VT = N->getValueType(0); 2490 2491 // Desired VT is same as promoted type. Use promoted float directly. 2492 if (VT == Op->getValueType(0)) { 2493 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 2494 return Op; 2495 } 2496 2497 // Else, extend the promoted float value to the desired VT. 2498 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(), 2499 N->getOperand(0), Op); 2500 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2501 return Res; 2502 } 2503 2504 // Promote the float operands used for comparison. The true- and false- 2505 // operands have the same type as the result and are promoted, if needed, by 2506 // PromoteFloatRes_SELECT_CC 2507 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2508 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2509 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2510 2511 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2512 LHS, RHS, N->getOperand(2), N->getOperand(3), 2513 N->getOperand(4)); 2514 } 2515 2516 // Construct a SETCC that compares the promoted values and sets the conditional 2517 // code. 2518 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2519 EVT VT = N->getValueType(0); 2520 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2521 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2522 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2523 2524 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2525 2526 } 2527 2528 // Lower the promoted Float down to the integer value of same size and construct 2529 // a STORE of the integer value. 2530 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2531 StoreSDNode *ST = cast<StoreSDNode>(N); 2532 SDValue Val = ST->getValue(); 2533 SDLoc DL(N); 2534 2535 SDValue Promoted = GetPromotedFloat(Val); 2536 EVT VT = ST->getOperand(1).getValueType(); 2537 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2538 2539 SDValue NewVal; 2540 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2541 IVT, Promoted); 2542 2543 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2544 ST->getMemOperand()); 2545 } 2546 2547 SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N, 2548 unsigned OpNo) { 2549 AtomicSDNode *ST = cast<AtomicSDNode>(N); 2550 SDValue Val = ST->getVal(); 2551 SDLoc DL(N); 2552 2553 SDValue Promoted = GetPromotedFloat(Val); 2554 EVT VT = ST->getOperand(1).getValueType(); 2555 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2556 2557 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), 2558 DL, IVT, Promoted); 2559 2560 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal, 2561 ST->getBasePtr(), ST->getMemOperand()); 2562 } 2563 2564 //===----------------------------------------------------------------------===// 2565 // Float Result Promotion 2566 //===----------------------------------------------------------------------===// 2567 2568 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2569 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG)); 2570 SDValue R = SDValue(); 2571 2572 // See if the target wants to custom expand this node. 2573 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2574 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2575 return; 2576 } 2577 2578 switch (N->getOpcode()) { 2579 // These opcodes cannot appear if promotion of FP16 is done in the backend 2580 // instead of Clang 2581 case ISD::FP16_TO_FP: 2582 case ISD::FP_TO_FP16: 2583 default: 2584 #ifndef NDEBUG 2585 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2586 N->dump(&DAG); dbgs() << "\n"; 2587 #endif 2588 report_fatal_error("Do not know how to promote this operator's result!"); 2589 2590 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2591 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2592 case ISD::EXTRACT_VECTOR_ELT: 2593 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2594 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2595 2596 // Unary FP Operations 2597 case ISD::FABS: 2598 case ISD::FACOS: 2599 case ISD::FASIN: 2600 case ISD::FATAN: 2601 case ISD::FCBRT: 2602 case ISD::FCEIL: 2603 case ISD::FCOS: 2604 case ISD::FCOSH: 2605 case ISD::FEXP: 2606 case ISD::FEXP2: 2607 case ISD::FEXP10: 2608 case ISD::FFLOOR: 2609 case ISD::FLOG: 2610 case ISD::FLOG2: 2611 case ISD::FLOG10: 2612 case ISD::FNEARBYINT: 2613 case ISD::FNEG: 2614 case ISD::FRINT: 2615 case ISD::FROUND: 2616 case ISD::FROUNDEVEN: 2617 case ISD::FSIN: 2618 case ISD::FSINH: 2619 case ISD::FSQRT: 2620 case ISD::FTRUNC: 2621 case ISD::FTAN: 2622 case ISD::FTANH: 2623 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2624 2625 // Binary FP Operations 2626 case ISD::FADD: 2627 case ISD::FDIV: 2628 case ISD::FMAXIMUM: 2629 case ISD::FMINIMUM: 2630 case ISD::FMAXNUM: 2631 case ISD::FMINNUM: 2632 case ISD::FMAXNUM_IEEE: 2633 case ISD::FMINNUM_IEEE: 2634 case ISD::FMUL: 2635 case ISD::FPOW: 2636 case ISD::FREM: 2637 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2638 2639 case ISD::FMA: // FMA is same as FMAD 2640 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2641 2642 case ISD::FPOWI: 2643 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break; 2644 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; 2645 2646 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2647 case ISD::STRICT_FP_ROUND: 2648 R = PromoteFloatRes_STRICT_FP_ROUND(N); 2649 break; 2650 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2651 case ISD::ATOMIC_LOAD: 2652 R = PromoteFloatRes_ATOMIC_LOAD(N); 2653 break; 2654 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2655 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2656 2657 case ISD::SINT_TO_FP: 2658 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2659 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2660 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2661 case ISD::VECREDUCE_FADD: 2662 case ISD::VECREDUCE_FMUL: 2663 case ISD::VECREDUCE_FMIN: 2664 case ISD::VECREDUCE_FMAX: 2665 case ISD::VECREDUCE_FMAXIMUM: 2666 case ISD::VECREDUCE_FMINIMUM: 2667 R = PromoteFloatRes_VECREDUCE(N); 2668 break; 2669 case ISD::VECREDUCE_SEQ_FADD: 2670 case ISD::VECREDUCE_SEQ_FMUL: 2671 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2672 break; 2673 } 2674 2675 if (R.getNode()) 2676 SetPromotedFloat(SDValue(N, ResNo), R); 2677 } 2678 2679 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2680 // At this point, it is not possible to determine if the bitcast value is 2681 // eventually stored to memory or promoted to f32 or promoted to a floating 2682 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2683 // STORE promotion handlers. 2684 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2685 EVT VT = N->getValueType(0); 2686 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2687 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2688 // bitcast will be legalized further if necessary. 2689 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2690 N->getOperand(0).getValueType().getSizeInBits()); 2691 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2692 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2693 } 2694 2695 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2696 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2697 EVT VT = N->getValueType(0); 2698 SDLoc DL(N); 2699 2700 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2701 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2702 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2703 IVT); 2704 2705 // Convert the Constant to the desired FP type 2706 // FIXME We might be able to do the conversion during compilation and get rid 2707 // of it from the object code 2708 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2709 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2710 } 2711 2712 // If the Index operand is a constant, try to redirect the extract operation to 2713 // the correct legalized vector. If not, bit-convert the input vector to 2714 // equivalent integer vector. Extract the element as an (bit-cast) integer 2715 // value and convert it to the promoted type. 2716 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2717 SDLoc DL(N); 2718 2719 // If the index is constant, try to extract the value from the legalized 2720 // vector type. 2721 if (isa<ConstantSDNode>(N->getOperand(1))) { 2722 SDValue Vec = N->getOperand(0); 2723 SDValue Idx = N->getOperand(1); 2724 EVT VecVT = Vec->getValueType(0); 2725 EVT EltVT = VecVT.getVectorElementType(); 2726 2727 uint64_t IdxVal = Idx->getAsZExtVal(); 2728 2729 switch (getTypeAction(VecVT)) { 2730 default: break; 2731 case TargetLowering::TypeScalarizeVector: { 2732 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2733 ReplaceValueWith(SDValue(N, 0), Res); 2734 return SDValue(); 2735 } 2736 case TargetLowering::TypeWidenVector: { 2737 Vec = GetWidenedVector(Vec); 2738 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2739 ReplaceValueWith(SDValue(N, 0), Res); 2740 return SDValue(); 2741 } 2742 case TargetLowering::TypeSplitVector: { 2743 SDValue Lo, Hi; 2744 GetSplitVector(Vec, Lo, Hi); 2745 2746 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2747 SDValue Res; 2748 if (IdxVal < LoElts) 2749 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2750 else 2751 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2752 DAG.getConstant(IdxVal - LoElts, DL, 2753 Idx.getValueType())); 2754 ReplaceValueWith(SDValue(N, 0), Res); 2755 return SDValue(); 2756 } 2757 2758 } 2759 } 2760 2761 // Bit-convert the input vector to the equivalent integer vector 2762 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2763 EVT IVT = NewOp.getValueType().getVectorElementType(); 2764 2765 // Extract the element as an (bit-cast) integer value 2766 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2767 NewOp, N->getOperand(1)); 2768 2769 // Convert the element to the desired FP type 2770 EVT VT = N->getValueType(0); 2771 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2772 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2773 } 2774 2775 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2776 // needs promotion, so does the argument X. Note that Y, if needed, will be 2777 // handled during operand promotion. 2778 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2779 EVT VT = N->getValueType(0); 2780 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2781 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2782 2783 SDValue Op1 = N->getOperand(1); 2784 2785 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2786 } 2787 2788 // Unary operation where the result and the operand have PromoteFloat type 2789 // action. Construct a new SDNode with the promoted float value of the old 2790 // operand. 2791 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2792 EVT VT = N->getValueType(0); 2793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2794 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2795 2796 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2797 } 2798 2799 // Binary operations where the result and both operands have PromoteFloat type 2800 // action. Construct a new SDNode with the promoted float values of the old 2801 // operands. 2802 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2803 EVT VT = N->getValueType(0); 2804 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2805 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2806 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2807 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2808 } 2809 2810 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2811 EVT VT = N->getValueType(0); 2812 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2813 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2814 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2815 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2816 2817 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2818 } 2819 2820 // Promote the Float (first) operand and retain the Integer (second) operand 2821 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) { 2822 EVT VT = N->getValueType(0); 2823 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2824 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2825 SDValue Op1 = N->getOperand(1); 2826 2827 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2828 } 2829 2830 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) { 2831 EVT VT = N->getValueType(0); 2832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2833 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2834 SDValue Res = 2835 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op); 2836 2837 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2838 return Res; 2839 } 2840 2841 // Explicit operation to reduce precision. Reduce the value to half precision 2842 // and promote it back to the legal type. 2843 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2844 SDLoc DL(N); 2845 2846 SDValue Op = N->getOperand(0); 2847 EVT VT = N->getValueType(0); 2848 EVT OpVT = Op->getValueType(0); 2849 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2850 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2851 2852 // Round promoted float to desired precision 2853 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2854 // Promote it back to the legal output type 2855 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2856 } 2857 2858 // Explicit operation to reduce precision. Reduce the value to half precision 2859 // and promote it back to the legal type. 2860 SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { 2861 SDLoc DL(N); 2862 2863 SDValue Chain = N->getOperand(0); 2864 SDValue Op = N->getOperand(1); 2865 EVT VT = N->getValueType(0); 2866 EVT OpVT = Op->getValueType(0); 2867 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2868 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2869 2870 // Round promoted float to desired precision 2871 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, 2872 DAG.getVTList(IVT, MVT::Other), Chain, Op); 2873 // Promote it back to the legal output type 2874 SDValue Res = 2875 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, 2876 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); 2877 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2878 return Res; 2879 } 2880 2881 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 2882 LoadSDNode *L = cast<LoadSDNode>(N); 2883 EVT VT = N->getValueType(0); 2884 2885 // Load the value as an integer value with the same number of bits. 2886 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2887 SDValue newL = DAG.getLoad( 2888 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 2889 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 2890 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 2891 // Legalize the chain result by replacing uses of the old value chain with the 2892 // new one 2893 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2894 2895 // Convert the integer value to the desired FP type 2896 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2897 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 2898 } 2899 2900 SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) { 2901 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2902 EVT VT = AM->getValueType(0); 2903 2904 // Load the value as an integer value with the same number of bits. 2905 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2906 SDValue newL = DAG.getAtomic( 2907 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other), 2908 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand()); 2909 2910 // Legalize the chain result by replacing uses of the old value chain with the 2911 // new one 2912 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 2913 2914 // Convert the integer value to the desired FP type 2915 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2916 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL); 2917 } 2918 2919 // Construct a new SELECT node with the promoted true- and false- values. 2920 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 2921 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 2922 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 2923 2924 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 2925 N->getOperand(0), TrueVal, FalseVal); 2926 } 2927 2928 // Construct a new SELECT_CC node with the promoted true- and false- values. 2929 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 2930 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 2931 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 2932 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 2933 2934 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 2935 TrueVal.getNode()->getValueType(0), N->getOperand(0), 2936 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 2937 } 2938 2939 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 2940 // float type. 2941 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 2942 SDLoc DL(N); 2943 EVT VT = N->getValueType(0); 2944 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2945 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 2946 // Round the value to the desired precision (that of the source type). 2947 return DAG.getNode( 2948 ISD::FP_EXTEND, DL, NVT, 2949 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, 2950 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true))); 2951 } 2952 2953 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 2954 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 2955 N->getValueType(0))); 2956 } 2957 2958 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 2959 // Expand and promote recursively. 2960 // TODO: This is non-optimal, but dealing with the concurrently happening 2961 // vector-legalization is non-trivial. We could do something similar to 2962 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 2963 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 2964 return SDValue(); 2965 } 2966 2967 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 2968 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 2969 return SDValue(); 2970 } 2971 2972 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 2973 EVT VT = N->getValueType(0); 2974 2975 AtomicSDNode *AM = cast<AtomicSDNode>(N); 2976 SDLoc SL(N); 2977 2978 SDValue CastVal = BitConvertToInteger(AM->getVal()); 2979 EVT CastVT = CastVal.getValueType(); 2980 2981 SDValue NewAtomic 2982 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 2983 DAG.getVTList(CastVT, MVT::Other), 2984 { AM->getChain(), AM->getBasePtr(), CastVal }, 2985 AM->getMemOperand()); 2986 2987 SDValue Result = NewAtomic; 2988 2989 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 2990 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2991 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 2992 NewAtomic); 2993 } 2994 2995 // Legalize the chain result by replacing uses of the old value chain with the 2996 // new one 2997 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 2998 2999 return Result; 3000 3001 } 3002 3003 //===----------------------------------------------------------------------===// 3004 // Half Result Soft Promotion 3005 //===----------------------------------------------------------------------===// 3006 3007 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 3008 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 3009 N->dump(&DAG)); 3010 SDValue R = SDValue(); 3011 3012 // See if the target wants to custom expand this node. 3013 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 3014 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 3015 return; 3016 } 3017 3018 switch (N->getOpcode()) { 3019 default: 3020 #ifndef NDEBUG 3021 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 3022 N->dump(&DAG); dbgs() << "\n"; 3023 #endif 3024 report_fatal_error("Do not know how to soft promote this operator's " 3025 "result!"); 3026 3027 case ISD::ARITH_FENCE: 3028 R = SoftPromoteHalfRes_ARITH_FENCE(N); break; 3029 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 3030 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 3031 case ISD::EXTRACT_VECTOR_ELT: 3032 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 3033 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 3034 case ISD::STRICT_FP_ROUND: 3035 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 3036 3037 // Unary FP Operations 3038 case ISD::FABS: 3039 case ISD::FACOS: 3040 case ISD::FASIN: 3041 case ISD::FATAN: 3042 case ISD::FCBRT: 3043 case ISD::FCEIL: 3044 case ISD::FCOS: 3045 case ISD::FCOSH: 3046 case ISD::FEXP: 3047 case ISD::FEXP2: 3048 case ISD::FEXP10: 3049 case ISD::FFLOOR: 3050 case ISD::FLOG: 3051 case ISD::FLOG2: 3052 case ISD::FLOG10: 3053 case ISD::FNEARBYINT: 3054 case ISD::FNEG: 3055 case ISD::FREEZE: 3056 case ISD::FRINT: 3057 case ISD::FROUND: 3058 case ISD::FROUNDEVEN: 3059 case ISD::FSIN: 3060 case ISD::FSINH: 3061 case ISD::FSQRT: 3062 case ISD::FTRUNC: 3063 case ISD::FTAN: 3064 case ISD::FTANH: 3065 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 3066 3067 // Binary FP Operations 3068 case ISD::FADD: 3069 case ISD::FDIV: 3070 case ISD::FMAXIMUM: 3071 case ISD::FMINIMUM: 3072 case ISD::FMAXNUM: 3073 case ISD::FMINNUM: 3074 case ISD::FMUL: 3075 case ISD::FPOW: 3076 case ISD::FREM: 3077 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 3078 3079 case ISD::FMA: // FMA is same as FMAD 3080 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 3081 3082 case ISD::FPOWI: 3083 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; 3084 3085 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break; 3086 3087 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 3088 case ISD::ATOMIC_LOAD: 3089 R = SoftPromoteHalfRes_ATOMIC_LOAD(N); 3090 break; 3091 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 3092 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 3093 case ISD::SINT_TO_FP: 3094 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 3095 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 3096 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 3097 case ISD::VECREDUCE_FADD: 3098 case ISD::VECREDUCE_FMUL: 3099 case ISD::VECREDUCE_FMIN: 3100 case ISD::VECREDUCE_FMAX: 3101 case ISD::VECREDUCE_FMAXIMUM: 3102 case ISD::VECREDUCE_FMINIMUM: 3103 R = SoftPromoteHalfRes_VECREDUCE(N); 3104 break; 3105 case ISD::VECREDUCE_SEQ_FADD: 3106 case ISD::VECREDUCE_SEQ_FMUL: 3107 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 3108 break; 3109 } 3110 3111 if (R.getNode()) 3112 SetSoftPromotedHalf(SDValue(N, ResNo), R); 3113 } 3114 3115 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) { 3116 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16, 3117 BitConvertToInteger(N->getOperand(0))); 3118 } 3119 3120 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 3121 return BitConvertToInteger(N->getOperand(0)); 3122 } 3123 3124 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 3125 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 3126 3127 // Get the (bit-cast) APInt of the APFloat and build an integer constant 3128 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 3129 MVT::i16); 3130 } 3131 3132 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 3133 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 3134 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 3135 NewOp.getValueType().getVectorElementType(), NewOp, 3136 N->getOperand(1)); 3137 } 3138 3139 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 3140 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 3141 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 3142 SDLoc dl(N); 3143 3144 EVT LVT = LHS.getValueType(); 3145 EVT RVT = RHS.getValueType(); 3146 3147 unsigned LSize = LVT.getSizeInBits(); 3148 unsigned RSize = RVT.getSizeInBits(); 3149 3150 // First get the sign bit of second operand. 3151 SDValue SignBit = DAG.getNode( 3152 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 3153 DAG.getConstant(RSize - 1, dl, 3154 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 3155 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 3156 3157 // Shift right or sign-extend it if the two operands have different types. 3158 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 3159 if (SizeDiff > 0) { 3160 SignBit = 3161 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 3162 DAG.getConstant(SizeDiff, dl, 3163 TLI.getShiftAmountTy(SignBit.getValueType(), 3164 DAG.getDataLayout()))); 3165 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 3166 } else if (SizeDiff < 0) { 3167 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 3168 SignBit = 3169 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 3170 DAG.getConstant(-SizeDiff, dl, 3171 TLI.getShiftAmountTy(SignBit.getValueType(), 3172 DAG.getDataLayout()))); 3173 } 3174 3175 // Clear the sign bit of the first operand. 3176 SDValue Mask = DAG.getNode( 3177 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 3178 DAG.getConstant(LSize - 1, dl, 3179 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 3180 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 3181 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 3182 3183 // Or the value with the sign bit. 3184 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 3185 } 3186 3187 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 3188 EVT OVT = N->getValueType(0); 3189 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3190 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3191 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3192 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3193 SDLoc dl(N); 3194 3195 // Promote to the larger FP type. 3196 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 3197 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3198 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3199 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2); 3200 3201 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 3202 3203 // Convert back to FP16 as an integer. 3204 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3205 } 3206 3207 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { 3208 EVT OVT = N->getValueType(0); 3209 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3210 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3211 SDValue Op1 = N->getOperand(1); 3212 SDLoc dl(N); 3213 3214 // Promote to the larger FP type. 3215 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0); 3216 3217 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 3218 3219 // Convert back to FP16 as an integer. 3220 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3221 } 3222 3223 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) { 3224 EVT OVT = N->getValueType(0); 3225 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3226 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 3227 SDLoc dl(N); 3228 3229 // Promote to the larger FP type. 3230 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 3231 3232 SDValue Res = DAG.getNode(N->getOpcode(), dl, 3233 DAG.getVTList(NVT, N->getValueType(1)), Op); 3234 3235 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3236 3237 // Convert back to FP16 as an integer. 3238 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3239 } 3240 3241 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 3242 EVT RVT = N->getValueType(0); 3243 EVT SVT = N->getOperand(0).getValueType(); 3244 3245 if (N->isStrictFPOpcode()) { 3246 // FIXME: assume we only have two f16 variants for now. 3247 unsigned Opcode; 3248 if (RVT == MVT::f16) 3249 Opcode = ISD::STRICT_FP_TO_FP16; 3250 else if (RVT == MVT::bf16) 3251 Opcode = ISD::STRICT_FP_TO_BF16; 3252 else 3253 llvm_unreachable("unknown half type"); 3254 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other}, 3255 {N->getOperand(0), N->getOperand(1)}); 3256 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3257 return Res; 3258 } 3259 3260 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16, 3261 N->getOperand(0)); 3262 } 3263 3264 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 3265 LoadSDNode *L = cast<LoadSDNode>(N); 3266 3267 // Load the value as an integer value with the same number of bits. 3268 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 3269 SDValue NewL = 3270 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 3271 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 3272 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 3273 L->getMemOperand()->getFlags(), L->getAAInfo()); 3274 // Legalize the chain result by replacing uses of the old value chain with the 3275 // new one 3276 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 3277 return NewL; 3278 } 3279 3280 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) { 3281 AtomicSDNode *AM = cast<AtomicSDNode>(N); 3282 3283 // Load the value as an integer value with the same number of bits. 3284 SDValue NewL = DAG.getAtomic( 3285 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other), 3286 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand()); 3287 3288 // Legalize the chain result by replacing uses of the old value chain with the 3289 // new one 3290 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 3291 return NewL; 3292 } 3293 3294 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 3295 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3296 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3297 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 3298 Op2); 3299 } 3300 3301 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 3302 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3303 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 3304 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 3305 N->getOperand(0), N->getOperand(1), Op2, Op3, 3306 N->getOperand(4)); 3307 } 3308 3309 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 3310 EVT OVT = N->getValueType(0); 3311 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3312 SDLoc dl(N); 3313 3314 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 3315 3316 // Round the value to the softened type. 3317 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3318 } 3319 3320 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 3321 return DAG.getUNDEF(MVT::i16); 3322 } 3323 3324 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 3325 EVT OVT = N->getValueType(0); 3326 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3327 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 3328 SDLoc dl(N); 3329 3330 // Promote to the larger FP type. 3331 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 3332 3333 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 3334 3335 // Convert back to FP16 as an integer. 3336 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3337 } 3338 3339 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 3340 EVT OVT = N->getValueType(0); 3341 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3342 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3343 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3344 SDLoc dl(N); 3345 3346 // Promote to the larger FP type. 3347 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 3348 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3349 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3350 3351 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 3352 3353 // Convert back to FP16 as an integer. 3354 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3355 } 3356 3357 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 3358 // Expand and soften recursively. 3359 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 3360 return SDValue(); 3361 } 3362 3363 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 3364 // Expand and soften. 3365 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 3366 return SDValue(); 3367 } 3368 3369 //===----------------------------------------------------------------------===// 3370 // Half Operand Soft Promotion 3371 //===----------------------------------------------------------------------===// 3372 3373 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 3374 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 3375 N->dump(&DAG)); 3376 SDValue Res = SDValue(); 3377 3378 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 3379 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 3380 return false; 3381 } 3382 3383 // Nodes that use a promotion-requiring floating point operand, but doesn't 3384 // produce a soft promotion-requiring floating point result, need to be 3385 // legalized to use the soft promoted float operand. Nodes that produce at 3386 // least one soft promotion-requiring floating point result have their 3387 // operands legalized as a part of PromoteFloatResult. 3388 switch (N->getOpcode()) { 3389 default: 3390 #ifndef NDEBUG 3391 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 3392 N->dump(&DAG); dbgs() << "\n"; 3393 #endif 3394 report_fatal_error("Do not know how to soft promote this operator's " 3395 "operand!"); 3396 3397 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 3398 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 3399 case ISD::FP_TO_SINT: 3400 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 3401 case ISD::FP_TO_SINT_SAT: 3402 case ISD::FP_TO_UINT_SAT: 3403 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 3404 case ISD::STRICT_FP_EXTEND: 3405 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 3406 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 3407 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 3408 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 3409 case ISD::ATOMIC_STORE: 3410 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo); 3411 break; 3412 case ISD::STACKMAP: 3413 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); 3414 break; 3415 case ISD::PATCHPOINT: 3416 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); 3417 break; 3418 } 3419 3420 if (!Res.getNode()) 3421 return false; 3422 3423 assert(Res.getNode() != N && "Expected a new node!"); 3424 3425 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3426 "Invalid operand expansion"); 3427 3428 ReplaceValueWith(SDValue(N, 0), Res); 3429 return false; 3430 } 3431 3432 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 3433 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3434 3435 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 3436 } 3437 3438 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 3439 unsigned OpNo) { 3440 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 3441 SDValue Op1 = N->getOperand(1); 3442 EVT RVT = Op1.getValueType(); 3443 SDLoc dl(N); 3444 3445 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 3446 3447 Op1 = GetSoftPromotedHalf(Op1); 3448 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1); 3449 3450 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 3451 Op1); 3452 } 3453 3454 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 3455 EVT RVT = N->getValueType(0); 3456 bool IsStrict = N->isStrictFPOpcode(); 3457 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3458 EVT SVT = Op.getValueType(); 3459 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 3460 3461 if (IsStrict) { 3462 unsigned Opcode; 3463 if (SVT == MVT::f16) 3464 Opcode = ISD::STRICT_FP16_TO_FP; 3465 else if (SVT == MVT::bf16) 3466 Opcode = ISD::STRICT_BF16_TO_FP; 3467 else 3468 llvm_unreachable("unknown half type"); 3469 SDValue Res = 3470 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other}, 3471 {N->getOperand(0), Op}); 3472 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3473 ReplaceValueWith(SDValue(N, 0), Res); 3474 return SDValue(); 3475 } 3476 3477 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op); 3478 } 3479 3480 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 3481 EVT RVT = N->getValueType(0); 3482 SDValue Op = N->getOperand(0); 3483 EVT SVT = Op.getValueType(); 3484 SDLoc dl(N); 3485 3486 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3487 3488 Op = GetSoftPromotedHalf(Op); 3489 3490 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3491 3492 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res); 3493 } 3494 3495 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 3496 EVT RVT = N->getValueType(0); 3497 SDValue Op = N->getOperand(0); 3498 EVT SVT = Op.getValueType(); 3499 SDLoc dl(N); 3500 3501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3502 3503 Op = GetSoftPromotedHalf(Op); 3504 3505 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3506 3507 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 3508 N->getOperand(1)); 3509 } 3510 3511 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 3512 unsigned OpNo) { 3513 assert(OpNo == 0 && "Can only soften the comparison values"); 3514 SDValue Op0 = N->getOperand(0); 3515 SDValue Op1 = N->getOperand(1); 3516 SDLoc dl(N); 3517 3518 EVT SVT = Op0.getValueType(); 3519 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT); 3520 3521 Op0 = GetSoftPromotedHalf(Op0); 3522 Op1 = GetSoftPromotedHalf(Op1); 3523 3524 // Promote to the larger FP type. 3525 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3526 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3527 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3528 3529 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 3530 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 3531 } 3532 3533 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 3534 SDValue Op0 = N->getOperand(0); 3535 SDValue Op1 = N->getOperand(1); 3536 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3537 SDLoc dl(N); 3538 3539 EVT SVT = Op0.getValueType(); 3540 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3541 3542 Op0 = GetSoftPromotedHalf(Op0); 3543 Op1 = GetSoftPromotedHalf(Op1); 3544 3545 // Promote to the larger FP type. 3546 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3547 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3548 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3549 3550 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3551 } 3552 3553 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3554 assert(OpNo == 1 && "Can only soften the stored value!"); 3555 StoreSDNode *ST = cast<StoreSDNode>(N); 3556 SDValue Val = ST->getValue(); 3557 SDLoc dl(N); 3558 3559 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3560 SDValue Promoted = GetSoftPromotedHalf(Val); 3561 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3562 ST->getMemOperand()); 3563 } 3564 3565 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, 3566 unsigned OpNo) { 3567 assert(OpNo == 1 && "Can only soften the stored value!"); 3568 AtomicSDNode *ST = cast<AtomicSDNode>(N); 3569 SDValue Val = ST->getVal(); 3570 SDLoc dl(N); 3571 3572 SDValue Promoted = GetSoftPromotedHalf(Val); 3573 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(), 3574 ST->getChain(), Promoted, ST->getBasePtr(), 3575 ST->getMemOperand()); 3576 } 3577 3578 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { 3579 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 3580 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3581 SDValue Op = N->getOperand(OpNo); 3582 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3583 SDValue NewNode = 3584 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3585 3586 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3587 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3588 3589 return SDValue(); // Signal that we replaced the node ourselves. 3590 } 3591 3592 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, 3593 unsigned OpNo) { 3594 assert(OpNo >= 7); 3595 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end()); 3596 SDValue Op = N->getOperand(OpNo); 3597 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3598 SDValue NewNode = 3599 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3600 3601 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3602 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3603 3604 return SDValue(); // Signal that we replaced the node ourselves. 3605 } 3606