1 //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file implements float type expansion and softening for LegalizeTypes. 10 // Softening is the act of turning a computation in an illegal floating point 11 // type into a computation in an integer type of the same size; also known as 12 // "soft float". For example, turning f32 arithmetic into operations using i32. 13 // The resulting integer value is the same as what you would get by performing 14 // the floating point operation and bitcasting the result to the integer type. 15 // Expansion is the act of changing a computation in an illegal type to be a 16 // computation in two identical registers of a smaller type. For example, 17 // implementing ppcf128 arithmetic in two f64 registers. 18 // 19 //===----------------------------------------------------------------------===// 20 21 #include "LegalizeTypes.h" 22 #include "llvm/Analysis/TargetLibraryInfo.h" 23 #include "llvm/Support/ErrorHandling.h" 24 #include "llvm/Support/raw_ostream.h" 25 using namespace llvm; 26 27 #define DEBUG_TYPE "legalize-types" 28 29 /// GetFPLibCall - Return the right libcall for the given floating point type. 30 /// FIXME: This is a local version of RTLIB::getFPLibCall that should be 31 /// refactored away (see RTLIB::getPOWI for an example). 32 static RTLIB::Libcall GetFPLibCall(EVT VT, 33 RTLIB::Libcall Call_F32, 34 RTLIB::Libcall Call_F64, 35 RTLIB::Libcall Call_F80, 36 RTLIB::Libcall Call_F128, 37 RTLIB::Libcall Call_PPCF128) { 38 return 39 VT == MVT::f32 ? Call_F32 : 40 VT == MVT::f64 ? Call_F64 : 41 VT == MVT::f80 ? Call_F80 : 42 VT == MVT::f128 ? Call_F128 : 43 VT == MVT::ppcf128 ? Call_PPCF128 : 44 RTLIB::UNKNOWN_LIBCALL; 45 } 46 47 //===----------------------------------------------------------------------===// 48 // Convert Float Results to Integer 49 //===----------------------------------------------------------------------===// 50 51 void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { 52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG)); 53 SDValue R = SDValue(); 54 55 switch (N->getOpcode()) { 56 // clang-format off 57 default: 58 #ifndef NDEBUG 59 dbgs() << "SoftenFloatResult #" << ResNo << ": "; 60 N->dump(&DAG); dbgs() << "\n"; 61 #endif 62 report_fatal_error("Do not know how to soften the result of this " 63 "operator!"); 64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break; 65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break; 66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break; 67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break; 68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break; 69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break; 70 case ISD::EXTRACT_VECTOR_ELT: 71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break; 72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break; 73 case ISD::STRICT_FMINNUM: 74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break; 75 case ISD::STRICT_FMAXNUM: 76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break; 77 case ISD::FMINIMUMNUM: R = SoftenFloatRes_FMINIMUMNUM(N); break; 78 case ISD::FMAXIMUMNUM: R = SoftenFloatRes_FMAXIMUMNUM(N); break; 79 case ISD::FMINIMUM: R = SoftenFloatRes_FMINIMUM(N); break; 80 case ISD::FMAXIMUM: R = SoftenFloatRes_FMAXIMUM(N); break; 81 case ISD::STRICT_FADD: 82 case ISD::FADD: R = SoftenFloatRes_FADD(N); break; 83 case ISD::STRICT_FACOS: 84 case ISD::FACOS: R = SoftenFloatRes_FACOS(N); break; 85 case ISD::STRICT_FASIN: 86 case ISD::FASIN: R = SoftenFloatRes_FASIN(N); break; 87 case ISD::STRICT_FATAN: 88 case ISD::FATAN: R = SoftenFloatRes_FATAN(N); break; 89 case ISD::STRICT_FATAN2: 90 case ISD::FATAN2: R = SoftenFloatRes_FATAN2(N); break; 91 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break; 92 case ISD::STRICT_FCEIL: 93 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break; 94 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break; 95 case ISD::STRICT_FCOS: 96 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break; 97 case ISD::STRICT_FCOSH: 98 case ISD::FCOSH: R = SoftenFloatRes_FCOSH(N); break; 99 case ISD::STRICT_FDIV: 100 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break; 101 case ISD::STRICT_FEXP: 102 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break; 103 case ISD::STRICT_FEXP2: 104 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break; 105 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break; 106 case ISD::STRICT_FFLOOR: 107 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break; 108 case ISD::STRICT_FLOG: 109 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break; 110 case ISD::STRICT_FLOG2: 111 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break; 112 case ISD::STRICT_FLOG10: 113 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break; 114 case ISD::STRICT_FMA: 115 case ISD::FMA: R = SoftenFloatRes_FMA(N); break; 116 case ISD::STRICT_FMUL: 117 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break; 118 case ISD::STRICT_FNEARBYINT: 119 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break; 120 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break; 121 case ISD::STRICT_FP_EXTEND: 122 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break; 123 case ISD::STRICT_FP_ROUND: 124 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break; 125 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break; 126 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break; 127 case ISD::STRICT_FPOW: 128 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break; 129 case ISD::STRICT_FPOWI: 130 case ISD::FPOWI: 131 case ISD::FLDEXP: 132 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break; 133 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break; 134 case ISD::FSINCOS: R = SoftenFloatRes_FSINCOS(N); break; 135 case ISD::STRICT_FREM: 136 case ISD::FREM: R = SoftenFloatRes_FREM(N); break; 137 case ISD::STRICT_FRINT: 138 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break; 139 case ISD::STRICT_FROUND: 140 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break; 141 case ISD::STRICT_FROUNDEVEN: 142 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break; 143 case ISD::STRICT_FSIN: 144 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break; 145 case ISD::STRICT_FSINH: 146 case ISD::FSINH: R = SoftenFloatRes_FSINH(N); break; 147 case ISD::STRICT_FSQRT: 148 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break; 149 case ISD::STRICT_FSUB: 150 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; 151 case ISD::STRICT_FTAN: 152 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break; 153 case ISD::STRICT_FTANH: 154 case ISD::FTANH: R = SoftenFloatRes_FTANH(N); break; 155 case ISD::STRICT_FTRUNC: 156 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; 157 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break; 158 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break; 159 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 160 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break; 161 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break; 162 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break; 163 case ISD::STRICT_SINT_TO_FP: 164 case ISD::STRICT_UINT_TO_FP: 165 case ISD::SINT_TO_FP: 166 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break; 167 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break; 168 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break; 169 case ISD::VECREDUCE_FADD: 170 case ISD::VECREDUCE_FMUL: 171 case ISD::VECREDUCE_FMIN: 172 case ISD::VECREDUCE_FMAX: 173 case ISD::VECREDUCE_FMAXIMUM: 174 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break; 175 case ISD::VECREDUCE_SEQ_FADD: 176 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break; 177 // clang-format on 178 } 179 180 // If R is null, the sub-method took care of registering the result. 181 if (R.getNode()) { 182 assert(R.getNode() != N); 183 SetSoftenedFloat(SDValue(N, ResNo), R); 184 } 185 } 186 187 SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) { 188 bool IsStrict = N->isStrictFPOpcode(); 189 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 190 unsigned Offset = IsStrict ? 1 : 0; 191 assert(N->getNumOperands() == (1 + Offset) && 192 "Unexpected number of operands!"); 193 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 194 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 195 TargetLowering::MakeLibCallOptions CallOptions; 196 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 197 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 198 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 199 CallOptions, SDLoc(N), 200 Chain); 201 if (IsStrict) 202 ReplaceValueWith(SDValue(N, 1), Tmp.second); 203 return Tmp.first; 204 } 205 206 SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) { 207 bool IsStrict = N->isStrictFPOpcode(); 208 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 209 unsigned Offset = IsStrict ? 1 : 0; 210 assert(N->getNumOperands() == (2 + Offset) && 211 "Unexpected number of operands!"); 212 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 213 GetSoftenedFloat(N->getOperand(1 + Offset)) }; 214 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 215 TargetLowering::MakeLibCallOptions CallOptions; 216 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 217 N->getOperand(1 + Offset).getValueType() }; 218 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 219 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 220 CallOptions, SDLoc(N), 221 Chain); 222 if (IsStrict) 223 ReplaceValueWith(SDValue(N, 1), Tmp.second); 224 return Tmp.first; 225 } 226 227 SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) { 228 return BitConvertToInteger(N->getOperand(0)); 229 } 230 231 SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) { 232 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 233 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty, 234 GetSoftenedFloat(N->getOperand(0))); 235 } 236 237 SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) { 238 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 239 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty, 240 GetSoftenedFloat(N->getOperand(0))); 241 return NewFence; 242 } 243 244 SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N, 245 unsigned ResNo) { 246 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); 247 return BitConvertToInteger(Op); 248 } 249 250 SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) { 251 // Convert the inputs to integers, and build a new pair out of them. 252 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N), 253 TLI.getTypeToTransformTo(*DAG.getContext(), 254 N->getValueType(0)), 255 BitConvertToInteger(N->getOperand(0)), 256 BitConvertToInteger(N->getOperand(1))); 257 } 258 259 SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) { 260 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 261 // In ppcf128, the high 64 bits are always first in memory regardless 262 // of Endianness. LLVM's APFloat representation is not Endian sensitive, 263 // and so always converts into a 128-bit APInt in a non-Endian-sensitive 264 // way. However, APInt's are serialized in an Endian-sensitive fashion, 265 // so on big-Endian targets, the two doubles are output in the wrong 266 // order. Fix this by manually flipping the order of the high 64 bits 267 // and the low 64 bits here. 268 if (DAG.getDataLayout().isBigEndian() && 269 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) { 270 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1], 271 CN->getValueAPF().bitcastToAPInt().getRawData()[0] }; 272 APInt Val(128, words); 273 return DAG.getConstant(Val, SDLoc(CN), 274 TLI.getTypeToTransformTo(*DAG.getContext(), 275 CN->getValueType(0))); 276 } else { 277 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 278 TLI.getTypeToTransformTo(*DAG.getContext(), 279 CN->getValueType(0))); 280 } 281 } 282 283 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) { 284 SDValue Src = N->getOperand(0); 285 assert(Src.getValueType() == MVT::ppcf128 && 286 "In floats only ppcf128 can be extracted by element!"); 287 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N), 288 N->getValueType(0).changeTypeToInteger(), 289 DAG.getBitcast(MVT::i128, Src), N->getOperand(1)); 290 } 291 292 SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) { 293 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 294 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 295 NewOp.getValueType().getVectorElementType(), 296 NewOp, N->getOperand(1)); 297 } 298 299 SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) { 300 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 301 unsigned Size = NVT.getSizeInBits(); 302 303 // Mask = ~(1 << (Size-1)) 304 APInt API = APInt::getAllOnes(Size); 305 API.clearBit(Size - 1); 306 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT); 307 SDValue Op = GetSoftenedFloat(N->getOperand(0)); 308 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask); 309 } 310 311 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) { 312 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 313 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 314 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 315 RTLIB::FMIN_F32, 316 RTLIB::FMIN_F64, 317 RTLIB::FMIN_F80, 318 RTLIB::FMIN_F128, 319 RTLIB::FMIN_PPCF128)); 320 } 321 322 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) { 323 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG)) 324 return SoftenFloatRes_SELECT_CC(SelCC.getNode()); 325 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 326 RTLIB::FMAX_F32, 327 RTLIB::FMAX_F64, 328 RTLIB::FMAX_F80, 329 RTLIB::FMAX_F128, 330 RTLIB::FMAX_PPCF128)); 331 } 332 333 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUMNUM(SDNode *N) { 334 return SoftenFloatRes_Binary( 335 N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUMNUM_F32, 336 RTLIB::FMINIMUMNUM_F64, RTLIB::FMINIMUMNUM_F80, 337 RTLIB::FMINIMUMNUM_F128, RTLIB::FMINIMUMNUM_PPCF128)); 338 } 339 340 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUMNUM(SDNode *N) { 341 return SoftenFloatRes_Binary( 342 N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUMNUM_F32, 343 RTLIB::FMAXIMUMNUM_F64, RTLIB::FMAXIMUMNUM_F80, 344 RTLIB::FMAXIMUMNUM_F128, RTLIB::FMAXIMUMNUM_PPCF128)); 345 } 346 347 SDValue DAGTypeLegalizer::SoftenFloatRes_FMINIMUM(SDNode *N) { 348 return SoftenFloatRes_Binary( 349 N, GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUM_F32, 350 RTLIB::FMINIMUM_F64, RTLIB::FMINIMUM_F80, 351 RTLIB::FMINIMUM_F128, RTLIB::FMINIMUM_PPCF128)); 352 } 353 354 SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXIMUM(SDNode *N) { 355 return SoftenFloatRes_Binary( 356 N, GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUM_F32, 357 RTLIB::FMAXIMUM_F64, RTLIB::FMAXIMUM_F80, 358 RTLIB::FMAXIMUM_F128, RTLIB::FMAXIMUM_PPCF128)); 359 } 360 361 SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) { 362 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 363 RTLIB::ADD_F32, 364 RTLIB::ADD_F64, 365 RTLIB::ADD_F80, 366 RTLIB::ADD_F128, 367 RTLIB::ADD_PPCF128)); 368 } 369 370 SDValue DAGTypeLegalizer::SoftenFloatRes_FACOS(SDNode *N) { 371 return SoftenFloatRes_Unary( 372 N, GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, RTLIB::ACOS_F64, 373 RTLIB::ACOS_F80, RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128)); 374 } 375 376 SDValue DAGTypeLegalizer::SoftenFloatRes_FASIN(SDNode *N) { 377 return SoftenFloatRes_Unary( 378 N, GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, RTLIB::ASIN_F64, 379 RTLIB::ASIN_F80, RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128)); 380 } 381 382 SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN(SDNode *N) { 383 return SoftenFloatRes_Unary( 384 N, GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, RTLIB::ATAN_F64, 385 RTLIB::ATAN_F80, RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128)); 386 } 387 388 SDValue DAGTypeLegalizer::SoftenFloatRes_FATAN2(SDNode *N) { 389 return SoftenFloatRes_Binary( 390 N, 391 GetFPLibCall(N->getValueType(0), RTLIB::ATAN2_F32, RTLIB::ATAN2_F64, 392 RTLIB::ATAN2_F80, RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128)); 393 } 394 395 SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) { 396 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 397 RTLIB::CBRT_F32, 398 RTLIB::CBRT_F64, 399 RTLIB::CBRT_F80, 400 RTLIB::CBRT_F128, 401 RTLIB::CBRT_PPCF128)); 402 } 403 404 SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) { 405 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 406 RTLIB::CEIL_F32, 407 RTLIB::CEIL_F64, 408 RTLIB::CEIL_F80, 409 RTLIB::CEIL_F128, 410 RTLIB::CEIL_PPCF128)); 411 } 412 413 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) { 414 SDValue LHS = GetSoftenedFloat(N->getOperand(0)); 415 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 416 SDLoc dl(N); 417 418 EVT LVT = LHS.getValueType(); 419 EVT RVT = RHS.getValueType(); 420 421 unsigned LSize = LVT.getSizeInBits(); 422 unsigned RSize = RVT.getSizeInBits(); 423 424 // First get the sign bit of second operand. 425 SDValue SignBit = DAG.getNode( 426 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 427 DAG.getConstant(RSize - 1, dl, 428 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 429 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 430 431 // Shift right or sign-extend it if the two operands have different types. 432 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 433 if (SizeDiff > 0) { 434 SignBit = 435 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 436 DAG.getConstant(SizeDiff, dl, 437 TLI.getShiftAmountTy(SignBit.getValueType(), 438 DAG.getDataLayout()))); 439 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 440 } else if (SizeDiff < 0) { 441 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 442 SignBit = 443 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 444 DAG.getConstant(-SizeDiff, dl, 445 TLI.getShiftAmountTy(SignBit.getValueType(), 446 DAG.getDataLayout()))); 447 } 448 449 // Clear the sign bit of the first operand. 450 SDValue Mask = DAG.getNode( 451 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 452 DAG.getConstant(LSize - 1, dl, 453 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 454 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 455 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 456 457 // Or the value with the sign bit. 458 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 459 } 460 461 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) { 462 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 463 RTLIB::COS_F32, 464 RTLIB::COS_F64, 465 RTLIB::COS_F80, 466 RTLIB::COS_F128, 467 RTLIB::COS_PPCF128)); 468 } 469 470 SDValue DAGTypeLegalizer::SoftenFloatRes_FCOSH(SDNode *N) { 471 return SoftenFloatRes_Unary( 472 N, GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, RTLIB::COSH_F64, 473 RTLIB::COSH_F80, RTLIB::COSH_F128, RTLIB::COSH_PPCF128)); 474 } 475 476 SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) { 477 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 478 RTLIB::DIV_F32, 479 RTLIB::DIV_F64, 480 RTLIB::DIV_F80, 481 RTLIB::DIV_F128, 482 RTLIB::DIV_PPCF128)); 483 } 484 485 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) { 486 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 487 RTLIB::EXP_F32, 488 RTLIB::EXP_F64, 489 RTLIB::EXP_F80, 490 RTLIB::EXP_F128, 491 RTLIB::EXP_PPCF128)); 492 } 493 494 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) { 495 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 496 RTLIB::EXP2_F32, 497 RTLIB::EXP2_F64, 498 RTLIB::EXP2_F80, 499 RTLIB::EXP2_F128, 500 RTLIB::EXP2_PPCF128)); 501 } 502 503 SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) { 504 return SoftenFloatRes_Unary( 505 N, 506 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64, 507 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128)); 508 } 509 510 SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) { 511 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 512 RTLIB::FLOOR_F32, 513 RTLIB::FLOOR_F64, 514 RTLIB::FLOOR_F80, 515 RTLIB::FLOOR_F128, 516 RTLIB::FLOOR_PPCF128)); 517 } 518 519 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) { 520 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 521 RTLIB::LOG_F32, 522 RTLIB::LOG_F64, 523 RTLIB::LOG_F80, 524 RTLIB::LOG_F128, 525 RTLIB::LOG_PPCF128)); 526 } 527 528 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) { 529 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 530 RTLIB::LOG2_F32, 531 RTLIB::LOG2_F64, 532 RTLIB::LOG2_F80, 533 RTLIB::LOG2_F128, 534 RTLIB::LOG2_PPCF128)); 535 } 536 537 SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) { 538 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 539 RTLIB::LOG10_F32, 540 RTLIB::LOG10_F64, 541 RTLIB::LOG10_F80, 542 RTLIB::LOG10_F128, 543 RTLIB::LOG10_PPCF128)); 544 } 545 546 SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) { 547 bool IsStrict = N->isStrictFPOpcode(); 548 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 549 unsigned Offset = IsStrict ? 1 : 0; 550 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 551 GetSoftenedFloat(N->getOperand(1 + Offset)), 552 GetSoftenedFloat(N->getOperand(2 + Offset)) }; 553 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 554 TargetLowering::MakeLibCallOptions CallOptions; 555 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(), 556 N->getOperand(1 + Offset).getValueType(), 557 N->getOperand(2 + Offset).getValueType() }; 558 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 559 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, 560 GetFPLibCall(N->getValueType(0), 561 RTLIB::FMA_F32, 562 RTLIB::FMA_F64, 563 RTLIB::FMA_F80, 564 RTLIB::FMA_F128, 565 RTLIB::FMA_PPCF128), 566 NVT, Ops, CallOptions, SDLoc(N), Chain); 567 if (IsStrict) 568 ReplaceValueWith(SDValue(N, 1), Tmp.second); 569 return Tmp.first; 570 } 571 572 SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) { 573 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 574 RTLIB::MUL_F32, 575 RTLIB::MUL_F64, 576 RTLIB::MUL_F80, 577 RTLIB::MUL_F128, 578 RTLIB::MUL_PPCF128)); 579 } 580 581 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) { 582 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 583 RTLIB::NEARBYINT_F32, 584 RTLIB::NEARBYINT_F64, 585 RTLIB::NEARBYINT_F80, 586 RTLIB::NEARBYINT_F128, 587 RTLIB::NEARBYINT_PPCF128)); 588 } 589 590 SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) { 591 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 592 SDLoc dl(N); 593 594 // Expand Y = FNEG(X) -> Y = X ^ sign mask 595 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); 596 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), 597 DAG.getConstant(SignMask, dl, NVT)); 598 } 599 600 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) { 601 bool IsStrict = N->isStrictFPOpcode(); 602 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 603 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 604 605 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 606 607 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) { 608 Op = GetPromotedFloat(Op); 609 // If the promotion did the FP_EXTEND to the destination type for us, 610 // there's nothing left to do here. 611 if (Op.getValueType() == N->getValueType(0)) { 612 if (IsStrict) 613 ReplaceValueWith(SDValue(N, 1), Chain); 614 return BitConvertToInteger(Op); 615 } 616 } 617 618 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16 619 // -> f32, so proceed in two stages. Also, it's entirely possible for both 620 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather 621 // than FP16_TO_FP. 622 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) && 623 N->getValueType(0) != MVT::f32) { 624 if (IsStrict) { 625 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), 626 { MVT::f32, MVT::Other }, { Chain, Op }); 627 Chain = Op.getValue(1); 628 } else { 629 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op); 630 } 631 } 632 633 if (Op.getValueType() == MVT::bf16) { 634 // FIXME: Need ReplaceValueWith on chain in strict case 635 return SoftenFloatRes_BF16_TO_FP(N); 636 } 637 638 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0)); 639 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 640 TargetLowering::MakeLibCallOptions CallOptions; 641 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 642 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 643 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 644 CallOptions, SDLoc(N), 645 Chain); 646 if (IsStrict) 647 ReplaceValueWith(SDValue(N, 1), Tmp.second); 648 return Tmp.first; 649 } 650 651 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 652 // nodes? 653 SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) { 654 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 655 SDValue Op = N->getOperand(0); 656 TargetLowering::MakeLibCallOptions CallOptions; 657 EVT OpsVT[1] = { N->getOperand(0).getValueType() }; 658 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 659 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op, 660 CallOptions, SDLoc(N)).first; 661 if (N->getValueType(0) == MVT::f32) 662 return Res32; 663 664 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 665 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0)); 666 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!"); 667 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first; 668 } 669 670 // FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special 671 // nodes? 672 SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) { 673 assert(N->getValueType(0) == MVT::f32 && 674 "Can only soften BF16_TO_FP with f32 result"); 675 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32); 676 SDValue Op = N->getOperand(0); 677 SDLoc DL(N); 678 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT, 679 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op)); 680 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op, 681 DAG.getShiftAmountConstant(16, NVT, DL)); 682 return Res; 683 } 684 685 SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) { 686 bool IsStrict = N->isStrictFPOpcode(); 687 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 688 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 689 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 690 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0)); 691 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!"); 692 TargetLowering::MakeLibCallOptions CallOptions; 693 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 694 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 695 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 696 CallOptions, SDLoc(N), 697 Chain); 698 if (IsStrict) 699 ReplaceValueWith(SDValue(N, 1), Tmp.second); 700 return Tmp.first; 701 } 702 703 SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) { 704 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 705 RTLIB::POW_F32, 706 RTLIB::POW_F64, 707 RTLIB::POW_F80, 708 RTLIB::POW_F128, 709 RTLIB::POW_PPCF128)); 710 } 711 712 SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) { 713 bool IsStrict = N->isStrictFPOpcode(); 714 unsigned Offset = IsStrict ? 1 : 0; 715 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 || 716 N->getOperand(1 + Offset).getValueType() == MVT::i32) && 717 "Unsupported power type!"); 718 bool IsPowI = 719 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI; 720 721 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0)) 722 : RTLIB::getLDEXP(N->getValueType(0)); 723 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi."); 724 if (!TLI.getLibcallName(LC)) { 725 // Some targets don't have a powi libcall; use pow instead. 726 // FIXME: Implement this if some target needs it. 727 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow"); 728 return DAG.getUNDEF(N->getValueType(0)); 729 } 730 731 if (DAG.getLibInfo().getIntSize() != 732 N->getOperand(1 + Offset).getValueType().getSizeInBits()) { 733 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI 734 // would use the wrong type for the argument. 735 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)"); 736 return DAG.getUNDEF(N->getValueType(0)); 737 } 738 739 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 740 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)), 741 N->getOperand(1 + Offset) }; 742 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 743 TargetLowering::MakeLibCallOptions CallOptions; 744 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(), 745 N->getOperand(1 + Offset).getValueType() }; 746 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true); 747 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops, 748 CallOptions, SDLoc(N), 749 Chain); 750 if (IsStrict) 751 ReplaceValueWith(SDValue(N, 1), Tmp.second); 752 return Tmp.first; 753 } 754 755 SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) { 756 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp"); 757 EVT VT0 = N->getValueType(0); 758 EVT VT1 = N->getValueType(1); 759 RTLIB::Libcall LC = RTLIB::getFREXP(VT0); 760 761 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) { 762 // If the exponent does not match with sizeof(int) a libcall would use the 763 // wrong type for the argument. 764 // TODO: Should be able to handle mismatches. 765 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)"); 766 return DAG.getUNDEF(N->getValueType(0)); 767 } 768 769 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0); 770 SDValue StackSlot = DAG.CreateStackTemporary(VT1); 771 772 SDLoc DL(N); 773 774 TargetLowering::MakeLibCallOptions CallOptions; 775 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot}; 776 EVT OpsVT[2] = {VT0, StackSlot.getValueType()}; 777 778 // TODO: setTypeListBeforeSoften can't properly express multiple return types, 779 // but we only really need to handle the 0th one for softening anyway. 780 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true); 781 782 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL, 783 /*Chain=*/SDValue()); 784 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 785 auto PtrInfo = 786 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 787 788 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo); 789 790 ReplaceValueWith(SDValue(N, 1), LoadExp); 791 return ReturnVal; 792 } 793 794 SDValue 795 DAGTypeLegalizer::SoftenFloatRes_UnaryWithTwoFPResults(SDNode *N, 796 RTLIB::Libcall LC) { 797 assert(!N->isStrictFPOpcode() && "strictfp not implemented"); 798 EVT VT = N->getValueType(0); 799 800 if (!TLI.getLibcallName(LC)) 801 return SDValue(); 802 803 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 804 SDValue FirstResultSlot = DAG.CreateStackTemporary(NVT); 805 SDValue SecondResultSlot = DAG.CreateStackTemporary(NVT); 806 807 SDLoc DL(N); 808 809 TargetLowering::MakeLibCallOptions CallOptions; 810 std::array Ops{GetSoftenedFloat(N->getOperand(0)), FirstResultSlot, 811 SecondResultSlot}; 812 std::array OpsVT{VT, FirstResultSlot.getValueType(), 813 SecondResultSlot.getValueType()}; 814 815 // TODO: setTypeListBeforeSoften can't properly express multiple return types, 816 // but since both returns have the same type it should be okay. 817 CallOptions.setTypeListBeforeSoften({OpsVT}, VT, true); 818 819 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT, Ops, CallOptions, DL, 820 /*Chain=*/SDValue()); 821 822 auto CreateStackLoad = [&, Chain = Chain](SDValue StackSlot) { 823 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex(); 824 auto PtrInfo = 825 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx); 826 return DAG.getLoad(NVT, DL, Chain, StackSlot, PtrInfo); 827 }; 828 SetSoftenedFloat(SDValue(N, 0), CreateStackLoad(FirstResultSlot)); 829 SetSoftenedFloat(SDValue(N, 1), CreateStackLoad(SecondResultSlot)); 830 831 return SDValue(); 832 } 833 834 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINCOS(SDNode *N) { 835 return SoftenFloatRes_UnaryWithTwoFPResults( 836 N, RTLIB::getFSINCOS(N->getValueType(0))); 837 } 838 839 SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) { 840 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 841 RTLIB::REM_F32, 842 RTLIB::REM_F64, 843 RTLIB::REM_F80, 844 RTLIB::REM_F128, 845 RTLIB::REM_PPCF128)); 846 } 847 848 SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) { 849 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 850 RTLIB::RINT_F32, 851 RTLIB::RINT_F64, 852 RTLIB::RINT_F80, 853 RTLIB::RINT_F128, 854 RTLIB::RINT_PPCF128)); 855 } 856 857 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) { 858 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 859 RTLIB::ROUND_F32, 860 RTLIB::ROUND_F64, 861 RTLIB::ROUND_F80, 862 RTLIB::ROUND_F128, 863 RTLIB::ROUND_PPCF128)); 864 } 865 866 SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) { 867 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 868 RTLIB::ROUNDEVEN_F32, 869 RTLIB::ROUNDEVEN_F64, 870 RTLIB::ROUNDEVEN_F80, 871 RTLIB::ROUNDEVEN_F128, 872 RTLIB::ROUNDEVEN_PPCF128)); 873 } 874 875 SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) { 876 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 877 RTLIB::SIN_F32, 878 RTLIB::SIN_F64, 879 RTLIB::SIN_F80, 880 RTLIB::SIN_F128, 881 RTLIB::SIN_PPCF128)); 882 } 883 884 SDValue DAGTypeLegalizer::SoftenFloatRes_FSINH(SDNode *N) { 885 return SoftenFloatRes_Unary( 886 N, GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, RTLIB::SINH_F64, 887 RTLIB::SINH_F80, RTLIB::SINH_F128, RTLIB::SINH_PPCF128)); 888 } 889 890 SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) { 891 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 892 RTLIB::SQRT_F32, 893 RTLIB::SQRT_F64, 894 RTLIB::SQRT_F80, 895 RTLIB::SQRT_F128, 896 RTLIB::SQRT_PPCF128)); 897 } 898 899 SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) { 900 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 901 RTLIB::SUB_F32, 902 RTLIB::SUB_F64, 903 RTLIB::SUB_F80, 904 RTLIB::SUB_F128, 905 RTLIB::SUB_PPCF128)); 906 } 907 908 SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) { 909 return SoftenFloatRes_Unary( 910 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64, 911 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128)); 912 } 913 914 SDValue DAGTypeLegalizer::SoftenFloatRes_FTANH(SDNode *N) { 915 return SoftenFloatRes_Unary( 916 N, GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, RTLIB::TANH_F64, 917 RTLIB::TANH_F80, RTLIB::TANH_F128, RTLIB::TANH_PPCF128)); 918 } 919 920 SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) { 921 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 922 RTLIB::TRUNC_F32, 923 RTLIB::TRUNC_F64, 924 RTLIB::TRUNC_F80, 925 RTLIB::TRUNC_F128, 926 RTLIB::TRUNC_PPCF128)); 927 } 928 929 SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) { 930 LoadSDNode *L = cast<LoadSDNode>(N); 931 EVT VT = N->getValueType(0); 932 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 933 SDLoc dl(N); 934 935 auto MMOFlags = 936 L->getMemOperand()->getFlags() & 937 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); 938 SDValue NewL; 939 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 940 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, 941 L->getChain(), L->getBasePtr(), L->getOffset(), 942 L->getPointerInfo(), NVT, L->getOriginalAlign(), 943 MMOFlags, L->getAAInfo()); 944 // Legalized the chain result - switch anything that used the old chain to 945 // use the new one. 946 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 947 return NewL; 948 } 949 950 // Do a non-extending load followed by FP_EXTEND. 951 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(), 952 dl, L->getChain(), L->getBasePtr(), L->getOffset(), 953 L->getPointerInfo(), L->getMemoryVT(), 954 L->getOriginalAlign(), MMOFlags, L->getAAInfo()); 955 // Legalized the chain result - switch anything that used the old chain to 956 // use the new one. 957 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 958 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL); 959 return BitConvertToInteger(ExtendNode); 960 } 961 962 SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) { 963 AtomicSDNode *L = cast<AtomicSDNode>(N); 964 EVT VT = N->getValueType(0); 965 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 966 SDLoc dl(N); 967 968 if (L->getExtensionType() == ISD::NON_EXTLOAD) { 969 SDValue NewL = 970 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other), 971 {L->getChain(), L->getBasePtr()}, L->getMemOperand()); 972 973 // Legalized the chain result - switch anything that used the old chain to 974 // use the new one. 975 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 976 return NewL; 977 } 978 979 report_fatal_error("softening fp extending atomic load not handled"); 980 } 981 982 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) { 983 SDValue LHS = GetSoftenedFloat(N->getOperand(1)); 984 SDValue RHS = GetSoftenedFloat(N->getOperand(2)); 985 return DAG.getSelect(SDLoc(N), 986 LHS.getValueType(), N->getOperand(0), LHS, RHS); 987 } 988 989 SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) { 990 SDValue LHS = GetSoftenedFloat(N->getOperand(2)); 991 SDValue RHS = GetSoftenedFloat(N->getOperand(3)); 992 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 993 LHS.getValueType(), N->getOperand(0), 994 N->getOperand(1), LHS, RHS, N->getOperand(4)); 995 } 996 997 SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) { 998 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 999 N->getValueType(0))); 1000 } 1001 1002 SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) { 1003 SDValue Chain = N->getOperand(0); // Get the chain. 1004 SDValue Ptr = N->getOperand(1); // Get the pointer. 1005 EVT VT = N->getValueType(0); 1006 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 1007 SDLoc dl(N); 1008 1009 SDValue NewVAARG; 1010 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), 1011 N->getConstantOperandVal(3)); 1012 1013 // Legalized the chain result - switch anything that used the old chain to 1014 // use the new one. 1015 if (N != NewVAARG.getValue(1).getNode()) 1016 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1)); 1017 return NewVAARG; 1018 } 1019 1020 SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) { 1021 bool IsStrict = N->isStrictFPOpcode(); 1022 bool Signed = N->getOpcode() == ISD::SINT_TO_FP || 1023 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 1024 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType(); 1025 EVT RVT = N->getValueType(0); 1026 EVT NVT = EVT(); 1027 SDLoc dl(N); 1028 1029 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to 1030 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly 1031 // match. Look for an appropriate libcall. 1032 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1033 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE; 1034 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) { 1035 NVT = (MVT::SimpleValueType)t; 1036 // The source needs to big enough to hold the operand. 1037 if (NVT.bitsGE(SVT)) 1038 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT); 1039 } 1040 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 1041 1042 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1043 // Sign/zero extend the argument if the libcall takes a larger type. 1044 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 1045 NVT, N->getOperand(IsStrict ? 1 : 0)); 1046 TargetLowering::MakeLibCallOptions CallOptions; 1047 CallOptions.setIsSigned(Signed); 1048 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1049 std::pair<SDValue, SDValue> Tmp = 1050 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT), 1051 Op, CallOptions, dl, Chain); 1052 1053 if (IsStrict) 1054 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1055 return Tmp.first; 1056 } 1057 1058 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) { 1059 // Expand and soften recursively. 1060 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 1061 return SDValue(); 1062 } 1063 1064 SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) { 1065 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 1066 return SDValue(); 1067 } 1068 1069 //===----------------------------------------------------------------------===// 1070 // Convert Float Operand to Integer 1071 //===----------------------------------------------------------------------===// 1072 1073 bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { 1074 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG)); 1075 SDValue Res = SDValue(); 1076 1077 switch (N->getOpcode()) { 1078 default: 1079 #ifndef NDEBUG 1080 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": "; 1081 N->dump(&DAG); dbgs() << "\n"; 1082 #endif 1083 report_fatal_error("Do not know how to soften this operator's operand!"); 1084 1085 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break; 1086 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break; 1087 case ISD::STRICT_FP_TO_FP16: 1088 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes 1089 case ISD::FP_TO_BF16: 1090 case ISD::STRICT_FP_TO_BF16: 1091 case ISD::STRICT_FP_ROUND: 1092 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; 1093 case ISD::STRICT_FP_TO_SINT: 1094 case ISD::STRICT_FP_TO_UINT: 1095 case ISD::FP_TO_SINT: 1096 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; 1097 case ISD::FP_TO_SINT_SAT: 1098 case ISD::FP_TO_UINT_SAT: 1099 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break; 1100 case ISD::STRICT_LROUND: 1101 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; 1102 case ISD::STRICT_LLROUND: 1103 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; 1104 case ISD::STRICT_LRINT: 1105 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; 1106 case ISD::STRICT_LLRINT: 1107 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; 1108 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; 1109 case ISD::STRICT_FSETCC: 1110 case ISD::STRICT_FSETCCS: 1111 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; 1112 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break; 1113 case ISD::ATOMIC_STORE: 1114 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo); 1115 break; 1116 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break; 1117 } 1118 1119 // If the result is null, the sub-method took care of registering results etc. 1120 if (!Res.getNode()) return false; 1121 1122 // If the result is N, the sub-method updated N in place. Tell the legalizer 1123 // core about this to re-analyze. 1124 if (Res.getNode() == N) 1125 return true; 1126 1127 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 1128 "Invalid operand softening"); 1129 1130 ReplaceValueWith(SDValue(N, 0), Res); 1131 return false; 1132 } 1133 1134 SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) { 1135 SDValue Op0 = GetSoftenedFloat(N->getOperand(0)); 1136 1137 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 1138 } 1139 1140 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) { 1141 // We actually deal with the partially-softened FP_TO_FP16 node too, which 1142 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND. 1143 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 || 1144 N->getOpcode() == ISD::STRICT_FP_TO_FP16 || 1145 N->getOpcode() == ISD::FP_TO_BF16 || 1146 N->getOpcode() == ISD::STRICT_FP_TO_BF16 || 1147 N->getOpcode() == ISD::STRICT_FP_ROUND); 1148 1149 bool IsStrict = N->isStrictFPOpcode(); 1150 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1151 EVT SVT = Op.getValueType(); 1152 EVT RVT = N->getValueType(0); 1153 EVT FloatRVT = RVT; 1154 if (N->getOpcode() == ISD::FP_TO_FP16 || 1155 N->getOpcode() == ISD::STRICT_FP_TO_FP16) 1156 FloatRVT = MVT::f16; 1157 else if (N->getOpcode() == ISD::FP_TO_BF16 || 1158 N->getOpcode() == ISD::STRICT_FP_TO_BF16) 1159 FloatRVT = MVT::bf16; 1160 1161 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT); 1162 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 1163 1164 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1165 Op = GetSoftenedFloat(Op); 1166 TargetLowering::MakeLibCallOptions CallOptions; 1167 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1168 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op, 1169 CallOptions, SDLoc(N), 1170 Chain); 1171 if (IsStrict) { 1172 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1173 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1174 return SDValue(); 1175 } 1176 return Tmp.first; 1177 } 1178 1179 SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) { 1180 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 1181 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 1182 1183 EVT VT = NewLHS.getValueType(); 1184 NewLHS = GetSoftenedFloat(NewLHS); 1185 NewRHS = GetSoftenedFloat(NewRHS); 1186 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1187 N->getOperand(2), N->getOperand(3)); 1188 1189 // If softenSetCCOperands returned a scalar, we need to compare the result 1190 // against zero to select between true and false values. 1191 if (!NewRHS.getNode()) { 1192 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1193 CCCode = ISD::SETNE; 1194 } 1195 1196 // Update N to have the operands specified. 1197 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 1198 DAG.getCondCode(CCCode), NewLHS, NewRHS, 1199 N->getOperand(4)), 1200 0); 1201 } 1202 1203 // Even if the result type is legal, no libcall may exactly match. (e.g. We 1204 // don't have FP-i8 conversions) This helper method looks for an appropriate 1205 // promoted libcall. 1206 static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, 1207 bool Signed) { 1208 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 1209 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE; 1210 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; 1211 ++IntVT) { 1212 Promoted = (MVT::SimpleValueType)IntVT; 1213 // The type needs to big enough to hold the result. 1214 if (Promoted.bitsGE(RetVT)) 1215 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted) 1216 : RTLIB::getFPTOUINT(SrcVT, Promoted); 1217 } 1218 return LC; 1219 } 1220 1221 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) { 1222 bool IsStrict = N->isStrictFPOpcode(); 1223 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 1224 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 1225 1226 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 1227 EVT SVT = Op.getValueType(); 1228 EVT RVT = N->getValueType(0); 1229 EVT NVT = EVT(); 1230 SDLoc dl(N); 1231 1232 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to 1233 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly 1234 // match, eg. we don't have fp -> i8 conversions. 1235 // Look for an appropriate libcall. 1236 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed); 1237 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 1238 "Unsupported FP_TO_XINT!"); 1239 1240 Op = GetSoftenedFloat(Op); 1241 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1242 TargetLowering::MakeLibCallOptions CallOptions; 1243 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 1244 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1245 CallOptions, dl, Chain); 1246 1247 // Truncate the result if the libcall returns a larger type. 1248 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first); 1249 1250 if (!IsStrict) 1251 return Res; 1252 1253 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1254 ReplaceValueWith(SDValue(N, 0), Res); 1255 return SDValue(); 1256 } 1257 1258 SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) { 1259 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG); 1260 return Res; 1261 } 1262 1263 SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) { 1264 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 1265 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 1266 1267 EVT VT = NewLHS.getValueType(); 1268 NewLHS = GetSoftenedFloat(NewLHS); 1269 NewRHS = GetSoftenedFloat(NewRHS); 1270 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), 1271 N->getOperand(0), N->getOperand(1)); 1272 1273 // If softenSetCCOperands returned a scalar, we need to compare the result 1274 // against zero to select between true and false values. 1275 if (!NewRHS.getNode()) { 1276 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 1277 CCCode = ISD::SETNE; 1278 } 1279 1280 // Update N to have the operands specified. 1281 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1282 N->getOperand(2), N->getOperand(3), 1283 DAG.getCondCode(CCCode)), 1284 0); 1285 } 1286 1287 SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) { 1288 bool IsStrict = N->isStrictFPOpcode(); 1289 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0); 1290 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1); 1291 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1292 ISD::CondCode CCCode = 1293 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 1294 1295 EVT VT = Op0.getValueType(); 1296 SDValue NewLHS = GetSoftenedFloat(Op0); 1297 SDValue NewRHS = GetSoftenedFloat(Op1); 1298 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1, 1299 Chain, N->getOpcode() == ISD::STRICT_FSETCCS); 1300 1301 // Update N to have the operands specified. 1302 if (NewRHS.getNode()) { 1303 if (IsStrict) 1304 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS, 1305 NewRHS, DAG.getCondCode(CCCode)); 1306 else 1307 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 1308 DAG.getCondCode(CCCode)), 0); 1309 } 1310 1311 // Otherwise, softenSetCCOperands returned a scalar, use it. 1312 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) && 1313 "Unexpected setcc expansion!"); 1314 1315 if (IsStrict) { 1316 ReplaceValueWith(SDValue(N, 0), NewLHS); 1317 ReplaceValueWith(SDValue(N, 1), Chain); 1318 return SDValue(); 1319 } 1320 return NewLHS; 1321 } 1322 1323 SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { 1324 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 1325 assert(OpNo == 1 && "Can only soften the stored value!"); 1326 StoreSDNode *ST = cast<StoreSDNode>(N); 1327 SDValue Val = ST->getValue(); 1328 SDLoc dl(N); 1329 1330 if (ST->isTruncatingStore()) 1331 // Do an FP_ROUND followed by a non-truncating store. 1332 Val = BitConvertToInteger( 1333 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val, 1334 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true))); 1335 else 1336 Val = GetSoftenedFloat(Val); 1337 1338 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(), 1339 ST->getMemOperand()); 1340 } 1341 1342 SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) { 1343 assert(OpNo == 1 && "Can only soften the stored value!"); 1344 AtomicSDNode *ST = cast<AtomicSDNode>(N); 1345 SDValue Val = ST->getVal(); 1346 EVT VT = Val.getValueType(); 1347 SDLoc dl(N); 1348 1349 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled"); 1350 1351 SDValue NewVal = GetSoftenedFloat(Val); 1352 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal, 1353 ST->getBasePtr(), ST->getMemOperand()); 1354 } 1355 1356 SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) { 1357 SDValue LHS = N->getOperand(0); 1358 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 1359 SDLoc dl(N); 1360 1361 EVT LVT = LHS.getValueType(); 1362 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits()); 1363 EVT RVT = RHS.getValueType(); 1364 1365 unsigned LSize = LVT.getSizeInBits(); 1366 unsigned RSize = RVT.getSizeInBits(); 1367 1368 // Shift right or sign-extend it if the two operands have different types. 1369 int SizeDiff = RSize - LSize; 1370 if (SizeDiff > 0) { 1371 RHS = 1372 DAG.getNode(ISD::SRL, dl, RVT, RHS, 1373 DAG.getConstant(SizeDiff, dl, 1374 TLI.getShiftAmountTy(RHS.getValueType(), 1375 DAG.getDataLayout()))); 1376 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS); 1377 } else if (SizeDiff < 0) { 1378 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS); 1379 RHS = 1380 DAG.getNode(ISD::SHL, dl, ILVT, RHS, 1381 DAG.getConstant(-SizeDiff, dl, 1382 TLI.getShiftAmountTy(RHS.getValueType(), 1383 DAG.getDataLayout()))); 1384 } 1385 1386 RHS = DAG.getBitcast(LVT, RHS); 1387 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS); 1388 } 1389 1390 SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) { 1391 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1392 bool IsStrict = N->isStrictFPOpcode(); 1393 unsigned Offset = IsStrict ? 1 : 0; 1394 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset)); 1395 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1396 TargetLowering::MakeLibCallOptions CallOptions; 1397 EVT OpVT = N->getOperand(0 + Offset).getValueType(); 1398 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true); 1399 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op, 1400 CallOptions, SDLoc(N), 1401 Chain); 1402 if (IsStrict) { 1403 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1404 ReplaceValueWith(SDValue(N, 0), Tmp.first); 1405 return SDValue(); 1406 } 1407 1408 return Tmp.first; 1409 } 1410 1411 SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { 1412 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1413 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1414 RTLIB::LROUND_F32, 1415 RTLIB::LROUND_F64, 1416 RTLIB::LROUND_F80, 1417 RTLIB::LROUND_F128, 1418 RTLIB::LROUND_PPCF128)); 1419 } 1420 1421 SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { 1422 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1423 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1424 RTLIB::LLROUND_F32, 1425 RTLIB::LLROUND_F64, 1426 RTLIB::LLROUND_F80, 1427 RTLIB::LLROUND_F128, 1428 RTLIB::LLROUND_PPCF128)); 1429 } 1430 1431 SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { 1432 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1433 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1434 RTLIB::LRINT_F32, 1435 RTLIB::LRINT_F64, 1436 RTLIB::LRINT_F80, 1437 RTLIB::LRINT_F128, 1438 RTLIB::LRINT_PPCF128)); 1439 } 1440 1441 SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { 1442 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType(); 1443 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT, 1444 RTLIB::LLRINT_F32, 1445 RTLIB::LLRINT_F64, 1446 RTLIB::LLRINT_F80, 1447 RTLIB::LLRINT_F128, 1448 RTLIB::LLRINT_PPCF128)); 1449 } 1450 1451 //===----------------------------------------------------------------------===// 1452 // Float Result Expansion 1453 //===----------------------------------------------------------------------===// 1454 1455 /// ExpandFloatResult - This method is called when the specified result of the 1456 /// specified node is found to need expansion. At this point, the node may also 1457 /// have invalid operands or may have other results that need promotion, we just 1458 /// know that (at least) one result needs expansion. 1459 void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) { 1460 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG)); 1461 SDValue Lo, Hi; 1462 Lo = Hi = SDValue(); 1463 1464 // See if the target wants to custom expand this node. 1465 if (CustomLowerNode(N, N->getValueType(ResNo), true)) 1466 return; 1467 1468 switch (N->getOpcode()) { 1469 default: 1470 #ifndef NDEBUG 1471 dbgs() << "ExpandFloatResult #" << ResNo << ": "; 1472 N->dump(&DAG); dbgs() << "\n"; 1473 #endif 1474 report_fatal_error("Do not know how to expand the result of this " 1475 "operator!"); 1476 // clang-format off 1477 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break; 1478 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break; 1479 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break; 1480 1481 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; 1482 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break; 1483 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break; 1484 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break; 1485 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break; 1486 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break; 1487 1488 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break; 1489 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break; 1490 case ISD::STRICT_FMINNUM: 1491 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break; 1492 case ISD::STRICT_FMAXNUM: 1493 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break; 1494 case ISD::FMINIMUMNUM: ExpandFloatRes_FMINIMUMNUM(N, Lo, Hi); break; 1495 case ISD::FMAXIMUMNUM: ExpandFloatRes_FMAXIMUMNUM(N, Lo, Hi); break; 1496 case ISD::STRICT_FADD: 1497 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break; 1498 case ISD::STRICT_FACOS: 1499 case ISD::FACOS: ExpandFloatRes_FACOS(N, Lo, Hi); break; 1500 case ISD::STRICT_FASIN: 1501 case ISD::FASIN: ExpandFloatRes_FASIN(N, Lo, Hi); break; 1502 case ISD::STRICT_FATAN: 1503 case ISD::FATAN: ExpandFloatRes_FATAN(N, Lo, Hi); break; 1504 case ISD::STRICT_FATAN2: 1505 case ISD::FATAN2: ExpandFloatRes_FATAN2(N, Lo, Hi); break; 1506 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break; 1507 case ISD::STRICT_FCEIL: 1508 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break; 1509 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break; 1510 case ISD::STRICT_FCOS: 1511 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break; 1512 case ISD::STRICT_FCOSH: 1513 case ISD::FCOSH: ExpandFloatRes_FCOSH(N, Lo, Hi); break; 1514 case ISD::STRICT_FDIV: 1515 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break; 1516 case ISD::STRICT_FEXP: 1517 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break; 1518 case ISD::STRICT_FEXP2: 1519 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break; 1520 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break; 1521 case ISD::STRICT_FFLOOR: 1522 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break; 1523 case ISD::STRICT_FLOG: 1524 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break; 1525 case ISD::STRICT_FLOG2: 1526 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break; 1527 case ISD::STRICT_FLOG10: 1528 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break; 1529 case ISD::STRICT_FMA: 1530 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break; 1531 case ISD::STRICT_FMUL: 1532 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break; 1533 case ISD::STRICT_FNEARBYINT: 1534 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break; 1535 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break; 1536 case ISD::STRICT_FP_EXTEND: 1537 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break; 1538 case ISD::STRICT_FPOW: 1539 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break; 1540 case ISD::STRICT_FPOWI: 1541 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break; 1542 case ISD::FLDEXP: 1543 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break; 1544 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break; 1545 case ISD::STRICT_FRINT: 1546 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break; 1547 case ISD::STRICT_FROUND: 1548 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break; 1549 case ISD::STRICT_FROUNDEVEN: 1550 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break; 1551 case ISD::STRICT_FSIN: 1552 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break; 1553 case ISD::STRICT_FSINH: 1554 case ISD::FSINH: ExpandFloatRes_FSINH(N, Lo, Hi); break; 1555 case ISD::STRICT_FSQRT: 1556 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break; 1557 case ISD::STRICT_FSUB: 1558 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break; 1559 case ISD::STRICT_FTAN: 1560 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break; 1561 case ISD::STRICT_FTANH: 1562 case ISD::FTANH: ExpandFloatRes_FTANH(N, Lo, Hi); break; 1563 case ISD::STRICT_FTRUNC: 1564 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break; 1565 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break; 1566 case ISD::STRICT_SINT_TO_FP: 1567 case ISD::STRICT_UINT_TO_FP: 1568 case ISD::SINT_TO_FP: 1569 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break; 1570 case ISD::STRICT_FREM: 1571 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break; 1572 // clang-format on 1573 } 1574 1575 // If Lo/Hi is null, the sub-method took care of registering results etc. 1576 if (Lo.getNode()) 1577 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi); 1578 } 1579 1580 void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, 1581 SDValue &Hi) { 1582 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1583 assert(NVT.getSizeInBits() == 64 && 1584 "Do not know how to expand this float constant!"); 1585 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt(); 1586 SDLoc dl(N); 1587 const fltSemantics &Sem = NVT.getFltSemantics(); 1588 Lo = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 64)), dl, NVT); 1589 Hi = DAG.getConstantFP(APFloat(Sem, C.extractBits(64, 0)), dl, NVT); 1590 } 1591 1592 void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC, 1593 SDValue &Lo, SDValue &Hi) { 1594 bool IsStrict = N->isStrictFPOpcode(); 1595 unsigned Offset = IsStrict ? 1 : 0; 1596 SDValue Op = N->getOperand(0 + Offset); 1597 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1598 TargetLowering::MakeLibCallOptions CallOptions; 1599 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1600 Op, CallOptions, SDLoc(N), 1601 Chain); 1602 if (IsStrict) 1603 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1604 GetPairElements(Tmp.first, Lo, Hi); 1605 } 1606 1607 void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC, 1608 SDValue &Lo, SDValue &Hi) { 1609 bool IsStrict = N->isStrictFPOpcode(); 1610 unsigned Offset = IsStrict ? 1 : 0; 1611 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) }; 1612 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1613 TargetLowering::MakeLibCallOptions CallOptions; 1614 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0), 1615 Ops, CallOptions, SDLoc(N), 1616 Chain); 1617 if (IsStrict) 1618 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1619 GetPairElements(Tmp.first, Lo, Hi); 1620 } 1621 1622 void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo, 1623 SDValue &Hi) { 1624 assert(N->getValueType(0) == MVT::ppcf128 && 1625 "Logic only correct for ppcf128!"); 1626 SDLoc dl(N); 1627 SDValue Tmp; 1628 GetExpandedFloat(N->getOperand(0), Lo, Tmp); 1629 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp); 1630 // Lo = Hi==fabs(Hi) ? Lo : -Lo; 1631 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo, 1632 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo), 1633 ISD::SETEQ); 1634 } 1635 1636 void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo, 1637 SDValue &Hi) { 1638 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1639 RTLIB::FMIN_F32, RTLIB::FMIN_F64, 1640 RTLIB::FMIN_F80, RTLIB::FMIN_F128, 1641 RTLIB::FMIN_PPCF128), Lo, Hi); 1642 } 1643 1644 void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo, 1645 SDValue &Hi) { 1646 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1647 RTLIB::FMAX_F32, RTLIB::FMAX_F64, 1648 RTLIB::FMAX_F80, RTLIB::FMAX_F128, 1649 RTLIB::FMAX_PPCF128), Lo, Hi); 1650 } 1651 1652 void DAGTypeLegalizer::ExpandFloatRes_FMINIMUMNUM(SDNode *N, SDValue &Lo, 1653 SDValue &Hi) { 1654 ExpandFloatRes_Binary( 1655 N, 1656 GetFPLibCall(N->getValueType(0), RTLIB::FMINIMUMNUM_F32, 1657 RTLIB::FMINIMUMNUM_F64, RTLIB::FMINIMUMNUM_F80, 1658 RTLIB::FMINIMUMNUM_F128, RTLIB::FMINIMUMNUM_PPCF128), 1659 Lo, Hi); 1660 } 1661 1662 void DAGTypeLegalizer::ExpandFloatRes_FMAXIMUMNUM(SDNode *N, SDValue &Lo, 1663 SDValue &Hi) { 1664 ExpandFloatRes_Binary( 1665 N, 1666 GetFPLibCall(N->getValueType(0), RTLIB::FMAXIMUMNUM_F32, 1667 RTLIB::FMAXIMUMNUM_F64, RTLIB::FMAXIMUMNUM_F80, 1668 RTLIB::FMAXIMUMNUM_F128, RTLIB::FMAXIMUMNUM_PPCF128), 1669 Lo, Hi); 1670 } 1671 1672 void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo, 1673 SDValue &Hi) { 1674 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1675 RTLIB::ADD_F32, RTLIB::ADD_F64, 1676 RTLIB::ADD_F80, RTLIB::ADD_F128, 1677 RTLIB::ADD_PPCF128), Lo, Hi); 1678 } 1679 1680 void DAGTypeLegalizer::ExpandFloatRes_FACOS(SDNode *N, SDValue &Lo, 1681 SDValue &Hi) { 1682 ExpandFloatRes_Unary(N, 1683 GetFPLibCall(N->getValueType(0), RTLIB::ACOS_F32, 1684 RTLIB::ACOS_F64, RTLIB::ACOS_F80, 1685 RTLIB::ACOS_F128, RTLIB::ACOS_PPCF128), 1686 Lo, Hi); 1687 } 1688 1689 void DAGTypeLegalizer::ExpandFloatRes_FASIN(SDNode *N, SDValue &Lo, 1690 SDValue &Hi) { 1691 ExpandFloatRes_Unary(N, 1692 GetFPLibCall(N->getValueType(0), RTLIB::ASIN_F32, 1693 RTLIB::ASIN_F64, RTLIB::ASIN_F80, 1694 RTLIB::ASIN_F128, RTLIB::ASIN_PPCF128), 1695 Lo, Hi); 1696 } 1697 1698 void DAGTypeLegalizer::ExpandFloatRes_FATAN(SDNode *N, SDValue &Lo, 1699 SDValue &Hi) { 1700 ExpandFloatRes_Unary(N, 1701 GetFPLibCall(N->getValueType(0), RTLIB::ATAN_F32, 1702 RTLIB::ATAN_F64, RTLIB::ATAN_F80, 1703 RTLIB::ATAN_F128, RTLIB::ATAN_PPCF128), 1704 Lo, Hi); 1705 } 1706 1707 void DAGTypeLegalizer::ExpandFloatRes_FATAN2(SDNode *N, SDValue &Lo, 1708 SDValue &Hi) { 1709 ExpandFloatRes_Binary(N, 1710 GetFPLibCall(N->getValueType(0), RTLIB::ATAN2_F32, 1711 RTLIB::ATAN2_F64, RTLIB::ATAN2_F80, 1712 RTLIB::ATAN2_F128, RTLIB::ATAN2_PPCF128), 1713 Lo, Hi); 1714 } 1715 1716 void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo, 1717 SDValue &Hi) { 1718 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32, 1719 RTLIB::CBRT_F64, RTLIB::CBRT_F80, 1720 RTLIB::CBRT_F128, 1721 RTLIB::CBRT_PPCF128), Lo, Hi); 1722 } 1723 1724 void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N, 1725 SDValue &Lo, SDValue &Hi) { 1726 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1727 RTLIB::CEIL_F32, RTLIB::CEIL_F64, 1728 RTLIB::CEIL_F80, RTLIB::CEIL_F128, 1729 RTLIB::CEIL_PPCF128), Lo, Hi); 1730 } 1731 1732 void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N, 1733 SDValue &Lo, SDValue &Hi) { 1734 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1735 RTLIB::COPYSIGN_F32, 1736 RTLIB::COPYSIGN_F64, 1737 RTLIB::COPYSIGN_F80, 1738 RTLIB::COPYSIGN_F128, 1739 RTLIB::COPYSIGN_PPCF128), Lo, Hi); 1740 } 1741 1742 void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N, 1743 SDValue &Lo, SDValue &Hi) { 1744 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1745 RTLIB::COS_F32, RTLIB::COS_F64, 1746 RTLIB::COS_F80, RTLIB::COS_F128, 1747 RTLIB::COS_PPCF128), Lo, Hi); 1748 } 1749 1750 void DAGTypeLegalizer::ExpandFloatRes_FCOSH(SDNode *N, SDValue &Lo, 1751 SDValue &Hi) { 1752 ExpandFloatRes_Unary(N, 1753 GetFPLibCall(N->getValueType(0), RTLIB::COSH_F32, 1754 RTLIB::COSH_F64, RTLIB::COSH_F80, 1755 RTLIB::COSH_F128, RTLIB::COSH_PPCF128), 1756 Lo, Hi); 1757 } 1758 1759 void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo, 1760 SDValue &Hi) { 1761 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1762 RTLIB::DIV_F32, 1763 RTLIB::DIV_F64, 1764 RTLIB::DIV_F80, 1765 RTLIB::DIV_F128, 1766 RTLIB::DIV_PPCF128), Lo, Hi); 1767 } 1768 1769 void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N, 1770 SDValue &Lo, SDValue &Hi) { 1771 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1772 RTLIB::EXP_F32, RTLIB::EXP_F64, 1773 RTLIB::EXP_F80, RTLIB::EXP_F128, 1774 RTLIB::EXP_PPCF128), Lo, Hi); 1775 } 1776 1777 void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N, 1778 SDValue &Lo, SDValue &Hi) { 1779 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1780 RTLIB::EXP2_F32, RTLIB::EXP2_F64, 1781 RTLIB::EXP2_F80, RTLIB::EXP2_F128, 1782 RTLIB::EXP2_PPCF128), Lo, Hi); 1783 } 1784 1785 void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo, 1786 SDValue &Hi) { 1787 ExpandFloatRes_Unary(N, 1788 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, 1789 RTLIB::EXP10_F64, RTLIB::EXP10_F80, 1790 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128), 1791 Lo, Hi); 1792 } 1793 1794 void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N, 1795 SDValue &Lo, SDValue &Hi) { 1796 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1797 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, 1798 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, 1799 RTLIB::FLOOR_PPCF128), Lo, Hi); 1800 } 1801 1802 void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N, 1803 SDValue &Lo, SDValue &Hi) { 1804 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1805 RTLIB::LOG_F32, RTLIB::LOG_F64, 1806 RTLIB::LOG_F80, RTLIB::LOG_F128, 1807 RTLIB::LOG_PPCF128), Lo, Hi); 1808 } 1809 1810 void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N, 1811 SDValue &Lo, SDValue &Hi) { 1812 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1813 RTLIB::LOG2_F32, RTLIB::LOG2_F64, 1814 RTLIB::LOG2_F80, RTLIB::LOG2_F128, 1815 RTLIB::LOG2_PPCF128), Lo, Hi); 1816 } 1817 1818 void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N, 1819 SDValue &Lo, SDValue &Hi) { 1820 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1821 RTLIB::LOG10_F32, RTLIB::LOG10_F64, 1822 RTLIB::LOG10_F80, RTLIB::LOG10_F128, 1823 RTLIB::LOG10_PPCF128), Lo, Hi); 1824 } 1825 1826 void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo, 1827 SDValue &Hi) { 1828 bool IsStrict = N->isStrictFPOpcode(); 1829 unsigned Offset = IsStrict ? 1 : 0; 1830 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset), 1831 N->getOperand(2 + Offset) }; 1832 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 1833 TargetLowering::MakeLibCallOptions CallOptions; 1834 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0), 1835 RTLIB::FMA_F32, 1836 RTLIB::FMA_F64, 1837 RTLIB::FMA_F80, 1838 RTLIB::FMA_F128, 1839 RTLIB::FMA_PPCF128), 1840 N->getValueType(0), Ops, CallOptions, 1841 SDLoc(N), Chain); 1842 if (IsStrict) 1843 ReplaceValueWith(SDValue(N, 1), Tmp.second); 1844 GetPairElements(Tmp.first, Lo, Hi); 1845 } 1846 1847 void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo, 1848 SDValue &Hi) { 1849 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1850 RTLIB::MUL_F32, 1851 RTLIB::MUL_F64, 1852 RTLIB::MUL_F80, 1853 RTLIB::MUL_F128, 1854 RTLIB::MUL_PPCF128), Lo, Hi); 1855 } 1856 1857 void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N, 1858 SDValue &Lo, SDValue &Hi) { 1859 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1860 RTLIB::NEARBYINT_F32, 1861 RTLIB::NEARBYINT_F64, 1862 RTLIB::NEARBYINT_F80, 1863 RTLIB::NEARBYINT_F128, 1864 RTLIB::NEARBYINT_PPCF128), Lo, Hi); 1865 } 1866 1867 void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo, 1868 SDValue &Hi) { 1869 SDLoc dl(N); 1870 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1871 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo); 1872 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi); 1873 } 1874 1875 void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo, 1876 SDValue &Hi) { 1877 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 1878 SDLoc dl(N); 1879 bool IsStrict = N->isStrictFPOpcode(); 1880 1881 SDValue Chain; 1882 if (IsStrict) { 1883 // If the expanded type is the same as the input type, just bypass the node. 1884 if (NVT == N->getOperand(1).getValueType()) { 1885 Hi = N->getOperand(1); 1886 Chain = N->getOperand(0); 1887 } else { 1888 // Other we need to extend. 1889 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other }, 1890 { N->getOperand(0), N->getOperand(1) }); 1891 Chain = Hi.getValue(1); 1892 } 1893 } else { 1894 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0)); 1895 } 1896 1897 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT); 1898 1899 if (IsStrict) 1900 ReplaceValueWith(SDValue(N, 1), Chain); 1901 } 1902 1903 void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N, 1904 SDValue &Lo, SDValue &Hi) { 1905 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1906 RTLIB::POW_F32, RTLIB::POW_F64, 1907 RTLIB::POW_F80, RTLIB::POW_F128, 1908 RTLIB::POW_PPCF128), Lo, Hi); 1909 } 1910 1911 void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N, 1912 SDValue &Lo, SDValue &Hi) { 1913 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi); 1914 } 1915 1916 void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo, 1917 SDValue &Hi) { 1918 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi); 1919 } 1920 1921 void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N, 1922 SDValue &Lo, SDValue &Hi) { 1923 assert(N->getValueType(0) == MVT::ppcf128 && 1924 "Logic only correct for ppcf128!"); 1925 1926 SDLoc dl(N); 1927 GetExpandedFloat(N->getOperand(0), Lo, Hi); 1928 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo); 1929 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi); 1930 } 1931 1932 void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N, 1933 SDValue &Lo, SDValue &Hi) { 1934 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1935 RTLIB::REM_F32, RTLIB::REM_F64, 1936 RTLIB::REM_F80, RTLIB::REM_F128, 1937 RTLIB::REM_PPCF128), Lo, Hi); 1938 } 1939 1940 void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N, 1941 SDValue &Lo, SDValue &Hi) { 1942 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1943 RTLIB::RINT_F32, RTLIB::RINT_F64, 1944 RTLIB::RINT_F80, RTLIB::RINT_F128, 1945 RTLIB::RINT_PPCF128), Lo, Hi); 1946 } 1947 1948 void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N, 1949 SDValue &Lo, SDValue &Hi) { 1950 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1951 RTLIB::ROUND_F32, 1952 RTLIB::ROUND_F64, 1953 RTLIB::ROUND_F80, 1954 RTLIB::ROUND_F128, 1955 RTLIB::ROUND_PPCF128), Lo, Hi); 1956 } 1957 1958 void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N, 1959 SDValue &Lo, SDValue &Hi) { 1960 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1961 RTLIB::ROUNDEVEN_F32, 1962 RTLIB::ROUNDEVEN_F64, 1963 RTLIB::ROUNDEVEN_F80, 1964 RTLIB::ROUNDEVEN_F128, 1965 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi); 1966 } 1967 1968 void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N, 1969 SDValue &Lo, SDValue &Hi) { 1970 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1971 RTLIB::SIN_F32, RTLIB::SIN_F64, 1972 RTLIB::SIN_F80, RTLIB::SIN_F128, 1973 RTLIB::SIN_PPCF128), Lo, Hi); 1974 } 1975 1976 void DAGTypeLegalizer::ExpandFloatRes_FSINH(SDNode *N, SDValue &Lo, 1977 SDValue &Hi) { 1978 ExpandFloatRes_Unary(N, 1979 GetFPLibCall(N->getValueType(0), RTLIB::SINH_F32, 1980 RTLIB::SINH_F64, RTLIB::SINH_F80, 1981 RTLIB::SINH_F128, RTLIB::SINH_PPCF128), 1982 Lo, Hi); 1983 } 1984 1985 void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N, 1986 SDValue &Lo, SDValue &Hi) { 1987 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 1988 RTLIB::SQRT_F32, RTLIB::SQRT_F64, 1989 RTLIB::SQRT_F80, RTLIB::SQRT_F128, 1990 RTLIB::SQRT_PPCF128), Lo, Hi); 1991 } 1992 1993 void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo, 1994 SDValue &Hi) { 1995 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0), 1996 RTLIB::SUB_F32, 1997 RTLIB::SUB_F64, 1998 RTLIB::SUB_F80, 1999 RTLIB::SUB_F128, 2000 RTLIB::SUB_PPCF128), Lo, Hi); 2001 } 2002 2003 void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo, 2004 SDValue &Hi) { 2005 ExpandFloatRes_Unary(N, 2006 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, 2007 RTLIB::TAN_F64, RTLIB::TAN_F80, 2008 RTLIB::TAN_F128, RTLIB::TAN_PPCF128), 2009 Lo, Hi); 2010 } 2011 2012 void DAGTypeLegalizer::ExpandFloatRes_FTANH(SDNode *N, SDValue &Lo, 2013 SDValue &Hi) { 2014 ExpandFloatRes_Unary(N, 2015 GetFPLibCall(N->getValueType(0), RTLIB::TANH_F32, 2016 RTLIB::TANH_F64, RTLIB::TANH_F80, 2017 RTLIB::TANH_F128, RTLIB::TANH_PPCF128), 2018 Lo, Hi); 2019 } 2020 2021 void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N, 2022 SDValue &Lo, SDValue &Hi) { 2023 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), 2024 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, 2025 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, 2026 RTLIB::TRUNC_PPCF128), Lo, Hi); 2027 } 2028 2029 void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo, 2030 SDValue &Hi) { 2031 if (ISD::isNormalLoad(N)) { 2032 ExpandRes_NormalLoad(N, Lo, Hi); 2033 return; 2034 } 2035 2036 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!"); 2037 LoadSDNode *LD = cast<LoadSDNode>(N); 2038 SDValue Chain = LD->getChain(); 2039 SDValue Ptr = LD->getBasePtr(); 2040 SDLoc dl(N); 2041 2042 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0)); 2043 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2044 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2045 2046 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr, 2047 LD->getMemoryVT(), LD->getMemOperand()); 2048 2049 // Remember the chain. 2050 Chain = Hi.getValue(1); 2051 2052 // The low part is zero. 2053 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT); 2054 2055 // Modified the chain - switch anything that used the old chain to use the 2056 // new one. 2057 ReplaceValueWith(SDValue(LD, 1), Chain); 2058 } 2059 2060 void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, 2061 SDValue &Hi) { 2062 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!"); 2063 EVT VT = N->getValueType(0); 2064 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2065 bool Strict = N->isStrictFPOpcode(); 2066 SDValue Src = N->getOperand(Strict ? 1 : 0); 2067 EVT SrcVT = Src.getValueType(); 2068 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP || 2069 N->getOpcode() == ISD::STRICT_SINT_TO_FP; 2070 SDLoc dl(N); 2071 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode(); 2072 2073 // TODO: Any other flags to propagate? 2074 SDNodeFlags Flags; 2075 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept()); 2076 2077 // First do an SINT_TO_FP, whether the original was signed or unsigned. 2078 // When promoting partial word types to i32 we must honor the signedness, 2079 // though. 2080 if (SrcVT.bitsLE(MVT::i32)) { 2081 // The integer can be represented exactly in an f64. 2082 Lo = DAG.getConstantFP(APFloat::getZero(NVT.getFltSemantics()), dl, NVT); 2083 if (Strict) { 2084 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other), 2085 {Chain, Src}, Flags); 2086 Chain = Hi.getValue(1); 2087 } else 2088 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src); 2089 } else { 2090 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; 2091 if (SrcVT.bitsLE(MVT::i64)) { 2092 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl, 2093 MVT::i64, Src); 2094 LC = RTLIB::SINTTOFP_I64_PPCF128; 2095 } else if (SrcVT.bitsLE(MVT::i128)) { 2096 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src); 2097 LC = RTLIB::SINTTOFP_I128_PPCF128; 2098 } 2099 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!"); 2100 2101 TargetLowering::MakeLibCallOptions CallOptions; 2102 CallOptions.setIsSigned(true); 2103 std::pair<SDValue, SDValue> Tmp = 2104 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain); 2105 if (Strict) 2106 Chain = Tmp.second; 2107 GetPairElements(Tmp.first, Lo, Hi); 2108 } 2109 2110 // No need to complement for unsigned 32-bit integers 2111 if (isSigned || SrcVT.bitsLE(MVT::i32)) { 2112 if (Strict) 2113 ReplaceValueWith(SDValue(N, 1), Chain); 2114 2115 return; 2116 } 2117 2118 // Unsigned - fix up the SINT_TO_FP value just calculated. 2119 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully 2120 // keep semantics correctness if the integer is not exactly representable 2121 // here. See ExpandLegalINT_TO_FP. 2122 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi); 2123 SrcVT = Src.getValueType(); 2124 2125 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128. 2126 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 }; 2127 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 }; 2128 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 }; 2129 ArrayRef<uint64_t> Parts; 2130 2131 switch (SrcVT.getSimpleVT().SimpleTy) { 2132 default: 2133 llvm_unreachable("Unsupported UINT_TO_FP!"); 2134 case MVT::i32: 2135 Parts = TwoE32; 2136 break; 2137 case MVT::i64: 2138 Parts = TwoE64; 2139 break; 2140 case MVT::i128: 2141 Parts = TwoE128; 2142 break; 2143 } 2144 2145 // TODO: Are there other fast-math-flags to propagate to this FADD? 2146 SDValue NewLo = DAG.getConstantFP( 2147 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128); 2148 if (Strict) { 2149 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other), 2150 {Chain, Hi, NewLo}, Flags); 2151 Chain = Lo.getValue(1); 2152 ReplaceValueWith(SDValue(N, 1), Chain); 2153 } else 2154 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo); 2155 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT), 2156 Lo, Hi, ISD::SETLT); 2157 GetPairElements(Lo, Lo, Hi); 2158 } 2159 2160 2161 //===----------------------------------------------------------------------===// 2162 // Float Operand Expansion 2163 //===----------------------------------------------------------------------===// 2164 2165 /// ExpandFloatOperand - This method is called when the specified operand of the 2166 /// specified node is found to need expansion. At this point, all of the result 2167 /// types of the node are known to be legal, but other operands of the node may 2168 /// need promotion or expansion as well as the specified one. 2169 bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { 2170 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG)); 2171 SDValue Res = SDValue(); 2172 2173 // See if the target wants to custom expand this node. 2174 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) 2175 return false; 2176 2177 switch (N->getOpcode()) { 2178 default: 2179 #ifndef NDEBUG 2180 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": "; 2181 N->dump(&DAG); dbgs() << "\n"; 2182 #endif 2183 report_fatal_error("Do not know how to expand this operator's operand!"); 2184 2185 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; 2186 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break; 2187 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break; 2188 2189 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break; 2190 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break; 2191 case ISD::STRICT_FP_ROUND: 2192 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; 2193 case ISD::STRICT_FP_TO_SINT: 2194 case ISD::STRICT_FP_TO_UINT: 2195 case ISD::FP_TO_SINT: 2196 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break; 2197 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; 2198 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; 2199 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; 2200 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; 2201 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; 2202 case ISD::STRICT_FSETCC: 2203 case ISD::STRICT_FSETCCS: 2204 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; 2205 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N), 2206 OpNo); break; 2207 } 2208 2209 // If the result is null, the sub-method took care of registering results etc. 2210 if (!Res.getNode()) return false; 2211 2212 // If the result is N, the sub-method updated N in place. Tell the legalizer 2213 // core about this. 2214 if (Res.getNode() == N) 2215 return true; 2216 2217 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 2218 "Invalid operand expansion"); 2219 2220 ReplaceValueWith(SDValue(N, 0), Res); 2221 return false; 2222 } 2223 2224 /// FloatExpandSetCCOperands - Expand the operands of a comparison. This code 2225 /// is shared among BR_CC, SELECT_CC, and SETCC handlers. 2226 void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS, 2227 SDValue &NewRHS, 2228 ISD::CondCode &CCCode, 2229 const SDLoc &dl, SDValue &Chain, 2230 bool IsSignaling) { 2231 SDValue LHSLo, LHSHi, RHSLo, RHSHi; 2232 GetExpandedFloat(NewLHS, LHSLo, LHSHi); 2233 GetExpandedFloat(NewRHS, RHSLo, RHSHi); 2234 2235 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!"); 2236 2237 // FIXME: This generated code sucks. We want to generate 2238 // FCMPU crN, hi1, hi2 2239 // BNE crN, L: 2240 // FCMPU crN, lo1, lo2 2241 // The following can be improved, but not that much. 2242 SDValue Tmp1, Tmp2, Tmp3, OutputChain; 2243 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 2244 RHSHi, ISD::SETOEQ, Chain, IsSignaling); 2245 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 2246 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo, 2247 RHSLo, CCCode, OutputChain, IsSignaling); 2248 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 2249 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 2250 Tmp1 = 2251 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi, 2252 ISD::SETUNE, OutputChain, IsSignaling); 2253 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue(); 2254 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, 2255 RHSHi, CCCode, OutputChain, IsSignaling); 2256 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue(); 2257 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2); 2258 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3); 2259 NewRHS = SDValue(); // LHS is the result, not a compare. 2260 Chain = OutputChain; 2261 } 2262 2263 SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) { 2264 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3); 2265 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get(); 2266 SDValue Chain; 2267 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 2268 2269 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2270 // against zero to select between true and false values. 2271 if (!NewRHS.getNode()) { 2272 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2273 CCCode = ISD::SETNE; 2274 } 2275 2276 // Update N to have the operands specified. 2277 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), 2278 DAG.getCondCode(CCCode), NewLHS, NewRHS, 2279 N->getOperand(4)), 0); 2280 } 2281 2282 SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) { 2283 assert(N->getOperand(1).getValueType() == MVT::ppcf128 && 2284 "Logic only correct for ppcf128!"); 2285 SDValue Lo, Hi; 2286 GetExpandedFloat(N->getOperand(1), Lo, Hi); 2287 // The ppcf128 value is providing only the sign; take it from the 2288 // higher-order double (which must have the larger magnitude). 2289 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N), 2290 N->getValueType(0), N->getOperand(0), Hi); 2291 } 2292 2293 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) { 2294 bool IsStrict = N->isStrictFPOpcode(); 2295 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 && 2296 "Logic only correct for ppcf128!"); 2297 SDValue Lo, Hi; 2298 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi); 2299 2300 if (!IsStrict) 2301 // Round it the rest of the way (e.g. to f32) if needed. 2302 return DAG.getNode(ISD::FP_ROUND, SDLoc(N), 2303 N->getValueType(0), Hi, N->getOperand(1)); 2304 2305 // Eliminate the node if the input float type is the same as the output float 2306 // type. 2307 if (Hi.getValueType() == N->getValueType(0)) { 2308 // Connect the output chain to the input chain, unlinking the node. 2309 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 2310 ReplaceValueWith(SDValue(N, 0), Hi); 2311 return SDValue(); 2312 } 2313 2314 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), 2315 {N->getValueType(0), MVT::Other}, 2316 {N->getOperand(0), Hi, N->getOperand(2)}); 2317 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1)); 2318 ReplaceValueWith(SDValue(N, 0), Expansion); 2319 return SDValue(); 2320 } 2321 2322 SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) { 2323 EVT RVT = N->getValueType(0); 2324 SDLoc dl(N); 2325 2326 bool IsStrict = N->isStrictFPOpcode(); 2327 bool Signed = N->getOpcode() == ISD::FP_TO_SINT || 2328 N->getOpcode() == ISD::STRICT_FP_TO_SINT; 2329 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 2330 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2331 2332 EVT NVT; 2333 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed); 2334 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() && 2335 "Unsupported FP_TO_XINT!"); 2336 TargetLowering::MakeLibCallOptions CallOptions; 2337 std::pair<SDValue, SDValue> Tmp = 2338 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain); 2339 if (!IsStrict) 2340 return Tmp.first; 2341 2342 ReplaceValueWith(SDValue(N, 1), Tmp.second); 2343 ReplaceValueWith(SDValue(N, 0), Tmp.first); 2344 return SDValue(); 2345 } 2346 2347 SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) { 2348 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1); 2349 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get(); 2350 SDValue Chain; 2351 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain); 2352 2353 // If ExpandSetCCOperands returned a scalar, we need to compare the result 2354 // against zero to select between true and false values. 2355 if (!NewRHS.getNode()) { 2356 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType()); 2357 CCCode = ISD::SETNE; 2358 } 2359 2360 // Update N to have the operands specified. 2361 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS, 2362 N->getOperand(2), N->getOperand(3), 2363 DAG.getCondCode(CCCode)), 0); 2364 } 2365 2366 SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) { 2367 bool IsStrict = N->isStrictFPOpcode(); 2368 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0); 2369 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1); 2370 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 2371 ISD::CondCode CCCode = 2372 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get(); 2373 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain, 2374 N->getOpcode() == ISD::STRICT_FSETCCS); 2375 2376 // FloatExpandSetCCOperands always returned a scalar. 2377 assert(!NewRHS.getNode() && "Expect to return scalar"); 2378 assert(NewLHS.getValueType() == N->getValueType(0) && 2379 "Unexpected setcc expansion!"); 2380 if (Chain) { 2381 ReplaceValueWith(SDValue(N, 0), NewLHS); 2382 ReplaceValueWith(SDValue(N, 1), Chain); 2383 return SDValue(); 2384 } 2385 return NewLHS; 2386 } 2387 2388 SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { 2389 if (ISD::isNormalStore(N)) 2390 return ExpandOp_NormalStore(N, OpNo); 2391 2392 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!"); 2393 assert(OpNo == 1 && "Can only expand the stored value so far"); 2394 StoreSDNode *ST = cast<StoreSDNode>(N); 2395 2396 SDValue Chain = ST->getChain(); 2397 SDValue Ptr = ST->getBasePtr(); 2398 2399 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), 2400 ST->getValue().getValueType()); 2401 assert(NVT.isByteSized() && "Expanded type not byte sized!"); 2402 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?"); 2403 (void)NVT; 2404 2405 SDValue Lo, Hi; 2406 GetExpandedOp(ST->getValue(), Lo, Hi); 2407 2408 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr, 2409 ST->getMemoryVT(), ST->getMemOperand()); 2410 } 2411 2412 SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { 2413 EVT RVT = N->getValueType(0); 2414 EVT RetVT = N->getOperand(0).getValueType(); 2415 TargetLowering::MakeLibCallOptions CallOptions; 2416 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2417 RTLIB::LROUND_F32, 2418 RTLIB::LROUND_F64, 2419 RTLIB::LROUND_F80, 2420 RTLIB::LROUND_F128, 2421 RTLIB::LROUND_PPCF128), 2422 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2423 } 2424 2425 SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { 2426 EVT RVT = N->getValueType(0); 2427 EVT RetVT = N->getOperand(0).getValueType(); 2428 TargetLowering::MakeLibCallOptions CallOptions; 2429 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2430 RTLIB::LLROUND_F32, 2431 RTLIB::LLROUND_F64, 2432 RTLIB::LLROUND_F80, 2433 RTLIB::LLROUND_F128, 2434 RTLIB::LLROUND_PPCF128), 2435 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2436 } 2437 2438 SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { 2439 EVT RVT = N->getValueType(0); 2440 EVT RetVT = N->getOperand(0).getValueType(); 2441 TargetLowering::MakeLibCallOptions CallOptions; 2442 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2443 RTLIB::LRINT_F32, 2444 RTLIB::LRINT_F64, 2445 RTLIB::LRINT_F80, 2446 RTLIB::LRINT_F128, 2447 RTLIB::LRINT_PPCF128), 2448 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2449 } 2450 2451 SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { 2452 EVT RVT = N->getValueType(0); 2453 EVT RetVT = N->getOperand(0).getValueType(); 2454 TargetLowering::MakeLibCallOptions CallOptions; 2455 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, 2456 RTLIB::LLRINT_F32, 2457 RTLIB::LLRINT_F64, 2458 RTLIB::LLRINT_F80, 2459 RTLIB::LLRINT_F128, 2460 RTLIB::LLRINT_PPCF128), 2461 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first; 2462 } 2463 2464 //===----------------------------------------------------------------------===// 2465 // Float Operand Promotion 2466 //===----------------------------------------------------------------------===// 2467 // 2468 2469 static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { 2470 if (OpVT == MVT::f16) 2471 return ISD::FP16_TO_FP; 2472 if (RetVT == MVT::f16) 2473 return ISD::FP_TO_FP16; 2474 if (OpVT == MVT::bf16) 2475 return ISD::BF16_TO_FP; 2476 if (RetVT == MVT::bf16) 2477 return ISD::FP_TO_BF16; 2478 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2479 } 2480 2481 static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT) { 2482 if (OpVT == MVT::f16) 2483 return ISD::STRICT_FP16_TO_FP; 2484 if (RetVT == MVT::f16) 2485 return ISD::STRICT_FP_TO_FP16; 2486 if (OpVT == MVT::bf16) 2487 return ISD::STRICT_BF16_TO_FP; 2488 if (RetVT == MVT::bf16) 2489 return ISD::STRICT_FP_TO_BF16; 2490 report_fatal_error("Attempt at an invalid promotion-related conversion"); 2491 } 2492 2493 bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { 2494 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG)); 2495 SDValue R = SDValue(); 2496 2497 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 2498 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 2499 return false; 2500 } 2501 2502 // Nodes that use a promotion-requiring floating point operand, but doesn't 2503 // produce a promotion-requiring floating point result, need to be legalized 2504 // to use the promoted float operand. Nodes that produce at least one 2505 // promotion-requiring floating point result have their operands legalized as 2506 // a part of PromoteFloatResult. 2507 // clang-format off 2508 switch (N->getOpcode()) { 2509 default: 2510 #ifndef NDEBUG 2511 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; 2512 N->dump(&DAG); dbgs() << "\n"; 2513 #endif 2514 report_fatal_error("Do not know how to promote this operator's operand!"); 2515 2516 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; 2517 case ISD::FAKE_USE: 2518 R = PromoteFloatOp_FAKE_USE(N, OpNo); 2519 break; 2520 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break; 2521 case ISD::FP_TO_SINT: 2522 case ISD::FP_TO_UINT: 2523 case ISD::LROUND: 2524 case ISD::LLROUND: 2525 case ISD::LRINT: 2526 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break; 2527 case ISD::FP_TO_SINT_SAT: 2528 case ISD::FP_TO_UINT_SAT: 2529 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break; 2530 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break; 2531 case ISD::STRICT_FP_EXTEND: 2532 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo); 2533 break; 2534 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break; 2535 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break; 2536 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break; 2537 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break; 2538 } 2539 // clang-format on 2540 2541 if (R.getNode()) 2542 ReplaceValueWith(SDValue(N, 0), R); 2543 return false; 2544 } 2545 2546 SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) { 2547 SDValue Op = N->getOperand(0); 2548 EVT OpVT = Op->getValueType(0); 2549 2550 SDValue Promoted = GetPromotedFloat(N->getOperand(0)); 2551 EVT PromotedVT = Promoted->getValueType(0); 2552 2553 // Convert the promoted float value to the desired IVT. 2554 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits()); 2555 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N), 2556 IVT, Promoted); 2557 // The final result type might not be an scalar so we need a bitcast. The 2558 // bitcast will be further legalized if needed. 2559 return DAG.getBitcast(N->getValueType(0), Convert); 2560 } 2561 2562 SDValue DAGTypeLegalizer::PromoteFloatOp_FAKE_USE(SDNode *N, unsigned OpNo) { 2563 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 2564 SDValue Op = GetPromotedFloat(N->getOperand(OpNo)); 2565 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0), 2566 Op); 2567 } 2568 2569 // Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by 2570 // PromoteFloatRes_FCOPYSIGN. 2571 SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) { 2572 assert (OpNo == 1 && "Only Operand 1 must need promotion here"); 2573 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2574 2575 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), 2576 N->getOperand(0), Op1); 2577 } 2578 2579 // Convert the promoted float value to the desired integer type 2580 SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) { 2581 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2582 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op); 2583 } 2584 2585 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N, 2586 unsigned OpNo) { 2587 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2588 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op, 2589 N->getOperand(1)); 2590 } 2591 2592 SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) { 2593 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2594 EVT VT = N->getValueType(0); 2595 2596 // Desired VT is same as promoted type. Use promoted float directly. 2597 if (VT == Op->getValueType(0)) 2598 return Op; 2599 2600 // Else, extend the promoted float value to the desired VT. 2601 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op); 2602 } 2603 2604 SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N, 2605 unsigned OpNo) { 2606 assert(OpNo == 1 && "Promoting unpromotable operand"); 2607 2608 SDValue Op = GetPromotedFloat(N->getOperand(1)); 2609 EVT VT = N->getValueType(0); 2610 2611 // Desired VT is same as promoted type. Use promoted float directly. 2612 if (VT == Op->getValueType(0)) { 2613 ReplaceValueWith(SDValue(N, 1), N->getOperand(0)); 2614 return Op; 2615 } 2616 2617 // Else, extend the promoted float value to the desired VT. 2618 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(), 2619 N->getOperand(0), Op); 2620 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2621 return Res; 2622 } 2623 2624 // Promote the float operands used for comparison. The true- and false- 2625 // operands have the same type as the result and are promoted, if needed, by 2626 // PromoteFloatRes_SELECT_CC 2627 SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) { 2628 SDValue LHS = GetPromotedFloat(N->getOperand(0)); 2629 SDValue RHS = GetPromotedFloat(N->getOperand(1)); 2630 2631 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), 2632 LHS, RHS, N->getOperand(2), N->getOperand(3), 2633 N->getOperand(4)); 2634 } 2635 2636 // Construct a SETCC that compares the promoted values and sets the conditional 2637 // code. 2638 SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) { 2639 EVT VT = N->getValueType(0); 2640 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2641 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2642 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 2643 2644 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode); 2645 2646 } 2647 2648 // Lower the promoted Float down to the integer value of same size and construct 2649 // a STORE of the integer value. 2650 SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { 2651 StoreSDNode *ST = cast<StoreSDNode>(N); 2652 SDValue Val = ST->getValue(); 2653 SDLoc DL(N); 2654 2655 SDValue Promoted = GetPromotedFloat(Val); 2656 EVT VT = ST->getOperand(1).getValueType(); 2657 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2658 2659 SDValue NewVal; 2660 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL, 2661 IVT, Promoted); 2662 2663 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(), 2664 ST->getMemOperand()); 2665 } 2666 2667 SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N, 2668 unsigned OpNo) { 2669 AtomicSDNode *ST = cast<AtomicSDNode>(N); 2670 SDValue Val = ST->getVal(); 2671 SDLoc DL(N); 2672 2673 SDValue Promoted = GetPromotedFloat(Val); 2674 EVT VT = ST->getOperand(1).getValueType(); 2675 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2676 2677 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), 2678 DL, IVT, Promoted); 2679 2680 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal, 2681 ST->getBasePtr(), ST->getMemOperand()); 2682 } 2683 2684 //===----------------------------------------------------------------------===// 2685 // Float Result Promotion 2686 //===----------------------------------------------------------------------===// 2687 2688 void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { 2689 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG)); 2690 SDValue R = SDValue(); 2691 2692 // See if the target wants to custom expand this node. 2693 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 2694 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 2695 return; 2696 } 2697 2698 switch (N->getOpcode()) { 2699 // These opcodes cannot appear if promotion of FP16 is done in the backend 2700 // instead of Clang 2701 case ISD::FP16_TO_FP: 2702 case ISD::FP_TO_FP16: 2703 default: 2704 #ifndef NDEBUG 2705 dbgs() << "PromoteFloatResult #" << ResNo << ": "; 2706 N->dump(&DAG); dbgs() << "\n"; 2707 #endif 2708 report_fatal_error("Do not know how to promote this operator's result!"); 2709 2710 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; 2711 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break; 2712 case ISD::EXTRACT_VECTOR_ELT: 2713 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break; 2714 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break; 2715 2716 // Unary FP Operations 2717 case ISD::FABS: 2718 case ISD::FACOS: 2719 case ISD::FASIN: 2720 case ISD::FATAN: 2721 case ISD::FCBRT: 2722 case ISD::FCEIL: 2723 case ISD::FCOS: 2724 case ISD::FCOSH: 2725 case ISD::FEXP: 2726 case ISD::FEXP2: 2727 case ISD::FEXP10: 2728 case ISD::FFLOOR: 2729 case ISD::FLOG: 2730 case ISD::FLOG2: 2731 case ISD::FLOG10: 2732 case ISD::FNEARBYINT: 2733 case ISD::FNEG: 2734 case ISD::FRINT: 2735 case ISD::FROUND: 2736 case ISD::FROUNDEVEN: 2737 case ISD::FSIN: 2738 case ISD::FSINH: 2739 case ISD::FSQRT: 2740 case ISD::FTRUNC: 2741 case ISD::FTAN: 2742 case ISD::FTANH: 2743 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break; 2744 2745 // Binary FP Operations 2746 case ISD::FADD: 2747 case ISD::FDIV: 2748 case ISD::FMAXIMUM: 2749 case ISD::FMINIMUM: 2750 case ISD::FMAXIMUMNUM: 2751 case ISD::FMINIMUMNUM: 2752 case ISD::FMAXNUM: 2753 case ISD::FMINNUM: 2754 case ISD::FMAXNUM_IEEE: 2755 case ISD::FMINNUM_IEEE: 2756 case ISD::FMUL: 2757 case ISD::FPOW: 2758 case ISD::FATAN2: 2759 case ISD::FREM: 2760 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break; 2761 2762 case ISD::FMA: // FMA is same as FMAD 2763 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break; 2764 2765 case ISD::FPOWI: 2766 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break; 2767 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break; 2768 2769 case ISD::FSINCOS: 2770 R = PromoteFloatRes_UnaryWithTwoFPResults(N); 2771 break; 2772 2773 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break; 2774 case ISD::STRICT_FP_ROUND: 2775 R = PromoteFloatRes_STRICT_FP_ROUND(N); 2776 break; 2777 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break; 2778 case ISD::ATOMIC_LOAD: 2779 R = PromoteFloatRes_ATOMIC_LOAD(N); 2780 break; 2781 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break; 2782 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break; 2783 2784 case ISD::SINT_TO_FP: 2785 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; 2786 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; 2787 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 2788 case ISD::VECREDUCE_FADD: 2789 case ISD::VECREDUCE_FMUL: 2790 case ISD::VECREDUCE_FMIN: 2791 case ISD::VECREDUCE_FMAX: 2792 case ISD::VECREDUCE_FMAXIMUM: 2793 case ISD::VECREDUCE_FMINIMUM: 2794 R = PromoteFloatRes_VECREDUCE(N); 2795 break; 2796 case ISD::VECREDUCE_SEQ_FADD: 2797 case ISD::VECREDUCE_SEQ_FMUL: 2798 R = PromoteFloatRes_VECREDUCE_SEQ(N); 2799 break; 2800 } 2801 2802 if (R.getNode()) 2803 SetPromotedFloat(SDValue(N, ResNo), R); 2804 } 2805 2806 // Bitcast from i16 to f16: convert the i16 to a f32 value instead. 2807 // At this point, it is not possible to determine if the bitcast value is 2808 // eventually stored to memory or promoted to f32 or promoted to a floating 2809 // point at a higher precision. Some of these cases are handled by FP_EXTEND, 2810 // STORE promotion handlers. 2811 SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) { 2812 EVT VT = N->getValueType(0); 2813 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2814 // Input type isn't guaranteed to be a scalar int so bitcast if not. The 2815 // bitcast will be legalized further if necessary. 2816 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), 2817 N->getOperand(0).getValueType().getSizeInBits()); 2818 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0)); 2819 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast); 2820 } 2821 2822 SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) { 2823 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N); 2824 EVT VT = N->getValueType(0); 2825 SDLoc DL(N); 2826 2827 // Get the (bit-cast) APInt of the APFloat and build an integer constant 2828 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2829 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL, 2830 IVT); 2831 2832 // Convert the Constant to the desired FP type 2833 // FIXME We might be able to do the conversion during compilation and get rid 2834 // of it from the object code 2835 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2836 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C); 2837 } 2838 2839 // If the Index operand is a constant, try to redirect the extract operation to 2840 // the correct legalized vector. If not, bit-convert the input vector to 2841 // equivalent integer vector. Extract the element as an (bit-cast) integer 2842 // value and convert it to the promoted type. 2843 SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) { 2844 SDLoc DL(N); 2845 2846 // If the index is constant, try to extract the value from the legalized 2847 // vector type. 2848 if (isa<ConstantSDNode>(N->getOperand(1))) { 2849 SDValue Vec = N->getOperand(0); 2850 SDValue Idx = N->getOperand(1); 2851 EVT VecVT = Vec->getValueType(0); 2852 EVT EltVT = VecVT.getVectorElementType(); 2853 2854 uint64_t IdxVal = Idx->getAsZExtVal(); 2855 2856 switch (getTypeAction(VecVT)) { 2857 default: break; 2858 case TargetLowering::TypeScalarizeVector: { 2859 SDValue Res = GetScalarizedVector(N->getOperand(0)); 2860 ReplaceValueWith(SDValue(N, 0), Res); 2861 return SDValue(); 2862 } 2863 case TargetLowering::TypeWidenVector: { 2864 Vec = GetWidenedVector(Vec); 2865 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx); 2866 ReplaceValueWith(SDValue(N, 0), Res); 2867 return SDValue(); 2868 } 2869 case TargetLowering::TypeSplitVector: { 2870 SDValue Lo, Hi; 2871 GetSplitVector(Vec, Lo, Hi); 2872 2873 uint64_t LoElts = Lo.getValueType().getVectorNumElements(); 2874 SDValue Res; 2875 if (IdxVal < LoElts) 2876 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx); 2877 else 2878 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi, 2879 DAG.getConstant(IdxVal - LoElts, DL, 2880 Idx.getValueType())); 2881 ReplaceValueWith(SDValue(N, 0), Res); 2882 return SDValue(); 2883 } 2884 2885 } 2886 } 2887 2888 // Bit-convert the input vector to the equivalent integer vector 2889 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 2890 EVT IVT = NewOp.getValueType().getVectorElementType(); 2891 2892 // Extract the element as an (bit-cast) integer value 2893 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT, 2894 NewOp, N->getOperand(1)); 2895 2896 // Convert the element to the desired FP type 2897 EVT VT = N->getValueType(0); 2898 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2899 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal); 2900 } 2901 2902 // FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result 2903 // needs promotion, so does the argument X. Note that Y, if needed, will be 2904 // handled during operand promotion. 2905 SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) { 2906 EVT VT = N->getValueType(0); 2907 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2908 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2909 2910 SDValue Op1 = N->getOperand(1); 2911 2912 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2913 } 2914 2915 // Unary operation where the result and the operand have PromoteFloat type 2916 // action. Construct a new SDNode with the promoted float value of the old 2917 // operand. 2918 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) { 2919 EVT VT = N->getValueType(0); 2920 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2921 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2922 2923 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op); 2924 } 2925 2926 // Binary operations where the result and both operands have PromoteFloat type 2927 // action. Construct a new SDNode with the promoted float values of the old 2928 // operands. 2929 SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) { 2930 EVT VT = N->getValueType(0); 2931 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2932 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2933 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2934 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags()); 2935 } 2936 2937 SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) { 2938 EVT VT = N->getValueType(0); 2939 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2940 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2941 SDValue Op1 = GetPromotedFloat(N->getOperand(1)); 2942 SDValue Op2 = GetPromotedFloat(N->getOperand(2)); 2943 2944 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2); 2945 } 2946 2947 // Promote the Float (first) operand and retain the Integer (second) operand 2948 SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) { 2949 EVT VT = N->getValueType(0); 2950 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2951 SDValue Op0 = GetPromotedFloat(N->getOperand(0)); 2952 SDValue Op1 = N->getOperand(1); 2953 2954 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1); 2955 } 2956 2957 SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) { 2958 EVT VT = N->getValueType(0); 2959 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2960 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2961 SDValue Res = 2962 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op); 2963 2964 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 2965 return Res; 2966 } 2967 2968 SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryWithTwoFPResults(SDNode *N) { 2969 EVT VT = N->getValueType(0); 2970 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 2971 SDValue Op = GetPromotedFloat(N->getOperand(0)); 2972 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, NVT}, Op); 2973 2974 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues; 2975 ++ResNum) { 2976 SetPromotedFloat(SDValue(N, ResNum), Res.getValue(ResNum)); 2977 } 2978 2979 return SDValue(); 2980 } 2981 2982 // Explicit operation to reduce precision. Reduce the value to half precision 2983 // and promote it back to the legal type. 2984 SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) { 2985 SDLoc DL(N); 2986 2987 SDValue Op = N->getOperand(0); 2988 EVT VT = N->getValueType(0); 2989 EVT OpVT = Op->getValueType(0); 2990 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 2991 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 2992 2993 // Round promoted float to desired precision 2994 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op); 2995 // Promote it back to the legal output type 2996 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round); 2997 } 2998 2999 // Explicit operation to reduce precision. Reduce the value to half precision 3000 // and promote it back to the legal type. 3001 SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) { 3002 SDLoc DL(N); 3003 3004 SDValue Chain = N->getOperand(0); 3005 SDValue Op = N->getOperand(1); 3006 EVT VT = N->getValueType(0); 3007 EVT OpVT = Op->getValueType(0); 3008 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); 3009 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 3010 3011 // Round promoted float to desired precision 3012 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL, 3013 DAG.getVTList(IVT, MVT::Other), Chain, Op); 3014 // Promote it back to the legal output type 3015 SDValue Res = 3016 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL, 3017 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round); 3018 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3019 return Res; 3020 } 3021 3022 SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { 3023 LoadSDNode *L = cast<LoadSDNode>(N); 3024 EVT VT = N->getValueType(0); 3025 3026 // Load the value as an integer value with the same number of bits. 3027 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 3028 SDValue newL = DAG.getLoad( 3029 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), 3030 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, 3031 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo()); 3032 // Legalize the chain result by replacing uses of the old value chain with the 3033 // new one 3034 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 3035 3036 // Convert the integer value to the desired FP type 3037 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3038 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL); 3039 } 3040 3041 SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) { 3042 AtomicSDNode *AM = cast<AtomicSDNode>(N); 3043 EVT VT = AM->getValueType(0); 3044 3045 // Load the value as an integer value with the same number of bits. 3046 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); 3047 SDValue newL = DAG.getAtomic( 3048 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other), 3049 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand()); 3050 3051 // Legalize the chain result by replacing uses of the old value chain with the 3052 // new one 3053 ReplaceValueWith(SDValue(N, 1), newL.getValue(1)); 3054 3055 // Convert the integer value to the desired FP type 3056 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3057 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL); 3058 } 3059 3060 // Construct a new SELECT node with the promoted true- and false- values. 3061 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) { 3062 SDValue TrueVal = GetPromotedFloat(N->getOperand(1)); 3063 SDValue FalseVal = GetPromotedFloat(N->getOperand(2)); 3064 3065 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0), 3066 N->getOperand(0), TrueVal, FalseVal); 3067 } 3068 3069 // Construct a new SELECT_CC node with the promoted true- and false- values. 3070 // The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC. 3071 SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) { 3072 SDValue TrueVal = GetPromotedFloat(N->getOperand(2)); 3073 SDValue FalseVal = GetPromotedFloat(N->getOperand(3)); 3074 3075 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), 3076 TrueVal.getNode()->getValueType(0), N->getOperand(0), 3077 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4)); 3078 } 3079 3080 // Construct a SDNode that transforms the SINT or UINT operand to the promoted 3081 // float type. 3082 SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) { 3083 SDLoc DL(N); 3084 EVT VT = N->getValueType(0); 3085 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3086 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0)); 3087 // Round the value to the desired precision (that of the source type). 3088 return DAG.getNode( 3089 ISD::FP_EXTEND, DL, NVT, 3090 DAG.getNode(ISD::FP_ROUND, DL, VT, NV, 3091 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true))); 3092 } 3093 3094 SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { 3095 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(), 3096 N->getValueType(0))); 3097 } 3098 3099 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) { 3100 // Expand and promote recursively. 3101 // TODO: This is non-optimal, but dealing with the concurrently happening 3102 // vector-legalization is non-trivial. We could do something similar to 3103 // PromoteFloatRes_EXTRACT_VECTOR_ELT here. 3104 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 3105 return SDValue(); 3106 } 3107 3108 SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) { 3109 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 3110 return SDValue(); 3111 } 3112 3113 SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { 3114 EVT VT = N->getValueType(0); 3115 3116 AtomicSDNode *AM = cast<AtomicSDNode>(N); 3117 SDLoc SL(N); 3118 3119 SDValue CastVal = BitConvertToInteger(AM->getVal()); 3120 EVT CastVT = CastVal.getValueType(); 3121 3122 SDValue NewAtomic 3123 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, 3124 DAG.getVTList(CastVT, MVT::Other), 3125 { AM->getChain(), AM->getBasePtr(), CastVal }, 3126 AM->getMemOperand()); 3127 3128 SDValue Result = NewAtomic; 3129 3130 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) { 3131 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); 3132 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, 3133 NewAtomic); 3134 } 3135 3136 // Legalize the chain result by replacing uses of the old value chain with the 3137 // new one 3138 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); 3139 3140 return Result; 3141 3142 } 3143 3144 //===----------------------------------------------------------------------===// 3145 // Half Result Soft Promotion 3146 //===----------------------------------------------------------------------===// 3147 3148 void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { 3149 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": "; 3150 N->dump(&DAG)); 3151 SDValue R = SDValue(); 3152 3153 // See if the target wants to custom expand this node. 3154 if (CustomLowerNode(N, N->getValueType(ResNo), true)) { 3155 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n"); 3156 return; 3157 } 3158 3159 switch (N->getOpcode()) { 3160 default: 3161 #ifndef NDEBUG 3162 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": "; 3163 N->dump(&DAG); dbgs() << "\n"; 3164 #endif 3165 report_fatal_error("Do not know how to soft promote this operator's " 3166 "result!"); 3167 3168 case ISD::ARITH_FENCE: 3169 R = SoftPromoteHalfRes_ARITH_FENCE(N); break; 3170 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break; 3171 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break; 3172 case ISD::EXTRACT_VECTOR_ELT: 3173 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break; 3174 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break; 3175 case ISD::STRICT_FP_ROUND: 3176 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break; 3177 3178 // Unary FP Operations 3179 case ISD::FABS: 3180 case ISD::FACOS: 3181 case ISD::FASIN: 3182 case ISD::FATAN: 3183 case ISD::FCBRT: 3184 case ISD::FCEIL: 3185 case ISD::FCOS: 3186 case ISD::FCOSH: 3187 case ISD::FEXP: 3188 case ISD::FEXP2: 3189 case ISD::FEXP10: 3190 case ISD::FFLOOR: 3191 case ISD::FLOG: 3192 case ISD::FLOG2: 3193 case ISD::FLOG10: 3194 case ISD::FNEARBYINT: 3195 case ISD::FNEG: 3196 case ISD::FREEZE: 3197 case ISD::FRINT: 3198 case ISD::FROUND: 3199 case ISD::FROUNDEVEN: 3200 case ISD::FSIN: 3201 case ISD::FSINH: 3202 case ISD::FSQRT: 3203 case ISD::FTRUNC: 3204 case ISD::FTAN: 3205 case ISD::FTANH: 3206 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break; 3207 3208 // Binary FP Operations 3209 case ISD::FADD: 3210 case ISD::FDIV: 3211 case ISD::FMAXIMUM: 3212 case ISD::FMINIMUM: 3213 case ISD::FMAXIMUMNUM: 3214 case ISD::FMINIMUMNUM: 3215 case ISD::FMAXNUM: 3216 case ISD::FMINNUM: 3217 case ISD::FMUL: 3218 case ISD::FPOW: 3219 case ISD::FATAN2: 3220 case ISD::FREM: 3221 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break; 3222 3223 case ISD::FMA: // FMA is same as FMAD 3224 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break; 3225 3226 case ISD::FPOWI: 3227 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; 3228 3229 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break; 3230 3231 case ISD::FSINCOS: 3232 R = SoftPromoteHalfRes_UnaryWithTwoFPResults(N); 3233 break; 3234 3235 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; 3236 case ISD::ATOMIC_LOAD: 3237 R = SoftPromoteHalfRes_ATOMIC_LOAD(N); 3238 break; 3239 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; 3240 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; 3241 case ISD::STRICT_SINT_TO_FP: 3242 case ISD::STRICT_UINT_TO_FP: 3243 case ISD::SINT_TO_FP: 3244 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break; 3245 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break; 3246 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; 3247 case ISD::VECREDUCE_FADD: 3248 case ISD::VECREDUCE_FMUL: 3249 case ISD::VECREDUCE_FMIN: 3250 case ISD::VECREDUCE_FMAX: 3251 case ISD::VECREDUCE_FMAXIMUM: 3252 case ISD::VECREDUCE_FMINIMUM: 3253 R = SoftPromoteHalfRes_VECREDUCE(N); 3254 break; 3255 case ISD::VECREDUCE_SEQ_FADD: 3256 case ISD::VECREDUCE_SEQ_FMUL: 3257 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N); 3258 break; 3259 } 3260 3261 if (R.getNode()) 3262 SetSoftPromotedHalf(SDValue(N, ResNo), R); 3263 } 3264 3265 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) { 3266 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16, 3267 BitConvertToInteger(N->getOperand(0))); 3268 } 3269 3270 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) { 3271 return BitConvertToInteger(N->getOperand(0)); 3272 } 3273 3274 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) { 3275 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N); 3276 3277 // Get the (bit-cast) APInt of the APFloat and build an integer constant 3278 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN), 3279 MVT::i16); 3280 } 3281 3282 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) { 3283 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0)); 3284 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), 3285 NewOp.getValueType().getVectorElementType(), NewOp, 3286 N->getOperand(1)); 3287 } 3288 3289 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) { 3290 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0)); 3291 SDValue RHS = BitConvertToInteger(N->getOperand(1)); 3292 SDLoc dl(N); 3293 3294 EVT LVT = LHS.getValueType(); 3295 EVT RVT = RHS.getValueType(); 3296 3297 unsigned LSize = LVT.getSizeInBits(); 3298 unsigned RSize = RVT.getSizeInBits(); 3299 3300 // First get the sign bit of second operand. 3301 SDValue SignBit = DAG.getNode( 3302 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT), 3303 DAG.getConstant(RSize - 1, dl, 3304 TLI.getShiftAmountTy(RVT, DAG.getDataLayout()))); 3305 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit); 3306 3307 // Shift right or sign-extend it if the two operands have different types. 3308 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits(); 3309 if (SizeDiff > 0) { 3310 SignBit = 3311 DAG.getNode(ISD::SRL, dl, RVT, SignBit, 3312 DAG.getConstant(SizeDiff, dl, 3313 TLI.getShiftAmountTy(SignBit.getValueType(), 3314 DAG.getDataLayout()))); 3315 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit); 3316 } else if (SizeDiff < 0) { 3317 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit); 3318 SignBit = 3319 DAG.getNode(ISD::SHL, dl, LVT, SignBit, 3320 DAG.getConstant(-SizeDiff, dl, 3321 TLI.getShiftAmountTy(SignBit.getValueType(), 3322 DAG.getDataLayout()))); 3323 } 3324 3325 // Clear the sign bit of the first operand. 3326 SDValue Mask = DAG.getNode( 3327 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT), 3328 DAG.getConstant(LSize - 1, dl, 3329 TLI.getShiftAmountTy(LVT, DAG.getDataLayout()))); 3330 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT)); 3331 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask); 3332 3333 // Or the value with the sign bit. 3334 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit); 3335 } 3336 3337 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) { 3338 EVT OVT = N->getValueType(0); 3339 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3340 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3341 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3342 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3343 SDLoc dl(N); 3344 3345 // Promote to the larger FP type. 3346 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 3347 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3348 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3349 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2); 3350 3351 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2); 3352 3353 // Convert back to FP16 as an integer. 3354 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3355 } 3356 3357 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { 3358 EVT OVT = N->getValueType(0); 3359 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3360 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3361 SDValue Op1 = N->getOperand(1); 3362 SDLoc dl(N); 3363 3364 // Promote to the larger FP type. 3365 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0); 3366 3367 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 3368 3369 // Convert back to FP16 as an integer. 3370 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3371 } 3372 3373 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) { 3374 EVT OVT = N->getValueType(0); 3375 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3376 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 3377 SDLoc dl(N); 3378 3379 // Promote to the larger FP type. 3380 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 3381 3382 SDValue Res = DAG.getNode(N->getOpcode(), dl, 3383 DAG.getVTList(NVT, N->getValueType(1)), Op); 3384 3385 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3386 3387 // Convert back to FP16 as an integer. 3388 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3389 } 3390 3391 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryWithTwoFPResults(SDNode *N) { 3392 EVT OVT = N->getValueType(0); 3393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3394 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 3395 SDLoc dl(N); 3396 3397 // Promote to the larger FP type. 3398 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 3399 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, NVT), Op); 3400 3401 // Convert back to FP16 as an integer. 3402 ISD::NodeType Truncate = GetPromotionOpcode(NVT, OVT); 3403 for (unsigned ResNum = 0, NumValues = N->getNumValues(); ResNum < NumValues; 3404 ++ResNum) { 3405 SDValue Trunc = DAG.getNode(Truncate, dl, MVT::i16, Res.getValue(ResNum)); 3406 SetSoftPromotedHalf(SDValue(N, ResNum), Trunc); 3407 } 3408 3409 return SDValue(); 3410 } 3411 3412 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { 3413 EVT RVT = N->getValueType(0); 3414 bool IsStrict = N->isStrictFPOpcode(); 3415 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3416 EVT SVT = Op.getValueType(); 3417 3418 // If the input type needs to be softened, do that now so that call lowering 3419 // will see the f16 type. 3420 if (getTypeAction(SVT) == TargetLowering::TypeSoftenFloat) { 3421 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, RVT); 3422 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall"); 3423 3424 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue(); 3425 Op = GetSoftenedFloat(Op); 3426 TargetLowering::MakeLibCallOptions CallOptions; 3427 CallOptions.setTypeListBeforeSoften(SVT, RVT, true); 3428 std::pair<SDValue, SDValue> Tmp = 3429 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, SDLoc(N), Chain); 3430 if (IsStrict) 3431 ReplaceValueWith(SDValue(N, 1), Tmp.second); 3432 return DAG.getNode(ISD::BITCAST, SDLoc(N), MVT::i16, Tmp.first); 3433 } 3434 3435 if (IsStrict) { 3436 SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N), 3437 {MVT::i16, MVT::Other}, {N->getOperand(0), Op}); 3438 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3439 return Res; 3440 } 3441 3442 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16, 3443 N->getOperand(0)); 3444 } 3445 3446 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) { 3447 LoadSDNode *L = cast<LoadSDNode>(N); 3448 3449 // Load the value as an integer value with the same number of bits. 3450 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!"); 3451 SDValue NewL = 3452 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16, 3453 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), 3454 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(), 3455 L->getMemOperand()->getFlags(), L->getAAInfo()); 3456 // Legalize the chain result by replacing uses of the old value chain with the 3457 // new one 3458 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 3459 return NewL; 3460 } 3461 3462 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) { 3463 AtomicSDNode *AM = cast<AtomicSDNode>(N); 3464 3465 // Load the value as an integer value with the same number of bits. 3466 SDValue NewL = DAG.getAtomic( 3467 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other), 3468 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand()); 3469 3470 // Legalize the chain result by replacing uses of the old value chain with the 3471 // new one 3472 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1)); 3473 return NewL; 3474 } 3475 3476 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) { 3477 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3478 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3479 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1, 3480 Op2); 3481 } 3482 3483 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) { 3484 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2)); 3485 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3)); 3486 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(), 3487 N->getOperand(0), N->getOperand(1), Op2, Op3, 3488 N->getOperand(4)); 3489 } 3490 3491 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) { 3492 EVT OVT = N->getValueType(0); 3493 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3494 SDLoc dl(N); 3495 3496 if (N->isStrictFPOpcode()) { 3497 SDValue Op = DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, 3498 {N->getOperand(0), N->getOperand(1)}); 3499 Op = DAG.getNode(GetPromotionOpcodeStrict(NVT, OVT), dl, 3500 {MVT::i16, MVT::Other}, {Op.getValue(1), Op}); 3501 ReplaceValueWith(SDValue(N, 1), Op.getValue(1)); 3502 return Op; 3503 } 3504 3505 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); 3506 3507 // Round the value to the softened type. 3508 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3509 } 3510 3511 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) { 3512 return DAG.getUNDEF(MVT::i16); 3513 } 3514 3515 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) { 3516 EVT OVT = N->getValueType(0); 3517 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3518 SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); 3519 SDLoc dl(N); 3520 3521 // Promote to the larger FP type. 3522 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); 3523 3524 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op); 3525 3526 // Convert back to FP16 as an integer. 3527 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3528 } 3529 3530 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) { 3531 EVT OVT = N->getValueType(0); 3532 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); 3533 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3534 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1)); 3535 SDLoc dl(N); 3536 3537 // Promote to the larger FP type. 3538 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT); 3539 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3540 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3541 3542 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1); 3543 3544 // Convert back to FP16 as an integer. 3545 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); 3546 } 3547 3548 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) { 3549 // Expand and soften recursively. 3550 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG)); 3551 return SDValue(); 3552 } 3553 3554 SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) { 3555 // Expand and soften. 3556 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG)); 3557 return SDValue(); 3558 } 3559 3560 //===----------------------------------------------------------------------===// 3561 // Half Operand Soft Promotion 3562 //===----------------------------------------------------------------------===// 3563 3564 bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) { 3565 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": "; 3566 N->dump(&DAG)); 3567 SDValue Res = SDValue(); 3568 3569 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { 3570 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n"); 3571 return false; 3572 } 3573 3574 // Nodes that use a promotion-requiring floating point operand, but doesn't 3575 // produce a soft promotion-requiring floating point result, need to be 3576 // legalized to use the soft promoted float operand. Nodes that produce at 3577 // least one soft promotion-requiring floating point result have their 3578 // operands legalized as a part of PromoteFloatResult. 3579 switch (N->getOpcode()) { 3580 default: 3581 #ifndef NDEBUG 3582 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": "; 3583 N->dump(&DAG); dbgs() << "\n"; 3584 #endif 3585 report_fatal_error("Do not know how to soft promote this operator's " 3586 "operand!"); 3587 3588 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break; 3589 case ISD::FAKE_USE: 3590 Res = SoftPromoteHalfOp_FAKE_USE(N, OpNo); 3591 break; 3592 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break; 3593 case ISD::STRICT_FP_TO_SINT: 3594 case ISD::STRICT_FP_TO_UINT: 3595 case ISD::FP_TO_SINT: 3596 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break; 3597 case ISD::FP_TO_SINT_SAT: 3598 case ISD::FP_TO_UINT_SAT: 3599 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break; 3600 case ISD::STRICT_FP_EXTEND: 3601 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break; 3602 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break; 3603 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break; 3604 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break; 3605 case ISD::ATOMIC_STORE: 3606 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo); 3607 break; 3608 case ISD::STACKMAP: 3609 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo); 3610 break; 3611 case ISD::PATCHPOINT: 3612 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo); 3613 break; 3614 } 3615 3616 if (!Res.getNode()) 3617 return false; 3618 3619 assert(Res.getNode() != N && "Expected a new node!"); 3620 3621 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && 3622 "Invalid operand expansion"); 3623 3624 ReplaceValueWith(SDValue(N, 0), Res); 3625 return false; 3626 } 3627 3628 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) { 3629 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0)); 3630 3631 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0); 3632 } 3633 3634 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FAKE_USE(SDNode *N, unsigned OpNo) { 3635 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 3636 SDValue Op = GetSoftPromotedHalf(N->getOperand(OpNo)); 3637 return DAG.getNode(N->getOpcode(), SDLoc(N), MVT::Other, N->getOperand(0), 3638 Op); 3639 } 3640 3641 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N, 3642 unsigned OpNo) { 3643 assert(OpNo == 1 && "Only Operand 1 must need promotion here"); 3644 SDValue Op1 = N->getOperand(1); 3645 EVT RVT = Op1.getValueType(); 3646 SDLoc dl(N); 3647 3648 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType()); 3649 3650 Op1 = GetSoftPromotedHalf(Op1); 3651 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1); 3652 3653 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0), 3654 Op1); 3655 } 3656 3657 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) { 3658 EVT RVT = N->getValueType(0); 3659 bool IsStrict = N->isStrictFPOpcode(); 3660 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3661 EVT SVT = Op.getValueType(); 3662 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0)); 3663 3664 if (IsStrict) { 3665 SDValue Res = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), SDLoc(N), 3666 {RVT, MVT::Other}, {N->getOperand(0), Op}); 3667 ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); 3668 ReplaceValueWith(SDValue(N, 0), Res); 3669 return SDValue(); 3670 } 3671 3672 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op); 3673 } 3674 3675 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) { 3676 EVT RVT = N->getValueType(0); 3677 bool IsStrict = N->isStrictFPOpcode(); 3678 SDValue Op = N->getOperand(IsStrict ? 1 : 0); 3679 EVT SVT = Op.getValueType(); 3680 SDLoc dl(N); 3681 3682 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT); 3683 Op = GetSoftPromotedHalf(Op); 3684 3685 if (IsStrict) { 3686 Op = DAG.getNode(GetPromotionOpcodeStrict(SVT, RVT), dl, {NVT, MVT::Other}, 3687 {N->getOperand(0), Op}); 3688 Op = DAG.getNode(N->getOpcode(), dl, {RVT, MVT::Other}, 3689 {Op.getValue(1), Op}); 3690 ReplaceValueWith(SDValue(N, 1), Op.getValue(1)); 3691 ReplaceValueWith(SDValue(N, 0), Op); 3692 return SDValue(); 3693 } 3694 3695 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3696 return DAG.getNode(N->getOpcode(), dl, RVT, Res); 3697 } 3698 3699 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) { 3700 EVT RVT = N->getValueType(0); 3701 SDValue Op = N->getOperand(0); 3702 EVT SVT = Op.getValueType(); 3703 SDLoc dl(N); 3704 3705 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType()); 3706 3707 Op = GetSoftPromotedHalf(Op); 3708 3709 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op); 3710 3711 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res, 3712 N->getOperand(1)); 3713 } 3714 3715 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N, 3716 unsigned OpNo) { 3717 assert(OpNo == 0 && "Can only soften the comparison values"); 3718 SDValue Op0 = N->getOperand(0); 3719 SDValue Op1 = N->getOperand(1); 3720 SDLoc dl(N); 3721 3722 EVT SVT = Op0.getValueType(); 3723 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT); 3724 3725 Op0 = GetSoftPromotedHalf(Op0); 3726 Op1 = GetSoftPromotedHalf(Op1); 3727 3728 // Promote to the larger FP type. 3729 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3730 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3731 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3732 3733 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1, 3734 N->getOperand(2), N->getOperand(3), N->getOperand(4)); 3735 } 3736 3737 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) { 3738 SDValue Op0 = N->getOperand(0); 3739 SDValue Op1 = N->getOperand(1); 3740 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get(); 3741 SDLoc dl(N); 3742 3743 EVT SVT = Op0.getValueType(); 3744 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType()); 3745 3746 Op0 = GetSoftPromotedHalf(Op0); 3747 Op1 = GetSoftPromotedHalf(Op1); 3748 3749 // Promote to the larger FP type. 3750 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT); 3751 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0); 3752 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1); 3753 3754 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode); 3755 } 3756 3757 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) { 3758 assert(OpNo == 1 && "Can only soften the stored value!"); 3759 StoreSDNode *ST = cast<StoreSDNode>(N); 3760 SDValue Val = ST->getValue(); 3761 SDLoc dl(N); 3762 3763 assert(!ST->isTruncatingStore() && "Unexpected truncating store."); 3764 SDValue Promoted = GetSoftPromotedHalf(Val); 3765 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(), 3766 ST->getMemOperand()); 3767 } 3768 3769 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N, 3770 unsigned OpNo) { 3771 assert(OpNo == 1 && "Can only soften the stored value!"); 3772 AtomicSDNode *ST = cast<AtomicSDNode>(N); 3773 SDValue Val = ST->getVal(); 3774 SDLoc dl(N); 3775 3776 SDValue Promoted = GetSoftPromotedHalf(Val); 3777 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(), 3778 ST->getChain(), Promoted, ST->getBasePtr(), 3779 ST->getMemOperand()); 3780 } 3781 3782 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) { 3783 assert(OpNo > 1); // Because the first two arguments are guaranteed legal. 3784 SmallVector<SDValue> NewOps(N->ops()); 3785 SDValue Op = N->getOperand(OpNo); 3786 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3787 SDValue NewNode = 3788 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3789 3790 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3791 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3792 3793 return SDValue(); // Signal that we replaced the node ourselves. 3794 } 3795 3796 SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N, 3797 unsigned OpNo) { 3798 assert(OpNo >= 7); 3799 SmallVector<SDValue> NewOps(N->ops()); 3800 SDValue Op = N->getOperand(OpNo); 3801 NewOps[OpNo] = GetSoftPromotedHalf(Op); 3802 SDValue NewNode = 3803 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps); 3804 3805 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++) 3806 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum)); 3807 3808 return SDValue(); // Signal that we replaced the node ourselves. 3809 } 3810