1 //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file performs vector type splitting and scalarization for LegalizeTypes.
10 // Scalarization is the act of changing a computation in an illegal one-element
11 // vector type to be a computation in its scalar element type. For example,
12 // implementing <1 x f32> arithmetic in a scalar f32 register. This is needed
13 // as a base case when scalarizing vector arithmetic like <4 x f32>, which
14 // eventually decomposes to scalars if the target doesn't support v4f32 or v2f32
15 // types.
16 // Splitting is the act of changing a computation in an invalid vector type to
17 // be a computation in two vectors of half the size. For example, implementing
18 // <128 x f32> operations in terms of two <64 x f32> operations.
19 //
20 //===----------------------------------------------------------------------===//
21
22 #include "LegalizeTypes.h"
23 #include "llvm/Analysis/MemoryLocation.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/Support/ErrorHandling.h"
26 #include "llvm/Support/TypeSize.h"
27 #include "llvm/Support/raw_ostream.h"
28 using namespace llvm;
29
30 #define DEBUG_TYPE "legalize-types"
31
32 //===----------------------------------------------------------------------===//
33 // Result Vector Scalarization: <1 x ty> -> ty.
34 //===----------------------------------------------------------------------===//
35
ScalarizeVectorResult(SDNode * N,unsigned ResNo)36 void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
37 LLVM_DEBUG(dbgs() << "Scalarize node result " << ResNo << ": "; N->dump(&DAG);
38 dbgs() << "\n");
39 SDValue R = SDValue();
40
41 switch (N->getOpcode()) {
42 default:
43 #ifndef NDEBUG
44 dbgs() << "ScalarizeVectorResult #" << ResNo << ": ";
45 N->dump(&DAG);
46 dbgs() << "\n";
47 #endif
48 report_fatal_error("Do not know how to scalarize the result of this "
49 "operator!\n");
50
51 case ISD::MERGE_VALUES: R = ScalarizeVecRes_MERGE_VALUES(N, ResNo);break;
52 case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break;
53 case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break;
54 case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break;
55 case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break;
56 case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break;
57 case ISD::INSERT_VECTOR_ELT: R = ScalarizeVecRes_INSERT_VECTOR_ELT(N); break;
58 case ISD::LOAD: R = ScalarizeVecRes_LOAD(cast<LoadSDNode>(N));break;
59 case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
60 case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
61 case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
62 case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
63 case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
64 case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
65 case ISD::UNDEF: R = ScalarizeVecRes_UNDEF(N); break;
66 case ISD::VECTOR_SHUFFLE: R = ScalarizeVecRes_VECTOR_SHUFFLE(N); break;
67 case ISD::ANY_EXTEND_VECTOR_INREG:
68 case ISD::SIGN_EXTEND_VECTOR_INREG:
69 case ISD::ZERO_EXTEND_VECTOR_INREG:
70 R = ScalarizeVecRes_VecInregOp(N);
71 break;
72 case ISD::ABS:
73 case ISD::ANY_EXTEND:
74 case ISD::BITREVERSE:
75 case ISD::BSWAP:
76 case ISD::CTLZ:
77 case ISD::CTLZ_ZERO_UNDEF:
78 case ISD::CTPOP:
79 case ISD::CTTZ:
80 case ISD::CTTZ_ZERO_UNDEF:
81 case ISD::FABS:
82 case ISD::FCEIL:
83 case ISD::FCOS:
84 case ISD::FEXP:
85 case ISD::FEXP2:
86 case ISD::FFLOOR:
87 case ISD::FLOG:
88 case ISD::FLOG10:
89 case ISD::FLOG2:
90 case ISD::FNEARBYINT:
91 case ISD::FNEG:
92 case ISD::FREEZE:
93 case ISD::FP_EXTEND:
94 case ISD::FP_TO_SINT:
95 case ISD::FP_TO_UINT:
96 case ISD::FRINT:
97 case ISD::FROUND:
98 case ISD::FROUNDEVEN:
99 case ISD::FSIN:
100 case ISD::FSQRT:
101 case ISD::FTRUNC:
102 case ISD::SIGN_EXTEND:
103 case ISD::SINT_TO_FP:
104 case ISD::TRUNCATE:
105 case ISD::UINT_TO_FP:
106 case ISD::ZERO_EXTEND:
107 case ISD::FCANONICALIZE:
108 R = ScalarizeVecRes_UnaryOp(N);
109 break;
110
111 case ISD::ADD:
112 case ISD::AND:
113 case ISD::FADD:
114 case ISD::FCOPYSIGN:
115 case ISD::FDIV:
116 case ISD::FMUL:
117 case ISD::FMINNUM:
118 case ISD::FMAXNUM:
119 case ISD::FMINNUM_IEEE:
120 case ISD::FMAXNUM_IEEE:
121 case ISD::FMINIMUM:
122 case ISD::FMAXIMUM:
123 case ISD::SMIN:
124 case ISD::SMAX:
125 case ISD::UMIN:
126 case ISD::UMAX:
127
128 case ISD::SADDSAT:
129 case ISD::UADDSAT:
130 case ISD::SSUBSAT:
131 case ISD::USUBSAT:
132 case ISD::SSHLSAT:
133 case ISD::USHLSAT:
134
135 case ISD::FPOW:
136 case ISD::FREM:
137 case ISD::FSUB:
138 case ISD::MUL:
139 case ISD::OR:
140 case ISD::SDIV:
141 case ISD::SREM:
142 case ISD::SUB:
143 case ISD::UDIV:
144 case ISD::UREM:
145 case ISD::XOR:
146 case ISD::SHL:
147 case ISD::SRA:
148 case ISD::SRL:
149 case ISD::ROTL:
150 case ISD::ROTR:
151 R = ScalarizeVecRes_BinOp(N);
152 break;
153 case ISD::FMA:
154 case ISD::FSHL:
155 case ISD::FSHR:
156 R = ScalarizeVecRes_TernaryOp(N);
157 break;
158
159 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
160 case ISD::STRICT_##DAGN:
161 #include "llvm/IR/ConstrainedOps.def"
162 R = ScalarizeVecRes_StrictFPOp(N);
163 break;
164
165 case ISD::FP_TO_UINT_SAT:
166 case ISD::FP_TO_SINT_SAT:
167 R = ScalarizeVecRes_FP_TO_XINT_SAT(N);
168 break;
169
170 case ISD::UADDO:
171 case ISD::SADDO:
172 case ISD::USUBO:
173 case ISD::SSUBO:
174 case ISD::UMULO:
175 case ISD::SMULO:
176 R = ScalarizeVecRes_OverflowOp(N, ResNo);
177 break;
178 case ISD::SMULFIX:
179 case ISD::SMULFIXSAT:
180 case ISD::UMULFIX:
181 case ISD::UMULFIXSAT:
182 case ISD::SDIVFIX:
183 case ISD::SDIVFIXSAT:
184 case ISD::UDIVFIX:
185 case ISD::UDIVFIXSAT:
186 R = ScalarizeVecRes_FIX(N);
187 break;
188 }
189
190 // If R is null, the sub-method took care of registering the result.
191 if (R.getNode())
192 SetScalarizedVector(SDValue(N, ResNo), R);
193 }
194
ScalarizeVecRes_BinOp(SDNode * N)195 SDValue DAGTypeLegalizer::ScalarizeVecRes_BinOp(SDNode *N) {
196 SDValue LHS = GetScalarizedVector(N->getOperand(0));
197 SDValue RHS = GetScalarizedVector(N->getOperand(1));
198 return DAG.getNode(N->getOpcode(), SDLoc(N),
199 LHS.getValueType(), LHS, RHS, N->getFlags());
200 }
201
ScalarizeVecRes_TernaryOp(SDNode * N)202 SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) {
203 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
204 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
205 SDValue Op2 = GetScalarizedVector(N->getOperand(2));
206 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
207 Op2, N->getFlags());
208 }
209
ScalarizeVecRes_FIX(SDNode * N)210 SDValue DAGTypeLegalizer::ScalarizeVecRes_FIX(SDNode *N) {
211 SDValue Op0 = GetScalarizedVector(N->getOperand(0));
212 SDValue Op1 = GetScalarizedVector(N->getOperand(1));
213 SDValue Op2 = N->getOperand(2);
214 return DAG.getNode(N->getOpcode(), SDLoc(N), Op0.getValueType(), Op0, Op1,
215 Op2, N->getFlags());
216 }
217
ScalarizeVecRes_StrictFPOp(SDNode * N)218 SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) {
219 EVT VT = N->getValueType(0).getVectorElementType();
220 unsigned NumOpers = N->getNumOperands();
221 SDValue Chain = N->getOperand(0);
222 EVT ValueVTs[] = {VT, MVT::Other};
223 SDLoc dl(N);
224
225 SmallVector<SDValue, 4> Opers(NumOpers);
226
227 // The Chain is the first operand.
228 Opers[0] = Chain;
229
230 // Now process the remaining operands.
231 for (unsigned i = 1; i < NumOpers; ++i) {
232 SDValue Oper = N->getOperand(i);
233
234 if (Oper.getValueType().isVector())
235 Oper = GetScalarizedVector(Oper);
236
237 Opers[i] = Oper;
238 }
239
240 SDValue Result = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(ValueVTs),
241 Opers, N->getFlags());
242
243 // Legalize the chain result - switch anything that used the old chain to
244 // use the new one.
245 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
246 return Result;
247 }
248
ScalarizeVecRes_OverflowOp(SDNode * N,unsigned ResNo)249 SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N,
250 unsigned ResNo) {
251 SDLoc DL(N);
252 EVT ResVT = N->getValueType(0);
253 EVT OvVT = N->getValueType(1);
254
255 SDValue ScalarLHS, ScalarRHS;
256 if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) {
257 ScalarLHS = GetScalarizedVector(N->getOperand(0));
258 ScalarRHS = GetScalarizedVector(N->getOperand(1));
259 } else {
260 SmallVector<SDValue, 1> ElemsLHS, ElemsRHS;
261 DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS);
262 DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS);
263 ScalarLHS = ElemsLHS[0];
264 ScalarRHS = ElemsRHS[0];
265 }
266
267 SDVTList ScalarVTs = DAG.getVTList(
268 ResVT.getVectorElementType(), OvVT.getVectorElementType());
269 SDNode *ScalarNode = DAG.getNode(
270 N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode();
271 ScalarNode->setFlags(N->getFlags());
272
273 // Replace the other vector result not being explicitly scalarized here.
274 unsigned OtherNo = 1 - ResNo;
275 EVT OtherVT = N->getValueType(OtherNo);
276 if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) {
277 SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo));
278 } else {
279 SDValue OtherVal = DAG.getNode(
280 ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo));
281 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
282 }
283
284 return SDValue(ScalarNode, ResNo);
285 }
286
ScalarizeVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)287 SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N,
288 unsigned ResNo) {
289 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
290 return GetScalarizedVector(Op);
291 }
292
ScalarizeVecRes_BITCAST(SDNode * N)293 SDValue DAGTypeLegalizer::ScalarizeVecRes_BITCAST(SDNode *N) {
294 SDValue Op = N->getOperand(0);
295 if (Op.getValueType().isVector()
296 && Op.getValueType().getVectorNumElements() == 1
297 && !isSimpleLegalType(Op.getValueType()))
298 Op = GetScalarizedVector(Op);
299 EVT NewVT = N->getValueType(0).getVectorElementType();
300 return DAG.getNode(ISD::BITCAST, SDLoc(N),
301 NewVT, Op);
302 }
303
ScalarizeVecRes_BUILD_VECTOR(SDNode * N)304 SDValue DAGTypeLegalizer::ScalarizeVecRes_BUILD_VECTOR(SDNode *N) {
305 EVT EltVT = N->getValueType(0).getVectorElementType();
306 SDValue InOp = N->getOperand(0);
307 // The BUILD_VECTOR operands may be of wider element types and
308 // we may need to truncate them back to the requested return type.
309 if (EltVT.isInteger())
310 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
311 return InOp;
312 }
313
ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode * N)314 SDValue DAGTypeLegalizer::ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
315 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
316 N->getValueType(0).getVectorElementType(),
317 N->getOperand(0), N->getOperand(1));
318 }
319
ScalarizeVecRes_FP_ROUND(SDNode * N)320 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) {
321 SDLoc DL(N);
322 SDValue Op = N->getOperand(0);
323 EVT OpVT = Op.getValueType();
324 // The result needs scalarizing, but it's not a given that the source does.
325 // See similar logic in ScalarizeVecRes_UnaryOp.
326 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
327 Op = GetScalarizedVector(Op);
328 } else {
329 EVT VT = OpVT.getVectorElementType();
330 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
331 DAG.getVectorIdxConstant(0, DL));
332 }
333 return DAG.getNode(ISD::FP_ROUND, DL,
334 N->getValueType(0).getVectorElementType(), Op,
335 N->getOperand(1));
336 }
337
ScalarizeVecRes_FPOWI(SDNode * N)338 SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) {
339 SDValue Op = GetScalarizedVector(N->getOperand(0));
340 return DAG.getNode(ISD::FPOWI, SDLoc(N),
341 Op.getValueType(), Op, N->getOperand(1));
342 }
343
ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode * N)344 SDValue DAGTypeLegalizer::ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N) {
345 // The value to insert may have a wider type than the vector element type,
346 // so be sure to truncate it to the element type if necessary.
347 SDValue Op = N->getOperand(1);
348 EVT EltVT = N->getValueType(0).getVectorElementType();
349 if (Op.getValueType() != EltVT)
350 // FIXME: Can this happen for floating point types?
351 Op = DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, Op);
352 return Op;
353 }
354
ScalarizeVecRes_LOAD(LoadSDNode * N)355 SDValue DAGTypeLegalizer::ScalarizeVecRes_LOAD(LoadSDNode *N) {
356 assert(N->isUnindexed() && "Indexed vector load?");
357
358 SDValue Result = DAG.getLoad(
359 ISD::UNINDEXED, N->getExtensionType(),
360 N->getValueType(0).getVectorElementType(), SDLoc(N), N->getChain(),
361 N->getBasePtr(), DAG.getUNDEF(N->getBasePtr().getValueType()),
362 N->getPointerInfo(), N->getMemoryVT().getVectorElementType(),
363 N->getOriginalAlign(), N->getMemOperand()->getFlags(), N->getAAInfo());
364
365 // Legalize the chain result - switch anything that used the old chain to
366 // use the new one.
367 ReplaceValueWith(SDValue(N, 1), Result.getValue(1));
368 return Result;
369 }
370
ScalarizeVecRes_UnaryOp(SDNode * N)371 SDValue DAGTypeLegalizer::ScalarizeVecRes_UnaryOp(SDNode *N) {
372 // Get the dest type - it doesn't always match the input type, e.g. int_to_fp.
373 EVT DestVT = N->getValueType(0).getVectorElementType();
374 SDValue Op = N->getOperand(0);
375 EVT OpVT = Op.getValueType();
376 SDLoc DL(N);
377 // The result needs scalarizing, but it's not a given that the source does.
378 // This is a workaround for targets where it's impossible to scalarize the
379 // result of a conversion, because the source type is legal.
380 // For instance, this happens on AArch64: v1i1 is illegal but v1i{8,16,32}
381 // are widened to v8i8, v4i16, and v2i32, which is legal, because v1i64 is
382 // legal and was not scalarized.
383 // See the similar logic in ScalarizeVecRes_SETCC
384 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
385 Op = GetScalarizedVector(Op);
386 } else {
387 EVT VT = OpVT.getVectorElementType();
388 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op,
389 DAG.getVectorIdxConstant(0, DL));
390 }
391 return DAG.getNode(N->getOpcode(), SDLoc(N), DestVT, Op, N->getFlags());
392 }
393
ScalarizeVecRes_InregOp(SDNode * N)394 SDValue DAGTypeLegalizer::ScalarizeVecRes_InregOp(SDNode *N) {
395 EVT EltVT = N->getValueType(0).getVectorElementType();
396 EVT ExtVT = cast<VTSDNode>(N->getOperand(1))->getVT().getVectorElementType();
397 SDValue LHS = GetScalarizedVector(N->getOperand(0));
398 return DAG.getNode(N->getOpcode(), SDLoc(N), EltVT,
399 LHS, DAG.getValueType(ExtVT));
400 }
401
ScalarizeVecRes_VecInregOp(SDNode * N)402 SDValue DAGTypeLegalizer::ScalarizeVecRes_VecInregOp(SDNode *N) {
403 SDLoc DL(N);
404 SDValue Op = N->getOperand(0);
405
406 EVT OpVT = Op.getValueType();
407 EVT OpEltVT = OpVT.getVectorElementType();
408 EVT EltVT = N->getValueType(0).getVectorElementType();
409
410 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
411 Op = GetScalarizedVector(Op);
412 } else {
413 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpEltVT, Op,
414 DAG.getVectorIdxConstant(0, DL));
415 }
416
417 switch (N->getOpcode()) {
418 case ISD::ANY_EXTEND_VECTOR_INREG:
419 return DAG.getNode(ISD::ANY_EXTEND, DL, EltVT, Op);
420 case ISD::SIGN_EXTEND_VECTOR_INREG:
421 return DAG.getNode(ISD::SIGN_EXTEND, DL, EltVT, Op);
422 case ISD::ZERO_EXTEND_VECTOR_INREG:
423 return DAG.getNode(ISD::ZERO_EXTEND, DL, EltVT, Op);
424 }
425
426 llvm_unreachable("Illegal extend_vector_inreg opcode");
427 }
428
ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode * N)429 SDValue DAGTypeLegalizer::ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N) {
430 // If the operand is wider than the vector element type then it is implicitly
431 // truncated. Make that explicit here.
432 EVT EltVT = N->getValueType(0).getVectorElementType();
433 SDValue InOp = N->getOperand(0);
434 if (InOp.getValueType() != EltVT)
435 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), EltVT, InOp);
436 return InOp;
437 }
438
ScalarizeVecRes_VSELECT(SDNode * N)439 SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
440 SDValue Cond = N->getOperand(0);
441 EVT OpVT = Cond.getValueType();
442 SDLoc DL(N);
443 // The vselect result and true/value operands needs scalarizing, but it's
444 // not a given that the Cond does. For instance, in AVX512 v1i1 is legal.
445 // See the similar logic in ScalarizeVecRes_SETCC
446 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
447 Cond = GetScalarizedVector(Cond);
448 } else {
449 EVT VT = OpVT.getVectorElementType();
450 Cond = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Cond,
451 DAG.getVectorIdxConstant(0, DL));
452 }
453
454 SDValue LHS = GetScalarizedVector(N->getOperand(1));
455 TargetLowering::BooleanContent ScalarBool =
456 TLI.getBooleanContents(false, false);
457 TargetLowering::BooleanContent VecBool = TLI.getBooleanContents(true, false);
458
459 // If integer and float booleans have different contents then we can't
460 // reliably optimize in all cases. There is a full explanation for this in
461 // DAGCombiner::visitSELECT() where the same issue affects folding
462 // (select C, 0, 1) to (xor C, 1).
463 if (TLI.getBooleanContents(false, false) !=
464 TLI.getBooleanContents(false, true)) {
465 // At least try the common case where the boolean is generated by a
466 // comparison.
467 if (Cond->getOpcode() == ISD::SETCC) {
468 EVT OpVT = Cond->getOperand(0).getValueType();
469 ScalarBool = TLI.getBooleanContents(OpVT.getScalarType());
470 VecBool = TLI.getBooleanContents(OpVT);
471 } else
472 ScalarBool = TargetLowering::UndefinedBooleanContent;
473 }
474
475 EVT CondVT = Cond.getValueType();
476 if (ScalarBool != VecBool) {
477 switch (ScalarBool) {
478 case TargetLowering::UndefinedBooleanContent:
479 break;
480 case TargetLowering::ZeroOrOneBooleanContent:
481 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
482 VecBool == TargetLowering::ZeroOrNegativeOneBooleanContent);
483 // Vector read from all ones, scalar expects a single 1 so mask.
484 Cond = DAG.getNode(ISD::AND, SDLoc(N), CondVT,
485 Cond, DAG.getConstant(1, SDLoc(N), CondVT));
486 break;
487 case TargetLowering::ZeroOrNegativeOneBooleanContent:
488 assert(VecBool == TargetLowering::UndefinedBooleanContent ||
489 VecBool == TargetLowering::ZeroOrOneBooleanContent);
490 // Vector reads from a one, scalar from all ones so sign extend.
491 Cond = DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N), CondVT,
492 Cond, DAG.getValueType(MVT::i1));
493 break;
494 }
495 }
496
497 // Truncate the condition if needed
498 auto BoolVT = getSetCCResultType(CondVT);
499 if (BoolVT.bitsLT(CondVT))
500 Cond = DAG.getNode(ISD::TRUNCATE, SDLoc(N), BoolVT, Cond);
501
502 return DAG.getSelect(SDLoc(N),
503 LHS.getValueType(), Cond, LHS,
504 GetScalarizedVector(N->getOperand(2)));
505 }
506
ScalarizeVecRes_SELECT(SDNode * N)507 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
508 SDValue LHS = GetScalarizedVector(N->getOperand(1));
509 return DAG.getSelect(SDLoc(N),
510 LHS.getValueType(), N->getOperand(0), LHS,
511 GetScalarizedVector(N->getOperand(2)));
512 }
513
ScalarizeVecRes_SELECT_CC(SDNode * N)514 SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
515 SDValue LHS = GetScalarizedVector(N->getOperand(2));
516 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
517 N->getOperand(0), N->getOperand(1),
518 LHS, GetScalarizedVector(N->getOperand(3)),
519 N->getOperand(4));
520 }
521
ScalarizeVecRes_UNDEF(SDNode * N)522 SDValue DAGTypeLegalizer::ScalarizeVecRes_UNDEF(SDNode *N) {
523 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
524 }
525
ScalarizeVecRes_VECTOR_SHUFFLE(SDNode * N)526 SDValue DAGTypeLegalizer::ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N) {
527 // Figure out if the scalar is the LHS or RHS and return it.
528 SDValue Arg = N->getOperand(2).getOperand(0);
529 if (Arg.isUndef())
530 return DAG.getUNDEF(N->getValueType(0).getVectorElementType());
531 unsigned Op = !cast<ConstantSDNode>(Arg)->isNullValue();
532 return GetScalarizedVector(N->getOperand(Op));
533 }
534
ScalarizeVecRes_FP_TO_XINT_SAT(SDNode * N)535 SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_TO_XINT_SAT(SDNode *N) {
536 SDValue Src = N->getOperand(0);
537 EVT SrcVT = Src.getValueType();
538 SDLoc dl(N);
539
540 // Handle case where result is scalarized but operand is not
541 if (getTypeAction(SrcVT) == TargetLowering::TypeScalarizeVector)
542 Src = GetScalarizedVector(Src);
543 else
544 Src = DAG.getNode(
545 ISD::EXTRACT_VECTOR_ELT, dl, SrcVT.getVectorElementType(), Src,
546 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
547
548 EVT DstVT = N->getValueType(0).getVectorElementType();
549 return DAG.getNode(N->getOpcode(), dl, DstVT, Src, N->getOperand(1));
550 }
551
ScalarizeVecRes_SETCC(SDNode * N)552 SDValue DAGTypeLegalizer::ScalarizeVecRes_SETCC(SDNode *N) {
553 assert(N->getValueType(0).isVector() &&
554 N->getOperand(0).getValueType().isVector() &&
555 "Operand types must be vectors");
556 SDValue LHS = N->getOperand(0);
557 SDValue RHS = N->getOperand(1);
558 EVT OpVT = LHS.getValueType();
559 EVT NVT = N->getValueType(0).getVectorElementType();
560 SDLoc DL(N);
561
562 // The result needs scalarizing, but it's not a given that the source does.
563 if (getTypeAction(OpVT) == TargetLowering::TypeScalarizeVector) {
564 LHS = GetScalarizedVector(LHS);
565 RHS = GetScalarizedVector(RHS);
566 } else {
567 EVT VT = OpVT.getVectorElementType();
568 LHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, LHS,
569 DAG.getVectorIdxConstant(0, DL));
570 RHS = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, RHS,
571 DAG.getVectorIdxConstant(0, DL));
572 }
573
574 // Turn it into a scalar SETCC.
575 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
576 N->getOperand(2));
577 // Vectors may have a different boolean contents to scalars. Promote the
578 // value appropriately.
579 ISD::NodeType ExtendCode =
580 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
581 return DAG.getNode(ExtendCode, DL, NVT, Res);
582 }
583
584
585 //===----------------------------------------------------------------------===//
586 // Operand Vector Scalarization <1 x ty> -> ty.
587 //===----------------------------------------------------------------------===//
588
ScalarizeVectorOperand(SDNode * N,unsigned OpNo)589 bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) {
590 LLVM_DEBUG(dbgs() << "Scalarize node operand " << OpNo << ": "; N->dump(&DAG);
591 dbgs() << "\n");
592 SDValue Res = SDValue();
593
594 switch (N->getOpcode()) {
595 default:
596 #ifndef NDEBUG
597 dbgs() << "ScalarizeVectorOperand Op #" << OpNo << ": ";
598 N->dump(&DAG);
599 dbgs() << "\n";
600 #endif
601 report_fatal_error("Do not know how to scalarize this operator's "
602 "operand!\n");
603 case ISD::BITCAST:
604 Res = ScalarizeVecOp_BITCAST(N);
605 break;
606 case ISD::ANY_EXTEND:
607 case ISD::ZERO_EXTEND:
608 case ISD::SIGN_EXTEND:
609 case ISD::TRUNCATE:
610 case ISD::FP_TO_SINT:
611 case ISD::FP_TO_UINT:
612 case ISD::SINT_TO_FP:
613 case ISD::UINT_TO_FP:
614 Res = ScalarizeVecOp_UnaryOp(N);
615 break;
616 case ISD::STRICT_SINT_TO_FP:
617 case ISD::STRICT_UINT_TO_FP:
618 case ISD::STRICT_FP_TO_SINT:
619 case ISD::STRICT_FP_TO_UINT:
620 Res = ScalarizeVecOp_UnaryOp_StrictFP(N);
621 break;
622 case ISD::CONCAT_VECTORS:
623 Res = ScalarizeVecOp_CONCAT_VECTORS(N);
624 break;
625 case ISD::EXTRACT_VECTOR_ELT:
626 Res = ScalarizeVecOp_EXTRACT_VECTOR_ELT(N);
627 break;
628 case ISD::VSELECT:
629 Res = ScalarizeVecOp_VSELECT(N);
630 break;
631 case ISD::SETCC:
632 Res = ScalarizeVecOp_VSETCC(N);
633 break;
634 case ISD::STORE:
635 Res = ScalarizeVecOp_STORE(cast<StoreSDNode>(N), OpNo);
636 break;
637 case ISD::STRICT_FP_ROUND:
638 Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo);
639 break;
640 case ISD::FP_ROUND:
641 Res = ScalarizeVecOp_FP_ROUND(N, OpNo);
642 break;
643 case ISD::STRICT_FP_EXTEND:
644 Res = ScalarizeVecOp_STRICT_FP_EXTEND(N);
645 break;
646 case ISD::FP_EXTEND:
647 Res = ScalarizeVecOp_FP_EXTEND(N);
648 break;
649 case ISD::VECREDUCE_FADD:
650 case ISD::VECREDUCE_FMUL:
651 case ISD::VECREDUCE_ADD:
652 case ISD::VECREDUCE_MUL:
653 case ISD::VECREDUCE_AND:
654 case ISD::VECREDUCE_OR:
655 case ISD::VECREDUCE_XOR:
656 case ISD::VECREDUCE_SMAX:
657 case ISD::VECREDUCE_SMIN:
658 case ISD::VECREDUCE_UMAX:
659 case ISD::VECREDUCE_UMIN:
660 case ISD::VECREDUCE_FMAX:
661 case ISD::VECREDUCE_FMIN:
662 Res = ScalarizeVecOp_VECREDUCE(N);
663 break;
664 case ISD::VECREDUCE_SEQ_FADD:
665 case ISD::VECREDUCE_SEQ_FMUL:
666 Res = ScalarizeVecOp_VECREDUCE_SEQ(N);
667 break;
668 }
669
670 // If the result is null, the sub-method took care of registering results etc.
671 if (!Res.getNode()) return false;
672
673 // If the result is N, the sub-method updated N in place. Tell the legalizer
674 // core about this.
675 if (Res.getNode() == N)
676 return true;
677
678 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
679 "Invalid operand expansion");
680
681 ReplaceValueWith(SDValue(N, 0), Res);
682 return false;
683 }
684
685 /// If the value to convert is a vector that needs to be scalarized, it must be
686 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_BITCAST(SDNode * N)687 SDValue DAGTypeLegalizer::ScalarizeVecOp_BITCAST(SDNode *N) {
688 SDValue Elt = GetScalarizedVector(N->getOperand(0));
689 return DAG.getNode(ISD::BITCAST, SDLoc(N),
690 N->getValueType(0), Elt);
691 }
692
693 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
694 /// Do the operation on the element instead.
ScalarizeVecOp_UnaryOp(SDNode * N)695 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp(SDNode *N) {
696 assert(N->getValueType(0).getVectorNumElements() == 1 &&
697 "Unexpected vector type!");
698 SDValue Elt = GetScalarizedVector(N->getOperand(0));
699 SDValue Op = DAG.getNode(N->getOpcode(), SDLoc(N),
700 N->getValueType(0).getScalarType(), Elt);
701 // Revectorize the result so the types line up with what the uses of this
702 // expression expect.
703 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Op);
704 }
705
706 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>.
707 /// Do the strict FP operation on the element instead.
ScalarizeVecOp_UnaryOp_StrictFP(SDNode * N)708 SDValue DAGTypeLegalizer::ScalarizeVecOp_UnaryOp_StrictFP(SDNode *N) {
709 assert(N->getValueType(0).getVectorNumElements() == 1 &&
710 "Unexpected vector type!");
711 SDValue Elt = GetScalarizedVector(N->getOperand(1));
712 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N),
713 { N->getValueType(0).getScalarType(), MVT::Other },
714 { N->getOperand(0), Elt });
715 // Legalize the chain result - switch anything that used the old chain to
716 // use the new one.
717 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
718 // Revectorize the result so the types line up with what the uses of this
719 // expression expect.
720 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
721
722 // Do our own replacement and return SDValue() to tell the caller that we
723 // handled all replacements since caller can only handle a single result.
724 ReplaceValueWith(SDValue(N, 0), Res);
725 return SDValue();
726 }
727
728 /// The vectors to concatenate have length one - use a BUILD_VECTOR instead.
ScalarizeVecOp_CONCAT_VECTORS(SDNode * N)729 SDValue DAGTypeLegalizer::ScalarizeVecOp_CONCAT_VECTORS(SDNode *N) {
730 SmallVector<SDValue, 8> Ops(N->getNumOperands());
731 for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i)
732 Ops[i] = GetScalarizedVector(N->getOperand(i));
733 return DAG.getBuildVector(N->getValueType(0), SDLoc(N), Ops);
734 }
735
736 /// If the input is a vector that needs to be scalarized, it must be <1 x ty>,
737 /// so just return the element, ignoring the index.
ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode * N)738 SDValue DAGTypeLegalizer::ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
739 EVT VT = N->getValueType(0);
740 SDValue Res = GetScalarizedVector(N->getOperand(0));
741 if (Res.getValueType() != VT)
742 Res = VT.isFloatingPoint()
743 ? DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Res)
744 : DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), VT, Res);
745 return Res;
746 }
747
748 /// If the input condition is a vector that needs to be scalarized, it must be
749 /// <1 x i1>, so just convert to a normal ISD::SELECT
750 /// (still with vector output type since that was acceptable if we got here).
ScalarizeVecOp_VSELECT(SDNode * N)751 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSELECT(SDNode *N) {
752 SDValue ScalarCond = GetScalarizedVector(N->getOperand(0));
753 EVT VT = N->getValueType(0);
754
755 return DAG.getNode(ISD::SELECT, SDLoc(N), VT, ScalarCond, N->getOperand(1),
756 N->getOperand(2));
757 }
758
759 /// If the operand is a vector that needs to be scalarized then the
760 /// result must be v1i1, so just convert to a scalar SETCC and wrap
761 /// with a scalar_to_vector since the res type is legal if we got here
ScalarizeVecOp_VSETCC(SDNode * N)762 SDValue DAGTypeLegalizer::ScalarizeVecOp_VSETCC(SDNode *N) {
763 assert(N->getValueType(0).isVector() &&
764 N->getOperand(0).getValueType().isVector() &&
765 "Operand types must be vectors");
766 assert(N->getValueType(0) == MVT::v1i1 && "Expected v1i1 type");
767
768 EVT VT = N->getValueType(0);
769 SDValue LHS = GetScalarizedVector(N->getOperand(0));
770 SDValue RHS = GetScalarizedVector(N->getOperand(1));
771
772 EVT OpVT = N->getOperand(0).getValueType();
773 EVT NVT = VT.getVectorElementType();
774 SDLoc DL(N);
775 // Turn it into a scalar SETCC.
776 SDValue Res = DAG.getNode(ISD::SETCC, DL, MVT::i1, LHS, RHS,
777 N->getOperand(2));
778
779 // Vectors may have a different boolean contents to scalars. Promote the
780 // value appropriately.
781 ISD::NodeType ExtendCode =
782 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
783
784 Res = DAG.getNode(ExtendCode, DL, NVT, Res);
785
786 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Res);
787 }
788
789 /// If the value to store is a vector that needs to be scalarized, it must be
790 /// <1 x ty>. Just store the element.
ScalarizeVecOp_STORE(StoreSDNode * N,unsigned OpNo)791 SDValue DAGTypeLegalizer::ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo){
792 assert(N->isUnindexed() && "Indexed store of one-element vector?");
793 assert(OpNo == 1 && "Do not know how to scalarize this operand!");
794 SDLoc dl(N);
795
796 if (N->isTruncatingStore())
797 return DAG.getTruncStore(
798 N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
799 N->getBasePtr(), N->getPointerInfo(),
800 N->getMemoryVT().getVectorElementType(), N->getOriginalAlign(),
801 N->getMemOperand()->getFlags(), N->getAAInfo());
802
803 return DAG.getStore(N->getChain(), dl, GetScalarizedVector(N->getOperand(1)),
804 N->getBasePtr(), N->getPointerInfo(),
805 N->getOriginalAlign(), N->getMemOperand()->getFlags(),
806 N->getAAInfo());
807 }
808
809 /// If the value to round is a vector that needs to be scalarized, it must be
810 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_ROUND(SDNode * N,unsigned OpNo)811 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) {
812 assert(OpNo == 0 && "Wrong operand for scalarization!");
813 SDValue Elt = GetScalarizedVector(N->getOperand(0));
814 SDValue Res = DAG.getNode(ISD::FP_ROUND, SDLoc(N),
815 N->getValueType(0).getVectorElementType(), Elt,
816 N->getOperand(1));
817 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
818 }
819
ScalarizeVecOp_STRICT_FP_ROUND(SDNode * N,unsigned OpNo)820 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N,
821 unsigned OpNo) {
822 assert(OpNo == 1 && "Wrong operand for scalarization!");
823 SDValue Elt = GetScalarizedVector(N->getOperand(1));
824 SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
825 { N->getValueType(0).getVectorElementType(),
826 MVT::Other },
827 { N->getOperand(0), Elt, N->getOperand(2) });
828 // Legalize the chain result - switch anything that used the old chain to
829 // use the new one.
830 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
831
832 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
833
834 // Do our own replacement and return SDValue() to tell the caller that we
835 // handled all replacements since caller can only handle a single result.
836 ReplaceValueWith(SDValue(N, 0), Res);
837 return SDValue();
838 }
839
840 /// If the value to extend is a vector that needs to be scalarized, it must be
841 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_FP_EXTEND(SDNode * N)842 SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_EXTEND(SDNode *N) {
843 SDValue Elt = GetScalarizedVector(N->getOperand(0));
844 SDValue Res = DAG.getNode(ISD::FP_EXTEND, SDLoc(N),
845 N->getValueType(0).getVectorElementType(), Elt);
846 return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
847 }
848
849 /// If the value to extend is a vector that needs to be scalarized, it must be
850 /// <1 x ty>. Convert the element instead.
ScalarizeVecOp_STRICT_FP_EXTEND(SDNode * N)851 SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_EXTEND(SDNode *N) {
852 SDValue Elt = GetScalarizedVector(N->getOperand(1));
853 SDValue Res =
854 DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
855 {N->getValueType(0).getVectorElementType(), MVT::Other},
856 {N->getOperand(0), Elt});
857 // Legalize the chain result - switch anything that used the old chain to
858 // use the new one.
859 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
860
861 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res);
862
863 // Do our own replacement and return SDValue() to tell the caller that we
864 // handled all replacements since caller can only handle a single result.
865 ReplaceValueWith(SDValue(N, 0), Res);
866 return SDValue();
867 }
868
ScalarizeVecOp_VECREDUCE(SDNode * N)869 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) {
870 SDValue Res = GetScalarizedVector(N->getOperand(0));
871 // Result type may be wider than element type.
872 if (Res.getValueType() != N->getValueType(0))
873 Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res);
874 return Res;
875 }
876
ScalarizeVecOp_VECREDUCE_SEQ(SDNode * N)877 SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE_SEQ(SDNode *N) {
878 SDValue AccOp = N->getOperand(0);
879 SDValue VecOp = N->getOperand(1);
880
881 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
882
883 SDValue Op = GetScalarizedVector(VecOp);
884 return DAG.getNode(BaseOpc, SDLoc(N), N->getValueType(0),
885 AccOp, Op, N->getFlags());
886 }
887
888 //===----------------------------------------------------------------------===//
889 // Result Vector Splitting
890 //===----------------------------------------------------------------------===//
891
892 /// This method is called when the specified result of the specified node is
893 /// found to need vector splitting. At this point, the node may also have
894 /// invalid operands or may have other results that need legalization, we just
895 /// know that (at least) one result needs vector splitting.
SplitVectorResult(SDNode * N,unsigned ResNo)896 void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
897 LLVM_DEBUG(dbgs() << "Split node result: "; N->dump(&DAG); dbgs() << "\n");
898 SDValue Lo, Hi;
899
900 // See if the target wants to custom expand this node.
901 if (CustomLowerNode(N, N->getValueType(ResNo), true))
902 return;
903
904 switch (N->getOpcode()) {
905 default:
906 #ifndef NDEBUG
907 dbgs() << "SplitVectorResult #" << ResNo << ": ";
908 N->dump(&DAG);
909 dbgs() << "\n";
910 #endif
911 report_fatal_error("Do not know how to split the result of this "
912 "operator!\n");
913
914 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
915 case ISD::VSELECT:
916 case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
917 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
918 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
919 case ISD::BITCAST: SplitVecRes_BITCAST(N, Lo, Hi); break;
920 case ISD::BUILD_VECTOR: SplitVecRes_BUILD_VECTOR(N, Lo, Hi); break;
921 case ISD::CONCAT_VECTORS: SplitVecRes_CONCAT_VECTORS(N, Lo, Hi); break;
922 case ISD::EXTRACT_SUBVECTOR: SplitVecRes_EXTRACT_SUBVECTOR(N, Lo, Hi); break;
923 case ISD::INSERT_SUBVECTOR: SplitVecRes_INSERT_SUBVECTOR(N, Lo, Hi); break;
924 case ISD::FPOWI: SplitVecRes_FPOWI(N, Lo, Hi); break;
925 case ISD::FCOPYSIGN: SplitVecRes_FCOPYSIGN(N, Lo, Hi); break;
926 case ISD::INSERT_VECTOR_ELT: SplitVecRes_INSERT_VECTOR_ELT(N, Lo, Hi); break;
927 case ISD::SPLAT_VECTOR:
928 case ISD::SCALAR_TO_VECTOR:
929 SplitVecRes_ScalarOp(N, Lo, Hi);
930 break;
931 case ISD::STEP_VECTOR:
932 SplitVecRes_STEP_VECTOR(N, Lo, Hi);
933 break;
934 case ISD::SIGN_EXTEND_INREG: SplitVecRes_InregOp(N, Lo, Hi); break;
935 case ISD::LOAD:
936 SplitVecRes_LOAD(cast<LoadSDNode>(N), Lo, Hi);
937 break;
938 case ISD::MLOAD:
939 SplitVecRes_MLOAD(cast<MaskedLoadSDNode>(N), Lo, Hi);
940 break;
941 case ISD::MGATHER:
942 SplitVecRes_MGATHER(cast<MaskedGatherSDNode>(N), Lo, Hi);
943 break;
944 case ISD::SETCC:
945 SplitVecRes_SETCC(N, Lo, Hi);
946 break;
947 case ISD::VECTOR_REVERSE:
948 SplitVecRes_VECTOR_REVERSE(N, Lo, Hi);
949 break;
950 case ISD::VECTOR_SHUFFLE:
951 SplitVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N), Lo, Hi);
952 break;
953 case ISD::VECTOR_SPLICE:
954 SplitVecRes_VECTOR_SPLICE(N, Lo, Hi);
955 break;
956 case ISD::VAARG:
957 SplitVecRes_VAARG(N, Lo, Hi);
958 break;
959
960 case ISD::ANY_EXTEND_VECTOR_INREG:
961 case ISD::SIGN_EXTEND_VECTOR_INREG:
962 case ISD::ZERO_EXTEND_VECTOR_INREG:
963 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
964 break;
965
966 case ISD::ABS:
967 case ISD::BITREVERSE:
968 case ISD::BSWAP:
969 case ISD::CTLZ:
970 case ISD::CTTZ:
971 case ISD::CTLZ_ZERO_UNDEF:
972 case ISD::CTTZ_ZERO_UNDEF:
973 case ISD::CTPOP:
974 case ISD::FABS:
975 case ISD::FCEIL:
976 case ISD::FCOS:
977 case ISD::FEXP:
978 case ISD::FEXP2:
979 case ISD::FFLOOR:
980 case ISD::FLOG:
981 case ISD::FLOG10:
982 case ISD::FLOG2:
983 case ISD::FNEARBYINT:
984 case ISD::FNEG:
985 case ISD::FREEZE:
986 case ISD::FP_EXTEND:
987 case ISD::FP_ROUND:
988 case ISD::FP_TO_SINT:
989 case ISD::FP_TO_UINT:
990 case ISD::FRINT:
991 case ISD::FROUND:
992 case ISD::FROUNDEVEN:
993 case ISD::FSIN:
994 case ISD::FSQRT:
995 case ISD::FTRUNC:
996 case ISD::SINT_TO_FP:
997 case ISD::TRUNCATE:
998 case ISD::UINT_TO_FP:
999 case ISD::FCANONICALIZE:
1000 SplitVecRes_UnaryOp(N, Lo, Hi);
1001 break;
1002
1003 case ISD::ANY_EXTEND:
1004 case ISD::SIGN_EXTEND:
1005 case ISD::ZERO_EXTEND:
1006 SplitVecRes_ExtendOp(N, Lo, Hi);
1007 break;
1008
1009 case ISD::ADD:
1010 case ISD::SUB:
1011 case ISD::MUL:
1012 case ISD::MULHS:
1013 case ISD::MULHU:
1014 case ISD::FADD:
1015 case ISD::FSUB:
1016 case ISD::FMUL:
1017 case ISD::FMINNUM:
1018 case ISD::FMAXNUM:
1019 case ISD::FMINIMUM:
1020 case ISD::FMAXIMUM:
1021 case ISD::SDIV:
1022 case ISD::UDIV:
1023 case ISD::FDIV:
1024 case ISD::FPOW:
1025 case ISD::AND:
1026 case ISD::OR:
1027 case ISD::XOR:
1028 case ISD::SHL:
1029 case ISD::SRA:
1030 case ISD::SRL:
1031 case ISD::UREM:
1032 case ISD::SREM:
1033 case ISD::FREM:
1034 case ISD::SMIN:
1035 case ISD::SMAX:
1036 case ISD::UMIN:
1037 case ISD::UMAX:
1038 case ISD::SADDSAT:
1039 case ISD::UADDSAT:
1040 case ISD::SSUBSAT:
1041 case ISD::USUBSAT:
1042 case ISD::SSHLSAT:
1043 case ISD::USHLSAT:
1044 case ISD::ROTL:
1045 case ISD::ROTR:
1046 SplitVecRes_BinOp(N, Lo, Hi);
1047 break;
1048 case ISD::FMA:
1049 case ISD::FSHL:
1050 case ISD::FSHR:
1051 SplitVecRes_TernaryOp(N, Lo, Hi);
1052 break;
1053
1054 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1055 case ISD::STRICT_##DAGN:
1056 #include "llvm/IR/ConstrainedOps.def"
1057 SplitVecRes_StrictFPOp(N, Lo, Hi);
1058 break;
1059
1060 case ISD::FP_TO_UINT_SAT:
1061 case ISD::FP_TO_SINT_SAT:
1062 SplitVecRes_FP_TO_XINT_SAT(N, Lo, Hi);
1063 break;
1064
1065 case ISD::UADDO:
1066 case ISD::SADDO:
1067 case ISD::USUBO:
1068 case ISD::SSUBO:
1069 case ISD::UMULO:
1070 case ISD::SMULO:
1071 SplitVecRes_OverflowOp(N, ResNo, Lo, Hi);
1072 break;
1073 case ISD::SMULFIX:
1074 case ISD::SMULFIXSAT:
1075 case ISD::UMULFIX:
1076 case ISD::UMULFIXSAT:
1077 case ISD::SDIVFIX:
1078 case ISD::SDIVFIXSAT:
1079 case ISD::UDIVFIX:
1080 case ISD::UDIVFIXSAT:
1081 SplitVecRes_FIX(N, Lo, Hi);
1082 break;
1083 }
1084
1085 // If Lo/Hi is null, the sub-method took care of registering results etc.
1086 if (Lo.getNode())
1087 SetSplitVector(SDValue(N, ResNo), Lo, Hi);
1088 }
1089
IncrementPointer(MemSDNode * N,EVT MemVT,MachinePointerInfo & MPI,SDValue & Ptr,uint64_t * ScaledOffset)1090 void DAGTypeLegalizer::IncrementPointer(MemSDNode *N, EVT MemVT,
1091 MachinePointerInfo &MPI, SDValue &Ptr,
1092 uint64_t *ScaledOffset) {
1093 SDLoc DL(N);
1094 unsigned IncrementSize = MemVT.getSizeInBits().getKnownMinSize() / 8;
1095
1096 if (MemVT.isScalableVector()) {
1097 SDNodeFlags Flags;
1098 SDValue BytesIncrement = DAG.getVScale(
1099 DL, Ptr.getValueType(),
1100 APInt(Ptr.getValueSizeInBits().getFixedSize(), IncrementSize));
1101 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
1102 Flags.setNoUnsignedWrap(true);
1103 if (ScaledOffset)
1104 *ScaledOffset += IncrementSize;
1105 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr, BytesIncrement,
1106 Flags);
1107 } else {
1108 MPI = N->getPointerInfo().getWithOffset(IncrementSize);
1109 // Increment the pointer to the other half.
1110 Ptr = DAG.getObjectPtrOffset(DL, Ptr, TypeSize::Fixed(IncrementSize));
1111 }
1112 }
1113
SplitVecRes_BinOp(SDNode * N,SDValue & Lo,SDValue & Hi)1114 void DAGTypeLegalizer::SplitVecRes_BinOp(SDNode *N, SDValue &Lo,
1115 SDValue &Hi) {
1116 SDValue LHSLo, LHSHi;
1117 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1118 SDValue RHSLo, RHSHi;
1119 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1120 SDLoc dl(N);
1121
1122 const SDNodeFlags Flags = N->getFlags();
1123 unsigned Opcode = N->getOpcode();
1124 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Flags);
1125 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Flags);
1126 }
1127
SplitVecRes_TernaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1128 void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo,
1129 SDValue &Hi) {
1130 SDValue Op0Lo, Op0Hi;
1131 GetSplitVector(N->getOperand(0), Op0Lo, Op0Hi);
1132 SDValue Op1Lo, Op1Hi;
1133 GetSplitVector(N->getOperand(1), Op1Lo, Op1Hi);
1134 SDValue Op2Lo, Op2Hi;
1135 GetSplitVector(N->getOperand(2), Op2Lo, Op2Hi);
1136 SDLoc dl(N);
1137
1138 Lo = DAG.getNode(N->getOpcode(), dl, Op0Lo.getValueType(), Op0Lo, Op1Lo,
1139 Op2Lo, N->getFlags());
1140 Hi = DAG.getNode(N->getOpcode(), dl, Op0Hi.getValueType(), Op0Hi, Op1Hi,
1141 Op2Hi, N->getFlags());
1142 }
1143
SplitVecRes_FIX(SDNode * N,SDValue & Lo,SDValue & Hi)1144 void DAGTypeLegalizer::SplitVecRes_FIX(SDNode *N, SDValue &Lo, SDValue &Hi) {
1145 SDValue LHSLo, LHSHi;
1146 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1147 SDValue RHSLo, RHSHi;
1148 GetSplitVector(N->getOperand(1), RHSLo, RHSHi);
1149 SDLoc dl(N);
1150 SDValue Op2 = N->getOperand(2);
1151
1152 unsigned Opcode = N->getOpcode();
1153 Lo = DAG.getNode(Opcode, dl, LHSLo.getValueType(), LHSLo, RHSLo, Op2,
1154 N->getFlags());
1155 Hi = DAG.getNode(Opcode, dl, LHSHi.getValueType(), LHSHi, RHSHi, Op2,
1156 N->getFlags());
1157 }
1158
SplitVecRes_BITCAST(SDNode * N,SDValue & Lo,SDValue & Hi)1159 void DAGTypeLegalizer::SplitVecRes_BITCAST(SDNode *N, SDValue &Lo,
1160 SDValue &Hi) {
1161 // We know the result is a vector. The input may be either a vector or a
1162 // scalar value.
1163 EVT LoVT, HiVT;
1164 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1165 SDLoc dl(N);
1166
1167 SDValue InOp = N->getOperand(0);
1168 EVT InVT = InOp.getValueType();
1169
1170 // Handle some special cases efficiently.
1171 switch (getTypeAction(InVT)) {
1172 case TargetLowering::TypeLegal:
1173 case TargetLowering::TypePromoteInteger:
1174 case TargetLowering::TypePromoteFloat:
1175 case TargetLowering::TypeSoftPromoteHalf:
1176 case TargetLowering::TypeSoftenFloat:
1177 case TargetLowering::TypeScalarizeVector:
1178 case TargetLowering::TypeWidenVector:
1179 break;
1180 case TargetLowering::TypeExpandInteger:
1181 case TargetLowering::TypeExpandFloat:
1182 // A scalar to vector conversion, where the scalar needs expansion.
1183 // If the vector is being split in two then we can just convert the
1184 // expanded pieces.
1185 if (LoVT == HiVT) {
1186 GetExpandedOp(InOp, Lo, Hi);
1187 if (DAG.getDataLayout().isBigEndian())
1188 std::swap(Lo, Hi);
1189 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1190 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1191 return;
1192 }
1193 break;
1194 case TargetLowering::TypeSplitVector:
1195 // If the input is a vector that needs to be split, convert each split
1196 // piece of the input now.
1197 GetSplitVector(InOp, Lo, Hi);
1198 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1199 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1200 return;
1201 case TargetLowering::TypeScalarizeScalableVector:
1202 report_fatal_error("Scalarization of scalable vectors is not supported.");
1203 }
1204
1205 // In the general case, convert the input to an integer and split it by hand.
1206 EVT LoIntVT = EVT::getIntegerVT(*DAG.getContext(), LoVT.getSizeInBits());
1207 EVT HiIntVT = EVT::getIntegerVT(*DAG.getContext(), HiVT.getSizeInBits());
1208 if (DAG.getDataLayout().isBigEndian())
1209 std::swap(LoIntVT, HiIntVT);
1210
1211 SplitInteger(BitConvertToInteger(InOp), LoIntVT, HiIntVT, Lo, Hi);
1212
1213 if (DAG.getDataLayout().isBigEndian())
1214 std::swap(Lo, Hi);
1215 Lo = DAG.getNode(ISD::BITCAST, dl, LoVT, Lo);
1216 Hi = DAG.getNode(ISD::BITCAST, dl, HiVT, Hi);
1217 }
1218
SplitVecRes_BUILD_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1219 void DAGTypeLegalizer::SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo,
1220 SDValue &Hi) {
1221 EVT LoVT, HiVT;
1222 SDLoc dl(N);
1223 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1224 unsigned LoNumElts = LoVT.getVectorNumElements();
1225 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+LoNumElts);
1226 Lo = DAG.getBuildVector(LoVT, dl, LoOps);
1227
1228 SmallVector<SDValue, 8> HiOps(N->op_begin()+LoNumElts, N->op_end());
1229 Hi = DAG.getBuildVector(HiVT, dl, HiOps);
1230 }
1231
SplitVecRes_CONCAT_VECTORS(SDNode * N,SDValue & Lo,SDValue & Hi)1232 void DAGTypeLegalizer::SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo,
1233 SDValue &Hi) {
1234 assert(!(N->getNumOperands() & 1) && "Unsupported CONCAT_VECTORS");
1235 SDLoc dl(N);
1236 unsigned NumSubvectors = N->getNumOperands() / 2;
1237 if (NumSubvectors == 1) {
1238 Lo = N->getOperand(0);
1239 Hi = N->getOperand(1);
1240 return;
1241 }
1242
1243 EVT LoVT, HiVT;
1244 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1245
1246 SmallVector<SDValue, 8> LoOps(N->op_begin(), N->op_begin()+NumSubvectors);
1247 Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, LoVT, LoOps);
1248
1249 SmallVector<SDValue, 8> HiOps(N->op_begin()+NumSubvectors, N->op_end());
1250 Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HiVT, HiOps);
1251 }
1252
SplitVecRes_EXTRACT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1253 void DAGTypeLegalizer::SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo,
1254 SDValue &Hi) {
1255 SDValue Vec = N->getOperand(0);
1256 SDValue Idx = N->getOperand(1);
1257 SDLoc dl(N);
1258
1259 EVT LoVT, HiVT;
1260 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1261
1262 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, LoVT, Vec, Idx);
1263 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1264 Hi = DAG.getNode(
1265 ISD::EXTRACT_SUBVECTOR, dl, HiVT, Vec,
1266 DAG.getVectorIdxConstant(IdxVal + LoVT.getVectorMinNumElements(), dl));
1267 }
1268
SplitVecRes_INSERT_SUBVECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1269 void DAGTypeLegalizer::SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo,
1270 SDValue &Hi) {
1271 SDValue Vec = N->getOperand(0);
1272 SDValue SubVec = N->getOperand(1);
1273 SDValue Idx = N->getOperand(2);
1274 SDLoc dl(N);
1275 GetSplitVector(Vec, Lo, Hi);
1276
1277 EVT VecVT = Vec.getValueType();
1278 EVT LoVT = Lo.getValueType();
1279 EVT SubVecVT = SubVec.getValueType();
1280 unsigned VecElems = VecVT.getVectorMinNumElements();
1281 unsigned SubElems = SubVecVT.getVectorMinNumElements();
1282 unsigned LoElems = LoVT.getVectorMinNumElements();
1283
1284 // If we know the index is in the first half, and we know the subvector
1285 // doesn't cross the boundary between the halves, we can avoid spilling the
1286 // vector, and insert into the lower half of the split vector directly.
1287 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
1288 if (IdxVal + SubElems <= LoElems) {
1289 Lo = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, LoVT, Lo, SubVec, Idx);
1290 return;
1291 }
1292 // Similarly if the subvector is fully in the high half, but mind that we
1293 // can't tell whether a fixed-length subvector is fully within the high half
1294 // of a scalable vector.
1295 if (VecVT.isScalableVector() == SubVecVT.isScalableVector() &&
1296 IdxVal >= LoElems && IdxVal + SubElems <= VecElems) {
1297 Hi = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, Hi.getValueType(), Hi, SubVec,
1298 DAG.getVectorIdxConstant(IdxVal - LoElems, dl));
1299 return;
1300 }
1301
1302 // Spill the vector to the stack.
1303 // In cases where the vector is illegal it will be broken down into parts
1304 // and stored in parts - we should use the alignment for the smallest part.
1305 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1306 SDValue StackPtr =
1307 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1308 auto &MF = DAG.getMachineFunction();
1309 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1310 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1311
1312 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1313 SmallestAlign);
1314
1315 // Store the new subvector into the specified index.
1316 SDValue SubVecPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1317 Store = DAG.getStore(Store, dl, SubVec, SubVecPtr,
1318 MachinePointerInfo::getUnknownStack(MF));
1319
1320 // Load the Lo part from the stack slot.
1321 Lo = DAG.getLoad(Lo.getValueType(), dl, Store, StackPtr, PtrInfo,
1322 SmallestAlign);
1323
1324 // Increment the pointer to the other part.
1325 auto *Load = cast<LoadSDNode>(Lo);
1326 MachinePointerInfo MPI = Load->getPointerInfo();
1327 IncrementPointer(Load, LoVT, MPI, StackPtr);
1328
1329 // Load the Hi part from the stack slot.
1330 Hi = DAG.getLoad(Hi.getValueType(), dl, Store, StackPtr, MPI, SmallestAlign);
1331 }
1332
SplitVecRes_FPOWI(SDNode * N,SDValue & Lo,SDValue & Hi)1333 void DAGTypeLegalizer::SplitVecRes_FPOWI(SDNode *N, SDValue &Lo,
1334 SDValue &Hi) {
1335 SDLoc dl(N);
1336 GetSplitVector(N->getOperand(0), Lo, Hi);
1337 Lo = DAG.getNode(ISD::FPOWI, dl, Lo.getValueType(), Lo, N->getOperand(1));
1338 Hi = DAG.getNode(ISD::FPOWI, dl, Hi.getValueType(), Hi, N->getOperand(1));
1339 }
1340
SplitVecRes_FCOPYSIGN(SDNode * N,SDValue & Lo,SDValue & Hi)1341 void DAGTypeLegalizer::SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo,
1342 SDValue &Hi) {
1343 SDValue LHSLo, LHSHi;
1344 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1345 SDLoc DL(N);
1346
1347 SDValue RHSLo, RHSHi;
1348 SDValue RHS = N->getOperand(1);
1349 EVT RHSVT = RHS.getValueType();
1350 if (getTypeAction(RHSVT) == TargetLowering::TypeSplitVector)
1351 GetSplitVector(RHS, RHSLo, RHSHi);
1352 else
1353 std::tie(RHSLo, RHSHi) = DAG.SplitVector(RHS, SDLoc(RHS));
1354
1355
1356 Lo = DAG.getNode(ISD::FCOPYSIGN, DL, LHSLo.getValueType(), LHSLo, RHSLo);
1357 Hi = DAG.getNode(ISD::FCOPYSIGN, DL, LHSHi.getValueType(), LHSHi, RHSHi);
1358 }
1359
SplitVecRes_InregOp(SDNode * N,SDValue & Lo,SDValue & Hi)1360 void DAGTypeLegalizer::SplitVecRes_InregOp(SDNode *N, SDValue &Lo,
1361 SDValue &Hi) {
1362 SDValue LHSLo, LHSHi;
1363 GetSplitVector(N->getOperand(0), LHSLo, LHSHi);
1364 SDLoc dl(N);
1365
1366 EVT LoVT, HiVT;
1367 std::tie(LoVT, HiVT) =
1368 DAG.GetSplitDestVTs(cast<VTSDNode>(N->getOperand(1))->getVT());
1369
1370 Lo = DAG.getNode(N->getOpcode(), dl, LHSLo.getValueType(), LHSLo,
1371 DAG.getValueType(LoVT));
1372 Hi = DAG.getNode(N->getOpcode(), dl, LHSHi.getValueType(), LHSHi,
1373 DAG.getValueType(HiVT));
1374 }
1375
SplitVecRes_ExtVecInRegOp(SDNode * N,SDValue & Lo,SDValue & Hi)1376 void DAGTypeLegalizer::SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo,
1377 SDValue &Hi) {
1378 unsigned Opcode = N->getOpcode();
1379 SDValue N0 = N->getOperand(0);
1380
1381 SDLoc dl(N);
1382 SDValue InLo, InHi;
1383
1384 if (getTypeAction(N0.getValueType()) == TargetLowering::TypeSplitVector)
1385 GetSplitVector(N0, InLo, InHi);
1386 else
1387 std::tie(InLo, InHi) = DAG.SplitVectorOperand(N, 0);
1388
1389 EVT InLoVT = InLo.getValueType();
1390 unsigned InNumElements = InLoVT.getVectorNumElements();
1391
1392 EVT OutLoVT, OutHiVT;
1393 std::tie(OutLoVT, OutHiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1394 unsigned OutNumElements = OutLoVT.getVectorNumElements();
1395 assert((2 * OutNumElements) <= InNumElements &&
1396 "Illegal extend vector in reg split");
1397
1398 // *_EXTEND_VECTOR_INREG instructions extend the lowest elements of the
1399 // input vector (i.e. we only use InLo):
1400 // OutLo will extend the first OutNumElements from InLo.
1401 // OutHi will extend the next OutNumElements from InLo.
1402
1403 // Shuffle the elements from InLo for OutHi into the bottom elements to
1404 // create a 'fake' InHi.
1405 SmallVector<int, 8> SplitHi(InNumElements, -1);
1406 for (unsigned i = 0; i != OutNumElements; ++i)
1407 SplitHi[i] = i + OutNumElements;
1408 InHi = DAG.getVectorShuffle(InLoVT, dl, InLo, DAG.getUNDEF(InLoVT), SplitHi);
1409
1410 Lo = DAG.getNode(Opcode, dl, OutLoVT, InLo);
1411 Hi = DAG.getNode(Opcode, dl, OutHiVT, InHi);
1412 }
1413
SplitVecRes_StrictFPOp(SDNode * N,SDValue & Lo,SDValue & Hi)1414 void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo,
1415 SDValue &Hi) {
1416 unsigned NumOps = N->getNumOperands();
1417 SDValue Chain = N->getOperand(0);
1418 EVT LoVT, HiVT;
1419 SDLoc dl(N);
1420 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1421
1422 SmallVector<SDValue, 4> OpsLo(NumOps);
1423 SmallVector<SDValue, 4> OpsHi(NumOps);
1424
1425 // The Chain is the first operand.
1426 OpsLo[0] = Chain;
1427 OpsHi[0] = Chain;
1428
1429 // Now process the remaining operands.
1430 for (unsigned i = 1; i < NumOps; ++i) {
1431 SDValue Op = N->getOperand(i);
1432 SDValue OpLo = Op;
1433 SDValue OpHi = Op;
1434
1435 EVT InVT = Op.getValueType();
1436 if (InVT.isVector()) {
1437 // If the input also splits, handle it directly for a
1438 // compile time speedup. Otherwise split it by hand.
1439 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1440 GetSplitVector(Op, OpLo, OpHi);
1441 else
1442 std::tie(OpLo, OpHi) = DAG.SplitVectorOperand(N, i);
1443 }
1444
1445 OpsLo[i] = OpLo;
1446 OpsHi[i] = OpHi;
1447 }
1448
1449 EVT LoValueVTs[] = {LoVT, MVT::Other};
1450 EVT HiValueVTs[] = {HiVT, MVT::Other};
1451 Lo = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(LoValueVTs), OpsLo,
1452 N->getFlags());
1453 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(HiValueVTs), OpsHi,
1454 N->getFlags());
1455
1456 // Build a factor node to remember that this Op is independent of the
1457 // other one.
1458 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
1459 Lo.getValue(1), Hi.getValue(1));
1460
1461 // Legalize the chain result - switch anything that used the old chain to
1462 // use the new one.
1463 ReplaceValueWith(SDValue(N, 1), Chain);
1464 }
1465
UnrollVectorOp_StrictFP(SDNode * N,unsigned ResNE)1466 SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) {
1467 SDValue Chain = N->getOperand(0);
1468 EVT VT = N->getValueType(0);
1469 unsigned NE = VT.getVectorNumElements();
1470 EVT EltVT = VT.getVectorElementType();
1471 SDLoc dl(N);
1472
1473 SmallVector<SDValue, 8> Scalars;
1474 SmallVector<SDValue, 4> Operands(N->getNumOperands());
1475
1476 // If ResNE is 0, fully unroll the vector op.
1477 if (ResNE == 0)
1478 ResNE = NE;
1479 else if (NE > ResNE)
1480 NE = ResNE;
1481
1482 //The results of each unrolled operation, including the chain.
1483 EVT ChainVTs[] = {EltVT, MVT::Other};
1484 SmallVector<SDValue, 8> Chains;
1485
1486 unsigned i;
1487 for (i = 0; i != NE; ++i) {
1488 Operands[0] = Chain;
1489 for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) {
1490 SDValue Operand = N->getOperand(j);
1491 EVT OperandVT = Operand.getValueType();
1492 if (OperandVT.isVector()) {
1493 EVT OperandEltVT = OperandVT.getVectorElementType();
1494 Operands[j] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
1495 Operand, DAG.getVectorIdxConstant(i, dl));
1496 } else {
1497 Operands[j] = Operand;
1498 }
1499 }
1500 SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands);
1501 Scalar.getNode()->setFlags(N->getFlags());
1502
1503 //Add in the scalar as well as its chain value to the
1504 //result vectors.
1505 Scalars.push_back(Scalar);
1506 Chains.push_back(Scalar.getValue(1));
1507 }
1508
1509 for (; i < ResNE; ++i)
1510 Scalars.push_back(DAG.getUNDEF(EltVT));
1511
1512 // Build a new factor node to connect the chain back together.
1513 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
1514 ReplaceValueWith(SDValue(N, 1), Chain);
1515
1516 // Create a new BUILD_VECTOR node
1517 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE);
1518 return DAG.getBuildVector(VecVT, dl, Scalars);
1519 }
1520
SplitVecRes_OverflowOp(SDNode * N,unsigned ResNo,SDValue & Lo,SDValue & Hi)1521 void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
1522 SDValue &Lo, SDValue &Hi) {
1523 SDLoc dl(N);
1524 EVT ResVT = N->getValueType(0);
1525 EVT OvVT = N->getValueType(1);
1526 EVT LoResVT, HiResVT, LoOvVT, HiOvVT;
1527 std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT);
1528 std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT);
1529
1530 SDValue LoLHS, HiLHS, LoRHS, HiRHS;
1531 if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) {
1532 GetSplitVector(N->getOperand(0), LoLHS, HiLHS);
1533 GetSplitVector(N->getOperand(1), LoRHS, HiRHS);
1534 } else {
1535 std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0);
1536 std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1);
1537 }
1538
1539 unsigned Opcode = N->getOpcode();
1540 SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT);
1541 SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT);
1542 SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode();
1543 SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode();
1544 LoNode->setFlags(N->getFlags());
1545 HiNode->setFlags(N->getFlags());
1546
1547 Lo = SDValue(LoNode, ResNo);
1548 Hi = SDValue(HiNode, ResNo);
1549
1550 // Replace the other vector result not being explicitly split here.
1551 unsigned OtherNo = 1 - ResNo;
1552 EVT OtherVT = N->getValueType(OtherNo);
1553 if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) {
1554 SetSplitVector(SDValue(N, OtherNo),
1555 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1556 } else {
1557 SDValue OtherVal = DAG.getNode(
1558 ISD::CONCAT_VECTORS, dl, OtherVT,
1559 SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo));
1560 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
1561 }
1562 }
1563
SplitVecRes_INSERT_VECTOR_ELT(SDNode * N,SDValue & Lo,SDValue & Hi)1564 void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
1565 SDValue &Hi) {
1566 SDValue Vec = N->getOperand(0);
1567 SDValue Elt = N->getOperand(1);
1568 SDValue Idx = N->getOperand(2);
1569 SDLoc dl(N);
1570 GetSplitVector(Vec, Lo, Hi);
1571
1572 if (ConstantSDNode *CIdx = dyn_cast<ConstantSDNode>(Idx)) {
1573 unsigned IdxVal = CIdx->getZExtValue();
1574 unsigned LoNumElts = Lo.getValueType().getVectorMinNumElements();
1575 if (IdxVal < LoNumElts) {
1576 Lo = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl,
1577 Lo.getValueType(), Lo, Elt, Idx);
1578 return;
1579 } else if (!Vec.getValueType().isScalableVector()) {
1580 Hi = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, Hi.getValueType(), Hi, Elt,
1581 DAG.getVectorIdxConstant(IdxVal - LoNumElts, dl));
1582 return;
1583 }
1584 }
1585
1586 // See if the target wants to custom expand this node.
1587 if (CustomLowerNode(N, N->getValueType(0), true))
1588 return;
1589
1590 // Make the vector elements byte-addressable if they aren't already.
1591 EVT VecVT = Vec.getValueType();
1592 EVT EltVT = VecVT.getVectorElementType();
1593 if (VecVT.getScalarSizeInBits() < 8) {
1594 EltVT = MVT::i8;
1595 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
1596 VecVT.getVectorElementCount());
1597 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
1598 // Extend the element type to match if needed.
1599 if (EltVT.bitsGT(Elt.getValueType()))
1600 Elt = DAG.getNode(ISD::ANY_EXTEND, dl, EltVT, Elt);
1601 }
1602
1603 // Spill the vector to the stack.
1604 // In cases where the vector is illegal it will be broken down into parts
1605 // and stored in parts - we should use the alignment for the smallest part.
1606 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
1607 SDValue StackPtr =
1608 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
1609 auto &MF = DAG.getMachineFunction();
1610 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1611 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
1612
1613 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
1614 SmallestAlign);
1615
1616 // Store the new element. This may be larger than the vector element type,
1617 // so use a truncating store.
1618 SDValue EltPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1619 Store = DAG.getTruncStore(
1620 Store, dl, Elt, EltPtr, MachinePointerInfo::getUnknownStack(MF), EltVT,
1621 commonAlignment(SmallestAlign,
1622 EltVT.getFixedSizeInBits() / 8));
1623
1624 EVT LoVT, HiVT;
1625 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VecVT);
1626
1627 // Load the Lo part from the stack slot.
1628 Lo = DAG.getLoad(LoVT, dl, Store, StackPtr, PtrInfo, SmallestAlign);
1629
1630 // Increment the pointer to the other part.
1631 auto Load = cast<LoadSDNode>(Lo);
1632 MachinePointerInfo MPI = Load->getPointerInfo();
1633 IncrementPointer(Load, LoVT, MPI, StackPtr);
1634
1635 Hi = DAG.getLoad(HiVT, dl, Store, StackPtr, MPI, SmallestAlign);
1636
1637 // If we adjusted the original type, we need to truncate the results.
1638 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1639 if (LoVT != Lo.getValueType())
1640 Lo = DAG.getNode(ISD::TRUNCATE, dl, LoVT, Lo);
1641 if (HiVT != Hi.getValueType())
1642 Hi = DAG.getNode(ISD::TRUNCATE, dl, HiVT, Hi);
1643 }
1644
SplitVecRes_STEP_VECTOR(SDNode * N,SDValue & Lo,SDValue & Hi)1645 void DAGTypeLegalizer::SplitVecRes_STEP_VECTOR(SDNode *N, SDValue &Lo,
1646 SDValue &Hi) {
1647 EVT LoVT, HiVT;
1648 SDLoc dl(N);
1649 assert(N->getValueType(0).isScalableVector() &&
1650 "Only scalable vectors are supported for STEP_VECTOR");
1651 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1652 SDValue Step = N->getOperand(0);
1653
1654 Lo = DAG.getNode(ISD::STEP_VECTOR, dl, LoVT, Step);
1655
1656 // Hi = Lo + (EltCnt * Step)
1657 EVT EltVT = Step.getValueType();
1658 APInt StepVal = cast<ConstantSDNode>(Step)->getAPIntValue();
1659 SDValue StartOfHi =
1660 DAG.getVScale(dl, EltVT, StepVal * LoVT.getVectorMinNumElements());
1661 StartOfHi = DAG.getSExtOrTrunc(StartOfHi, dl, HiVT.getVectorElementType());
1662 StartOfHi = DAG.getNode(ISD::SPLAT_VECTOR, dl, HiVT, StartOfHi);
1663
1664 Hi = DAG.getNode(ISD::STEP_VECTOR, dl, HiVT, Step);
1665 Hi = DAG.getNode(ISD::ADD, dl, HiVT, Hi, StartOfHi);
1666 }
1667
SplitVecRes_ScalarOp(SDNode * N,SDValue & Lo,SDValue & Hi)1668 void DAGTypeLegalizer::SplitVecRes_ScalarOp(SDNode *N, SDValue &Lo,
1669 SDValue &Hi) {
1670 EVT LoVT, HiVT;
1671 SDLoc dl(N);
1672 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1673 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, N->getOperand(0));
1674 if (N->getOpcode() == ISD::SCALAR_TO_VECTOR) {
1675 Hi = DAG.getUNDEF(HiVT);
1676 } else {
1677 assert(N->getOpcode() == ISD::SPLAT_VECTOR && "Unexpected opcode");
1678 Hi = Lo;
1679 }
1680 }
1681
SplitVecRes_LOAD(LoadSDNode * LD,SDValue & Lo,SDValue & Hi)1682 void DAGTypeLegalizer::SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo,
1683 SDValue &Hi) {
1684 assert(ISD::isUNINDEXEDLoad(LD) && "Indexed load during type legalization!");
1685 EVT LoVT, HiVT;
1686 SDLoc dl(LD);
1687 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(LD->getValueType(0));
1688
1689 ISD::LoadExtType ExtType = LD->getExtensionType();
1690 SDValue Ch = LD->getChain();
1691 SDValue Ptr = LD->getBasePtr();
1692 SDValue Offset = DAG.getUNDEF(Ptr.getValueType());
1693 EVT MemoryVT = LD->getMemoryVT();
1694 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
1695 AAMDNodes AAInfo = LD->getAAInfo();
1696
1697 EVT LoMemVT, HiMemVT;
1698 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1699
1700 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized()) {
1701 SDValue Value, NewChain;
1702 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
1703 std::tie(Lo, Hi) = DAG.SplitVector(Value, dl);
1704 ReplaceValueWith(SDValue(LD, 1), NewChain);
1705 return;
1706 }
1707
1708 Lo = DAG.getLoad(ISD::UNINDEXED, ExtType, LoVT, dl, Ch, Ptr, Offset,
1709 LD->getPointerInfo(), LoMemVT, LD->getOriginalAlign(),
1710 MMOFlags, AAInfo);
1711
1712 MachinePointerInfo MPI;
1713 IncrementPointer(LD, LoMemVT, MPI, Ptr);
1714
1715 Hi = DAG.getLoad(ISD::UNINDEXED, ExtType, HiVT, dl, Ch, Ptr, Offset, MPI,
1716 HiMemVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
1717
1718 // Build a factor node to remember that this load is independent of the
1719 // other one.
1720 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1721 Hi.getValue(1));
1722
1723 // Legalize the chain result - switch anything that used the old chain to
1724 // use the new one.
1725 ReplaceValueWith(SDValue(LD, 1), Ch);
1726 }
1727
SplitVecRes_MLOAD(MaskedLoadSDNode * MLD,SDValue & Lo,SDValue & Hi)1728 void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD,
1729 SDValue &Lo, SDValue &Hi) {
1730 assert(MLD->isUnindexed() && "Indexed masked load during type legalization!");
1731 EVT LoVT, HiVT;
1732 SDLoc dl(MLD);
1733 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MLD->getValueType(0));
1734
1735 SDValue Ch = MLD->getChain();
1736 SDValue Ptr = MLD->getBasePtr();
1737 SDValue Offset = MLD->getOffset();
1738 assert(Offset.isUndef() && "Unexpected indexed masked load offset");
1739 SDValue Mask = MLD->getMask();
1740 SDValue PassThru = MLD->getPassThru();
1741 Align Alignment = MLD->getOriginalAlign();
1742 ISD::LoadExtType ExtType = MLD->getExtensionType();
1743
1744 // Split Mask operand
1745 SDValue MaskLo, MaskHi;
1746 if (Mask.getOpcode() == ISD::SETCC) {
1747 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1748 } else {
1749 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1750 GetSplitVector(Mask, MaskLo, MaskHi);
1751 else
1752 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1753 }
1754
1755 EVT MemoryVT = MLD->getMemoryVT();
1756 EVT LoMemVT, HiMemVT;
1757 bool HiIsEmpty = false;
1758 std::tie(LoMemVT, HiMemVT) =
1759 DAG.GetDependentSplitDestVTs(MemoryVT, LoVT, &HiIsEmpty);
1760
1761 SDValue PassThruLo, PassThruHi;
1762 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1763 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1764 else
1765 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1766
1767 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
1768 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1769 MLD->getPointerInfo(), MachineMemOperand::MOLoad, LoSize, Alignment,
1770 MLD->getAAInfo(), MLD->getRanges());
1771
1772 Lo = DAG.getMaskedLoad(LoVT, dl, Ch, Ptr, Offset, MaskLo, PassThruLo, LoMemVT,
1773 MMO, MLD->getAddressingMode(), ExtType,
1774 MLD->isExpandingLoad());
1775
1776 if (HiIsEmpty) {
1777 // The hi masked load has zero storage size. We therefore simply set it to
1778 // the low masked load and rely on subsequent removal from the chain.
1779 Hi = Lo;
1780 } else {
1781 // Generate hi masked load.
1782 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, dl, LoMemVT, DAG,
1783 MLD->isExpandingLoad());
1784 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
1785
1786 MachinePointerInfo MPI;
1787 if (LoMemVT.isScalableVector())
1788 MPI = MachinePointerInfo(MLD->getPointerInfo().getAddrSpace());
1789 else
1790 MPI = MLD->getPointerInfo().getWithOffset(
1791 LoMemVT.getStoreSize().getFixedSize());
1792
1793 MMO = DAG.getMachineFunction().getMachineMemOperand(
1794 MPI, MachineMemOperand::MOLoad, HiSize, Alignment, MLD->getAAInfo(),
1795 MLD->getRanges());
1796
1797 Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, Offset, MaskHi, PassThruHi,
1798 HiMemVT, MMO, MLD->getAddressingMode(), ExtType,
1799 MLD->isExpandingLoad());
1800 }
1801
1802 // Build a factor node to remember that this load is independent of the
1803 // other one.
1804 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1805 Hi.getValue(1));
1806
1807 // Legalize the chain result - switch anything that used the old chain to
1808 // use the new one.
1809 ReplaceValueWith(SDValue(MLD, 1), Ch);
1810
1811 }
1812
SplitVecRes_MGATHER(MaskedGatherSDNode * MGT,SDValue & Lo,SDValue & Hi)1813 void DAGTypeLegalizer::SplitVecRes_MGATHER(MaskedGatherSDNode *MGT,
1814 SDValue &Lo, SDValue &Hi) {
1815 EVT LoVT, HiVT;
1816 SDLoc dl(MGT);
1817 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
1818
1819 SDValue Ch = MGT->getChain();
1820 SDValue Ptr = MGT->getBasePtr();
1821 SDValue Mask = MGT->getMask();
1822 SDValue PassThru = MGT->getPassThru();
1823 SDValue Index = MGT->getIndex();
1824 SDValue Scale = MGT->getScale();
1825 EVT MemoryVT = MGT->getMemoryVT();
1826 Align Alignment = MGT->getOriginalAlign();
1827 ISD::LoadExtType ExtType = MGT->getExtensionType();
1828
1829 // Split Mask operand
1830 SDValue MaskLo, MaskHi;
1831 if (Mask.getOpcode() == ISD::SETCC) {
1832 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
1833 } else {
1834 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
1835 GetSplitVector(Mask, MaskLo, MaskHi);
1836 else
1837 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
1838 }
1839
1840 EVT LoMemVT, HiMemVT;
1841 // Split MemoryVT
1842 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
1843
1844 SDValue PassThruLo, PassThruHi;
1845 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
1846 GetSplitVector(PassThru, PassThruLo, PassThruHi);
1847 else
1848 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
1849
1850 SDValue IndexHi, IndexLo;
1851 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
1852 GetSplitVector(Index, IndexLo, IndexHi);
1853 else
1854 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
1855
1856 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
1857 MGT->getPointerInfo(), MachineMemOperand::MOLoad,
1858 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
1859 MGT->getRanges());
1860
1861 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
1862 Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl, OpsLo,
1863 MMO, MGT->getIndexType(), ExtType);
1864
1865 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
1866 Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl, OpsHi,
1867 MMO, MGT->getIndexType(), ExtType);
1868
1869 // Build a factor node to remember that this load is independent of the
1870 // other one.
1871 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
1872 Hi.getValue(1));
1873
1874 // Legalize the chain result - switch anything that used the old chain to
1875 // use the new one.
1876 ReplaceValueWith(SDValue(MGT, 1), Ch);
1877 }
1878
1879
SplitVecRes_SETCC(SDNode * N,SDValue & Lo,SDValue & Hi)1880 void DAGTypeLegalizer::SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi) {
1881 assert(N->getValueType(0).isVector() &&
1882 N->getOperand(0).getValueType().isVector() &&
1883 "Operand types must be vectors");
1884
1885 EVT LoVT, HiVT;
1886 SDLoc DL(N);
1887 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1888
1889 // If the input also splits, handle it directly. Otherwise split it by hand.
1890 SDValue LL, LH, RL, RH;
1891 if (getTypeAction(N->getOperand(0).getValueType()) ==
1892 TargetLowering::TypeSplitVector)
1893 GetSplitVector(N->getOperand(0), LL, LH);
1894 else
1895 std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
1896
1897 if (getTypeAction(N->getOperand(1).getValueType()) ==
1898 TargetLowering::TypeSplitVector)
1899 GetSplitVector(N->getOperand(1), RL, RH);
1900 else
1901 std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
1902
1903 Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
1904 Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
1905 }
1906
SplitVecRes_UnaryOp(SDNode * N,SDValue & Lo,SDValue & Hi)1907 void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo,
1908 SDValue &Hi) {
1909 // Get the dest types - they may not match the input types, e.g. int_to_fp.
1910 EVT LoVT, HiVT;
1911 SDLoc dl(N);
1912 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
1913
1914 // If the input also splits, handle it directly for a compile time speedup.
1915 // Otherwise split it by hand.
1916 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1917 EVT InVT = N->getOperand(OpNo).getValueType();
1918 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector)
1919 GetSplitVector(N->getOperand(OpNo), Lo, Hi);
1920 else
1921 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo);
1922
1923 if (N->getOpcode() == ISD::FP_ROUND) {
1924 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1),
1925 N->getFlags());
1926 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1),
1927 N->getFlags());
1928 } else {
1929 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getFlags());
1930 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getFlags());
1931 }
1932 }
1933
SplitVecRes_ExtendOp(SDNode * N,SDValue & Lo,SDValue & Hi)1934 void DAGTypeLegalizer::SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo,
1935 SDValue &Hi) {
1936 SDLoc dl(N);
1937 EVT SrcVT = N->getOperand(0).getValueType();
1938 EVT DestVT = N->getValueType(0);
1939 EVT LoVT, HiVT;
1940 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(DestVT);
1941
1942 // We can do better than a generic split operation if the extend is doing
1943 // more than just doubling the width of the elements and the following are
1944 // true:
1945 // - The number of vector elements is even,
1946 // - the source type is legal,
1947 // - the type of a split source is illegal,
1948 // - the type of an extended (by doubling element size) source is legal, and
1949 // - the type of that extended source when split is legal.
1950 //
1951 // This won't necessarily completely legalize the operation, but it will
1952 // more effectively move in the right direction and prevent falling down
1953 // to scalarization in many cases due to the input vector being split too
1954 // far.
1955 if (SrcVT.getVectorElementCount().isKnownEven() &&
1956 SrcVT.getScalarSizeInBits() * 2 < DestVT.getScalarSizeInBits()) {
1957 LLVMContext &Ctx = *DAG.getContext();
1958 EVT NewSrcVT = SrcVT.widenIntegerVectorElementType(Ctx);
1959 EVT SplitSrcVT = SrcVT.getHalfNumVectorElementsVT(Ctx);
1960
1961 EVT SplitLoVT, SplitHiVT;
1962 std::tie(SplitLoVT, SplitHiVT) = DAG.GetSplitDestVTs(NewSrcVT);
1963 if (TLI.isTypeLegal(SrcVT) && !TLI.isTypeLegal(SplitSrcVT) &&
1964 TLI.isTypeLegal(NewSrcVT) && TLI.isTypeLegal(SplitLoVT)) {
1965 LLVM_DEBUG(dbgs() << "Split vector extend via incremental extend:";
1966 N->dump(&DAG); dbgs() << "\n");
1967 // Extend the source vector by one step.
1968 SDValue NewSrc =
1969 DAG.getNode(N->getOpcode(), dl, NewSrcVT, N->getOperand(0));
1970 // Get the low and high halves of the new, extended one step, vector.
1971 std::tie(Lo, Hi) = DAG.SplitVector(NewSrc, dl);
1972 // Extend those vector halves the rest of the way.
1973 Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo);
1974 Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi);
1975 return;
1976 }
1977 }
1978 // Fall back to the generic unary operator splitting otherwise.
1979 SplitVecRes_UnaryOp(N, Lo, Hi);
1980 }
1981
SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N,SDValue & Lo,SDValue & Hi)1982 void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N,
1983 SDValue &Lo, SDValue &Hi) {
1984 // The low and high parts of the original input give four input vectors.
1985 SDValue Inputs[4];
1986 SDLoc dl(N);
1987 GetSplitVector(N->getOperand(0), Inputs[0], Inputs[1]);
1988 GetSplitVector(N->getOperand(1), Inputs[2], Inputs[3]);
1989 EVT NewVT = Inputs[0].getValueType();
1990 unsigned NewElts = NewVT.getVectorNumElements();
1991
1992 // If Lo or Hi uses elements from at most two of the four input vectors, then
1993 // express it as a vector shuffle of those two inputs. Otherwise extract the
1994 // input elements by hand and construct the Lo/Hi output using a BUILD_VECTOR.
1995 SmallVector<int, 16> Ops;
1996 for (unsigned High = 0; High < 2; ++High) {
1997 SDValue &Output = High ? Hi : Lo;
1998
1999 // Build a shuffle mask for the output, discovering on the fly which
2000 // input vectors to use as shuffle operands (recorded in InputUsed).
2001 // If building a suitable shuffle vector proves too hard, then bail
2002 // out with useBuildVector set.
2003 unsigned InputUsed[2] = { -1U, -1U }; // Not yet discovered.
2004 unsigned FirstMaskIdx = High * NewElts;
2005 bool useBuildVector = false;
2006 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2007 // The mask element. This indexes into the input.
2008 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2009
2010 // The input vector this mask element indexes into.
2011 unsigned Input = (unsigned)Idx / NewElts;
2012
2013 if (Input >= array_lengthof(Inputs)) {
2014 // The mask element does not index into any input vector.
2015 Ops.push_back(-1);
2016 continue;
2017 }
2018
2019 // Turn the index into an offset from the start of the input vector.
2020 Idx -= Input * NewElts;
2021
2022 // Find or create a shuffle vector operand to hold this input.
2023 unsigned OpNo;
2024 for (OpNo = 0; OpNo < array_lengthof(InputUsed); ++OpNo) {
2025 if (InputUsed[OpNo] == Input) {
2026 // This input vector is already an operand.
2027 break;
2028 } else if (InputUsed[OpNo] == -1U) {
2029 // Create a new operand for this input vector.
2030 InputUsed[OpNo] = Input;
2031 break;
2032 }
2033 }
2034
2035 if (OpNo >= array_lengthof(InputUsed)) {
2036 // More than two input vectors used! Give up on trying to create a
2037 // shuffle vector. Insert all elements into a BUILD_VECTOR instead.
2038 useBuildVector = true;
2039 break;
2040 }
2041
2042 // Add the mask index for the new shuffle vector.
2043 Ops.push_back(Idx + OpNo * NewElts);
2044 }
2045
2046 if (useBuildVector) {
2047 EVT EltVT = NewVT.getVectorElementType();
2048 SmallVector<SDValue, 16> SVOps;
2049
2050 // Extract the input elements by hand.
2051 for (unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
2052 // The mask element. This indexes into the input.
2053 int Idx = N->getMaskElt(FirstMaskIdx + MaskOffset);
2054
2055 // The input vector this mask element indexes into.
2056 unsigned Input = (unsigned)Idx / NewElts;
2057
2058 if (Input >= array_lengthof(Inputs)) {
2059 // The mask element is "undef" or indexes off the end of the input.
2060 SVOps.push_back(DAG.getUNDEF(EltVT));
2061 continue;
2062 }
2063
2064 // Turn the index into an offset from the start of the input vector.
2065 Idx -= Input * NewElts;
2066
2067 // Extract the vector element by hand.
2068 SVOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT,
2069 Inputs[Input],
2070 DAG.getVectorIdxConstant(Idx, dl)));
2071 }
2072
2073 // Construct the Lo/Hi output using a BUILD_VECTOR.
2074 Output = DAG.getBuildVector(NewVT, dl, SVOps);
2075 } else if (InputUsed[0] == -1U) {
2076 // No input vectors were used! The result is undefined.
2077 Output = DAG.getUNDEF(NewVT);
2078 } else {
2079 SDValue Op0 = Inputs[InputUsed[0]];
2080 // If only one input was used, use an undefined vector for the other.
2081 SDValue Op1 = InputUsed[1] == -1U ?
2082 DAG.getUNDEF(NewVT) : Inputs[InputUsed[1]];
2083 // At least one input vector was used. Create a new shuffle vector.
2084 Output = DAG.getVectorShuffle(NewVT, dl, Op0, Op1, Ops);
2085 }
2086
2087 Ops.clear();
2088 }
2089 }
2090
SplitVecRes_VAARG(SDNode * N,SDValue & Lo,SDValue & Hi)2091 void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
2092 EVT OVT = N->getValueType(0);
2093 EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext());
2094 SDValue Chain = N->getOperand(0);
2095 SDValue Ptr = N->getOperand(1);
2096 SDValue SV = N->getOperand(2);
2097 SDLoc dl(N);
2098
2099 const Align Alignment =
2100 DAG.getDataLayout().getABITypeAlign(NVT.getTypeForEVT(*DAG.getContext()));
2101
2102 Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment.value());
2103 Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment.value());
2104 Chain = Hi.getValue(1);
2105
2106 // Modified the chain - switch anything that used the old chain to use
2107 // the new one.
2108 ReplaceValueWith(SDValue(N, 1), Chain);
2109 }
2110
SplitVecRes_FP_TO_XINT_SAT(SDNode * N,SDValue & Lo,SDValue & Hi)2111 void DAGTypeLegalizer::SplitVecRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
2112 SDValue &Hi) {
2113 EVT DstVTLo, DstVTHi;
2114 std::tie(DstVTLo, DstVTHi) = DAG.GetSplitDestVTs(N->getValueType(0));
2115 SDLoc dl(N);
2116
2117 SDValue SrcLo, SrcHi;
2118 EVT SrcVT = N->getOperand(0).getValueType();
2119 if (getTypeAction(SrcVT) == TargetLowering::TypeSplitVector)
2120 GetSplitVector(N->getOperand(0), SrcLo, SrcHi);
2121 else
2122 std::tie(SrcLo, SrcHi) = DAG.SplitVectorOperand(N, 0);
2123
2124 Lo = DAG.getNode(N->getOpcode(), dl, DstVTLo, SrcLo, N->getOperand(1));
2125 Hi = DAG.getNode(N->getOpcode(), dl, DstVTHi, SrcHi, N->getOperand(1));
2126 }
2127
2128 //===----------------------------------------------------------------------===//
2129 // Operand Vector Splitting
2130 //===----------------------------------------------------------------------===//
2131
2132 /// This method is called when the specified operand of the specified node is
2133 /// found to need vector splitting. At this point, all of the result types of
2134 /// the node are known to be legal, but other operands of the node may need
2135 /// legalization as well as the specified one.
SplitVectorOperand(SDNode * N,unsigned OpNo)2136 bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) {
2137 LLVM_DEBUG(dbgs() << "Split node operand: "; N->dump(&DAG); dbgs() << "\n");
2138 SDValue Res = SDValue();
2139
2140 // See if the target wants to custom split this node.
2141 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
2142 return false;
2143
2144 switch (N->getOpcode()) {
2145 default:
2146 #ifndef NDEBUG
2147 dbgs() << "SplitVectorOperand Op #" << OpNo << ": ";
2148 N->dump(&DAG);
2149 dbgs() << "\n";
2150 #endif
2151 report_fatal_error("Do not know how to split this operator's "
2152 "operand!\n");
2153
2154 case ISD::SETCC: Res = SplitVecOp_VSETCC(N); break;
2155 case ISD::BITCAST: Res = SplitVecOp_BITCAST(N); break;
2156 case ISD::EXTRACT_SUBVECTOR: Res = SplitVecOp_EXTRACT_SUBVECTOR(N); break;
2157 case ISD::INSERT_SUBVECTOR: Res = SplitVecOp_INSERT_SUBVECTOR(N, OpNo); break;
2158 case ISD::EXTRACT_VECTOR_ELT:Res = SplitVecOp_EXTRACT_VECTOR_ELT(N); break;
2159 case ISD::CONCAT_VECTORS: Res = SplitVecOp_CONCAT_VECTORS(N); break;
2160 case ISD::TRUNCATE:
2161 Res = SplitVecOp_TruncateHelper(N);
2162 break;
2163 case ISD::STRICT_FP_ROUND:
2164 case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break;
2165 case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break;
2166 case ISD::STORE:
2167 Res = SplitVecOp_STORE(cast<StoreSDNode>(N), OpNo);
2168 break;
2169 case ISD::MSTORE:
2170 Res = SplitVecOp_MSTORE(cast<MaskedStoreSDNode>(N), OpNo);
2171 break;
2172 case ISD::MSCATTER:
2173 Res = SplitVecOp_MSCATTER(cast<MaskedScatterSDNode>(N), OpNo);
2174 break;
2175 case ISD::MGATHER:
2176 Res = SplitVecOp_MGATHER(cast<MaskedGatherSDNode>(N), OpNo);
2177 break;
2178 case ISD::VSELECT:
2179 Res = SplitVecOp_VSELECT(N, OpNo);
2180 break;
2181 case ISD::STRICT_SINT_TO_FP:
2182 case ISD::STRICT_UINT_TO_FP:
2183 case ISD::SINT_TO_FP:
2184 case ISD::UINT_TO_FP:
2185 if (N->getValueType(0).bitsLT(
2186 N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType()))
2187 Res = SplitVecOp_TruncateHelper(N);
2188 else
2189 Res = SplitVecOp_UnaryOp(N);
2190 break;
2191 case ISD::FP_TO_SINT_SAT:
2192 case ISD::FP_TO_UINT_SAT:
2193 Res = SplitVecOp_FP_TO_XINT_SAT(N);
2194 break;
2195 case ISD::FP_TO_SINT:
2196 case ISD::FP_TO_UINT:
2197 case ISD::STRICT_FP_TO_SINT:
2198 case ISD::STRICT_FP_TO_UINT:
2199 case ISD::STRICT_FP_EXTEND:
2200 case ISD::FP_EXTEND:
2201 case ISD::SIGN_EXTEND:
2202 case ISD::ZERO_EXTEND:
2203 case ISD::ANY_EXTEND:
2204 case ISD::FTRUNC:
2205 Res = SplitVecOp_UnaryOp(N);
2206 break;
2207
2208 case ISD::ANY_EXTEND_VECTOR_INREG:
2209 case ISD::SIGN_EXTEND_VECTOR_INREG:
2210 case ISD::ZERO_EXTEND_VECTOR_INREG:
2211 Res = SplitVecOp_ExtVecInRegOp(N);
2212 break;
2213
2214 case ISD::VECREDUCE_FADD:
2215 case ISD::VECREDUCE_FMUL:
2216 case ISD::VECREDUCE_ADD:
2217 case ISD::VECREDUCE_MUL:
2218 case ISD::VECREDUCE_AND:
2219 case ISD::VECREDUCE_OR:
2220 case ISD::VECREDUCE_XOR:
2221 case ISD::VECREDUCE_SMAX:
2222 case ISD::VECREDUCE_SMIN:
2223 case ISD::VECREDUCE_UMAX:
2224 case ISD::VECREDUCE_UMIN:
2225 case ISD::VECREDUCE_FMAX:
2226 case ISD::VECREDUCE_FMIN:
2227 Res = SplitVecOp_VECREDUCE(N, OpNo);
2228 break;
2229 case ISD::VECREDUCE_SEQ_FADD:
2230 case ISD::VECREDUCE_SEQ_FMUL:
2231 Res = SplitVecOp_VECREDUCE_SEQ(N);
2232 break;
2233 }
2234
2235 // If the result is null, the sub-method took care of registering results etc.
2236 if (!Res.getNode()) return false;
2237
2238 // If the result is N, the sub-method updated N in place. Tell the legalizer
2239 // core about this.
2240 if (Res.getNode() == N)
2241 return true;
2242
2243 if (N->isStrictFPOpcode())
2244 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
2245 "Invalid operand expansion");
2246 else
2247 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2248 "Invalid operand expansion");
2249
2250 ReplaceValueWith(SDValue(N, 0), Res);
2251 return false;
2252 }
2253
SplitVecOp_VSELECT(SDNode * N,unsigned OpNo)2254 SDValue DAGTypeLegalizer::SplitVecOp_VSELECT(SDNode *N, unsigned OpNo) {
2255 // The only possibility for an illegal operand is the mask, since result type
2256 // legalization would have handled this node already otherwise.
2257 assert(OpNo == 0 && "Illegal operand must be mask");
2258
2259 SDValue Mask = N->getOperand(0);
2260 SDValue Src0 = N->getOperand(1);
2261 SDValue Src1 = N->getOperand(2);
2262 EVT Src0VT = Src0.getValueType();
2263 SDLoc DL(N);
2264 assert(Mask.getValueType().isVector() && "VSELECT without a vector mask?");
2265
2266 SDValue Lo, Hi;
2267 GetSplitVector(N->getOperand(0), Lo, Hi);
2268 assert(Lo.getValueType() == Hi.getValueType() &&
2269 "Lo and Hi have differing types");
2270
2271 EVT LoOpVT, HiOpVT;
2272 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(Src0VT);
2273 assert(LoOpVT == HiOpVT && "Asymmetric vector split?");
2274
2275 SDValue LoOp0, HiOp0, LoOp1, HiOp1, LoMask, HiMask;
2276 std::tie(LoOp0, HiOp0) = DAG.SplitVector(Src0, DL);
2277 std::tie(LoOp1, HiOp1) = DAG.SplitVector(Src1, DL);
2278 std::tie(LoMask, HiMask) = DAG.SplitVector(Mask, DL);
2279
2280 SDValue LoSelect =
2281 DAG.getNode(ISD::VSELECT, DL, LoOpVT, LoMask, LoOp0, LoOp1);
2282 SDValue HiSelect =
2283 DAG.getNode(ISD::VSELECT, DL, HiOpVT, HiMask, HiOp0, HiOp1);
2284
2285 return DAG.getNode(ISD::CONCAT_VECTORS, DL, Src0VT, LoSelect, HiSelect);
2286 }
2287
SplitVecOp_VECREDUCE(SDNode * N,unsigned OpNo)2288 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo) {
2289 EVT ResVT = N->getValueType(0);
2290 SDValue Lo, Hi;
2291 SDLoc dl(N);
2292
2293 SDValue VecOp = N->getOperand(OpNo);
2294 EVT VecVT = VecOp.getValueType();
2295 assert(VecVT.isVector() && "Can only split reduce vector operand");
2296 GetSplitVector(VecOp, Lo, Hi);
2297 EVT LoOpVT, HiOpVT;
2298 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2299
2300 // Use the appropriate scalar instruction on the split subvectors before
2301 // reducing the now partially reduced smaller vector.
2302 unsigned CombineOpc = ISD::getVecReduceBaseOpcode(N->getOpcode());
2303 SDValue Partial = DAG.getNode(CombineOpc, dl, LoOpVT, Lo, Hi, N->getFlags());
2304 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, N->getFlags());
2305 }
2306
SplitVecOp_VECREDUCE_SEQ(SDNode * N)2307 SDValue DAGTypeLegalizer::SplitVecOp_VECREDUCE_SEQ(SDNode *N) {
2308 EVT ResVT = N->getValueType(0);
2309 SDValue Lo, Hi;
2310 SDLoc dl(N);
2311
2312 SDValue AccOp = N->getOperand(0);
2313 SDValue VecOp = N->getOperand(1);
2314 SDNodeFlags Flags = N->getFlags();
2315
2316 EVT VecVT = VecOp.getValueType();
2317 assert(VecVT.isVector() && "Can only split reduce vector operand");
2318 GetSplitVector(VecOp, Lo, Hi);
2319 EVT LoOpVT, HiOpVT;
2320 std::tie(LoOpVT, HiOpVT) = DAG.GetSplitDestVTs(VecVT);
2321
2322 // Reduce low half.
2323 SDValue Partial = DAG.getNode(N->getOpcode(), dl, ResVT, AccOp, Lo, Flags);
2324
2325 // Reduce high half, using low half result as initial value.
2326 return DAG.getNode(N->getOpcode(), dl, ResVT, Partial, Hi, Flags);
2327 }
2328
SplitVecOp_UnaryOp(SDNode * N)2329 SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) {
2330 // The result has a legal vector type, but the input needs splitting.
2331 EVT ResVT = N->getValueType(0);
2332 SDValue Lo, Hi;
2333 SDLoc dl(N);
2334 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2335 EVT InVT = Lo.getValueType();
2336
2337 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2338 InVT.getVectorElementCount());
2339
2340 if (N->isStrictFPOpcode()) {
2341 Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2342 { N->getOperand(0), Lo });
2343 Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other },
2344 { N->getOperand(0), Hi });
2345
2346 // Build a factor node to remember that this operation is independent
2347 // of the other one.
2348 SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2349 Hi.getValue(1));
2350
2351 // Legalize the chain result - switch anything that used the old chain to
2352 // use the new one.
2353 ReplaceValueWith(SDValue(N, 1), Ch);
2354 } else {
2355 Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo);
2356 Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi);
2357 }
2358
2359 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2360 }
2361
SplitVecOp_BITCAST(SDNode * N)2362 SDValue DAGTypeLegalizer::SplitVecOp_BITCAST(SDNode *N) {
2363 // For example, i64 = BITCAST v4i16 on alpha. Typically the vector will
2364 // end up being split all the way down to individual components. Convert the
2365 // split pieces into integers and reassemble.
2366 SDValue Lo, Hi;
2367 GetSplitVector(N->getOperand(0), Lo, Hi);
2368 Lo = BitConvertToInteger(Lo);
2369 Hi = BitConvertToInteger(Hi);
2370
2371 if (DAG.getDataLayout().isBigEndian())
2372 std::swap(Lo, Hi);
2373
2374 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0),
2375 JoinIntegers(Lo, Hi));
2376 }
2377
SplitVecOp_INSERT_SUBVECTOR(SDNode * N,unsigned OpNo)2378 SDValue DAGTypeLegalizer::SplitVecOp_INSERT_SUBVECTOR(SDNode *N,
2379 unsigned OpNo) {
2380 assert(OpNo == 1 && "Invalid OpNo; can only split SubVec.");
2381 // We know that the result type is legal.
2382 EVT ResVT = N->getValueType(0);
2383
2384 SDValue Vec = N->getOperand(0);
2385 SDValue SubVec = N->getOperand(1);
2386 SDValue Idx = N->getOperand(2);
2387 SDLoc dl(N);
2388
2389 SDValue Lo, Hi;
2390 GetSplitVector(SubVec, Lo, Hi);
2391
2392 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2393 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2394
2395 SDValue FirstInsertion =
2396 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Lo, Idx);
2397 SDValue SecondInsertion =
2398 DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, FirstInsertion, Hi,
2399 DAG.getVectorIdxConstant(IdxVal + LoElts, dl));
2400
2401 return SecondInsertion;
2402 }
2403
SplitVecOp_EXTRACT_SUBVECTOR(SDNode * N)2404 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
2405 // We know that the extracted result type is legal.
2406 EVT SubVT = N->getValueType(0);
2407
2408 SDValue Idx = N->getOperand(1);
2409 SDLoc dl(N);
2410 SDValue Lo, Hi;
2411
2412 if (SubVT.isScalableVector() !=
2413 N->getOperand(0).getValueType().isScalableVector())
2414 report_fatal_error("Extracting a fixed-length vector from an illegal "
2415 "scalable vector is not yet supported");
2416
2417 GetSplitVector(N->getOperand(0), Lo, Hi);
2418
2419 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2420 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2421
2422 if (IdxVal < LoElts) {
2423 assert(IdxVal + SubVT.getVectorMinNumElements() <= LoElts &&
2424 "Extracted subvector crosses vector split!");
2425 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Lo, Idx);
2426 } else {
2427 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVT, Hi,
2428 DAG.getVectorIdxConstant(IdxVal - LoElts, dl));
2429 }
2430 }
2431
SplitVecOp_EXTRACT_VECTOR_ELT(SDNode * N)2432 SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
2433 SDValue Vec = N->getOperand(0);
2434 SDValue Idx = N->getOperand(1);
2435 EVT VecVT = Vec.getValueType();
2436
2437 if (isa<ConstantSDNode>(Idx)) {
2438 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2439
2440 SDValue Lo, Hi;
2441 GetSplitVector(Vec, Lo, Hi);
2442
2443 uint64_t LoElts = Lo.getValueType().getVectorMinNumElements();
2444
2445 if (IdxVal < LoElts)
2446 return SDValue(DAG.UpdateNodeOperands(N, Lo, Idx), 0);
2447 else if (!Vec.getValueType().isScalableVector())
2448 return SDValue(DAG.UpdateNodeOperands(N, Hi,
2449 DAG.getConstant(IdxVal - LoElts, SDLoc(N),
2450 Idx.getValueType())), 0);
2451 }
2452
2453 // See if the target wants to custom expand this node.
2454 if (CustomLowerNode(N, N->getValueType(0), true))
2455 return SDValue();
2456
2457 // Make the vector elements byte-addressable if they aren't already.
2458 SDLoc dl(N);
2459 EVT EltVT = VecVT.getVectorElementType();
2460 if (VecVT.getScalarSizeInBits() < 8) {
2461 EltVT = MVT::i8;
2462 VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
2463 VecVT.getVectorElementCount());
2464 Vec = DAG.getNode(ISD::ANY_EXTEND, dl, VecVT, Vec);
2465 }
2466
2467 // Store the vector to the stack.
2468 // In cases where the vector is illegal it will be broken down into parts
2469 // and stored in parts - we should use the alignment for the smallest part.
2470 Align SmallestAlign = DAG.getReducedAlign(VecVT, /*UseABI=*/false);
2471 SDValue StackPtr =
2472 DAG.CreateStackTemporary(VecVT.getStoreSize(), SmallestAlign);
2473 auto &MF = DAG.getMachineFunction();
2474 auto FrameIndex = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
2475 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FrameIndex);
2476 SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo,
2477 SmallestAlign);
2478
2479 // Load back the required element.
2480 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
2481
2482 // FIXME: This is to handle i1 vectors with elements promoted to i8.
2483 // i1 vector handling needs general improvement.
2484 if (N->getValueType(0).bitsLT(EltVT)) {
2485 SDValue Load = DAG.getLoad(EltVT, dl, Store, StackPtr,
2486 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
2487 return DAG.getZExtOrTrunc(Load, dl, N->getValueType(0));
2488 }
2489
2490 return DAG.getExtLoad(
2491 ISD::EXTLOAD, dl, N->getValueType(0), Store, StackPtr,
2492 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT,
2493 commonAlignment(SmallestAlign, EltVT.getFixedSizeInBits() / 8));
2494 }
2495
SplitVecOp_ExtVecInRegOp(SDNode * N)2496 SDValue DAGTypeLegalizer::SplitVecOp_ExtVecInRegOp(SDNode *N) {
2497 SDValue Lo, Hi;
2498
2499 // *_EXTEND_VECTOR_INREG only reference the lower half of the input, so
2500 // splitting the result has the same effect as splitting the input operand.
2501 SplitVecRes_ExtVecInRegOp(N, Lo, Hi);
2502
2503 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), N->getValueType(0), Lo, Hi);
2504 }
2505
SplitVecOp_MGATHER(MaskedGatherSDNode * MGT,unsigned OpNo)2506 SDValue DAGTypeLegalizer::SplitVecOp_MGATHER(MaskedGatherSDNode *MGT,
2507 unsigned OpNo) {
2508 EVT LoVT, HiVT;
2509 SDLoc dl(MGT);
2510 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MGT->getValueType(0));
2511
2512 SDValue Ch = MGT->getChain();
2513 SDValue Ptr = MGT->getBasePtr();
2514 SDValue Index = MGT->getIndex();
2515 SDValue Scale = MGT->getScale();
2516 SDValue Mask = MGT->getMask();
2517 SDValue PassThru = MGT->getPassThru();
2518 Align Alignment = MGT->getOriginalAlign();
2519 ISD::LoadExtType ExtType = MGT->getExtensionType();
2520
2521 SDValue MaskLo, MaskHi;
2522 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2523 // Split Mask operand
2524 GetSplitVector(Mask, MaskLo, MaskHi);
2525 else
2526 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, dl);
2527
2528 EVT MemoryVT = MGT->getMemoryVT();
2529 EVT LoMemVT, HiMemVT;
2530 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2531
2532 SDValue PassThruLo, PassThruHi;
2533 if (getTypeAction(PassThru.getValueType()) == TargetLowering::TypeSplitVector)
2534 GetSplitVector(PassThru, PassThruLo, PassThruHi);
2535 else
2536 std::tie(PassThruLo, PassThruHi) = DAG.SplitVector(PassThru, dl);
2537
2538 SDValue IndexHi, IndexLo;
2539 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2540 GetSplitVector(Index, IndexLo, IndexHi);
2541 else
2542 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, dl);
2543
2544 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2545 MGT->getPointerInfo(), MachineMemOperand::MOLoad,
2546 MemoryLocation::UnknownSize, Alignment, MGT->getAAInfo(),
2547 MGT->getRanges());
2548
2549 SDValue OpsLo[] = {Ch, PassThruLo, MaskLo, Ptr, IndexLo, Scale};
2550 SDValue Lo = DAG.getMaskedGather(DAG.getVTList(LoVT, MVT::Other), LoMemVT, dl,
2551 OpsLo, MMO, MGT->getIndexType(), ExtType);
2552
2553 SDValue OpsHi[] = {Ch, PassThruHi, MaskHi, Ptr, IndexHi, Scale};
2554 SDValue Hi = DAG.getMaskedGather(DAG.getVTList(HiVT, MVT::Other), HiMemVT, dl,
2555 OpsHi, MMO, MGT->getIndexType(), ExtType);
2556
2557 // Build a factor node to remember that this load is independent of the
2558 // other one.
2559 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2560 Hi.getValue(1));
2561
2562 // Legalize the chain result - switch anything that used the old chain to
2563 // use the new one.
2564 ReplaceValueWith(SDValue(MGT, 1), Ch);
2565
2566 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MGT->getValueType(0), Lo,
2567 Hi);
2568 ReplaceValueWith(SDValue(MGT, 0), Res);
2569 return SDValue();
2570 }
2571
SplitVecOp_MSTORE(MaskedStoreSDNode * N,unsigned OpNo)2572 SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N,
2573 unsigned OpNo) {
2574 assert(N->isUnindexed() && "Indexed masked store of vector?");
2575 SDValue Ch = N->getChain();
2576 SDValue Ptr = N->getBasePtr();
2577 SDValue Offset = N->getOffset();
2578 assert(Offset.isUndef() && "Unexpected indexed masked store offset");
2579 SDValue Mask = N->getMask();
2580 SDValue Data = N->getValue();
2581 Align Alignment = N->getOriginalAlign();
2582 SDLoc DL(N);
2583
2584 SDValue DataLo, DataHi;
2585 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2586 // Split Data operand
2587 GetSplitVector(Data, DataLo, DataHi);
2588 else
2589 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2590
2591 // Split Mask operand
2592 SDValue MaskLo, MaskHi;
2593 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2594 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2595 } else {
2596 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2597 GetSplitVector(Mask, MaskLo, MaskHi);
2598 else
2599 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2600 }
2601
2602 EVT MemoryVT = N->getMemoryVT();
2603 EVT LoMemVT, HiMemVT;
2604 bool HiIsEmpty = false;
2605 std::tie(LoMemVT, HiMemVT) =
2606 DAG.GetDependentSplitDestVTs(MemoryVT, DataLo.getValueType(), &HiIsEmpty);
2607
2608 SDValue Lo, Hi, Res;
2609 unsigned LoSize = MemoryLocation::getSizeOrUnknown(LoMemVT.getStoreSize());
2610 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2611 N->getPointerInfo(), MachineMemOperand::MOStore, LoSize, Alignment,
2612 N->getAAInfo(), N->getRanges());
2613
2614 Lo = DAG.getMaskedStore(Ch, DL, DataLo, Ptr, Offset, MaskLo, LoMemVT, MMO,
2615 N->getAddressingMode(), N->isTruncatingStore(),
2616 N->isCompressingStore());
2617
2618 if (HiIsEmpty) {
2619 // The hi masked store has zero storage size.
2620 // Only the lo masked store is needed.
2621 Res = Lo;
2622 } else {
2623
2624 Ptr = TLI.IncrementMemoryAddress(Ptr, MaskLo, DL, LoMemVT, DAG,
2625 N->isCompressingStore());
2626
2627 MachinePointerInfo MPI;
2628 if (LoMemVT.isScalableVector()) {
2629 Alignment = commonAlignment(
2630 Alignment, LoMemVT.getSizeInBits().getKnownMinSize() / 8);
2631 MPI = MachinePointerInfo(N->getPointerInfo().getAddrSpace());
2632 } else
2633 MPI = N->getPointerInfo().getWithOffset(
2634 LoMemVT.getStoreSize().getFixedSize());
2635
2636 unsigned HiSize = MemoryLocation::getSizeOrUnknown(HiMemVT.getStoreSize());
2637 MMO = DAG.getMachineFunction().getMachineMemOperand(
2638 MPI, MachineMemOperand::MOStore, HiSize, Alignment, N->getAAInfo(),
2639 N->getRanges());
2640
2641 Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, Offset, MaskHi, HiMemVT, MMO,
2642 N->getAddressingMode(), N->isTruncatingStore(),
2643 N->isCompressingStore());
2644
2645 // Build a factor node to remember that this store is independent of the
2646 // other one.
2647 Res = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2648 }
2649
2650 return Res;
2651 }
2652
SplitVecOp_MSCATTER(MaskedScatterSDNode * N,unsigned OpNo)2653 SDValue DAGTypeLegalizer::SplitVecOp_MSCATTER(MaskedScatterSDNode *N,
2654 unsigned OpNo) {
2655 SDValue Ch = N->getChain();
2656 SDValue Ptr = N->getBasePtr();
2657 SDValue Mask = N->getMask();
2658 SDValue Index = N->getIndex();
2659 SDValue Scale = N->getScale();
2660 SDValue Data = N->getValue();
2661 EVT MemoryVT = N->getMemoryVT();
2662 Align Alignment = N->getOriginalAlign();
2663 SDLoc DL(N);
2664
2665 // Split all operands
2666
2667 EVT LoMemVT, HiMemVT;
2668 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2669
2670 SDValue DataLo, DataHi;
2671 if (getTypeAction(Data.getValueType()) == TargetLowering::TypeSplitVector)
2672 // Split Data operand
2673 GetSplitVector(Data, DataLo, DataHi);
2674 else
2675 std::tie(DataLo, DataHi) = DAG.SplitVector(Data, DL);
2676
2677 // Split Mask operand
2678 SDValue MaskLo, MaskHi;
2679 if (OpNo == 1 && Mask.getOpcode() == ISD::SETCC) {
2680 SplitVecRes_SETCC(Mask.getNode(), MaskLo, MaskHi);
2681 } else {
2682 if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector)
2683 GetSplitVector(Mask, MaskLo, MaskHi);
2684 else
2685 std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL);
2686 }
2687
2688 SDValue IndexHi, IndexLo;
2689 if (getTypeAction(Index.getValueType()) == TargetLowering::TypeSplitVector)
2690 GetSplitVector(Index, IndexLo, IndexHi);
2691 else
2692 std::tie(IndexLo, IndexHi) = DAG.SplitVector(Index, DL);
2693
2694 SDValue Lo;
2695 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
2696 N->getPointerInfo(), MachineMemOperand::MOStore,
2697 MemoryLocation::UnknownSize, Alignment, N->getAAInfo(), N->getRanges());
2698
2699 SDValue OpsLo[] = {Ch, DataLo, MaskLo, Ptr, IndexLo, Scale};
2700 Lo = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), LoMemVT,
2701 DL, OpsLo, MMO, N->getIndexType(),
2702 N->isTruncatingStore());
2703
2704 // The order of the Scatter operation after split is well defined. The "Hi"
2705 // part comes after the "Lo". So these two operations should be chained one
2706 // after another.
2707 SDValue OpsHi[] = {Lo, DataHi, MaskHi, Ptr, IndexHi, Scale};
2708 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), HiMemVT,
2709 DL, OpsHi, MMO, N->getIndexType(),
2710 N->isTruncatingStore());
2711 }
2712
SplitVecOp_STORE(StoreSDNode * N,unsigned OpNo)2713 SDValue DAGTypeLegalizer::SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo) {
2714 assert(N->isUnindexed() && "Indexed store of vector?");
2715 assert(OpNo == 1 && "Can only split the stored value");
2716 SDLoc DL(N);
2717
2718 bool isTruncating = N->isTruncatingStore();
2719 SDValue Ch = N->getChain();
2720 SDValue Ptr = N->getBasePtr();
2721 EVT MemoryVT = N->getMemoryVT();
2722 Align Alignment = N->getOriginalAlign();
2723 MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2724 AAMDNodes AAInfo = N->getAAInfo();
2725 SDValue Lo, Hi;
2726 GetSplitVector(N->getOperand(1), Lo, Hi);
2727
2728 EVT LoMemVT, HiMemVT;
2729 std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT);
2730
2731 // Scalarize if the split halves are not byte-sized.
2732 if (!LoMemVT.isByteSized() || !HiMemVT.isByteSized())
2733 return TLI.scalarizeVectorStore(N, DAG);
2734
2735 if (isTruncating)
2736 Lo = DAG.getTruncStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), LoMemVT,
2737 Alignment, MMOFlags, AAInfo);
2738 else
2739 Lo = DAG.getStore(Ch, DL, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2740 AAInfo);
2741
2742 MachinePointerInfo MPI;
2743 IncrementPointer(N, LoMemVT, MPI, Ptr);
2744
2745 if (isTruncating)
2746 Hi = DAG.getTruncStore(Ch, DL, Hi, Ptr, MPI,
2747 HiMemVT, Alignment, MMOFlags, AAInfo);
2748 else
2749 Hi = DAG.getStore(Ch, DL, Hi, Ptr, MPI, Alignment, MMOFlags, AAInfo);
2750
2751 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Lo, Hi);
2752 }
2753
SplitVecOp_CONCAT_VECTORS(SDNode * N)2754 SDValue DAGTypeLegalizer::SplitVecOp_CONCAT_VECTORS(SDNode *N) {
2755 SDLoc DL(N);
2756
2757 // The input operands all must have the same type, and we know the result
2758 // type is valid. Convert this to a buildvector which extracts all the
2759 // input elements.
2760 // TODO: If the input elements are power-two vectors, we could convert this to
2761 // a new CONCAT_VECTORS node with elements that are half-wide.
2762 SmallVector<SDValue, 32> Elts;
2763 EVT EltVT = N->getValueType(0).getVectorElementType();
2764 for (const SDValue &Op : N->op_values()) {
2765 for (unsigned i = 0, e = Op.getValueType().getVectorNumElements();
2766 i != e; ++i) {
2767 Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Op,
2768 DAG.getVectorIdxConstant(i, DL)));
2769 }
2770 }
2771
2772 return DAG.getBuildVector(N->getValueType(0), DL, Elts);
2773 }
2774
SplitVecOp_TruncateHelper(SDNode * N)2775 SDValue DAGTypeLegalizer::SplitVecOp_TruncateHelper(SDNode *N) {
2776 // The result type is legal, but the input type is illegal. If splitting
2777 // ends up with the result type of each half still being legal, just
2778 // do that. If, however, that would result in an illegal result type,
2779 // we can try to get more clever with power-two vectors. Specifically,
2780 // split the input type, but also widen the result element size, then
2781 // concatenate the halves and truncate again. For example, consider a target
2782 // where v8i8 is legal and v8i32 is not (ARM, which doesn't have 256-bit
2783 // vectors). To perform a "%res = v8i8 trunc v8i32 %in" we do:
2784 // %inlo = v4i32 extract_subvector %in, 0
2785 // %inhi = v4i32 extract_subvector %in, 4
2786 // %lo16 = v4i16 trunc v4i32 %inlo
2787 // %hi16 = v4i16 trunc v4i32 %inhi
2788 // %in16 = v8i16 concat_vectors v4i16 %lo16, v4i16 %hi16
2789 // %res = v8i8 trunc v8i16 %in16
2790 //
2791 // Without this transform, the original truncate would end up being
2792 // scalarized, which is pretty much always a last resort.
2793 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
2794 SDValue InVec = N->getOperand(OpNo);
2795 EVT InVT = InVec->getValueType(0);
2796 EVT OutVT = N->getValueType(0);
2797 ElementCount NumElements = OutVT.getVectorElementCount();
2798 bool IsFloat = OutVT.isFloatingPoint();
2799
2800 unsigned InElementSize = InVT.getScalarSizeInBits();
2801 unsigned OutElementSize = OutVT.getScalarSizeInBits();
2802
2803 // Determine the split output VT. If its legal we can just split dirctly.
2804 EVT LoOutVT, HiOutVT;
2805 std::tie(LoOutVT, HiOutVT) = DAG.GetSplitDestVTs(OutVT);
2806 assert(LoOutVT == HiOutVT && "Unequal split?");
2807
2808 // If the input elements are only 1/2 the width of the result elements,
2809 // just use the normal splitting. Our trick only work if there's room
2810 // to split more than once.
2811 if (isTypeLegal(LoOutVT) ||
2812 InElementSize <= OutElementSize * 2)
2813 return SplitVecOp_UnaryOp(N);
2814 SDLoc DL(N);
2815
2816 // Don't touch if this will be scalarized.
2817 EVT FinalVT = InVT;
2818 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
2819 FinalVT = FinalVT.getHalfNumVectorElementsVT(*DAG.getContext());
2820
2821 if (getTypeAction(FinalVT) == TargetLowering::TypeScalarizeVector)
2822 return SplitVecOp_UnaryOp(N);
2823
2824 // Get the split input vector.
2825 SDValue InLoVec, InHiVec;
2826 GetSplitVector(InVec, InLoVec, InHiVec);
2827
2828 // Truncate them to 1/2 the element size.
2829 //
2830 // This assumes the number of elements is a power of two; any vector that
2831 // isn't should be widened, not split.
2832 EVT HalfElementVT = IsFloat ?
2833 EVT::getFloatingPointVT(InElementSize/2) :
2834 EVT::getIntegerVT(*DAG.getContext(), InElementSize/2);
2835 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT,
2836 NumElements.divideCoefficientBy(2));
2837
2838 SDValue HalfLo;
2839 SDValue HalfHi;
2840 SDValue Chain;
2841 if (N->isStrictFPOpcode()) {
2842 HalfLo = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2843 {N->getOperand(0), InLoVec});
2844 HalfHi = DAG.getNode(N->getOpcode(), DL, {HalfVT, MVT::Other},
2845 {N->getOperand(0), InHiVec});
2846 // Legalize the chain result - switch anything that used the old chain to
2847 // use the new one.
2848 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, HalfLo.getValue(1),
2849 HalfHi.getValue(1));
2850 } else {
2851 HalfLo = DAG.getNode(N->getOpcode(), DL, HalfVT, InLoVec);
2852 HalfHi = DAG.getNode(N->getOpcode(), DL, HalfVT, InHiVec);
2853 }
2854 // Concatenate them to get the full intermediate truncation result.
2855 EVT InterVT = EVT::getVectorVT(*DAG.getContext(), HalfElementVT, NumElements);
2856 SDValue InterVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InterVT, HalfLo,
2857 HalfHi);
2858 // Now finish up by truncating all the way down to the original result
2859 // type. This should normally be something that ends up being legal directly,
2860 // but in theory if a target has very wide vectors and an annoyingly
2861 // restricted set of legal types, this split can chain to build things up.
2862
2863 if (N->isStrictFPOpcode()) {
2864 SDValue Res = DAG.getNode(
2865 ISD::STRICT_FP_ROUND, DL, {OutVT, MVT::Other},
2866 {Chain, InterVec,
2867 DAG.getTargetConstant(0, DL, TLI.getPointerTy(DAG.getDataLayout()))});
2868 // Relink the chain
2869 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
2870 return Res;
2871 }
2872
2873 return IsFloat
2874 ? DAG.getNode(ISD::FP_ROUND, DL, OutVT, InterVec,
2875 DAG.getTargetConstant(
2876 0, DL, TLI.getPointerTy(DAG.getDataLayout())))
2877 : DAG.getNode(ISD::TRUNCATE, DL, OutVT, InterVec);
2878 }
2879
SplitVecOp_VSETCC(SDNode * N)2880 SDValue DAGTypeLegalizer::SplitVecOp_VSETCC(SDNode *N) {
2881 assert(N->getValueType(0).isVector() &&
2882 N->getOperand(0).getValueType().isVector() &&
2883 "Operand types must be vectors");
2884 // The result has a legal vector type, but the input needs splitting.
2885 SDValue Lo0, Hi0, Lo1, Hi1, LoRes, HiRes;
2886 SDLoc DL(N);
2887 GetSplitVector(N->getOperand(0), Lo0, Hi0);
2888 GetSplitVector(N->getOperand(1), Lo1, Hi1);
2889 auto PartEltCnt = Lo0.getValueType().getVectorElementCount();
2890
2891 LLVMContext &Context = *DAG.getContext();
2892 EVT PartResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt);
2893 EVT WideResVT = EVT::getVectorVT(Context, MVT::i1, PartEltCnt*2);
2894
2895 LoRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Lo0, Lo1, N->getOperand(2));
2896 HiRes = DAG.getNode(ISD::SETCC, DL, PartResVT, Hi0, Hi1, N->getOperand(2));
2897 SDValue Con = DAG.getNode(ISD::CONCAT_VECTORS, DL, WideResVT, LoRes, HiRes);
2898
2899 EVT OpVT = N->getOperand(0).getValueType();
2900 ISD::NodeType ExtendCode =
2901 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
2902 return DAG.getNode(ExtendCode, DL, N->getValueType(0), Con);
2903 }
2904
2905
SplitVecOp_FP_ROUND(SDNode * N)2906 SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) {
2907 // The result has a legal vector type, but the input needs splitting.
2908 EVT ResVT = N->getValueType(0);
2909 SDValue Lo, Hi;
2910 SDLoc DL(N);
2911 GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi);
2912 EVT InVT = Lo.getValueType();
2913
2914 EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2915 InVT.getVectorElementCount());
2916
2917 if (N->isStrictFPOpcode()) {
2918 Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2919 { N->getOperand(0), Lo, N->getOperand(2) });
2920 Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other },
2921 { N->getOperand(0), Hi, N->getOperand(2) });
2922 // Legalize the chain result - switch anything that used the old chain to
2923 // use the new one.
2924 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
2925 Lo.getValue(1), Hi.getValue(1));
2926 ReplaceValueWith(SDValue(N, 1), NewChain);
2927 } else {
2928 Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1));
2929 Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1));
2930 }
2931
2932 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
2933 }
2934
SplitVecOp_FCOPYSIGN(SDNode * N)2935 SDValue DAGTypeLegalizer::SplitVecOp_FCOPYSIGN(SDNode *N) {
2936 // The result (and the first input) has a legal vector type, but the second
2937 // input needs splitting.
2938 return DAG.UnrollVectorOp(N, N->getValueType(0).getVectorNumElements());
2939 }
2940
SplitVecOp_FP_TO_XINT_SAT(SDNode * N)2941 SDValue DAGTypeLegalizer::SplitVecOp_FP_TO_XINT_SAT(SDNode *N) {
2942 EVT ResVT = N->getValueType(0);
2943 SDValue Lo, Hi;
2944 SDLoc dl(N);
2945 GetSplitVector(N->getOperand(0), Lo, Hi);
2946 EVT InVT = Lo.getValueType();
2947
2948 EVT NewResVT =
2949 EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(),
2950 InVT.getVectorElementCount());
2951
2952 Lo = DAG.getNode(N->getOpcode(), dl, NewResVT, Lo, N->getOperand(1));
2953 Hi = DAG.getNode(N->getOpcode(), dl, NewResVT, Hi, N->getOperand(1));
2954
2955 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
2956 }
2957
2958 //===----------------------------------------------------------------------===//
2959 // Result Vector Widening
2960 //===----------------------------------------------------------------------===//
2961
WidenVectorResult(SDNode * N,unsigned ResNo)2962 void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
2963 LLVM_DEBUG(dbgs() << "Widen node result " << ResNo << ": "; N->dump(&DAG);
2964 dbgs() << "\n");
2965
2966 // See if the target wants to custom widen this node.
2967 if (CustomWidenLowerNode(N, N->getValueType(ResNo)))
2968 return;
2969
2970 SDValue Res = SDValue();
2971 switch (N->getOpcode()) {
2972 default:
2973 #ifndef NDEBUG
2974 dbgs() << "WidenVectorResult #" << ResNo << ": ";
2975 N->dump(&DAG);
2976 dbgs() << "\n";
2977 #endif
2978 llvm_unreachable("Do not know how to widen the result of this operator!");
2979
2980 case ISD::MERGE_VALUES: Res = WidenVecRes_MERGE_VALUES(N, ResNo); break;
2981 case ISD::BITCAST: Res = WidenVecRes_BITCAST(N); break;
2982 case ISD::BUILD_VECTOR: Res = WidenVecRes_BUILD_VECTOR(N); break;
2983 case ISD::CONCAT_VECTORS: Res = WidenVecRes_CONCAT_VECTORS(N); break;
2984 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecRes_EXTRACT_SUBVECTOR(N); break;
2985 case ISD::INSERT_VECTOR_ELT: Res = WidenVecRes_INSERT_VECTOR_ELT(N); break;
2986 case ISD::LOAD: Res = WidenVecRes_LOAD(N); break;
2987 case ISD::SPLAT_VECTOR:
2988 case ISD::SCALAR_TO_VECTOR:
2989 Res = WidenVecRes_ScalarOp(N);
2990 break;
2991 case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
2992 case ISD::VSELECT:
2993 case ISD::SELECT: Res = WidenVecRes_SELECT(N); break;
2994 case ISD::SELECT_CC: Res = WidenVecRes_SELECT_CC(N); break;
2995 case ISD::SETCC: Res = WidenVecRes_SETCC(N); break;
2996 case ISD::UNDEF: Res = WidenVecRes_UNDEF(N); break;
2997 case ISD::VECTOR_SHUFFLE:
2998 Res = WidenVecRes_VECTOR_SHUFFLE(cast<ShuffleVectorSDNode>(N));
2999 break;
3000 case ISD::MLOAD:
3001 Res = WidenVecRes_MLOAD(cast<MaskedLoadSDNode>(N));
3002 break;
3003 case ISD::MGATHER:
3004 Res = WidenVecRes_MGATHER(cast<MaskedGatherSDNode>(N));
3005 break;
3006
3007 case ISD::ADD:
3008 case ISD::AND:
3009 case ISD::MUL:
3010 case ISD::MULHS:
3011 case ISD::MULHU:
3012 case ISD::OR:
3013 case ISD::SUB:
3014 case ISD::XOR:
3015 case ISD::SHL:
3016 case ISD::SRA:
3017 case ISD::SRL:
3018 case ISD::FMINNUM:
3019 case ISD::FMAXNUM:
3020 case ISD::FMINIMUM:
3021 case ISD::FMAXIMUM:
3022 case ISD::SMIN:
3023 case ISD::SMAX:
3024 case ISD::UMIN:
3025 case ISD::UMAX:
3026 case ISD::UADDSAT:
3027 case ISD::SADDSAT:
3028 case ISD::USUBSAT:
3029 case ISD::SSUBSAT:
3030 case ISD::SSHLSAT:
3031 case ISD::USHLSAT:
3032 case ISD::ROTL:
3033 case ISD::ROTR:
3034 Res = WidenVecRes_Binary(N);
3035 break;
3036
3037 case ISD::FADD:
3038 case ISD::FMUL:
3039 case ISD::FPOW:
3040 case ISD::FSUB:
3041 case ISD::FDIV:
3042 case ISD::FREM:
3043 case ISD::SDIV:
3044 case ISD::UDIV:
3045 case ISD::SREM:
3046 case ISD::UREM:
3047 Res = WidenVecRes_BinaryCanTrap(N);
3048 break;
3049
3050 case ISD::SMULFIX:
3051 case ISD::SMULFIXSAT:
3052 case ISD::UMULFIX:
3053 case ISD::UMULFIXSAT:
3054 // These are binary operations, but with an extra operand that shouldn't
3055 // be widened (the scale).
3056 Res = WidenVecRes_BinaryWithExtraScalarOp(N);
3057 break;
3058
3059 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
3060 case ISD::STRICT_##DAGN:
3061 #include "llvm/IR/ConstrainedOps.def"
3062 Res = WidenVecRes_StrictFP(N);
3063 break;
3064
3065 case ISD::UADDO:
3066 case ISD::SADDO:
3067 case ISD::USUBO:
3068 case ISD::SSUBO:
3069 case ISD::UMULO:
3070 case ISD::SMULO:
3071 Res = WidenVecRes_OverflowOp(N, ResNo);
3072 break;
3073
3074 case ISD::FCOPYSIGN:
3075 Res = WidenVecRes_FCOPYSIGN(N);
3076 break;
3077
3078 case ISD::FPOWI:
3079 Res = WidenVecRes_POWI(N);
3080 break;
3081
3082 case ISD::ANY_EXTEND_VECTOR_INREG:
3083 case ISD::SIGN_EXTEND_VECTOR_INREG:
3084 case ISD::ZERO_EXTEND_VECTOR_INREG:
3085 Res = WidenVecRes_EXTEND_VECTOR_INREG(N);
3086 break;
3087
3088 case ISD::ANY_EXTEND:
3089 case ISD::FP_EXTEND:
3090 case ISD::FP_ROUND:
3091 case ISD::FP_TO_SINT:
3092 case ISD::FP_TO_UINT:
3093 case ISD::SIGN_EXTEND:
3094 case ISD::SINT_TO_FP:
3095 case ISD::TRUNCATE:
3096 case ISD::UINT_TO_FP:
3097 case ISD::ZERO_EXTEND:
3098 Res = WidenVecRes_Convert(N);
3099 break;
3100
3101 case ISD::FP_TO_SINT_SAT:
3102 case ISD::FP_TO_UINT_SAT:
3103 Res = WidenVecRes_FP_TO_XINT_SAT(N);
3104 break;
3105
3106 case ISD::FABS:
3107 case ISD::FCEIL:
3108 case ISD::FCOS:
3109 case ISD::FEXP:
3110 case ISD::FEXP2:
3111 case ISD::FFLOOR:
3112 case ISD::FLOG:
3113 case ISD::FLOG10:
3114 case ISD::FLOG2:
3115 case ISD::FNEARBYINT:
3116 case ISD::FRINT:
3117 case ISD::FROUND:
3118 case ISD::FROUNDEVEN:
3119 case ISD::FSIN:
3120 case ISD::FSQRT:
3121 case ISD::FTRUNC: {
3122 // We're going to widen this vector op to a legal type by padding with undef
3123 // elements. If the wide vector op is eventually going to be expanded to
3124 // scalar libcalls, then unroll into scalar ops now to avoid unnecessary
3125 // libcalls on the undef elements.
3126 EVT VT = N->getValueType(0);
3127 EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3128 if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) &&
3129 TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) {
3130 Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements());
3131 break;
3132 }
3133 }
3134 // If the target has custom/legal support for the scalar FP intrinsic ops
3135 // (they are probably not destined to become libcalls), then widen those like
3136 // any other unary ops.
3137 LLVM_FALLTHROUGH;
3138
3139 case ISD::ABS:
3140 case ISD::BITREVERSE:
3141 case ISD::BSWAP:
3142 case ISD::CTLZ:
3143 case ISD::CTLZ_ZERO_UNDEF:
3144 case ISD::CTPOP:
3145 case ISD::CTTZ:
3146 case ISD::CTTZ_ZERO_UNDEF:
3147 case ISD::FNEG:
3148 case ISD::FREEZE:
3149 case ISD::FCANONICALIZE:
3150 Res = WidenVecRes_Unary(N);
3151 break;
3152 case ISD::FMA:
3153 case ISD::FSHL:
3154 case ISD::FSHR:
3155 Res = WidenVecRes_Ternary(N);
3156 break;
3157 }
3158
3159 // If Res is null, the sub-method took care of registering the result.
3160 if (Res.getNode())
3161 SetWidenedVector(SDValue(N, ResNo), Res);
3162 }
3163
WidenVecRes_Ternary(SDNode * N)3164 SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
3165 // Ternary op widening.
3166 SDLoc dl(N);
3167 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3168 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3169 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3170 SDValue InOp3 = GetWidenedVector(N->getOperand(2));
3171 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
3172 }
3173
WidenVecRes_Binary(SDNode * N)3174 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
3175 // Binary op widening.
3176 SDLoc dl(N);
3177 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3178 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3179 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3180 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, N->getFlags());
3181 }
3182
WidenVecRes_BinaryWithExtraScalarOp(SDNode * N)3183 SDValue DAGTypeLegalizer::WidenVecRes_BinaryWithExtraScalarOp(SDNode *N) {
3184 // Binary op widening, but with an extra operand that shouldn't be widened.
3185 SDLoc dl(N);
3186 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3187 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3188 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3189 SDValue InOp3 = N->getOperand(2);
3190 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3,
3191 N->getFlags());
3192 }
3193
3194 // Given a vector of operations that have been broken up to widen, see
3195 // if we can collect them together into the next widest legal VT. This
3196 // implementation is trap-safe.
CollectOpsToWiden(SelectionDAG & DAG,const TargetLowering & TLI,SmallVectorImpl<SDValue> & ConcatOps,unsigned ConcatEnd,EVT VT,EVT MaxVT,EVT WidenVT)3197 static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI,
3198 SmallVectorImpl<SDValue> &ConcatOps,
3199 unsigned ConcatEnd, EVT VT, EVT MaxVT,
3200 EVT WidenVT) {
3201 // Check to see if we have a single operation with the widen type.
3202 if (ConcatEnd == 1) {
3203 VT = ConcatOps[0].getValueType();
3204 if (VT == WidenVT)
3205 return ConcatOps[0];
3206 }
3207
3208 SDLoc dl(ConcatOps[0]);
3209 EVT WidenEltVT = WidenVT.getVectorElementType();
3210
3211 // while (Some element of ConcatOps is not of type MaxVT) {
3212 // From the end of ConcatOps, collect elements of the same type and put
3213 // them into an op of the next larger supported type
3214 // }
3215 while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) {
3216 int Idx = ConcatEnd - 1;
3217 VT = ConcatOps[Idx--].getValueType();
3218 while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT)
3219 Idx--;
3220
3221 int NextSize = VT.isVector() ? VT.getVectorNumElements() : 1;
3222 EVT NextVT;
3223 do {
3224 NextSize *= 2;
3225 NextVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NextSize);
3226 } while (!TLI.isTypeLegal(NextVT));
3227
3228 if (!VT.isVector()) {
3229 // Scalar type, create an INSERT_VECTOR_ELEMENT of type NextVT
3230 SDValue VecOp = DAG.getUNDEF(NextVT);
3231 unsigned NumToInsert = ConcatEnd - Idx - 1;
3232 for (unsigned i = 0, OpIdx = Idx+1; i < NumToInsert; i++, OpIdx++) {
3233 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NextVT, VecOp,
3234 ConcatOps[OpIdx], DAG.getVectorIdxConstant(i, dl));
3235 }
3236 ConcatOps[Idx+1] = VecOp;
3237 ConcatEnd = Idx + 2;
3238 } else {
3239 // Vector type, create a CONCAT_VECTORS of type NextVT
3240 SDValue undefVec = DAG.getUNDEF(VT);
3241 unsigned OpsToConcat = NextSize/VT.getVectorNumElements();
3242 SmallVector<SDValue, 16> SubConcatOps(OpsToConcat);
3243 unsigned RealVals = ConcatEnd - Idx - 1;
3244 unsigned SubConcatEnd = 0;
3245 unsigned SubConcatIdx = Idx + 1;
3246 while (SubConcatEnd < RealVals)
3247 SubConcatOps[SubConcatEnd++] = ConcatOps[++Idx];
3248 while (SubConcatEnd < OpsToConcat)
3249 SubConcatOps[SubConcatEnd++] = undefVec;
3250 ConcatOps[SubConcatIdx] = DAG.getNode(ISD::CONCAT_VECTORS, dl,
3251 NextVT, SubConcatOps);
3252 ConcatEnd = SubConcatIdx + 1;
3253 }
3254 }
3255
3256 // Check to see if we have a single operation with the widen type.
3257 if (ConcatEnd == 1) {
3258 VT = ConcatOps[0].getValueType();
3259 if (VT == WidenVT)
3260 return ConcatOps[0];
3261 }
3262
3263 // add undefs of size MaxVT until ConcatOps grows to length of WidenVT
3264 unsigned NumOps = WidenVT.getVectorNumElements()/MaxVT.getVectorNumElements();
3265 if (NumOps != ConcatEnd ) {
3266 SDValue UndefVal = DAG.getUNDEF(MaxVT);
3267 for (unsigned j = ConcatEnd; j < NumOps; ++j)
3268 ConcatOps[j] = UndefVal;
3269 }
3270 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
3271 makeArrayRef(ConcatOps.data(), NumOps));
3272 }
3273
WidenVecRes_BinaryCanTrap(SDNode * N)3274 SDValue DAGTypeLegalizer::WidenVecRes_BinaryCanTrap(SDNode *N) {
3275 // Binary op widening for operations that can trap.
3276 unsigned Opcode = N->getOpcode();
3277 SDLoc dl(N);
3278 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3279 EVT WidenEltVT = WidenVT.getVectorElementType();
3280 EVT VT = WidenVT;
3281 unsigned NumElts = VT.getVectorNumElements();
3282 const SDNodeFlags Flags = N->getFlags();
3283 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3284 NumElts = NumElts / 2;
3285 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3286 }
3287
3288 if (NumElts != 1 && !TLI.canOpTrap(N->getOpcode(), VT)) {
3289 // Operation doesn't trap so just widen as normal.
3290 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3291 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3292 return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, Flags);
3293 }
3294
3295 // No legal vector version so unroll the vector operation and then widen.
3296 if (NumElts == 1)
3297 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3298
3299 // Since the operation can trap, apply operation on the original vector.
3300 EVT MaxVT = VT;
3301 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
3302 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
3303 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3304
3305 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3306 unsigned ConcatEnd = 0; // Current ConcatOps index.
3307 int Idx = 0; // Current Idx into input vectors.
3308
3309 // NumElts := greatest legal vector size (at most WidenVT)
3310 // while (orig. vector has unhandled elements) {
3311 // take munches of size NumElts from the beginning and add to ConcatOps
3312 // NumElts := next smaller supported vector size or 1
3313 // }
3314 while (CurNumElts != 0) {
3315 while (CurNumElts >= NumElts) {
3316 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp1,
3317 DAG.getVectorIdxConstant(Idx, dl));
3318 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, InOp2,
3319 DAG.getVectorIdxConstant(Idx, dl));
3320 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, VT, EOp1, EOp2, Flags);
3321 Idx += NumElts;
3322 CurNumElts -= NumElts;
3323 }
3324 do {
3325 NumElts = NumElts / 2;
3326 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3327 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3328
3329 if (NumElts == 1) {
3330 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3331 SDValue EOp1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3332 InOp1, DAG.getVectorIdxConstant(Idx, dl));
3333 SDValue EOp2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT,
3334 InOp2, DAG.getVectorIdxConstant(Idx, dl));
3335 ConcatOps[ConcatEnd++] = DAG.getNode(Opcode, dl, WidenEltVT,
3336 EOp1, EOp2, Flags);
3337 }
3338 CurNumElts = 0;
3339 }
3340 }
3341
3342 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3343 }
3344
WidenVecRes_StrictFP(SDNode * N)3345 SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) {
3346 switch (N->getOpcode()) {
3347 case ISD::STRICT_FSETCC:
3348 case ISD::STRICT_FSETCCS:
3349 return WidenVecRes_STRICT_FSETCC(N);
3350 case ISD::STRICT_FP_EXTEND:
3351 case ISD::STRICT_FP_ROUND:
3352 case ISD::STRICT_FP_TO_SINT:
3353 case ISD::STRICT_FP_TO_UINT:
3354 case ISD::STRICT_SINT_TO_FP:
3355 case ISD::STRICT_UINT_TO_FP:
3356 return WidenVecRes_Convert_StrictFP(N);
3357 default:
3358 break;
3359 }
3360
3361 // StrictFP op widening for operations that can trap.
3362 unsigned NumOpers = N->getNumOperands();
3363 unsigned Opcode = N->getOpcode();
3364 SDLoc dl(N);
3365 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3366 EVT WidenEltVT = WidenVT.getVectorElementType();
3367 EVT VT = WidenVT;
3368 unsigned NumElts = VT.getVectorNumElements();
3369 while (!TLI.isTypeLegal(VT) && NumElts != 1) {
3370 NumElts = NumElts / 2;
3371 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3372 }
3373
3374 // No legal vector version so unroll the vector operation and then widen.
3375 if (NumElts == 1)
3376 return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements());
3377
3378 // Since the operation can trap, apply operation on the original vector.
3379 EVT MaxVT = VT;
3380 SmallVector<SDValue, 4> InOps;
3381 unsigned CurNumElts = N->getValueType(0).getVectorNumElements();
3382
3383 SmallVector<SDValue, 16> ConcatOps(CurNumElts);
3384 SmallVector<SDValue, 16> Chains;
3385 unsigned ConcatEnd = 0; // Current ConcatOps index.
3386 int Idx = 0; // Current Idx into input vectors.
3387
3388 // The Chain is the first operand.
3389 InOps.push_back(N->getOperand(0));
3390
3391 // Now process the remaining operands.
3392 for (unsigned i = 1; i < NumOpers; ++i) {
3393 SDValue Oper = N->getOperand(i);
3394
3395 if (Oper.getValueType().isVector()) {
3396 assert(Oper.getValueType() == N->getValueType(0) &&
3397 "Invalid operand type to widen!");
3398 Oper = GetWidenedVector(Oper);
3399 }
3400
3401 InOps.push_back(Oper);
3402 }
3403
3404 // NumElts := greatest legal vector size (at most WidenVT)
3405 // while (orig. vector has unhandled elements) {
3406 // take munches of size NumElts from the beginning and add to ConcatOps
3407 // NumElts := next smaller supported vector size or 1
3408 // }
3409 while (CurNumElts != 0) {
3410 while (CurNumElts >= NumElts) {
3411 SmallVector<SDValue, 4> EOps;
3412
3413 for (unsigned i = 0; i < NumOpers; ++i) {
3414 SDValue Op = InOps[i];
3415
3416 if (Op.getValueType().isVector())
3417 Op = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Op,
3418 DAG.getVectorIdxConstant(Idx, dl));
3419
3420 EOps.push_back(Op);
3421 }
3422
3423 EVT OperVT[] = {VT, MVT::Other};
3424 SDValue Oper = DAG.getNode(Opcode, dl, OperVT, EOps);
3425 ConcatOps[ConcatEnd++] = Oper;
3426 Chains.push_back(Oper.getValue(1));
3427 Idx += NumElts;
3428 CurNumElts -= NumElts;
3429 }
3430 do {
3431 NumElts = NumElts / 2;
3432 VT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, NumElts);
3433 } while (!TLI.isTypeLegal(VT) && NumElts != 1);
3434
3435 if (NumElts == 1) {
3436 for (unsigned i = 0; i != CurNumElts; ++i, ++Idx) {
3437 SmallVector<SDValue, 4> EOps;
3438
3439 for (unsigned i = 0; i < NumOpers; ++i) {
3440 SDValue Op = InOps[i];
3441
3442 if (Op.getValueType().isVector())
3443 Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, WidenEltVT, Op,
3444 DAG.getVectorIdxConstant(Idx, dl));
3445
3446 EOps.push_back(Op);
3447 }
3448
3449 EVT WidenVT[] = {WidenEltVT, MVT::Other};
3450 SDValue Oper = DAG.getNode(Opcode, dl, WidenVT, EOps);
3451 ConcatOps[ConcatEnd++] = Oper;
3452 Chains.push_back(Oper.getValue(1));
3453 }
3454 CurNumElts = 0;
3455 }
3456 }
3457
3458 // Build a factor node to remember all the Ops that have been created.
3459 SDValue NewChain;
3460 if (Chains.size() == 1)
3461 NewChain = Chains[0];
3462 else
3463 NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
3464 ReplaceValueWith(SDValue(N, 1), NewChain);
3465
3466 return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT);
3467 }
3468
WidenVecRes_OverflowOp(SDNode * N,unsigned ResNo)3469 SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) {
3470 SDLoc DL(N);
3471 EVT ResVT = N->getValueType(0);
3472 EVT OvVT = N->getValueType(1);
3473 EVT WideResVT, WideOvVT;
3474 SDValue WideLHS, WideRHS;
3475
3476 // TODO: This might result in a widen/split loop.
3477 if (ResNo == 0) {
3478 WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT);
3479 WideOvVT = EVT::getVectorVT(
3480 *DAG.getContext(), OvVT.getVectorElementType(),
3481 WideResVT.getVectorNumElements());
3482
3483 WideLHS = GetWidenedVector(N->getOperand(0));
3484 WideRHS = GetWidenedVector(N->getOperand(1));
3485 } else {
3486 WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT);
3487 WideResVT = EVT::getVectorVT(
3488 *DAG.getContext(), ResVT.getVectorElementType(),
3489 WideOvVT.getVectorNumElements());
3490
3491 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3492 WideLHS = DAG.getNode(
3493 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3494 N->getOperand(0), Zero);
3495 WideRHS = DAG.getNode(
3496 ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT),
3497 N->getOperand(1), Zero);
3498 }
3499
3500 SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT);
3501 SDNode *WideNode = DAG.getNode(
3502 N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode();
3503
3504 // Replace the other vector result not being explicitly widened here.
3505 unsigned OtherNo = 1 - ResNo;
3506 EVT OtherVT = N->getValueType(OtherNo);
3507 if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) {
3508 SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo));
3509 } else {
3510 SDValue Zero = DAG.getVectorIdxConstant(0, DL);
3511 SDValue OtherVal = DAG.getNode(
3512 ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero);
3513 ReplaceValueWith(SDValue(N, OtherNo), OtherVal);
3514 }
3515
3516 return SDValue(WideNode, ResNo);
3517 }
3518
WidenVecRes_Convert(SDNode * N)3519 SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) {
3520 LLVMContext &Ctx = *DAG.getContext();
3521 SDValue InOp = N->getOperand(0);
3522 SDLoc DL(N);
3523
3524 EVT WidenVT = TLI.getTypeToTransformTo(Ctx, N->getValueType(0));
3525 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3526
3527 EVT InVT = InOp.getValueType();
3528
3529 unsigned Opcode = N->getOpcode();
3530 const SDNodeFlags Flags = N->getFlags();
3531
3532 // Handle the case of ZERO_EXTEND where the promoted InVT element size does
3533 // not equal that of WidenVT.
3534 if (N->getOpcode() == ISD::ZERO_EXTEND &&
3535 getTypeAction(InVT) == TargetLowering::TypePromoteInteger &&
3536 TLI.getTypeToTransformTo(Ctx, InVT).getScalarSizeInBits() !=
3537 WidenVT.getScalarSizeInBits()) {
3538 InOp = ZExtPromotedInteger(InOp);
3539 InVT = InOp.getValueType();
3540 if (WidenVT.getScalarSizeInBits() < InVT.getScalarSizeInBits())
3541 Opcode = ISD::TRUNCATE;
3542 }
3543
3544 EVT InEltVT = InVT.getVectorElementType();
3545 EVT InWidenVT = EVT::getVectorVT(Ctx, InEltVT, WidenNumElts);
3546 unsigned InVTNumElts = InVT.getVectorNumElements();
3547
3548 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3549 InOp = GetWidenedVector(N->getOperand(0));
3550 InVT = InOp.getValueType();
3551 InVTNumElts = InVT.getVectorNumElements();
3552 if (InVTNumElts == WidenNumElts) {
3553 if (N->getNumOperands() == 1)
3554 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3555 return DAG.getNode(Opcode, DL, WidenVT, InOp, N->getOperand(1), Flags);
3556 }
3557 if (WidenVT.getSizeInBits() == InVT.getSizeInBits()) {
3558 // If both input and result vector types are of same width, extend
3559 // operations should be done with SIGN/ZERO_EXTEND_VECTOR_INREG, which
3560 // accepts fewer elements in the result than in the input.
3561 if (Opcode == ISD::ANY_EXTEND)
3562 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3563 if (Opcode == ISD::SIGN_EXTEND)
3564 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3565 if (Opcode == ISD::ZERO_EXTEND)
3566 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, WidenVT, InOp);
3567 }
3568 }
3569
3570 if (TLI.isTypeLegal(InWidenVT)) {
3571 // Because the result and the input are different vector types, widening
3572 // the result could create a legal type but widening the input might make
3573 // it an illegal type that might lead to repeatedly splitting the input
3574 // and then widening it. To avoid this, we widen the input only if
3575 // it results in a legal type.
3576 if (WidenNumElts % InVTNumElts == 0) {
3577 // Widen the input and call convert on the widened input vector.
3578 unsigned NumConcat = WidenNumElts/InVTNumElts;
3579 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
3580 Ops[0] = InOp;
3581 SDValue InVec = DAG.getNode(ISD::CONCAT_VECTORS, DL, InWidenVT, Ops);
3582 if (N->getNumOperands() == 1)
3583 return DAG.getNode(Opcode, DL, WidenVT, InVec);
3584 return DAG.getNode(Opcode, DL, WidenVT, InVec, N->getOperand(1), Flags);
3585 }
3586
3587 if (InVTNumElts % WidenNumElts == 0) {
3588 SDValue InVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InWidenVT, InOp,
3589 DAG.getVectorIdxConstant(0, DL));
3590 // Extract the input and convert the shorten input vector.
3591 if (N->getNumOperands() == 1)
3592 return DAG.getNode(Opcode, DL, WidenVT, InVal);
3593 return DAG.getNode(Opcode, DL, WidenVT, InVal, N->getOperand(1), Flags);
3594 }
3595 }
3596
3597 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3598 EVT EltVT = WidenVT.getVectorElementType();
3599 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3600 // Use the original element count so we don't do more scalar opts than
3601 // necessary.
3602 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3603 for (unsigned i=0; i < MinElts; ++i) {
3604 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3605 DAG.getVectorIdxConstant(i, DL));
3606 if (N->getNumOperands() == 1)
3607 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val);
3608 else
3609 Ops[i] = DAG.getNode(Opcode, DL, EltVT, Val, N->getOperand(1), Flags);
3610 }
3611
3612 return DAG.getBuildVector(WidenVT, DL, Ops);
3613 }
3614
WidenVecRes_FP_TO_XINT_SAT(SDNode * N)3615 SDValue DAGTypeLegalizer::WidenVecRes_FP_TO_XINT_SAT(SDNode *N) {
3616 SDLoc dl(N);
3617 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3618 ElementCount WidenNumElts = WidenVT.getVectorElementCount();
3619
3620 SDValue Src = N->getOperand(0);
3621 EVT SrcVT = Src.getValueType();
3622
3623 // Also widen the input.
3624 if (getTypeAction(SrcVT) == TargetLowering::TypeWidenVector) {
3625 Src = GetWidenedVector(Src);
3626 SrcVT = Src.getValueType();
3627 }
3628
3629 // Input and output not widened to the same size, give up.
3630 if (WidenNumElts != SrcVT.getVectorElementCount())
3631 return DAG.UnrollVectorOp(N, WidenNumElts.getKnownMinValue());
3632
3633 return DAG.getNode(N->getOpcode(), dl, WidenVT, Src, N->getOperand(1));
3634 }
3635
WidenVecRes_Convert_StrictFP(SDNode * N)3636 SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) {
3637 SDValue InOp = N->getOperand(1);
3638 SDLoc DL(N);
3639 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
3640
3641 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3642 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3643
3644 EVT InVT = InOp.getValueType();
3645 EVT InEltVT = InVT.getVectorElementType();
3646
3647 unsigned Opcode = N->getOpcode();
3648
3649 // FIXME: Optimizations need to be implemented here.
3650
3651 // Otherwise unroll into some nasty scalar code and rebuild the vector.
3652 EVT EltVT = WidenVT.getVectorElementType();
3653 std::array<EVT, 2> EltVTs = {{EltVT, MVT::Other}};
3654 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
3655 SmallVector<SDValue, 32> OpChains;
3656 // Use the original element count so we don't do more scalar opts than
3657 // necessary.
3658 unsigned MinElts = N->getValueType(0).getVectorNumElements();
3659 for (unsigned i=0; i < MinElts; ++i) {
3660 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp,
3661 DAG.getVectorIdxConstant(i, DL));
3662 Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps);
3663 OpChains.push_back(Ops[i].getValue(1));
3664 }
3665 SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains);
3666 ReplaceValueWith(SDValue(N, 1), NewChain);
3667
3668 return DAG.getBuildVector(WidenVT, DL, Ops);
3669 }
3670
WidenVecRes_EXTEND_VECTOR_INREG(SDNode * N)3671 SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) {
3672 unsigned Opcode = N->getOpcode();
3673 SDValue InOp = N->getOperand(0);
3674 SDLoc DL(N);
3675
3676 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3677 EVT WidenSVT = WidenVT.getVectorElementType();
3678 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3679
3680 EVT InVT = InOp.getValueType();
3681 EVT InSVT = InVT.getVectorElementType();
3682 unsigned InVTNumElts = InVT.getVectorNumElements();
3683
3684 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
3685 InOp = GetWidenedVector(InOp);
3686 InVT = InOp.getValueType();
3687 if (InVT.getSizeInBits() == WidenVT.getSizeInBits()) {
3688 switch (Opcode) {
3689 case ISD::ANY_EXTEND_VECTOR_INREG:
3690 case ISD::SIGN_EXTEND_VECTOR_INREG:
3691 case ISD::ZERO_EXTEND_VECTOR_INREG:
3692 return DAG.getNode(Opcode, DL, WidenVT, InOp);
3693 }
3694 }
3695 }
3696
3697 // Unroll, extend the scalars and rebuild the vector.
3698 SmallVector<SDValue, 16> Ops;
3699 for (unsigned i = 0, e = std::min(InVTNumElts, WidenNumElts); i != e; ++i) {
3700 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, InSVT, InOp,
3701 DAG.getVectorIdxConstant(i, DL));
3702 switch (Opcode) {
3703 case ISD::ANY_EXTEND_VECTOR_INREG:
3704 Val = DAG.getNode(ISD::ANY_EXTEND, DL, WidenSVT, Val);
3705 break;
3706 case ISD::SIGN_EXTEND_VECTOR_INREG:
3707 Val = DAG.getNode(ISD::SIGN_EXTEND, DL, WidenSVT, Val);
3708 break;
3709 case ISD::ZERO_EXTEND_VECTOR_INREG:
3710 Val = DAG.getNode(ISD::ZERO_EXTEND, DL, WidenSVT, Val);
3711 break;
3712 default:
3713 llvm_unreachable("A *_EXTEND_VECTOR_INREG node was expected");
3714 }
3715 Ops.push_back(Val);
3716 }
3717
3718 while (Ops.size() != WidenNumElts)
3719 Ops.push_back(DAG.getUNDEF(WidenSVT));
3720
3721 return DAG.getBuildVector(WidenVT, DL, Ops);
3722 }
3723
WidenVecRes_FCOPYSIGN(SDNode * N)3724 SDValue DAGTypeLegalizer::WidenVecRes_FCOPYSIGN(SDNode *N) {
3725 // If this is an FCOPYSIGN with same input types, we can treat it as a
3726 // normal (can trap) binary op.
3727 if (N->getOperand(0).getValueType() == N->getOperand(1).getValueType())
3728 return WidenVecRes_BinaryCanTrap(N);
3729
3730 // If the types are different, fall back to unrolling.
3731 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3732 return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements());
3733 }
3734
WidenVecRes_POWI(SDNode * N)3735 SDValue DAGTypeLegalizer::WidenVecRes_POWI(SDNode *N) {
3736 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3737 SDValue InOp = GetWidenedVector(N->getOperand(0));
3738 SDValue ShOp = N->getOperand(1);
3739 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp, ShOp);
3740 }
3741
WidenVecRes_Unary(SDNode * N)3742 SDValue DAGTypeLegalizer::WidenVecRes_Unary(SDNode *N) {
3743 // Unary op widening.
3744 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3745 SDValue InOp = GetWidenedVector(N->getOperand(0));
3746 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, InOp);
3747 }
3748
WidenVecRes_InregOp(SDNode * N)3749 SDValue DAGTypeLegalizer::WidenVecRes_InregOp(SDNode *N) {
3750 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3751 EVT ExtVT = EVT::getVectorVT(*DAG.getContext(),
3752 cast<VTSDNode>(N->getOperand(1))->getVT()
3753 .getVectorElementType(),
3754 WidenVT.getVectorNumElements());
3755 SDValue WidenLHS = GetWidenedVector(N->getOperand(0));
3756 return DAG.getNode(N->getOpcode(), SDLoc(N),
3757 WidenVT, WidenLHS, DAG.getValueType(ExtVT));
3758 }
3759
WidenVecRes_MERGE_VALUES(SDNode * N,unsigned ResNo)3760 SDValue DAGTypeLegalizer::WidenVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) {
3761 SDValue WidenVec = DisintegrateMERGE_VALUES(N, ResNo);
3762 return GetWidenedVector(WidenVec);
3763 }
3764
WidenVecRes_BITCAST(SDNode * N)3765 SDValue DAGTypeLegalizer::WidenVecRes_BITCAST(SDNode *N) {
3766 SDValue InOp = N->getOperand(0);
3767 EVT InVT = InOp.getValueType();
3768 EVT VT = N->getValueType(0);
3769 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3770 SDLoc dl(N);
3771
3772 switch (getTypeAction(InVT)) {
3773 case TargetLowering::TypeLegal:
3774 break;
3775 case TargetLowering::TypeScalarizeScalableVector:
3776 report_fatal_error("Scalarization of scalable vectors is not supported.");
3777 case TargetLowering::TypePromoteInteger: {
3778 // If the incoming type is a vector that is being promoted, then
3779 // we know that the elements are arranged differently and that we
3780 // must perform the conversion using a stack slot.
3781 if (InVT.isVector())
3782 break;
3783
3784 // If the InOp is promoted to the same size, convert it. Otherwise,
3785 // fall out of the switch and widen the promoted input.
3786 SDValue NInOp = GetPromotedInteger(InOp);
3787 EVT NInVT = NInOp.getValueType();
3788 if (WidenVT.bitsEq(NInVT)) {
3789 // For big endian targets we need to shift the input integer or the
3790 // interesting bits will end up at the wrong place.
3791 if (DAG.getDataLayout().isBigEndian()) {
3792 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
3793 EVT ShiftAmtTy = TLI.getShiftAmountTy(NInVT, DAG.getDataLayout());
3794 assert(ShiftAmt < WidenVT.getSizeInBits() && "Too large shift amount!");
3795 NInOp = DAG.getNode(ISD::SHL, dl, NInVT, NInOp,
3796 DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
3797 }
3798 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NInOp);
3799 }
3800 InOp = NInOp;
3801 InVT = NInVT;
3802 break;
3803 }
3804 case TargetLowering::TypeSoftenFloat:
3805 case TargetLowering::TypePromoteFloat:
3806 case TargetLowering::TypeSoftPromoteHalf:
3807 case TargetLowering::TypeExpandInteger:
3808 case TargetLowering::TypeExpandFloat:
3809 case TargetLowering::TypeScalarizeVector:
3810 case TargetLowering::TypeSplitVector:
3811 break;
3812 case TargetLowering::TypeWidenVector:
3813 // If the InOp is widened to the same size, convert it. Otherwise, fall
3814 // out of the switch and widen the widened input.
3815 InOp = GetWidenedVector(InOp);
3816 InVT = InOp.getValueType();
3817 if (WidenVT.bitsEq(InVT))
3818 // The input widens to the same size. Convert to the widen value.
3819 return DAG.getNode(ISD::BITCAST, dl, WidenVT, InOp);
3820 break;
3821 }
3822
3823 unsigned WidenSize = WidenVT.getSizeInBits();
3824 unsigned InSize = InVT.getSizeInBits();
3825 // x86mmx is not an acceptable vector element type, so don't try.
3826 if (WidenSize % InSize == 0 && InVT != MVT::x86mmx) {
3827 // Determine new input vector type. The new input vector type will use
3828 // the same element type (if its a vector) or use the input type as a
3829 // vector. It is the same size as the type to widen to.
3830 EVT NewInVT;
3831 unsigned NewNumElts = WidenSize / InSize;
3832 if (InVT.isVector()) {
3833 EVT InEltVT = InVT.getVectorElementType();
3834 NewInVT = EVT::getVectorVT(*DAG.getContext(), InEltVT,
3835 WidenSize / InEltVT.getSizeInBits());
3836 } else {
3837 NewInVT = EVT::getVectorVT(*DAG.getContext(), InVT, NewNumElts);
3838 }
3839
3840 if (TLI.isTypeLegal(NewInVT)) {
3841 SDValue NewVec;
3842 if (InVT.isVector()) {
3843 // Because the result and the input are different vector types, widening
3844 // the result could create a legal type but widening the input might make
3845 // it an illegal type that might lead to repeatedly splitting the input
3846 // and then widening it. To avoid this, we widen the input only if
3847 // it results in a legal type.
3848 SmallVector<SDValue, 16> Ops(NewNumElts, DAG.getUNDEF(InVT));
3849 Ops[0] = InOp;
3850
3851 NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewInVT, Ops);
3852 } else {
3853 NewVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewInVT, InOp);
3854 }
3855 return DAG.getNode(ISD::BITCAST, dl, WidenVT, NewVec);
3856 }
3857 }
3858
3859 return CreateStackStoreLoad(InOp, WidenVT);
3860 }
3861
WidenVecRes_BUILD_VECTOR(SDNode * N)3862 SDValue DAGTypeLegalizer::WidenVecRes_BUILD_VECTOR(SDNode *N) {
3863 SDLoc dl(N);
3864 // Build a vector with undefined for the new nodes.
3865 EVT VT = N->getValueType(0);
3866
3867 // Integer BUILD_VECTOR operands may be larger than the node's vector element
3868 // type. The UNDEFs need to have the same type as the existing operands.
3869 EVT EltVT = N->getOperand(0).getValueType();
3870 unsigned NumElts = VT.getVectorNumElements();
3871
3872 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3873 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3874
3875 SmallVector<SDValue, 16> NewOps(N->op_begin(), N->op_end());
3876 assert(WidenNumElts >= NumElts && "Shrinking vector instead of widening!");
3877 NewOps.append(WidenNumElts - NumElts, DAG.getUNDEF(EltVT));
3878
3879 return DAG.getBuildVector(WidenVT, dl, NewOps);
3880 }
3881
WidenVecRes_CONCAT_VECTORS(SDNode * N)3882 SDValue DAGTypeLegalizer::WidenVecRes_CONCAT_VECTORS(SDNode *N) {
3883 EVT InVT = N->getOperand(0).getValueType();
3884 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3885 SDLoc dl(N);
3886 unsigned NumOperands = N->getNumOperands();
3887
3888 bool InputWidened = false; // Indicates we need to widen the input.
3889 if (getTypeAction(InVT) != TargetLowering::TypeWidenVector) {
3890 unsigned WidenNumElts = WidenVT.getVectorMinNumElements();
3891 unsigned NumInElts = InVT.getVectorMinNumElements();
3892 if (WidenNumElts % NumInElts == 0) {
3893 // Add undef vectors to widen to correct length.
3894 unsigned NumConcat = WidenNumElts / NumInElts;
3895 SDValue UndefVal = DAG.getUNDEF(InVT);
3896 SmallVector<SDValue, 16> Ops(NumConcat);
3897 for (unsigned i=0; i < NumOperands; ++i)
3898 Ops[i] = N->getOperand(i);
3899 for (unsigned i = NumOperands; i != NumConcat; ++i)
3900 Ops[i] = UndefVal;
3901 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, Ops);
3902 }
3903 } else {
3904 InputWidened = true;
3905 if (WidenVT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
3906 // The inputs and the result are widen to the same value.
3907 unsigned i;
3908 for (i=1; i < NumOperands; ++i)
3909 if (!N->getOperand(i).isUndef())
3910 break;
3911
3912 if (i == NumOperands)
3913 // Everything but the first operand is an UNDEF so just return the
3914 // widened first operand.
3915 return GetWidenedVector(N->getOperand(0));
3916
3917 if (NumOperands == 2) {
3918 assert(!WidenVT.isScalableVector() &&
3919 "Cannot use vector shuffles to widen CONCAT_VECTOR result");
3920 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3921 unsigned NumInElts = InVT.getVectorNumElements();
3922
3923 // Replace concat of two operands with a shuffle.
3924 SmallVector<int, 16> MaskOps(WidenNumElts, -1);
3925 for (unsigned i = 0; i < NumInElts; ++i) {
3926 MaskOps[i] = i;
3927 MaskOps[i + NumInElts] = i + WidenNumElts;
3928 }
3929 return DAG.getVectorShuffle(WidenVT, dl,
3930 GetWidenedVector(N->getOperand(0)),
3931 GetWidenedVector(N->getOperand(1)),
3932 MaskOps);
3933 }
3934 }
3935 }
3936
3937 assert(!WidenVT.isScalableVector() &&
3938 "Cannot use build vectors to widen CONCAT_VECTOR result");
3939 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3940 unsigned NumInElts = InVT.getVectorNumElements();
3941
3942 // Fall back to use extracts and build vector.
3943 EVT EltVT = WidenVT.getVectorElementType();
3944 SmallVector<SDValue, 16> Ops(WidenNumElts);
3945 unsigned Idx = 0;
3946 for (unsigned i=0; i < NumOperands; ++i) {
3947 SDValue InOp = N->getOperand(i);
3948 if (InputWidened)
3949 InOp = GetWidenedVector(InOp);
3950 for (unsigned j = 0; j < NumInElts; ++j)
3951 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3952 DAG.getVectorIdxConstant(j, dl));
3953 }
3954 SDValue UndefVal = DAG.getUNDEF(EltVT);
3955 for (; Idx < WidenNumElts; ++Idx)
3956 Ops[Idx] = UndefVal;
3957 return DAG.getBuildVector(WidenVT, dl, Ops);
3958 }
3959
WidenVecRes_EXTRACT_SUBVECTOR(SDNode * N)3960 SDValue DAGTypeLegalizer::WidenVecRes_EXTRACT_SUBVECTOR(SDNode *N) {
3961 EVT VT = N->getValueType(0);
3962 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3963 SDValue InOp = N->getOperand(0);
3964 SDValue Idx = N->getOperand(1);
3965 SDLoc dl(N);
3966
3967 if (getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector)
3968 InOp = GetWidenedVector(InOp);
3969
3970 EVT InVT = InOp.getValueType();
3971
3972 // Check if we can just return the input vector after widening.
3973 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
3974 if (IdxVal == 0 && InVT == WidenVT)
3975 return InOp;
3976
3977 if (VT.isScalableVector())
3978 report_fatal_error("Don't know how to widen the result of "
3979 "EXTRACT_SUBVECTOR for scalable vectors");
3980
3981 // Check if we can extract from the vector.
3982 unsigned WidenNumElts = WidenVT.getVectorNumElements();
3983 unsigned InNumElts = InVT.getVectorNumElements();
3984 if (IdxVal % WidenNumElts == 0 && IdxVal + WidenNumElts < InNumElts)
3985 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, WidenVT, InOp, Idx);
3986
3987 // We could try widening the input to the right length but for now, extract
3988 // the original elements, fill the rest with undefs and build a vector.
3989 SmallVector<SDValue, 16> Ops(WidenNumElts);
3990 EVT EltVT = VT.getVectorElementType();
3991 unsigned NumElts = VT.getVectorNumElements();
3992 unsigned i;
3993 for (i = 0; i < NumElts; ++i)
3994 Ops[i] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
3995 DAG.getVectorIdxConstant(IdxVal + i, dl));
3996
3997 SDValue UndefVal = DAG.getUNDEF(EltVT);
3998 for (; i < WidenNumElts; ++i)
3999 Ops[i] = UndefVal;
4000 return DAG.getBuildVector(WidenVT, dl, Ops);
4001 }
4002
WidenVecRes_INSERT_VECTOR_ELT(SDNode * N)4003 SDValue DAGTypeLegalizer::WidenVecRes_INSERT_VECTOR_ELT(SDNode *N) {
4004 SDValue InOp = GetWidenedVector(N->getOperand(0));
4005 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N),
4006 InOp.getValueType(), InOp,
4007 N->getOperand(1), N->getOperand(2));
4008 }
4009
WidenVecRes_LOAD(SDNode * N)4010 SDValue DAGTypeLegalizer::WidenVecRes_LOAD(SDNode *N) {
4011 LoadSDNode *LD = cast<LoadSDNode>(N);
4012 ISD::LoadExtType ExtType = LD->getExtensionType();
4013
4014 // A vector must always be stored in memory as-is, i.e. without any padding
4015 // between the elements, since various code depend on it, e.g. in the
4016 // handling of a bitcast of a vector type to int, which may be done with a
4017 // vector store followed by an integer load. A vector that does not have
4018 // elements that are byte-sized must therefore be stored as an integer
4019 // built out of the extracted vector elements.
4020 if (!LD->getMemoryVT().isByteSized()) {
4021 SDValue Value, NewChain;
4022 std::tie(Value, NewChain) = TLI.scalarizeVectorLoad(LD, DAG);
4023 ReplaceValueWith(SDValue(LD, 0), Value);
4024 ReplaceValueWith(SDValue(LD, 1), NewChain);
4025 return SDValue();
4026 }
4027
4028 SDValue Result;
4029 SmallVector<SDValue, 16> LdChain; // Chain for the series of load
4030 if (ExtType != ISD::NON_EXTLOAD)
4031 Result = GenWidenVectorExtLoads(LdChain, LD, ExtType);
4032 else
4033 Result = GenWidenVectorLoads(LdChain, LD);
4034
4035 // If we generate a single load, we can use that for the chain. Otherwise,
4036 // build a factor node to remember the multiple loads are independent and
4037 // chain to that.
4038 SDValue NewChain;
4039 if (LdChain.size() == 1)
4040 NewChain = LdChain[0];
4041 else
4042 NewChain = DAG.getNode(ISD::TokenFactor, SDLoc(LD), MVT::Other, LdChain);
4043
4044 // Modified the chain - switch anything that used the old chain to use
4045 // the new one.
4046 ReplaceValueWith(SDValue(N, 1), NewChain);
4047
4048 return Result;
4049 }
4050
WidenVecRes_MLOAD(MaskedLoadSDNode * N)4051 SDValue DAGTypeLegalizer::WidenVecRes_MLOAD(MaskedLoadSDNode *N) {
4052
4053 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),N->getValueType(0));
4054 SDValue Mask = N->getMask();
4055 EVT MaskVT = Mask.getValueType();
4056 SDValue PassThru = GetWidenedVector(N->getPassThru());
4057 ISD::LoadExtType ExtType = N->getExtensionType();
4058 SDLoc dl(N);
4059
4060 // The mask should be widened as well
4061 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4062 MaskVT.getVectorElementType(),
4063 WidenVT.getVectorNumElements());
4064 Mask = ModifyToType(Mask, WideMaskVT, true);
4065
4066 SDValue Res = DAG.getMaskedLoad(
4067 WidenVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
4068 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
4069 ExtType, N->isExpandingLoad());
4070 // Legalize the chain result - switch anything that used the old chain to
4071 // use the new one.
4072 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4073 return Res;
4074 }
4075
WidenVecRes_MGATHER(MaskedGatherSDNode * N)4076 SDValue DAGTypeLegalizer::WidenVecRes_MGATHER(MaskedGatherSDNode *N) {
4077
4078 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4079 SDValue Mask = N->getMask();
4080 EVT MaskVT = Mask.getValueType();
4081 SDValue PassThru = GetWidenedVector(N->getPassThru());
4082 SDValue Scale = N->getScale();
4083 unsigned NumElts = WideVT.getVectorNumElements();
4084 SDLoc dl(N);
4085
4086 // The mask should be widened as well
4087 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4088 MaskVT.getVectorElementType(),
4089 WideVT.getVectorNumElements());
4090 Mask = ModifyToType(Mask, WideMaskVT, true);
4091
4092 // Widen the Index operand
4093 SDValue Index = N->getIndex();
4094 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4095 Index.getValueType().getScalarType(),
4096 NumElts);
4097 Index = ModifyToType(Index, WideIndexVT);
4098 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
4099 Scale };
4100
4101 // Widen the MemoryType
4102 EVT WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4103 N->getMemoryVT().getScalarType(), NumElts);
4104 SDValue Res = DAG.getMaskedGather(DAG.getVTList(WideVT, MVT::Other),
4105 WideMemVT, dl, Ops, N->getMemOperand(),
4106 N->getIndexType(), N->getExtensionType());
4107
4108 // Legalize the chain result - switch anything that used the old chain to
4109 // use the new one.
4110 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4111 return Res;
4112 }
4113
WidenVecRes_ScalarOp(SDNode * N)4114 SDValue DAGTypeLegalizer::WidenVecRes_ScalarOp(SDNode *N) {
4115 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4116 return DAG.getNode(N->getOpcode(), SDLoc(N), WidenVT, N->getOperand(0));
4117 }
4118
4119 // Return true is this is a SETCC node or a strict version of it.
isSETCCOp(unsigned Opcode)4120 static inline bool isSETCCOp(unsigned Opcode) {
4121 switch (Opcode) {
4122 case ISD::SETCC:
4123 case ISD::STRICT_FSETCC:
4124 case ISD::STRICT_FSETCCS:
4125 return true;
4126 }
4127 return false;
4128 }
4129
4130 // Return true if this is a node that could have two SETCCs as operands.
isLogicalMaskOp(unsigned Opcode)4131 static inline bool isLogicalMaskOp(unsigned Opcode) {
4132 switch (Opcode) {
4133 case ISD::AND:
4134 case ISD::OR:
4135 case ISD::XOR:
4136 return true;
4137 }
4138 return false;
4139 }
4140
4141 // If N is a SETCC or a strict variant of it, return the type
4142 // of the compare operands.
getSETCCOperandType(SDValue N)4143 static inline EVT getSETCCOperandType(SDValue N) {
4144 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
4145 return N->getOperand(OpNo).getValueType();
4146 }
4147
4148 // This is used just for the assert in convertMask(). Check that this either
4149 // a SETCC or a previously handled SETCC by convertMask().
4150 #ifndef NDEBUG
isSETCCorConvertedSETCC(SDValue N)4151 static inline bool isSETCCorConvertedSETCC(SDValue N) {
4152 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR)
4153 N = N.getOperand(0);
4154 else if (N.getOpcode() == ISD::CONCAT_VECTORS) {
4155 for (unsigned i = 1; i < N->getNumOperands(); ++i)
4156 if (!N->getOperand(i)->isUndef())
4157 return false;
4158 N = N.getOperand(0);
4159 }
4160
4161 if (N.getOpcode() == ISD::TRUNCATE)
4162 N = N.getOperand(0);
4163 else if (N.getOpcode() == ISD::SIGN_EXTEND)
4164 N = N.getOperand(0);
4165
4166 if (isLogicalMaskOp(N.getOpcode()))
4167 return isSETCCorConvertedSETCC(N.getOperand(0)) &&
4168 isSETCCorConvertedSETCC(N.getOperand(1));
4169
4170 return (isSETCCOp(N.getOpcode()) ||
4171 ISD::isBuildVectorOfConstantSDNodes(N.getNode()));
4172 }
4173 #endif
4174
4175 // Return a mask of vector type MaskVT to replace InMask. Also adjust MaskVT
4176 // to ToMaskVT if needed with vector extension or truncation.
convertMask(SDValue InMask,EVT MaskVT,EVT ToMaskVT)4177 SDValue DAGTypeLegalizer::convertMask(SDValue InMask, EVT MaskVT,
4178 EVT ToMaskVT) {
4179 // Currently a SETCC or a AND/OR/XOR with two SETCCs are handled.
4180 // FIXME: This code seems to be too restrictive, we might consider
4181 // generalizing it or dropping it.
4182 assert(isSETCCorConvertedSETCC(InMask) && "Unexpected mask argument.");
4183
4184 // Make a new Mask node, with a legal result VT.
4185 SDValue Mask;
4186 SmallVector<SDValue, 4> Ops;
4187 for (unsigned i = 0, e = InMask->getNumOperands(); i < e; ++i)
4188 Ops.push_back(InMask->getOperand(i));
4189 if (InMask->isStrictFPOpcode()) {
4190 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask),
4191 { MaskVT, MVT::Other }, Ops);
4192 ReplaceValueWith(InMask.getValue(1), Mask.getValue(1));
4193 }
4194 else
4195 Mask = DAG.getNode(InMask->getOpcode(), SDLoc(InMask), MaskVT, Ops);
4196
4197 // If MaskVT has smaller or bigger elements than ToMaskVT, a vector sign
4198 // extend or truncate is needed.
4199 LLVMContext &Ctx = *DAG.getContext();
4200 unsigned MaskScalarBits = MaskVT.getScalarSizeInBits();
4201 unsigned ToMaskScalBits = ToMaskVT.getScalarSizeInBits();
4202 if (MaskScalarBits < ToMaskScalBits) {
4203 EVT ExtVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4204 MaskVT.getVectorNumElements());
4205 Mask = DAG.getNode(ISD::SIGN_EXTEND, SDLoc(Mask), ExtVT, Mask);
4206 } else if (MaskScalarBits > ToMaskScalBits) {
4207 EVT TruncVT = EVT::getVectorVT(Ctx, ToMaskVT.getVectorElementType(),
4208 MaskVT.getVectorNumElements());
4209 Mask = DAG.getNode(ISD::TRUNCATE, SDLoc(Mask), TruncVT, Mask);
4210 }
4211
4212 assert(Mask->getValueType(0).getScalarSizeInBits() ==
4213 ToMaskVT.getScalarSizeInBits() &&
4214 "Mask should have the right element size by now.");
4215
4216 // Adjust Mask to the right number of elements.
4217 unsigned CurrMaskNumEls = Mask->getValueType(0).getVectorNumElements();
4218 if (CurrMaskNumEls > ToMaskVT.getVectorNumElements()) {
4219 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, SDLoc(Mask));
4220 Mask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Mask), ToMaskVT, Mask,
4221 ZeroIdx);
4222 } else if (CurrMaskNumEls < ToMaskVT.getVectorNumElements()) {
4223 unsigned NumSubVecs = (ToMaskVT.getVectorNumElements() / CurrMaskNumEls);
4224 EVT SubVT = Mask->getValueType(0);
4225 SmallVector<SDValue, 16> SubOps(NumSubVecs, DAG.getUNDEF(SubVT));
4226 SubOps[0] = Mask;
4227 Mask = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Mask), ToMaskVT, SubOps);
4228 }
4229
4230 assert((Mask->getValueType(0) == ToMaskVT) &&
4231 "A mask of ToMaskVT should have been produced by now.");
4232
4233 return Mask;
4234 }
4235
4236 // This method tries to handle some special cases for the vselect mask
4237 // and if needed adjusting the mask vector type to match that of the VSELECT.
4238 // Without it, many cases end up with scalarization of the SETCC, with many
4239 // unnecessary instructions.
WidenVSELECTMask(SDNode * N)4240 SDValue DAGTypeLegalizer::WidenVSELECTMask(SDNode *N) {
4241 LLVMContext &Ctx = *DAG.getContext();
4242 SDValue Cond = N->getOperand(0);
4243
4244 if (N->getOpcode() != ISD::VSELECT)
4245 return SDValue();
4246
4247 if (!isSETCCOp(Cond->getOpcode()) && !isLogicalMaskOp(Cond->getOpcode()))
4248 return SDValue();
4249
4250 // If this is a splitted VSELECT that was previously already handled, do
4251 // nothing.
4252 EVT CondVT = Cond->getValueType(0);
4253 if (CondVT.getScalarSizeInBits() != 1)
4254 return SDValue();
4255
4256 EVT VSelVT = N->getValueType(0);
4257 // Only handle vector types which are a power of 2.
4258 if (!isPowerOf2_64(VSelVT.getSizeInBits()))
4259 return SDValue();
4260
4261 // Don't touch if this will be scalarized.
4262 EVT FinalVT = VSelVT;
4263 while (getTypeAction(FinalVT) == TargetLowering::TypeSplitVector)
4264 FinalVT = FinalVT.getHalfNumVectorElementsVT(Ctx);
4265
4266 if (FinalVT.getVectorNumElements() == 1)
4267 return SDValue();
4268
4269 // If there is support for an i1 vector mask, don't touch.
4270 if (isSETCCOp(Cond.getOpcode())) {
4271 EVT SetCCOpVT = getSETCCOperandType(Cond);
4272 while (TLI.getTypeAction(Ctx, SetCCOpVT) != TargetLowering::TypeLegal)
4273 SetCCOpVT = TLI.getTypeToTransformTo(Ctx, SetCCOpVT);
4274 EVT SetCCResVT = getSetCCResultType(SetCCOpVT);
4275 if (SetCCResVT.getScalarSizeInBits() == 1)
4276 return SDValue();
4277 } else if (CondVT.getScalarType() == MVT::i1) {
4278 // If there is support for an i1 vector mask (or only scalar i1 conditions),
4279 // don't touch.
4280 while (TLI.getTypeAction(Ctx, CondVT) != TargetLowering::TypeLegal)
4281 CondVT = TLI.getTypeToTransformTo(Ctx, CondVT);
4282
4283 if (CondVT.getScalarType() == MVT::i1)
4284 return SDValue();
4285 }
4286
4287 // Widen the vselect result type if needed.
4288 if (getTypeAction(VSelVT) == TargetLowering::TypeWidenVector)
4289 VSelVT = TLI.getTypeToTransformTo(Ctx, VSelVT);
4290
4291 // The mask of the VSELECT should have integer elements.
4292 EVT ToMaskVT = VSelVT;
4293 if (!ToMaskVT.getScalarType().isInteger())
4294 ToMaskVT = ToMaskVT.changeVectorElementTypeToInteger();
4295
4296 SDValue Mask;
4297 if (isSETCCOp(Cond->getOpcode())) {
4298 EVT MaskVT = getSetCCResultType(getSETCCOperandType(Cond));
4299 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4300 } else if (isLogicalMaskOp(Cond->getOpcode()) &&
4301 isSETCCOp(Cond->getOperand(0).getOpcode()) &&
4302 isSETCCOp(Cond->getOperand(1).getOpcode())) {
4303 // Cond is (AND/OR/XOR (SETCC, SETCC))
4304 SDValue SETCC0 = Cond->getOperand(0);
4305 SDValue SETCC1 = Cond->getOperand(1);
4306 EVT VT0 = getSetCCResultType(getSETCCOperandType(SETCC0));
4307 EVT VT1 = getSetCCResultType(getSETCCOperandType(SETCC1));
4308 unsigned ScalarBits0 = VT0.getScalarSizeInBits();
4309 unsigned ScalarBits1 = VT1.getScalarSizeInBits();
4310 unsigned ScalarBits_ToMask = ToMaskVT.getScalarSizeInBits();
4311 EVT MaskVT;
4312 // If the two SETCCs have different VTs, either extend/truncate one of
4313 // them to the other "towards" ToMaskVT, or truncate one and extend the
4314 // other to ToMaskVT.
4315 if (ScalarBits0 != ScalarBits1) {
4316 EVT NarrowVT = ((ScalarBits0 < ScalarBits1) ? VT0 : VT1);
4317 EVT WideVT = ((NarrowVT == VT0) ? VT1 : VT0);
4318 if (ScalarBits_ToMask >= WideVT.getScalarSizeInBits())
4319 MaskVT = WideVT;
4320 else if (ScalarBits_ToMask <= NarrowVT.getScalarSizeInBits())
4321 MaskVT = NarrowVT;
4322 else
4323 MaskVT = ToMaskVT;
4324 } else
4325 // If the two SETCCs have the same VT, don't change it.
4326 MaskVT = VT0;
4327
4328 // Make new SETCCs and logical nodes.
4329 SETCC0 = convertMask(SETCC0, VT0, MaskVT);
4330 SETCC1 = convertMask(SETCC1, VT1, MaskVT);
4331 Cond = DAG.getNode(Cond->getOpcode(), SDLoc(Cond), MaskVT, SETCC0, SETCC1);
4332
4333 // Convert the logical op for VSELECT if needed.
4334 Mask = convertMask(Cond, MaskVT, ToMaskVT);
4335 } else
4336 return SDValue();
4337
4338 return Mask;
4339 }
4340
WidenVecRes_SELECT(SDNode * N)4341 SDValue DAGTypeLegalizer::WidenVecRes_SELECT(SDNode *N) {
4342 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4343 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4344
4345 SDValue Cond1 = N->getOperand(0);
4346 EVT CondVT = Cond1.getValueType();
4347 if (CondVT.isVector()) {
4348 if (SDValue WideCond = WidenVSELECTMask(N)) {
4349 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4350 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4351 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4352 return DAG.getNode(N->getOpcode(), SDLoc(N),
4353 WidenVT, WideCond, InOp1, InOp2);
4354 }
4355
4356 EVT CondEltVT = CondVT.getVectorElementType();
4357 EVT CondWidenVT = EVT::getVectorVT(*DAG.getContext(),
4358 CondEltVT, WidenNumElts);
4359 if (getTypeAction(CondVT) == TargetLowering::TypeWidenVector)
4360 Cond1 = GetWidenedVector(Cond1);
4361
4362 // If we have to split the condition there is no point in widening the
4363 // select. This would result in an cycle of widening the select ->
4364 // widening the condition operand -> splitting the condition operand ->
4365 // splitting the select -> widening the select. Instead split this select
4366 // further and widen the resulting type.
4367 if (getTypeAction(CondVT) == TargetLowering::TypeSplitVector) {
4368 SDValue SplitSelect = SplitVecOp_VSELECT(N, 0);
4369 SDValue Res = ModifyToType(SplitSelect, WidenVT);
4370 return Res;
4371 }
4372
4373 if (Cond1.getValueType() != CondWidenVT)
4374 Cond1 = ModifyToType(Cond1, CondWidenVT);
4375 }
4376
4377 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4378 SDValue InOp2 = GetWidenedVector(N->getOperand(2));
4379 assert(InOp1.getValueType() == WidenVT && InOp2.getValueType() == WidenVT);
4380 return DAG.getNode(N->getOpcode(), SDLoc(N),
4381 WidenVT, Cond1, InOp1, InOp2);
4382 }
4383
WidenVecRes_SELECT_CC(SDNode * N)4384 SDValue DAGTypeLegalizer::WidenVecRes_SELECT_CC(SDNode *N) {
4385 SDValue InOp1 = GetWidenedVector(N->getOperand(2));
4386 SDValue InOp2 = GetWidenedVector(N->getOperand(3));
4387 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
4388 InOp1.getValueType(), N->getOperand(0),
4389 N->getOperand(1), InOp1, InOp2, N->getOperand(4));
4390 }
4391
WidenVecRes_UNDEF(SDNode * N)4392 SDValue DAGTypeLegalizer::WidenVecRes_UNDEF(SDNode *N) {
4393 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4394 return DAG.getUNDEF(WidenVT);
4395 }
4396
WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode * N)4397 SDValue DAGTypeLegalizer::WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N) {
4398 EVT VT = N->getValueType(0);
4399 SDLoc dl(N);
4400
4401 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4402 unsigned NumElts = VT.getVectorNumElements();
4403 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4404
4405 SDValue InOp1 = GetWidenedVector(N->getOperand(0));
4406 SDValue InOp2 = GetWidenedVector(N->getOperand(1));
4407
4408 // Adjust mask based on new input vector length.
4409 SmallVector<int, 16> NewMask;
4410 for (unsigned i = 0; i != NumElts; ++i) {
4411 int Idx = N->getMaskElt(i);
4412 if (Idx < (int)NumElts)
4413 NewMask.push_back(Idx);
4414 else
4415 NewMask.push_back(Idx - NumElts + WidenNumElts);
4416 }
4417 for (unsigned i = NumElts; i != WidenNumElts; ++i)
4418 NewMask.push_back(-1);
4419 return DAG.getVectorShuffle(WidenVT, dl, InOp1, InOp2, NewMask);
4420 }
4421
WidenVecRes_SETCC(SDNode * N)4422 SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) {
4423 assert(N->getValueType(0).isVector() &&
4424 N->getOperand(0).getValueType().isVector() &&
4425 "Operands must be vectors");
4426 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4427 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4428
4429 SDValue InOp1 = N->getOperand(0);
4430 EVT InVT = InOp1.getValueType();
4431 assert(InVT.isVector() && "can not widen non-vector type");
4432 EVT WidenInVT = EVT::getVectorVT(*DAG.getContext(),
4433 InVT.getVectorElementType(), WidenNumElts);
4434
4435 // The input and output types often differ here, and it could be that while
4436 // we'd prefer to widen the result type, the input operands have been split.
4437 // In this case, we also need to split the result of this node as well.
4438 if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) {
4439 SDValue SplitVSetCC = SplitVecOp_VSETCC(N);
4440 SDValue Res = ModifyToType(SplitVSetCC, WidenVT);
4441 return Res;
4442 }
4443
4444 // If the inputs also widen, handle them directly. Otherwise widen by hand.
4445 SDValue InOp2 = N->getOperand(1);
4446 if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) {
4447 InOp1 = GetWidenedVector(InOp1);
4448 InOp2 = GetWidenedVector(InOp2);
4449 } else {
4450 InOp1 = DAG.WidenVector(InOp1, SDLoc(N));
4451 InOp2 = DAG.WidenVector(InOp2, SDLoc(N));
4452 }
4453
4454 // Assume that the input and output will be widen appropriately. If not,
4455 // we will have to unroll it at some point.
4456 assert(InOp1.getValueType() == WidenInVT &&
4457 InOp2.getValueType() == WidenInVT &&
4458 "Input not widened to expected type!");
4459 (void)WidenInVT;
4460 return DAG.getNode(ISD::SETCC, SDLoc(N),
4461 WidenVT, InOp1, InOp2, N->getOperand(2));
4462 }
4463
WidenVecRes_STRICT_FSETCC(SDNode * N)4464 SDValue DAGTypeLegalizer::WidenVecRes_STRICT_FSETCC(SDNode *N) {
4465 assert(N->getValueType(0).isVector() &&
4466 N->getOperand(1).getValueType().isVector() &&
4467 "Operands must be vectors");
4468 EVT VT = N->getValueType(0);
4469 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4470 unsigned WidenNumElts = WidenVT.getVectorNumElements();
4471 unsigned NumElts = VT.getVectorNumElements();
4472 EVT EltVT = VT.getVectorElementType();
4473
4474 SDLoc dl(N);
4475 SDValue Chain = N->getOperand(0);
4476 SDValue LHS = N->getOperand(1);
4477 SDValue RHS = N->getOperand(2);
4478 SDValue CC = N->getOperand(3);
4479 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
4480
4481 // Fully unroll and reassemble.
4482 SmallVector<SDValue, 8> Scalars(WidenNumElts, DAG.getUNDEF(EltVT));
4483 SmallVector<SDValue, 8> Chains(NumElts);
4484 for (unsigned i = 0; i != NumElts; ++i) {
4485 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
4486 DAG.getVectorIdxConstant(i, dl));
4487 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
4488 DAG.getVectorIdxConstant(i, dl));
4489
4490 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
4491 {Chain, LHSElem, RHSElem, CC});
4492 Chains[i] = Scalars[i].getValue(1);
4493 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
4494 DAG.getBoolConstant(true, dl, EltVT, VT),
4495 DAG.getBoolConstant(false, dl, EltVT, VT));
4496 }
4497
4498 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
4499 ReplaceValueWith(SDValue(N, 1), NewChain);
4500
4501 return DAG.getBuildVector(WidenVT, dl, Scalars);
4502 }
4503
4504 //===----------------------------------------------------------------------===//
4505 // Widen Vector Operand
4506 //===----------------------------------------------------------------------===//
WidenVectorOperand(SDNode * N,unsigned OpNo)4507 bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
4508 LLVM_DEBUG(dbgs() << "Widen node operand " << OpNo << ": "; N->dump(&DAG);
4509 dbgs() << "\n");
4510 SDValue Res = SDValue();
4511
4512 // See if the target wants to custom widen this node.
4513 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
4514 return false;
4515
4516 switch (N->getOpcode()) {
4517 default:
4518 #ifndef NDEBUG
4519 dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
4520 N->dump(&DAG);
4521 dbgs() << "\n";
4522 #endif
4523 llvm_unreachable("Do not know how to widen this operator's operand!");
4524
4525 case ISD::BITCAST: Res = WidenVecOp_BITCAST(N); break;
4526 case ISD::CONCAT_VECTORS: Res = WidenVecOp_CONCAT_VECTORS(N); break;
4527 case ISD::INSERT_SUBVECTOR: Res = WidenVecOp_INSERT_SUBVECTOR(N); break;
4528 case ISD::EXTRACT_SUBVECTOR: Res = WidenVecOp_EXTRACT_SUBVECTOR(N); break;
4529 case ISD::EXTRACT_VECTOR_ELT: Res = WidenVecOp_EXTRACT_VECTOR_ELT(N); break;
4530 case ISD::STORE: Res = WidenVecOp_STORE(N); break;
4531 case ISD::MSTORE: Res = WidenVecOp_MSTORE(N, OpNo); break;
4532 case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break;
4533 case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break;
4534 case ISD::SETCC: Res = WidenVecOp_SETCC(N); break;
4535 case ISD::STRICT_FSETCC:
4536 case ISD::STRICT_FSETCCS: Res = WidenVecOp_STRICT_FSETCC(N); break;
4537 case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break;
4538 case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break;
4539
4540 case ISD::ANY_EXTEND:
4541 case ISD::SIGN_EXTEND:
4542 case ISD::ZERO_EXTEND:
4543 Res = WidenVecOp_EXTEND(N);
4544 break;
4545
4546 case ISD::FP_EXTEND:
4547 case ISD::STRICT_FP_EXTEND:
4548 case ISD::FP_ROUND:
4549 case ISD::STRICT_FP_ROUND:
4550 case ISD::FP_TO_SINT:
4551 case ISD::STRICT_FP_TO_SINT:
4552 case ISD::FP_TO_UINT:
4553 case ISD::STRICT_FP_TO_UINT:
4554 case ISD::SINT_TO_FP:
4555 case ISD::STRICT_SINT_TO_FP:
4556 case ISD::UINT_TO_FP:
4557 case ISD::STRICT_UINT_TO_FP:
4558 case ISD::TRUNCATE:
4559 Res = WidenVecOp_Convert(N);
4560 break;
4561
4562 case ISD::FP_TO_SINT_SAT:
4563 case ISD::FP_TO_UINT_SAT:
4564 Res = WidenVecOp_FP_TO_XINT_SAT(N);
4565 break;
4566
4567 case ISD::VECREDUCE_FADD:
4568 case ISD::VECREDUCE_FMUL:
4569 case ISD::VECREDUCE_ADD:
4570 case ISD::VECREDUCE_MUL:
4571 case ISD::VECREDUCE_AND:
4572 case ISD::VECREDUCE_OR:
4573 case ISD::VECREDUCE_XOR:
4574 case ISD::VECREDUCE_SMAX:
4575 case ISD::VECREDUCE_SMIN:
4576 case ISD::VECREDUCE_UMAX:
4577 case ISD::VECREDUCE_UMIN:
4578 case ISD::VECREDUCE_FMAX:
4579 case ISD::VECREDUCE_FMIN:
4580 Res = WidenVecOp_VECREDUCE(N);
4581 break;
4582 case ISD::VECREDUCE_SEQ_FADD:
4583 case ISD::VECREDUCE_SEQ_FMUL:
4584 Res = WidenVecOp_VECREDUCE_SEQ(N);
4585 break;
4586 }
4587
4588 // If Res is null, the sub-method took care of registering the result.
4589 if (!Res.getNode()) return false;
4590
4591 // If the result is N, the sub-method updated N in place. Tell the legalizer
4592 // core about this.
4593 if (Res.getNode() == N)
4594 return true;
4595
4596
4597 if (N->isStrictFPOpcode())
4598 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 &&
4599 "Invalid operand expansion");
4600 else
4601 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
4602 "Invalid operand expansion");
4603
4604 ReplaceValueWith(SDValue(N, 0), Res);
4605 return false;
4606 }
4607
WidenVecOp_EXTEND(SDNode * N)4608 SDValue DAGTypeLegalizer::WidenVecOp_EXTEND(SDNode *N) {
4609 SDLoc DL(N);
4610 EVT VT = N->getValueType(0);
4611
4612 SDValue InOp = N->getOperand(0);
4613 assert(getTypeAction(InOp.getValueType()) ==
4614 TargetLowering::TypeWidenVector &&
4615 "Unexpected type action");
4616 InOp = GetWidenedVector(InOp);
4617 assert(VT.getVectorNumElements() <
4618 InOp.getValueType().getVectorNumElements() &&
4619 "Input wasn't widened!");
4620
4621 // We may need to further widen the operand until it has the same total
4622 // vector size as the result.
4623 EVT InVT = InOp.getValueType();
4624 if (InVT.getSizeInBits() != VT.getSizeInBits()) {
4625 EVT InEltVT = InVT.getVectorElementType();
4626 for (int i = MVT::FIRST_VECTOR_VALUETYPE, e = MVT::LAST_VECTOR_VALUETYPE; i < e; ++i) {
4627 EVT FixedVT = (MVT::SimpleValueType)i;
4628 EVT FixedEltVT = FixedVT.getVectorElementType();
4629 if (TLI.isTypeLegal(FixedVT) &&
4630 FixedVT.getSizeInBits() == VT.getSizeInBits() &&
4631 FixedEltVT == InEltVT) {
4632 assert(FixedVT.getVectorNumElements() >= VT.getVectorNumElements() &&
4633 "Not enough elements in the fixed type for the operand!");
4634 assert(FixedVT.getVectorNumElements() != InVT.getVectorNumElements() &&
4635 "We can't have the same type as we started with!");
4636 if (FixedVT.getVectorNumElements() > InVT.getVectorNumElements())
4637 InOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, FixedVT,
4638 DAG.getUNDEF(FixedVT), InOp,
4639 DAG.getVectorIdxConstant(0, DL));
4640 else
4641 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, FixedVT, InOp,
4642 DAG.getVectorIdxConstant(0, DL));
4643 break;
4644 }
4645 }
4646 InVT = InOp.getValueType();
4647 if (InVT.getSizeInBits() != VT.getSizeInBits())
4648 // We couldn't find a legal vector type that was a widening of the input
4649 // and could be extended in-register to the result type, so we have to
4650 // scalarize.
4651 return WidenVecOp_Convert(N);
4652 }
4653
4654 // Use special DAG nodes to represent the operation of extending the
4655 // low lanes.
4656 switch (N->getOpcode()) {
4657 default:
4658 llvm_unreachable("Extend legalization on extend operation!");
4659 case ISD::ANY_EXTEND:
4660 return DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, InOp);
4661 case ISD::SIGN_EXTEND:
4662 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, VT, InOp);
4663 case ISD::ZERO_EXTEND:
4664 return DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, DL, VT, InOp);
4665 }
4666 }
4667
WidenVecOp_FCOPYSIGN(SDNode * N)4668 SDValue DAGTypeLegalizer::WidenVecOp_FCOPYSIGN(SDNode *N) {
4669 // The result (and first input) is legal, but the second input is illegal.
4670 // We can't do much to fix that, so just unroll and let the extracts off of
4671 // the second input be widened as needed later.
4672 return DAG.UnrollVectorOp(N);
4673 }
4674
WidenVecOp_Convert(SDNode * N)4675 SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) {
4676 // Since the result is legal and the input is illegal.
4677 EVT VT = N->getValueType(0);
4678 EVT EltVT = VT.getVectorElementType();
4679 SDLoc dl(N);
4680 unsigned NumElts = VT.getVectorNumElements();
4681 SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
4682 assert(getTypeAction(InOp.getValueType()) ==
4683 TargetLowering::TypeWidenVector &&
4684 "Unexpected type action");
4685 InOp = GetWidenedVector(InOp);
4686 EVT InVT = InOp.getValueType();
4687 unsigned Opcode = N->getOpcode();
4688
4689 // See if a widened result type would be legal, if so widen the node.
4690 // FIXME: This isn't safe for StrictFP. Other optimization here is needed.
4691 EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
4692 InVT.getVectorNumElements());
4693 if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) {
4694 SDValue Res;
4695 if (N->isStrictFPOpcode()) {
4696 if (Opcode == ISD::STRICT_FP_ROUND)
4697 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4698 { N->getOperand(0), InOp, N->getOperand(2) });
4699 else
4700 Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other },
4701 { N->getOperand(0), InOp });
4702 // Legalize the chain result - switch anything that used the old chain to
4703 // use the new one.
4704 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4705 } else {
4706 if (Opcode == ISD::FP_ROUND)
4707 Res = DAG.getNode(Opcode, dl, WideVT, InOp, N->getOperand(1));
4708 else
4709 Res = DAG.getNode(Opcode, dl, WideVT, InOp);
4710 }
4711 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
4712 DAG.getVectorIdxConstant(0, dl));
4713 }
4714
4715 EVT InEltVT = InVT.getVectorElementType();
4716
4717 // Unroll the convert into some scalar code and create a nasty build vector.
4718 SmallVector<SDValue, 16> Ops(NumElts);
4719 if (N->isStrictFPOpcode()) {
4720 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
4721 SmallVector<SDValue, 32> OpChains;
4722 for (unsigned i=0; i < NumElts; ++i) {
4723 NewOps[1] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp,
4724 DAG.getVectorIdxConstant(i, dl));
4725 Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps);
4726 OpChains.push_back(Ops[i].getValue(1));
4727 }
4728 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
4729 ReplaceValueWith(SDValue(N, 1), NewChain);
4730 } else {
4731 for (unsigned i = 0; i < NumElts; ++i)
4732 Ops[i] = DAG.getNode(Opcode, dl, EltVT,
4733 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT,
4734 InOp, DAG.getVectorIdxConstant(i, dl)));
4735 }
4736
4737 return DAG.getBuildVector(VT, dl, Ops);
4738 }
4739
WidenVecOp_FP_TO_XINT_SAT(SDNode * N)4740 SDValue DAGTypeLegalizer::WidenVecOp_FP_TO_XINT_SAT(SDNode *N) {
4741 EVT DstVT = N->getValueType(0);
4742 SDValue Src = GetWidenedVector(N->getOperand(0));
4743 EVT SrcVT = Src.getValueType();
4744 ElementCount WideNumElts = SrcVT.getVectorElementCount();
4745 SDLoc dl(N);
4746
4747 // See if a widened result type would be legal, if so widen the node.
4748 EVT WideDstVT = EVT::getVectorVT(*DAG.getContext(),
4749 DstVT.getVectorElementType(), WideNumElts);
4750 if (TLI.isTypeLegal(WideDstVT)) {
4751 SDValue Res =
4752 DAG.getNode(N->getOpcode(), dl, WideDstVT, Src, N->getOperand(1));
4753 return DAG.getNode(
4754 ISD::EXTRACT_SUBVECTOR, dl, DstVT, Res,
4755 DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4756 }
4757
4758 // Give up and unroll.
4759 return DAG.UnrollVectorOp(N);
4760 }
4761
WidenVecOp_BITCAST(SDNode * N)4762 SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) {
4763 EVT VT = N->getValueType(0);
4764 SDValue InOp = GetWidenedVector(N->getOperand(0));
4765 EVT InWidenVT = InOp.getValueType();
4766 SDLoc dl(N);
4767
4768 // Check if we can convert between two legal vector types and extract.
4769 unsigned InWidenSize = InWidenVT.getSizeInBits();
4770 unsigned Size = VT.getSizeInBits();
4771 // x86mmx is not an acceptable vector element type, so don't try.
4772 if (InWidenSize % Size == 0 && !VT.isVector() && VT != MVT::x86mmx) {
4773 unsigned NewNumElts = InWidenSize / Size;
4774 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), VT, NewNumElts);
4775 if (TLI.isTypeLegal(NewVT)) {
4776 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4777 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, BitOp,
4778 DAG.getVectorIdxConstant(0, dl));
4779 }
4780 }
4781
4782 // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened
4783 // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not,
4784 // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids
4785 // having to copy via memory.
4786 if (VT.isVector()) {
4787 EVT EltVT = VT.getVectorElementType();
4788 unsigned EltSize = EltVT.getSizeInBits();
4789 if (InWidenSize % EltSize == 0) {
4790 unsigned NewNumElts = InWidenSize / EltSize;
4791 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts);
4792 if (TLI.isTypeLegal(NewVT)) {
4793 SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp);
4794 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp,
4795 DAG.getVectorIdxConstant(0, dl));
4796 }
4797 }
4798 }
4799
4800 return CreateStackStoreLoad(InOp, VT);
4801 }
4802
WidenVecOp_CONCAT_VECTORS(SDNode * N)4803 SDValue DAGTypeLegalizer::WidenVecOp_CONCAT_VECTORS(SDNode *N) {
4804 EVT VT = N->getValueType(0);
4805 EVT EltVT = VT.getVectorElementType();
4806 EVT InVT = N->getOperand(0).getValueType();
4807 SDLoc dl(N);
4808
4809 // If the widen width for this operand is the same as the width of the concat
4810 // and all but the first operand is undef, just use the widened operand.
4811 unsigned NumOperands = N->getNumOperands();
4812 if (VT == TLI.getTypeToTransformTo(*DAG.getContext(), InVT)) {
4813 unsigned i;
4814 for (i = 1; i < NumOperands; ++i)
4815 if (!N->getOperand(i).isUndef())
4816 break;
4817
4818 if (i == NumOperands)
4819 return GetWidenedVector(N->getOperand(0));
4820 }
4821
4822 // Otherwise, fall back to a nasty build vector.
4823 unsigned NumElts = VT.getVectorNumElements();
4824 SmallVector<SDValue, 16> Ops(NumElts);
4825
4826 unsigned NumInElts = InVT.getVectorNumElements();
4827
4828 unsigned Idx = 0;
4829 for (unsigned i=0; i < NumOperands; ++i) {
4830 SDValue InOp = N->getOperand(i);
4831 assert(getTypeAction(InOp.getValueType()) ==
4832 TargetLowering::TypeWidenVector &&
4833 "Unexpected type action");
4834 InOp = GetWidenedVector(InOp);
4835 for (unsigned j = 0; j < NumInElts; ++j)
4836 Ops[Idx++] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
4837 DAG.getVectorIdxConstant(j, dl));
4838 }
4839 return DAG.getBuildVector(VT, dl, Ops);
4840 }
4841
WidenVecOp_INSERT_SUBVECTOR(SDNode * N)4842 SDValue DAGTypeLegalizer::WidenVecOp_INSERT_SUBVECTOR(SDNode *N) {
4843 SDValue SubVec = N->getOperand(1);
4844 SDValue InVec = N->getOperand(0);
4845
4846 if (getTypeAction(InVec.getValueType()) == TargetLowering::TypeWidenVector)
4847 InVec = GetWidenedVector(InVec);
4848
4849 if (getTypeAction(SubVec.getValueType()) == TargetLowering::TypeWidenVector)
4850 SubVec = GetWidenedVector(SubVec);
4851
4852 if (SubVec.getValueType() == InVec.getValueType() && InVec.isUndef() &&
4853 N->getConstantOperandVal(2) == 0)
4854 return SubVec;
4855
4856 report_fatal_error("Don't know how to widen the operands for "
4857 "INSERT_SUBVECTOR");
4858 }
4859
WidenVecOp_EXTRACT_SUBVECTOR(SDNode * N)4860 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N) {
4861 SDValue InOp = GetWidenedVector(N->getOperand(0));
4862 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N),
4863 N->getValueType(0), InOp, N->getOperand(1));
4864 }
4865
WidenVecOp_EXTRACT_VECTOR_ELT(SDNode * N)4866 SDValue DAGTypeLegalizer::WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4867 SDValue InOp = GetWidenedVector(N->getOperand(0));
4868 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
4869 N->getValueType(0), InOp, N->getOperand(1));
4870 }
4871
WidenVecOp_STORE(SDNode * N)4872 SDValue DAGTypeLegalizer::WidenVecOp_STORE(SDNode *N) {
4873 // We have to widen the value, but we want only to store the original
4874 // vector type.
4875 StoreSDNode *ST = cast<StoreSDNode>(N);
4876
4877 if (!ST->getMemoryVT().getScalarType().isByteSized())
4878 return TLI.scalarizeVectorStore(ST, DAG);
4879
4880 if (ST->isTruncatingStore())
4881 return TLI.scalarizeVectorStore(ST, DAG);
4882
4883 SmallVector<SDValue, 16> StChain;
4884 GenWidenVectorStores(StChain, ST);
4885
4886 if (StChain.size() == 1)
4887 return StChain[0];
4888 else
4889 return DAG.getNode(ISD::TokenFactor, SDLoc(ST), MVT::Other, StChain);
4890 }
4891
WidenVecOp_MSTORE(SDNode * N,unsigned OpNo)4892 SDValue DAGTypeLegalizer::WidenVecOp_MSTORE(SDNode *N, unsigned OpNo) {
4893 assert((OpNo == 1 || OpNo == 3) &&
4894 "Can widen only data or mask operand of mstore");
4895 MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
4896 SDValue Mask = MST->getMask();
4897 EVT MaskVT = Mask.getValueType();
4898 SDValue StVal = MST->getValue();
4899 SDLoc dl(N);
4900
4901 if (OpNo == 1) {
4902 // Widen the value.
4903 StVal = GetWidenedVector(StVal);
4904
4905 // The mask should be widened as well.
4906 EVT WideVT = StVal.getValueType();
4907 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4908 MaskVT.getVectorElementType(),
4909 WideVT.getVectorNumElements());
4910 Mask = ModifyToType(Mask, WideMaskVT, true);
4911 } else {
4912 // Widen the mask.
4913 EVT WideMaskVT = TLI.getTypeToTransformTo(*DAG.getContext(), MaskVT);
4914 Mask = ModifyToType(Mask, WideMaskVT, true);
4915
4916 EVT ValueVT = StVal.getValueType();
4917 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
4918 ValueVT.getVectorElementType(),
4919 WideMaskVT.getVectorNumElements());
4920 StVal = ModifyToType(StVal, WideVT);
4921 }
4922
4923 assert(Mask.getValueType().getVectorNumElements() ==
4924 StVal.getValueType().getVectorNumElements() &&
4925 "Mask and data vectors should have the same number of elements");
4926 return DAG.getMaskedStore(MST->getChain(), dl, StVal, MST->getBasePtr(),
4927 MST->getOffset(), Mask, MST->getMemoryVT(),
4928 MST->getMemOperand(), MST->getAddressingMode(),
4929 false, MST->isCompressingStore());
4930 }
4931
WidenVecOp_MGATHER(SDNode * N,unsigned OpNo)4932 SDValue DAGTypeLegalizer::WidenVecOp_MGATHER(SDNode *N, unsigned OpNo) {
4933 assert(OpNo == 4 && "Can widen only the index of mgather");
4934 auto *MG = cast<MaskedGatherSDNode>(N);
4935 SDValue DataOp = MG->getPassThru();
4936 SDValue Mask = MG->getMask();
4937 SDValue Scale = MG->getScale();
4938
4939 // Just widen the index. It's allowed to have extra elements.
4940 SDValue Index = GetWidenedVector(MG->getIndex());
4941
4942 SDLoc dl(N);
4943 SDValue Ops[] = {MG->getChain(), DataOp, Mask, MG->getBasePtr(), Index,
4944 Scale};
4945 SDValue Res = DAG.getMaskedGather(MG->getVTList(), MG->getMemoryVT(), dl, Ops,
4946 MG->getMemOperand(), MG->getIndexType(),
4947 MG->getExtensionType());
4948 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
4949 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
4950 return SDValue();
4951 }
4952
WidenVecOp_MSCATTER(SDNode * N,unsigned OpNo)4953 SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) {
4954 MaskedScatterSDNode *MSC = cast<MaskedScatterSDNode>(N);
4955 SDValue DataOp = MSC->getValue();
4956 SDValue Mask = MSC->getMask();
4957 SDValue Index = MSC->getIndex();
4958 SDValue Scale = MSC->getScale();
4959 EVT WideMemVT = MSC->getMemoryVT();
4960
4961 if (OpNo == 1) {
4962 DataOp = GetWidenedVector(DataOp);
4963 unsigned NumElts = DataOp.getValueType().getVectorNumElements();
4964
4965 // Widen index.
4966 EVT IndexVT = Index.getValueType();
4967 EVT WideIndexVT = EVT::getVectorVT(*DAG.getContext(),
4968 IndexVT.getVectorElementType(), NumElts);
4969 Index = ModifyToType(Index, WideIndexVT);
4970
4971 // The mask should be widened as well.
4972 EVT MaskVT = Mask.getValueType();
4973 EVT WideMaskVT = EVT::getVectorVT(*DAG.getContext(),
4974 MaskVT.getVectorElementType(), NumElts);
4975 Mask = ModifyToType(Mask, WideMaskVT, true);
4976
4977 // Widen the MemoryType
4978 WideMemVT = EVT::getVectorVT(*DAG.getContext(),
4979 MSC->getMemoryVT().getScalarType(), NumElts);
4980 } else if (OpNo == 4) {
4981 // Just widen the index. It's allowed to have extra elements.
4982 Index = GetWidenedVector(Index);
4983 } else
4984 llvm_unreachable("Can't widen this operand of mscatter");
4985
4986 SDValue Ops[] = {MSC->getChain(), DataOp, Mask, MSC->getBasePtr(), Index,
4987 Scale};
4988 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), WideMemVT, SDLoc(N),
4989 Ops, MSC->getMemOperand(), MSC->getIndexType(),
4990 MSC->isTruncatingStore());
4991 }
4992
WidenVecOp_SETCC(SDNode * N)4993 SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) {
4994 SDValue InOp0 = GetWidenedVector(N->getOperand(0));
4995 SDValue InOp1 = GetWidenedVector(N->getOperand(1));
4996 SDLoc dl(N);
4997 EVT VT = N->getValueType(0);
4998
4999 // WARNING: In this code we widen the compare instruction with garbage.
5000 // This garbage may contain denormal floats which may be slow. Is this a real
5001 // concern ? Should we zero the unused lanes if this is a float compare ?
5002
5003 // Get a new SETCC node to compare the newly widened operands.
5004 // Only some of the compared elements are legal.
5005 EVT SVT = getSetCCResultType(InOp0.getValueType());
5006 // The result type is legal, if its vXi1, keep vXi1 for the new SETCC.
5007 if (VT.getScalarType() == MVT::i1)
5008 SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
5009 SVT.getVectorNumElements());
5010
5011 SDValue WideSETCC = DAG.getNode(ISD::SETCC, SDLoc(N),
5012 SVT, InOp0, InOp1, N->getOperand(2));
5013
5014 // Extract the needed results from the result vector.
5015 EVT ResVT = EVT::getVectorVT(*DAG.getContext(),
5016 SVT.getVectorElementType(),
5017 VT.getVectorNumElements());
5018 SDValue CC = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, WideSETCC,
5019 DAG.getVectorIdxConstant(0, dl));
5020
5021 EVT OpVT = N->getOperand(0).getValueType();
5022 ISD::NodeType ExtendCode =
5023 TargetLowering::getExtendForContent(TLI.getBooleanContents(OpVT));
5024 return DAG.getNode(ExtendCode, dl, VT, CC);
5025 }
5026
WidenVecOp_STRICT_FSETCC(SDNode * N)5027 SDValue DAGTypeLegalizer::WidenVecOp_STRICT_FSETCC(SDNode *N) {
5028 SDValue Chain = N->getOperand(0);
5029 SDValue LHS = GetWidenedVector(N->getOperand(1));
5030 SDValue RHS = GetWidenedVector(N->getOperand(2));
5031 SDValue CC = N->getOperand(3);
5032 SDLoc dl(N);
5033
5034 EVT VT = N->getValueType(0);
5035 EVT EltVT = VT.getVectorElementType();
5036 EVT TmpEltVT = LHS.getValueType().getVectorElementType();
5037 unsigned NumElts = VT.getVectorNumElements();
5038
5039 // Unroll into a build vector.
5040 SmallVector<SDValue, 8> Scalars(NumElts);
5041 SmallVector<SDValue, 8> Chains(NumElts);
5042
5043 for (unsigned i = 0; i != NumElts; ++i) {
5044 SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
5045 DAG.getVectorIdxConstant(i, dl));
5046 SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
5047 DAG.getVectorIdxConstant(i, dl));
5048
5049 Scalars[i] = DAG.getNode(N->getOpcode(), dl, {MVT::i1, MVT::Other},
5050 {Chain, LHSElem, RHSElem, CC});
5051 Chains[i] = Scalars[i].getValue(1);
5052 Scalars[i] = DAG.getSelect(dl, EltVT, Scalars[i],
5053 DAG.getBoolConstant(true, dl, EltVT, VT),
5054 DAG.getBoolConstant(false, dl, EltVT, VT));
5055 }
5056
5057 SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
5058 ReplaceValueWith(SDValue(N, 1), NewChain);
5059
5060 return DAG.getBuildVector(VT, dl, Scalars);
5061 }
5062
WidenVecOp_VECREDUCE(SDNode * N)5063 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) {
5064 SDLoc dl(N);
5065 SDValue Op = GetWidenedVector(N->getOperand(0));
5066 EVT OrigVT = N->getOperand(0).getValueType();
5067 EVT WideVT = Op.getValueType();
5068 EVT ElemVT = OrigVT.getVectorElementType();
5069 SDNodeFlags Flags = N->getFlags();
5070
5071 unsigned Opc = N->getOpcode();
5072 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
5073 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
5074 assert(NeutralElem && "Neutral element must exist");
5075
5076 // Pad the vector with the neutral element.
5077 unsigned OrigElts = OrigVT.getVectorNumElements();
5078 unsigned WideElts = WideVT.getVectorNumElements();
5079 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
5080 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
5081 DAG.getVectorIdxConstant(Idx, dl));
5082
5083 return DAG.getNode(Opc, dl, N->getValueType(0), Op, Flags);
5084 }
5085
WidenVecOp_VECREDUCE_SEQ(SDNode * N)5086 SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE_SEQ(SDNode *N) {
5087 SDLoc dl(N);
5088 SDValue AccOp = N->getOperand(0);
5089 SDValue VecOp = N->getOperand(1);
5090 SDValue Op = GetWidenedVector(VecOp);
5091
5092 EVT OrigVT = VecOp.getValueType();
5093 EVT WideVT = Op.getValueType();
5094 EVT ElemVT = OrigVT.getVectorElementType();
5095 SDNodeFlags Flags = N->getFlags();
5096
5097 unsigned Opc = N->getOpcode();
5098 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(Opc);
5099 SDValue NeutralElem = DAG.getNeutralElement(BaseOpc, dl, ElemVT, Flags);
5100
5101 // Pad the vector with the neutral element.
5102 unsigned OrigElts = OrigVT.getVectorNumElements();
5103 unsigned WideElts = WideVT.getVectorNumElements();
5104 for (unsigned Idx = OrigElts; Idx < WideElts; Idx++)
5105 Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem,
5106 DAG.getVectorIdxConstant(Idx, dl));
5107
5108 return DAG.getNode(Opc, dl, N->getValueType(0), AccOp, Op, Flags);
5109 }
5110
WidenVecOp_VSELECT(SDNode * N)5111 SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) {
5112 // This only gets called in the case that the left and right inputs and
5113 // result are of a legal odd vector type, and the condition is illegal i1 of
5114 // the same odd width that needs widening.
5115 EVT VT = N->getValueType(0);
5116 assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT));
5117
5118 SDValue Cond = GetWidenedVector(N->getOperand(0));
5119 SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N));
5120 SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N));
5121 SDLoc DL(N);
5122
5123 SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond,
5124 LeftIn, RightIn);
5125 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Select,
5126 DAG.getVectorIdxConstant(0, DL));
5127 }
5128
5129 //===----------------------------------------------------------------------===//
5130 // Vector Widening Utilities
5131 //===----------------------------------------------------------------------===//
5132
5133 // Utility function to find the type to chop up a widen vector for load/store
5134 // TLI: Target lowering used to determine legal types.
5135 // Width: Width left need to load/store.
5136 // WidenVT: The widen vector type to load to/store from
5137 // Align: If 0, don't allow use of a wider type
5138 // WidenEx: If Align is not 0, the amount additional we can load/store from.
5139
FindMemType(SelectionDAG & DAG,const TargetLowering & TLI,unsigned Width,EVT WidenVT,unsigned Align=0,unsigned WidenEx=0)5140 static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI,
5141 unsigned Width, EVT WidenVT,
5142 unsigned Align = 0, unsigned WidenEx = 0) {
5143 EVT WidenEltVT = WidenVT.getVectorElementType();
5144 const bool Scalable = WidenVT.isScalableVector();
5145 unsigned WidenWidth = WidenVT.getSizeInBits().getKnownMinSize();
5146 unsigned WidenEltWidth = WidenEltVT.getSizeInBits();
5147 unsigned AlignInBits = Align*8;
5148
5149 // If we have one element to load/store, return it.
5150 EVT RetVT = WidenEltVT;
5151 if (!Scalable && Width == WidenEltWidth)
5152 return RetVT;
5153
5154 // See if there is larger legal integer than the element type to load/store.
5155 unsigned VT;
5156 // Don't bother looking for an integer type if the vector is scalable, skip
5157 // to vector types.
5158 if (!Scalable) {
5159 for (VT = (unsigned)MVT::LAST_INTEGER_VALUETYPE;
5160 VT >= (unsigned)MVT::FIRST_INTEGER_VALUETYPE; --VT) {
5161 EVT MemVT((MVT::SimpleValueType) VT);
5162 unsigned MemVTWidth = MemVT.getSizeInBits();
5163 if (MemVT.getSizeInBits() <= WidenEltWidth)
5164 break;
5165 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
5166 if ((Action == TargetLowering::TypeLegal ||
5167 Action == TargetLowering::TypePromoteInteger) &&
5168 (WidenWidth % MemVTWidth) == 0 &&
5169 isPowerOf2_32(WidenWidth / MemVTWidth) &&
5170 (MemVTWidth <= Width ||
5171 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
5172 if (MemVTWidth == WidenWidth)
5173 return MemVT;
5174 RetVT = MemVT;
5175 break;
5176 }
5177 }
5178 }
5179
5180 // See if there is a larger vector type to load/store that has the same vector
5181 // element type and is evenly divisible with the WidenVT.
5182 for (VT = (unsigned)MVT::LAST_VECTOR_VALUETYPE;
5183 VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) {
5184 EVT MemVT = (MVT::SimpleValueType) VT;
5185 // Skip vector MVTs which don't match the scalable property of WidenVT.
5186 if (Scalable != MemVT.isScalableVector())
5187 continue;
5188 unsigned MemVTWidth = MemVT.getSizeInBits().getKnownMinSize();
5189 auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT);
5190 if ((Action == TargetLowering::TypeLegal ||
5191 Action == TargetLowering::TypePromoteInteger) &&
5192 WidenEltVT == MemVT.getVectorElementType() &&
5193 (WidenWidth % MemVTWidth) == 0 &&
5194 isPowerOf2_32(WidenWidth / MemVTWidth) &&
5195 (MemVTWidth <= Width ||
5196 (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) {
5197 if (RetVT.getFixedSizeInBits() < MemVTWidth || MemVT == WidenVT)
5198 return MemVT;
5199 }
5200 }
5201
5202 if (Scalable)
5203 report_fatal_error("Using element-wise loads and stores for widening "
5204 "operations is not supported for scalable vectors");
5205 return RetVT;
5206 }
5207
5208 // Builds a vector type from scalar loads
5209 // VecTy: Resulting Vector type
5210 // LDOps: Load operators to build a vector type
5211 // [Start,End) the list of loads to use.
BuildVectorFromScalar(SelectionDAG & DAG,EVT VecTy,SmallVectorImpl<SDValue> & LdOps,unsigned Start,unsigned End)5212 static SDValue BuildVectorFromScalar(SelectionDAG& DAG, EVT VecTy,
5213 SmallVectorImpl<SDValue> &LdOps,
5214 unsigned Start, unsigned End) {
5215 SDLoc dl(LdOps[Start]);
5216 EVT LdTy = LdOps[Start].getValueType();
5217 unsigned Width = VecTy.getSizeInBits();
5218 unsigned NumElts = Width / LdTy.getSizeInBits();
5219 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), LdTy, NumElts);
5220
5221 unsigned Idx = 1;
5222 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT,LdOps[Start]);
5223
5224 for (unsigned i = Start + 1; i != End; ++i) {
5225 EVT NewLdTy = LdOps[i].getValueType();
5226 if (NewLdTy != LdTy) {
5227 NumElts = Width / NewLdTy.getSizeInBits();
5228 NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewLdTy, NumElts);
5229 VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, VecOp);
5230 // Readjust position and vector position based on new load type.
5231 Idx = Idx * LdTy.getSizeInBits() / NewLdTy.getSizeInBits();
5232 LdTy = NewLdTy;
5233 }
5234 VecOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, VecOp, LdOps[i],
5235 DAG.getVectorIdxConstant(Idx++, dl));
5236 }
5237 return DAG.getNode(ISD::BITCAST, dl, VecTy, VecOp);
5238 }
5239
GenWidenVectorLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD)5240 SDValue DAGTypeLegalizer::GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
5241 LoadSDNode *LD) {
5242 // The strategy assumes that we can efficiently load power-of-two widths.
5243 // The routine chops the vector into the largest vector loads with the same
5244 // element type or scalar loads and then recombines it to the widen vector
5245 // type.
5246 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
5247 EVT LdVT = LD->getMemoryVT();
5248 SDLoc dl(LD);
5249 assert(LdVT.isVector() && WidenVT.isVector());
5250 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
5251 assert(LdVT.getVectorElementType() == WidenVT.getVectorElementType());
5252
5253 // Load information
5254 SDValue Chain = LD->getChain();
5255 SDValue BasePtr = LD->getBasePtr();
5256 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
5257 AAMDNodes AAInfo = LD->getAAInfo();
5258
5259 TypeSize LdWidth = LdVT.getSizeInBits();
5260 TypeSize WidenWidth = WidenVT.getSizeInBits();
5261 TypeSize WidthDiff = WidenWidth - LdWidth;
5262 // Allow wider loads if they are sufficiently aligned to avoid memory faults
5263 // and if the original load is simple.
5264 unsigned LdAlign = (!LD->isSimple()) ? 0 : LD->getAlignment();
5265
5266 // Find the vector type that can load from.
5267 EVT NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
5268 WidthDiff.getKnownMinSize());
5269 TypeSize NewVTWidth = NewVT.getSizeInBits();
5270 SDValue LdOp = DAG.getLoad(NewVT, dl, Chain, BasePtr, LD->getPointerInfo(),
5271 LD->getOriginalAlign(), MMOFlags, AAInfo);
5272 LdChain.push_back(LdOp.getValue(1));
5273
5274 // Check if we can load the element with one instruction.
5275 if (TypeSize::isKnownLE(LdWidth, NewVTWidth)) {
5276 if (!NewVT.isVector()) {
5277 unsigned NumElts = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize();
5278 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5279 SDValue VecOp = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NewVecVT, LdOp);
5280 return DAG.getNode(ISD::BITCAST, dl, WidenVT, VecOp);
5281 }
5282 if (NewVT == WidenVT)
5283 return LdOp;
5284
5285 // TODO: We don't currently have any tests that exercise this code path.
5286 assert(WidenWidth.getFixedSize() % NewVTWidth.getFixedSize() == 0);
5287 unsigned NumConcat = WidenWidth.getFixedSize() / NewVTWidth.getFixedSize();
5288 SmallVector<SDValue, 16> ConcatOps(NumConcat);
5289 SDValue UndefVal = DAG.getUNDEF(NewVT);
5290 ConcatOps[0] = LdOp;
5291 for (unsigned i = 1; i != NumConcat; ++i)
5292 ConcatOps[i] = UndefVal;
5293 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, ConcatOps);
5294 }
5295
5296 // Load vector by using multiple loads from largest vector to scalar.
5297 SmallVector<SDValue, 16> LdOps;
5298 LdOps.push_back(LdOp);
5299
5300 uint64_t ScaledOffset = 0;
5301 MachinePointerInfo MPI = LD->getPointerInfo();
5302 do {
5303 LdWidth -= NewVTWidth;
5304 IncrementPointer(cast<LoadSDNode>(LdOp), NewVT, MPI, BasePtr,
5305 &ScaledOffset);
5306
5307 if (TypeSize::isKnownLT(LdWidth, NewVTWidth)) {
5308 // The current type we are using is too large. Find a better size.
5309 NewVT = FindMemType(DAG, TLI, LdWidth.getKnownMinSize(), WidenVT, LdAlign,
5310 WidthDiff.getKnownMinSize());
5311 NewVTWidth = NewVT.getSizeInBits();
5312 }
5313
5314 Align NewAlign = ScaledOffset == 0
5315 ? LD->getOriginalAlign()
5316 : commonAlignment(LD->getAlign(), ScaledOffset);
5317 SDValue L =
5318 DAG.getLoad(NewVT, dl, Chain, BasePtr, MPI, NewAlign, MMOFlags, AAInfo);
5319 LdChain.push_back(L.getValue(1));
5320
5321 LdOps.push_back(L);
5322 LdOp = L;
5323 } while (TypeSize::isKnownGT(LdWidth, NewVTWidth));
5324
5325 // Build the vector from the load operations.
5326 unsigned End = LdOps.size();
5327 if (!LdOps[0].getValueType().isVector())
5328 // All the loads are scalar loads.
5329 return BuildVectorFromScalar(DAG, WidenVT, LdOps, 0, End);
5330
5331 // If the load contains vectors, build the vector using concat vector.
5332 // All of the vectors used to load are power-of-2, and the scalar loads can be
5333 // combined to make a power-of-2 vector.
5334 SmallVector<SDValue, 16> ConcatOps(End);
5335 int i = End - 1;
5336 int Idx = End;
5337 EVT LdTy = LdOps[i].getValueType();
5338 // First, combine the scalar loads to a vector.
5339 if (!LdTy.isVector()) {
5340 for (--i; i >= 0; --i) {
5341 LdTy = LdOps[i].getValueType();
5342 if (LdTy.isVector())
5343 break;
5344 }
5345 ConcatOps[--Idx] = BuildVectorFromScalar(DAG, LdTy, LdOps, i + 1, End);
5346 }
5347
5348 ConcatOps[--Idx] = LdOps[i];
5349 for (--i; i >= 0; --i) {
5350 EVT NewLdTy = LdOps[i].getValueType();
5351 if (NewLdTy != LdTy) {
5352 // Create a larger vector.
5353 TypeSize LdTySize = LdTy.getSizeInBits();
5354 TypeSize NewLdTySize = NewLdTy.getSizeInBits();
5355 assert(NewLdTySize.isScalable() == LdTySize.isScalable() &&
5356 NewLdTySize.isKnownMultipleOf(LdTySize.getKnownMinSize()));
5357 unsigned NumOps =
5358 NewLdTySize.getKnownMinSize() / LdTySize.getKnownMinSize();
5359 SmallVector<SDValue, 16> WidenOps(NumOps);
5360 unsigned j = 0;
5361 for (; j != End-Idx; ++j)
5362 WidenOps[j] = ConcatOps[Idx+j];
5363 for (; j != NumOps; ++j)
5364 WidenOps[j] = DAG.getUNDEF(LdTy);
5365
5366 ConcatOps[End-1] = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewLdTy,
5367 WidenOps);
5368 Idx = End - 1;
5369 LdTy = NewLdTy;
5370 }
5371 ConcatOps[--Idx] = LdOps[i];
5372 }
5373
5374 if (WidenWidth == LdTy.getSizeInBits() * (End - Idx))
5375 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT,
5376 makeArrayRef(&ConcatOps[Idx], End - Idx));
5377
5378 // We need to fill the rest with undefs to build the vector.
5379 unsigned NumOps =
5380 WidenWidth.getKnownMinSize() / LdTy.getSizeInBits().getKnownMinSize();
5381 SmallVector<SDValue, 16> WidenOps(NumOps);
5382 SDValue UndefVal = DAG.getUNDEF(LdTy);
5383 {
5384 unsigned i = 0;
5385 for (; i != End-Idx; ++i)
5386 WidenOps[i] = ConcatOps[Idx+i];
5387 for (; i != NumOps; ++i)
5388 WidenOps[i] = UndefVal;
5389 }
5390 return DAG.getNode(ISD::CONCAT_VECTORS, dl, WidenVT, WidenOps);
5391 }
5392
5393 SDValue
GenWidenVectorExtLoads(SmallVectorImpl<SDValue> & LdChain,LoadSDNode * LD,ISD::LoadExtType ExtType)5394 DAGTypeLegalizer::GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
5395 LoadSDNode *LD,
5396 ISD::LoadExtType ExtType) {
5397 // For extension loads, it may not be more efficient to chop up the vector
5398 // and then extend it. Instead, we unroll the load and build a new vector.
5399 EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(),LD->getValueType(0));
5400 EVT LdVT = LD->getMemoryVT();
5401 SDLoc dl(LD);
5402 assert(LdVT.isVector() && WidenVT.isVector());
5403 assert(LdVT.isScalableVector() == WidenVT.isScalableVector());
5404
5405 // Load information
5406 SDValue Chain = LD->getChain();
5407 SDValue BasePtr = LD->getBasePtr();
5408 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
5409 AAMDNodes AAInfo = LD->getAAInfo();
5410
5411 if (LdVT.isScalableVector())
5412 report_fatal_error("Generating widen scalable extending vector loads is "
5413 "not yet supported");
5414
5415 EVT EltVT = WidenVT.getVectorElementType();
5416 EVT LdEltVT = LdVT.getVectorElementType();
5417 unsigned NumElts = LdVT.getVectorNumElements();
5418
5419 // Load each element and widen.
5420 unsigned WidenNumElts = WidenVT.getVectorNumElements();
5421 SmallVector<SDValue, 16> Ops(WidenNumElts);
5422 unsigned Increment = LdEltVT.getSizeInBits() / 8;
5423 Ops[0] =
5424 DAG.getExtLoad(ExtType, dl, EltVT, Chain, BasePtr, LD->getPointerInfo(),
5425 LdEltVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
5426 LdChain.push_back(Ops[0].getValue(1));
5427 unsigned i = 0, Offset = Increment;
5428 for (i=1; i < NumElts; ++i, Offset += Increment) {
5429 SDValue NewBasePtr =
5430 DAG.getObjectPtrOffset(dl, BasePtr, TypeSize::Fixed(Offset));
5431 Ops[i] = DAG.getExtLoad(ExtType, dl, EltVT, Chain, NewBasePtr,
5432 LD->getPointerInfo().getWithOffset(Offset), LdEltVT,
5433 LD->getOriginalAlign(), MMOFlags, AAInfo);
5434 LdChain.push_back(Ops[i].getValue(1));
5435 }
5436
5437 // Fill the rest with undefs.
5438 SDValue UndefVal = DAG.getUNDEF(EltVT);
5439 for (; i != WidenNumElts; ++i)
5440 Ops[i] = UndefVal;
5441
5442 return DAG.getBuildVector(WidenVT, dl, Ops);
5443 }
5444
GenWidenVectorStores(SmallVectorImpl<SDValue> & StChain,StoreSDNode * ST)5445 void DAGTypeLegalizer::GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain,
5446 StoreSDNode *ST) {
5447 // The strategy assumes that we can efficiently store power-of-two widths.
5448 // The routine chops the vector into the largest vector stores with the same
5449 // element type or scalar stores.
5450 SDValue Chain = ST->getChain();
5451 SDValue BasePtr = ST->getBasePtr();
5452 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
5453 AAMDNodes AAInfo = ST->getAAInfo();
5454 SDValue ValOp = GetWidenedVector(ST->getValue());
5455 SDLoc dl(ST);
5456
5457 EVT StVT = ST->getMemoryVT();
5458 TypeSize StWidth = StVT.getSizeInBits();
5459 EVT ValVT = ValOp.getValueType();
5460 TypeSize ValWidth = ValVT.getSizeInBits();
5461 EVT ValEltVT = ValVT.getVectorElementType();
5462 unsigned ValEltWidth = ValEltVT.getFixedSizeInBits();
5463 assert(StVT.getVectorElementType() == ValEltVT);
5464 assert(StVT.isScalableVector() == ValVT.isScalableVector() &&
5465 "Mismatch between store and value types");
5466
5467 int Idx = 0; // current index to store
5468
5469 MachinePointerInfo MPI = ST->getPointerInfo();
5470 uint64_t ScaledOffset = 0;
5471 while (StWidth.isNonZero()) {
5472 // Find the largest vector type we can store with.
5473 EVT NewVT = FindMemType(DAG, TLI, StWidth.getKnownMinSize(), ValVT);
5474 TypeSize NewVTWidth = NewVT.getSizeInBits();
5475
5476 if (NewVT.isVector()) {
5477 unsigned NumVTElts = NewVT.getVectorMinNumElements();
5478 do {
5479 Align NewAlign = ScaledOffset == 0
5480 ? ST->getOriginalAlign()
5481 : commonAlignment(ST->getAlign(), ScaledOffset);
5482 SDValue EOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NewVT, ValOp,
5483 DAG.getVectorIdxConstant(Idx, dl));
5484 SDValue PartStore = DAG.getStore(Chain, dl, EOp, BasePtr, MPI, NewAlign,
5485 MMOFlags, AAInfo);
5486 StChain.push_back(PartStore);
5487
5488 StWidth -= NewVTWidth;
5489 Idx += NumVTElts;
5490
5491 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr,
5492 &ScaledOffset);
5493 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
5494 } else {
5495 // Cast the vector to the scalar type we can store.
5496 unsigned NumElts = ValWidth.getFixedSize() / NewVTWidth.getFixedSize();
5497 EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NumElts);
5498 SDValue VecOp = DAG.getNode(ISD::BITCAST, dl, NewVecVT, ValOp);
5499 // Readjust index position based on new vector type.
5500 Idx = Idx * ValEltWidth / NewVTWidth.getFixedSize();
5501 do {
5502 SDValue EOp = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, VecOp,
5503 DAG.getVectorIdxConstant(Idx++, dl));
5504 SDValue PartStore =
5505 DAG.getStore(Chain, dl, EOp, BasePtr, MPI, ST->getOriginalAlign(),
5506 MMOFlags, AAInfo);
5507 StChain.push_back(PartStore);
5508
5509 StWidth -= NewVTWidth;
5510 IncrementPointer(cast<StoreSDNode>(PartStore), NewVT, MPI, BasePtr);
5511 } while (StWidth.isNonZero() && TypeSize::isKnownGE(StWidth, NewVTWidth));
5512 // Restore index back to be relative to the original widen element type.
5513 Idx = Idx * NewVTWidth.getFixedSize() / ValEltWidth;
5514 }
5515 }
5516 }
5517
5518 /// Modifies a vector input (widen or narrows) to a vector of NVT. The
5519 /// input vector must have the same element type as NVT.
5520 /// FillWithZeroes specifies that the vector should be widened with zeroes.
ModifyToType(SDValue InOp,EVT NVT,bool FillWithZeroes)5521 SDValue DAGTypeLegalizer::ModifyToType(SDValue InOp, EVT NVT,
5522 bool FillWithZeroes) {
5523 // Note that InOp might have been widened so it might already have
5524 // the right width or it might need be narrowed.
5525 EVT InVT = InOp.getValueType();
5526 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&
5527 "input and widen element type must match");
5528 SDLoc dl(InOp);
5529
5530 // Check if InOp already has the right width.
5531 if (InVT == NVT)
5532 return InOp;
5533
5534 unsigned InNumElts = InVT.getVectorNumElements();
5535 unsigned WidenNumElts = NVT.getVectorNumElements();
5536 if (WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0) {
5537 unsigned NumConcat = WidenNumElts / InNumElts;
5538 SmallVector<SDValue, 16> Ops(NumConcat);
5539 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, InVT) :
5540 DAG.getUNDEF(InVT);
5541 Ops[0] = InOp;
5542 for (unsigned i = 1; i != NumConcat; ++i)
5543 Ops[i] = FillVal;
5544
5545 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, Ops);
5546 }
5547
5548 if (WidenNumElts < InNumElts && InNumElts % WidenNumElts)
5549 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, InOp,
5550 DAG.getVectorIdxConstant(0, dl));
5551
5552 // Fall back to extract and build.
5553 SmallVector<SDValue, 16> Ops(WidenNumElts);
5554 EVT EltVT = NVT.getVectorElementType();
5555 unsigned MinNumElts = std::min(WidenNumElts, InNumElts);
5556 unsigned Idx;
5557 for (Idx = 0; Idx < MinNumElts; ++Idx)
5558 Ops[Idx] = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, InOp,
5559 DAG.getVectorIdxConstant(Idx, dl));
5560
5561 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
5562 DAG.getUNDEF(EltVT);
5563 for ( ; Idx < WidenNumElts; ++Idx)
5564 Ops[Idx] = FillVal;
5565 return DAG.getBuildVector(NVT, dl, Ops);
5566 }
5567
SplitVecRes_VECTOR_REVERSE(SDNode * N,SDValue & Lo,SDValue & Hi)5568 void DAGTypeLegalizer::SplitVecRes_VECTOR_REVERSE(SDNode *N, SDValue &Lo,
5569 SDValue &Hi) {
5570 SDValue InLo, InHi;
5571 GetSplitVector(N->getOperand(0), InLo, InHi);
5572 SDLoc DL(N);
5573
5574 Lo = DAG.getNode(ISD::VECTOR_REVERSE, DL, InHi.getValueType(), InHi);
5575 Hi = DAG.getNode(ISD::VECTOR_REVERSE, DL, InLo.getValueType(), InLo);
5576 }
5577
SplitVecRes_VECTOR_SPLICE(SDNode * N,SDValue & Lo,SDValue & Hi)5578 void DAGTypeLegalizer::SplitVecRes_VECTOR_SPLICE(SDNode *N, SDValue &Lo,
5579 SDValue &Hi) {
5580 EVT VT = N->getValueType(0);
5581 SDLoc DL(N);
5582
5583 EVT LoVT, HiVT;
5584 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
5585
5586 SDValue Expanded = TLI.expandVectorSplice(N, DAG);
5587 Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, LoVT, Expanded,
5588 DAG.getVectorIdxConstant(0, DL));
5589 Hi =
5590 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HiVT, Expanded,
5591 DAG.getVectorIdxConstant(LoVT.getVectorMinNumElements(), DL));
5592 }
5593