1 //===- MVELaneInterleaving.cpp - Inverleave for MVE instructions ----------===//
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
43 //===----------------------------------------------------------------------===//
77 #define DEBUG_TYPE "mve-laneinterleave"
80     "enable-mve-interleave", cl::Hidden, cl::init(true),
130     if (isa<FPExtInst>(E) || !isa<LoadInst>(E->getOperand(0))) {  in isProfitableToInterleave()
136     if (T->hasOneUse() && !isa<StoreInst>(*T->user_begin())) {  in isProfitableToInterleave()
145     if (!E->hasOneUse() ||  in isProfitableToInterleave()
146         cast<Instruction>(*E->user_begin())->getOpcode() != Instruction::Mul) {  in isProfitableToInterleave()
158   if (!isa<Instruction>(Start->getOperand(0)))  in tryInterleave()
164   Worklist.push_back(cast<Instruction>(Start->getOperand(0)));  in tryInterleave()
176     switch (I->getOpcode()) {  in tryInterleave()
191       for (auto *Use : I->users())  in tryInterleave()
201       if (II->getIntrinsicID() == Intrinsic::vector_reduce_add) {  in tryInterleave()
208       switch (II->getIntrinsicID()) {  in tryInterleave()
248       for (Use &Op : I->operands()) {  in tryInterleave()
249         if (!isa<FixedVectorType>(Op->getType()))  in tryInterleave()
251         if (isa<Instruction>(Op))  in tryInterleave()
257       for (auto *Use : I->users())  in tryInterleave()
263       if (cast<ShuffleVectorInst>(I)->isZeroEltSplat())  in tryInterleave()
285       dbgs() << "  " << *I->get() << " of " << *I->getUser() << "\n";  in tryInterleave()
300                  ? cast<FixedVectorType>(Truncs[0]->getType())  in tryInterleave()
301                  : cast<FixedVectorType>(Exts[0]->getOperand(0)->getType());  in tryInterleave()
305   unsigned NumElts = VT->getNumElements();  in tryInterleave()
306   unsigned BaseElts = VT->getScalarSizeInBits() == 16  in tryInterleave()
308                           : (VT->getScalarSizeInBits() == 8 ? 16 : 0);  in tryInterleave()
313   if (Start->getOperand(0)->getType()->getScalarSizeInBits() !=  in tryInterleave()
314       VT->getScalarSizeInBits() * 2) {  in tryInterleave()
319     if (I->getOperand(0)->getType() != VT) {  in tryInterleave()
324     if (I->getType() != VT) {  in tryInterleave()
333                              return I->getOpcode() == Instruction::Mul ||  in tryInterleave()
334                                     I->getOpcode() == Instruction::Select ||  in tryInterleave()
335                                     I->getOpcode() == Instruction::ICmp;  in tryInterleave()
348   for (unsigned Base = 0; Base < NumElts; Base += BaseElts) {  in tryInterleave()  local
350       LeafMask.push_back(Base + i * 2);  in tryInterleave()
352       LeafMask.push_back(Base + i * 2 + 1);  in tryInterleave()
354   for (unsigned Base = 0; Base < NumElts; Base += BaseElts) {  in tryInterleave()  local
356       TruncMask.push_back(Base + i);  in tryInterleave()
357       TruncMask.push_back(Base + i + BaseElts / 2);  in tryInterleave()
364     Value *Shuffle = Builder.CreateShuffleVector(I->getOperand(0), LeafMask);  in tryInterleave()
365     bool FPext = isa<FPExtInst>(I);  in tryInterleave()
366     bool Sext = isa<SExtInst>(I);  in tryInterleave()
367     Value *Ext = FPext ? Builder.CreateFPExt(Shuffle, I->getType())  in tryInterleave()
368                        : Sext ? Builder.CreateSExt(Shuffle, I->getType())  in tryInterleave()
369                               : Builder.CreateZExt(Shuffle, I->getType());  in tryInterleave()
370     I->replaceAllUsesWith(Ext);  in tryInterleave()
376     Builder.SetInsertPoint(cast<Instruction>(I->getUser()));  in tryInterleave()
377     Value *Shuffle = Builder.CreateShuffleVector(I->get(), LeafMask);  in tryInterleave()
378     I->getUser()->setOperand(I->getOperandNo(), Shuffle);  in tryInterleave()
385     Builder.SetInsertPoint(I->getParent(), ++I->getIterator());  in tryInterleave()
387     I->replaceAllUsesWith(Shuf);  in tryInterleave()
388     cast<Instruction>(Shuf)->setOperand(0, I);  in tryInterleave()
400     return II->getIntrinsicID() == Intrinsic::vector_reduce_add;  in isAddReduction()
410   if (!ST->hasMVEIntegerOps())  in runOnFunction()
417     if (((I.getType()->isVectorTy() &&  in runOnFunction()
418           (isa<TruncInst>(I) || isa<FPTruncInst>(I))) ||  in runOnFunction()