Lines Matching defs:MatMul

1329   /// Special case for MatMul lowering. Prevents scalar loads of row-major
1332   void lowerDotProduct(CallInst *MatMul,
1335     if (FusedInsts.contains(MatMul) ||
1338     ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1339     ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1344     Value *LHS = MatMul->getArgOperand(0);
1345     Value *RHS = MatMul->getArgOperand(1);
1456     FusedInsts.insert(MatMul);
1457     IRBuilder<> Builder(MatMul);
1491     LHS = MatMul->getArgOperand(0);
1509     Result = Builder.CreateInsertElement(PoisonValue::get(MatMul->getType()),
1511     MatMul->replaceAllUsesWith(Result);
1512     FusedInsts.insert(MatMul);
1513     ToRemove.push_back(MatMul);
1608                                CallInst *MatMul) {
1620     BasicBlock *Check0 = MatMul->getParent();
1629         SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI,
1632         SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI,
1635         SplitBlock(MatMul->getParent(), MatMul, (DomTreeUpdater *)nullptr, LI,
1641     IRBuilder<> Builder(MatMul);
1692   bool isFusionProfitable(CallInst *MatMul) {
1696     ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1697     ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1702     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1736   void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape,
1738     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1743     Instruction *InsertI = cast<Instruction>(MatMul);
1747     IRBuilder<> Builder(MatMul);
1751         FixedVectorType::get(MatMul->getType()->getScalarType(), TileSize);
1776                        getFastMathFlags(MatMul));
1795   void emitSIMDTiling(CallInst *MatMul, LoadInst *LoadOp0, LoadInst *LoadOp1,
1800     if (!isFusionProfitable(MatMul))
1803     ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1804     ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1809     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1811     Value *APtr = getNonAliasingPointer(LoadOp0, Store, MatMul);
1812     Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul);
1816       createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store);
1836                                getFastMathFlags(MatMul));
1846     FusedInsts.insert(MatMul);
1848     MatMul->eraseFromParent();
1864   LowerMatrixMultiplyFused(CallInst *MatMul,
1872     Value *A = MatMul->getArgOperand(0);
1873     Value *B = MatMul->getArgOperand(1);
1880       IRBuilder<> Builder(MatMul);
1881       auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
1882       ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
1883       ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
1906                          getFastMathFlags(MatMul));
1908       FusedInsts.insert(MatMul);
1916       finalizeLowering(MatMul, Result, Builder);
1920     if (!MatMul->hasOneUse() || MatrixLayout != MatrixLayoutTy::ColumnMajor)
1927     auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin());
1929       // The store address must dominate the MatMul instruction, otherwise
1941         if (DT->dominates(CurrI, MatMul))
1953         I->moveBefore(MatMul);
2004       emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts);
2010   void LowerMultiply(CallInst *MatMul) {
2011     IRBuilder<> Builder(MatMul);
2012     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
2013     ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
2014     ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
2016     const MatrixTy &Lhs = getMatrix(MatMul->getArgOperand(0), LShape, Builder);
2017     const MatrixTy &Rhs = getMatrix(MatMul->getArgOperand(1), RShape, Builder);
2031                        getFastMathFlags(MatMul));
2032     finalizeLowering(MatMul, Result, Builder);