Lines Matching defs:adaptor
336 matchAndRewrite(gpu::HostRegisterOp hostRegisterOp, OpAdaptor adaptor,
350 matchAndRewrite(gpu::HostUnregisterOp hostUnregisterOp, OpAdaptor adaptor,
364 matchAndRewrite(gpu::AllocOp allocOp, OpAdaptor adaptor,
379 matchAndRewrite(gpu::DeallocOp deallocOp, OpAdaptor adaptor,
392 matchAndRewrite(async::YieldOp yieldOp, OpAdaptor adaptor,
406 matchAndRewrite(gpu::WaitOp waitOp, OpAdaptor adaptor,
421 matchAndRewrite(gpu::WaitOp waitOp, OpAdaptor adaptor,
438 matchAndRewrite(gpu::LaunchFuncOp launchOp, OpAdaptor adaptor,
455 matchAndRewrite(gpu::MemcpyOp memcpyOp, OpAdaptor adaptor,
469 matchAndRewrite(gpu::MemsetOp memsetOp, OpAdaptor adaptor,
484 matchAndRewrite(gpu::SetDefaultDeviceOp op, OpAdaptor adaptor,
500 matchAndRewrite(gpu::op_name op, OpAdaptor adaptor, \
712 gpu::HostRegisterOp hostRegisterOp, OpAdaptor adaptor,
715 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)))
725 loc, op->getOperands(), adaptor.getOperands(), rewriter);
734 gpu::HostUnregisterOp hostUnregisterOp, OpAdaptor adaptor,
737 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)))
747 loc, op->getOperands(), adaptor.getOperands(), rewriter);
756 gpu::AllocOp allocOp, OpAdaptor adaptor,
761 if (failed(areAllLLVMTypes(allocOp, adaptor.getOperands(), rewriter)) ||
780 getMemRefDescriptorSizes(loc, memRefType, adaptor.getDynamicSizes(), rewriter,
786 Value stream = adaptor.getAsyncDependencies().empty()
788 : adaptor.getAsyncDependencies().front();
815 gpu::DeallocOp deallocOp, OpAdaptor adaptor,
817 if (failed(areAllLLVMTypes(deallocOp, adaptor.getOperands(), rewriter)) ||
824 MemRefDescriptor(adaptor.getMemref()).allocatedPtr(rewriter, loc);
825 Value stream = adaptor.getAsyncDependencies().front();
841 async::YieldOp yieldOp, OpAdaptor adaptor,
847 SmallVector<Value, 4> newOperands(adaptor.getOperands());
853 auto stream = adaptor.getOperands()[idx];
879 gpu::WaitOp waitOp, OpAdaptor adaptor,
886 for (auto operand : adaptor.getOperands()) {
909 gpu::WaitOp waitOp, OpAdaptor adaptor,
919 llvm::zip(waitOp.getAsyncDependencies(), adaptor.getOperands())) {
948 gpu::LaunchFuncOp launchOp, OpAdaptor adaptor,
950 if (failed(areAllLLVMTypes(launchOp, adaptor.getOperands(), rewriter)))
967 if (!adaptor.getAsyncDependencies().empty())
968 stream = adaptor.getAsyncDependencies().front();
979 loc, origArguments, adaptor.getKernelOperands(), rewriter,
1027 gpu::KernelDim3{adaptor.getClusterSizeX(), adaptor.getClusterSizeY(),
1028 adaptor.getClusterSizeZ()};
1032 gpu::KernelDim3{adaptor.getGridSizeX(), adaptor.getGridSizeY(),
1033 adaptor.getGridSizeZ()},
1034 gpu::KernelDim3{adaptor.getBlockSizeX(), adaptor.getBlockSizeY(),
1035 adaptor.getBlockSizeZ()},
1036 adaptor.getDynamicSharedMemorySize(),
1062 gpu::MemcpyOp memcpyOp, OpAdaptor adaptor,
1066 if (failed(areAllLLVMTypes(memcpyOp, adaptor.getOperands(), rewriter)) ||
1073 MemRefDescriptor srcDesc(adaptor.getSrc());
1090 MemRefDescriptor(adaptor.getDst()).alignedPtr(rewriter, loc),
1093 auto stream = adaptor.getAsyncDependencies().front();
1102 gpu::MemsetOp memsetOp, OpAdaptor adaptor,
1106 if (failed(areAllLLVMTypes(memsetOp, adaptor.getOperands(), rewriter)) ||
1113 Type valueType = adaptor.getValue().getType();
1124 MemRefDescriptor dstDesc(adaptor.getDst());
1128 rewriter.create<LLVM::BitcastOp>(loc, bitCastType, adaptor.getValue());
1133 auto stream = adaptor.getAsyncDependencies().front();
1143 gpu::SetDefaultDeviceOp op, OpAdaptor adaptor,
1147 {adaptor.getDevIndex()});
1168 gpu::CreateDnTensorOp op, OpAdaptor adaptor,
1170 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1174 auto stream = adaptor.getAsyncDependencies().front();
1176 MemRefDescriptor(adaptor.getMemref()).allocatedPtr(rewriter, loc);
1181 for (Value dim : adaptor.getDims()) {
1221 gpu::DestroyDnTensorOp op, OpAdaptor adaptor,
1223 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1227 auto stream = adaptor.getAsyncDependencies().front();
1238 {adaptor.getDnTensor(), stream});
1241 {adaptor.getDnTensor(), stream});
1246 {adaptor.getDnTensor(), stream});
1253 gpu::CreateCooOp op, OpAdaptor adaptor,
1255 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1259 auto stream = adaptor.getAsyncDependencies().front();
1261 MemRefDescriptor(adaptor.getRowIdxs()).allocatedPtr(rewriter, loc);
1263 MemRefDescriptor(adaptor.getColIdxs()).allocatedPtr(rewriter, loc);
1265 MemRefDescriptor(adaptor.getValues()).allocatedPtr(rewriter, loc);
1275 {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1283 gpu::CreateCooAoSOp op, OpAdaptor adaptor,
1285 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1289 auto stream = adaptor.getAsyncDependencies().front();
1290 Value pIdxs = MemRefDescriptor(adaptor.getIdxs()).allocatedPtr(rewriter, loc);
1292 MemRefDescriptor(adaptor.getValues()).allocatedPtr(rewriter, loc);
1301 {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1309 gpu::CreateCsrOp op, OpAdaptor adaptor,
1311 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1315 auto stream = adaptor.getAsyncDependencies().front();
1317 MemRefDescriptor(adaptor.getRowPos()).allocatedPtr(rewriter, loc);
1319 MemRefDescriptor(adaptor.getColIdxs()).allocatedPtr(rewriter, loc);
1321 MemRefDescriptor(adaptor.getValues()).allocatedPtr(rewriter, loc);
1334 {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1342 gpu::Create2To4SpMatOp op, OpAdaptor adaptor,
1344 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1348 auto stream = adaptor.getAsyncDependencies().front();
1350 MemRefDescriptor(adaptor.getMemref()).allocatedPtr(rewriter, loc);
1364 {handle, adaptor.getRows(), adaptor.getCols(), pMat, dtp, stream})
1371 gpu::DestroySpMatOp op, OpAdaptor adaptor,
1373 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1377 auto stream = adaptor.getAsyncDependencies().front();
1381 {adaptor.getSpmat(), stream});
1384 destroySpMatCallBuilder.create(loc, rewriter, {adaptor.getSpmat(), stream});
1391 gpu::SpMVBufferSizeOp op, OpAdaptor adaptor,
1393 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1399 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1400 auto stream = adaptor.getAsyncDependencies().front();
1403 {modeA, adaptor.getSpmatA(), adaptor.getDnX(),
1404 adaptor.getDnY(), computeType, stream})
1411 gpu::SpMVOp op, OpAdaptor adaptor,
1413 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1417 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1419 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1420 auto stream = adaptor.getAsyncDependencies().front();
1422 MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(rewriter, loc);
1424 {modeA, adaptor.getSpmatA(), adaptor.getDnX(),
1425 adaptor.getDnY(), computeType, pBuf, stream});
1431 gpu::SpMMBufferSizeOp op, OpAdaptor adaptor,
1433 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1437 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1438 auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
1439 auto stream = adaptor.getAsyncDependencies().front();
1445 rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType()));
1452 {bufferSize, modeA, modeB, adaptor.getSpmatA(),
1453 adaptor.getDnmatB(), adaptor.getDnmatC(), computeType,
1475 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1479 {modeA, modeB, adaptor.getSpmatA(), adaptor.getDnmatB(),
1480 adaptor.getDnmatC(), computeType, stream})
1488 gpu::SDDMMBufferSizeOp op, OpAdaptor adaptor,
1490 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1494 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1495 auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
1497 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1498 auto stream = adaptor.getAsyncDependencies().front();
1502 {modeA, modeB, adaptor.getDnmatA(), adaptor.getDnmatB(),
1503 adaptor.getSpmatC(), computeType, stream})
1510 gpu::SpMMOp op, OpAdaptor adaptor,
1512 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1516 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1517 auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
1519 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1521 auto stream = adaptor.getAsyncDependencies().front();
1526 for (Value buffer : adaptor.getBuffers()) {
1532 {adaptor.getSpmatA(), adaptor.getDnmatB(), adaptor.getDnmatC(),
1535 Value pBuf = MemRefDescriptor(adaptor.getBuffers().front())
1538 {modeA, modeB, adaptor.getSpmatA(),
1539 adaptor.getDnmatB(), adaptor.getDnmatC(),
1554 gpu::SDDMMOp op, OpAdaptor adaptor,
1556 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1561 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1562 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1563 auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
1564 auto stream = adaptor.getAsyncDependencies().front();
1566 MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(rewriter, loc);
1568 {modeA, modeB, adaptor.getDnmatA(),
1569 adaptor.getDnmatB(), adaptor.getSpmatC(),
1577 gpu::SpGEMMCreateDescrOp op, OpAdaptor adaptor,
1579 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1583 auto stream = adaptor.getAsyncDependencies().front();
1592 gpu::SpGEMMDestroyDescrOp op, OpAdaptor adaptor,
1594 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1598 auto stream = adaptor.getAsyncDependencies().front();
1600 {adaptor.getDesc(), stream});
1607 gpu::SpGEMMWorkEstimationOrComputeOp op, OpAdaptor adaptor,
1609 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1614 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1615 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1616 auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
1617 auto stream = adaptor.getAsyncDependencies().front();
1620 MemRefDescriptor(adaptor.getBuffer()).allocatedPtr(rewriter, loc);
1623 if (adaptor.getKind() ==
1628 {adaptor.getDesc(), modeA, modeB, adaptor.getSpmatA(),
1629 adaptor.getSpmatB(), adaptor.getSpmatC(), computeType,
1630 adaptor.getBufferSz(), pBuf, stream})
1636 {adaptor.getDesc(), modeA, modeB, adaptor.getSpmatA(),
1637 adaptor.getSpmatB(), adaptor.getSpmatC(), computeType,
1638 adaptor.getBufferSz(), pBuf, stream})
1646 gpu::SpGEMMCopyOp op, OpAdaptor adaptor,
1648 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1653 rewriter, loc, getCuSparseDataTypeFrom(adaptor.getComputeType()));
1654 auto modeA = genConstInt32From(rewriter, loc, adaptor.getModeA());
1655 auto modeB = genConstInt32From(rewriter, loc, adaptor.getModeB());
1656 auto stream = adaptor.getAsyncDependencies().front();
1658 {adaptor.getDesc(), modeA, modeB,
1659 adaptor.getSpmatA(), adaptor.getSpmatB(),
1660 adaptor.getSpmatC(), computeType, stream});
1666 gpu::SpMatGetSizeOp op, OpAdaptor adaptor,
1668 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1672 auto stream = adaptor.getAsyncDependencies().front();
1692 loc, rewriter, {adaptor.getSpmat(), rowsPtr, colsPtr, nnzsPtr, stream});
1702 gpu::SetCsrPointersOp op, OpAdaptor adaptor,
1704 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1708 auto stream = adaptor.getAsyncDependencies().front();
1710 MemRefDescriptor(adaptor.getPositions()).allocatedPtr(rewriter, loc);
1712 MemRefDescriptor(adaptor.getCoordinates()).allocatedPtr(rewriter, loc);
1714 MemRefDescriptor(adaptor.getValues()).allocatedPtr(rewriter, loc);
1716 loc, rewriter, {adaptor.getSpmat(), pPos, pCrd, pVal, stream});
1722 gpu::CreateCscOp op, OpAdaptor adaptor,
1724 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1728 auto stream = adaptor.getAsyncDependencies().front();
1730 MemRefDescriptor(adaptor.getColPos()).allocatedPtr(rewriter, loc);
1732 MemRefDescriptor(adaptor.getRowIdxs()).allocatedPtr(rewriter, loc);
1734 MemRefDescriptor(adaptor.getValues()).allocatedPtr(rewriter, loc);
1747 {adaptor.getRows(), adaptor.getCols(), adaptor.getNnz(),
1755 gpu::CreateBsrOp op, OpAdaptor adaptor,
1757 if (failed(areAllLLVMTypes(op, adaptor.getOperands(), rewriter)) ||
1761 auto stream = adaptor.getAsyncDependencies().front();
1763 MemRefDescriptor(adaptor.getBRowPos()).allocatedPtr(rewriter, loc);
1765 MemRefDescriptor(adaptor.getBColIdxs()).allocatedPtr(rewriter, loc);
1767 MemRefDescriptor(adaptor.getValues()).allocatedPtr(rewriter, loc);
1780 {adaptor.getBrows(), adaptor.getBcols(), adaptor.getBnnz(),
1781 adaptor.getRBlockSize(), adaptor.getCBlockSize(), pRowPos,