Lines Matching defs:hlfir
13 // The pass lowers these operations to regular hlfir.assign, loops and, if
34 namespace hlfir {
37 } // namespace hlfir
52 /// expressions are any expressions inside an hlfir.where. As described in
92 /// expression evaluation. These are the hlfir.elemental and
93 /// hlfir.elemental_addr that form the elemental tree producing
94 /// the expression value. hlfir.elemental that produce values
111 hlfir::OrderedAssignmentTreeOpInterface root)
115 void lowerRun(hlfir::Run &run) {
131 /// Lowered value for an expression, and the original hlfir.yield if any
133 using ValueAndCleanUp = std::pair<mlir::Value, std::optional<hlfir::YieldOp>>;
138 void walk(hlfir::OrderedAssignmentTreeOpInterface node);
141 void pre(hlfir::ForallOp forallOp);
142 void pre(hlfir::ForallIndexOp);
143 void pre(hlfir::ForallMaskOp);
144 void pre(hlfir::WhereOp whereOp);
145 void pre(hlfir::ElseWhereOp elseWhereOp);
146 void pre(hlfir::RegionAssignOp);
149 void post(hlfir::ForallOp);
150 void post(hlfir::ForallMaskOp);
151 void post(hlfir::WhereOp);
152 void post(hlfir::ElseWhereOp);
155 void enterElsewhere(hlfir::ElseWhereOp);
160 hlfir::OrderedAssignmentTreeOpInterface node,
161 llvm::SmallVectorImpl<hlfir::SaveEntity> &saveEntities) const;
165 isRequiredInCurrentRun(hlfir::OrderedAssignmentTreeOpInterface node) const;
191 std::optional<hlfir::YieldOp> elementalCleanup;
193 std::optional<hlfir::LoopNest> vectorSubscriptLoopNest;
198 /// subscripted (hlfir.elemental_addr), this will create a loop nest
203 std::optional<hlfir::Entity> loweredRhs = std::nullopt);
207 void generateCleanupIfAny(std::optional<hlfir::YieldOp> maybeYield);
211 /// set (When an hlfir.where is being visited).
225 void generateSaveEntity(hlfir::SaveEntity savedEntity,
227 void saveLeftHandSide(hlfir::SaveEntity savedEntity,
228 hlfir::RegionAssignOp regionAssignOp);
277 std::optional<hlfir::LoopNest> whereLoopNest;
293 hlfir::OrderedAssignmentTreeOpInterface root;
295 hlfir::Run *currentRun = nullptr;
305 hlfir::OrderedAssignmentTreeOpInterface node) {
307 isRequiredInCurrentRun(node) || mlir::isa<hlfir::ForallIndexOp>(node);
308 llvm::SmallVector<hlfir::SaveEntity> saveEntities;
312 if (auto elseWhereOp = mlir::dyn_cast<hlfir::ElseWhereOp>(nodeOp)) {
319 for (hlfir::SaveEntity saveEntity : saveEntities)
326 .Case<hlfir::ForallOp, hlfir::ForallIndexOp, hlfir::ForallMaskOp,
327 hlfir::RegionAssignOp, hlfir::WhereOp, hlfir::ElseWhereOp>(
333 mlir::dyn_cast<hlfir::OrderedAssignmentTreeOpInterface>(op))
336 .Case<hlfir::ForallOp, hlfir::ForallMaskOp, hlfir::WhereOp,
337 hlfir::ElseWhereOp>([&](auto concreteOp) { post(concreteOp); })
343 void OrderedAssignmentRewriter::pre(hlfir::ForallOp forallOp) {
344 /// Create a fir.do_loop given the hlfir.forall control values.
369 void OrderedAssignmentRewriter::post(hlfir::ForallOp) {
374 void OrderedAssignmentRewriter::pre(hlfir::ForallIndexOp forallIndexOp) {
384 void OrderedAssignmentRewriter::pre(hlfir::ForallMaskOp forallMaskOp) {
393 void OrderedAssignmentRewriter::post(hlfir::ForallMaskOp forallMaskOp) {
399 /// This is intended to help with cases where hlfir entity is a value while
403 /// while the block argument expects an hlfir.expr.
404 static hlfir::Entity
406 hlfir::Entity input, hlfir::Entity mold,
407 llvm::SmallVectorImpl<hlfir::CleanupFunction> &cleanups) {
415 return hlfir::Entity{builder.createConvert(loc, mold.getType(), load)};
417 // fir.ref<T> to hlfir.expr<T>.
418 mlir::Value asExpr = builder.create<hlfir::AsExprOp>(loc, input);
420 TODO(loc, "hlfir.expr conversion");
421 cleanups.emplace_back([=]() { b->create<hlfir::DestroyOp>(loc, asExpr); });
422 return hlfir::Entity{asExpr};
425 // T to fir.ref<T>, or hlfir.expr<T> to fir.ref<T>.
426 hlfir::AssociateOp associate = hlfir::genAssociateExpr(
429 [=]() { b->create<hlfir::EndAssociateOp>(loc, associate); });
430 return hlfir::Entity{associate.getBase()};
438 auto emboxed = hlfir::convertToBox(loc, builder, input, mold.getType());
440 input = hlfir::Entity{fir::getBase(emboxed.first)};
442 return hlfir::Entity{builder.createConvert(loc, mold.getType(), input)};
445 void OrderedAssignmentRewriter::pre(hlfir::RegionAssignOp regionAssignOp) {
447 std::optional<hlfir::LoopNest> elementalLoopNest;
450 hlfir::Entity rhsEntity{rhsValue};
453 hlfir::Entity lhsEntity{loweredLhs.lhs};
455 rhsEntity = hlfir::getElementAt(
459 hlfir::Entity userAssignLhs{regionAssignOp.getUserAssignmentLhs()};
460 hlfir::Entity userAssignRhs{regionAssignOp.getUserAssignmentRhs()};
461 std::optional<hlfir::LoopNest> elementalLoopNest;
465 mlir::Value shape = hlfir::genShape(loc, builder, lhsEntity);
466 elementalLoopNest = hlfir::genLoopNest(loc, builder, shape);
468 lhsEntity = hlfir::getElementAt(loc, builder, lhsEntity,
470 rhsEntity = hlfir::getElementAt(loc, builder, rhsEntity,
474 llvm::SmallVector<hlfir::CleanupFunction, 2> argConversionCleanups;
490 // they are conveyed in hlfir.region_assign.
491 builder.create<hlfir::AssignOp>(loc, rhsEntity, lhsEntity);
502 cdt = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{cdt});
510 void OrderedAssignmentRewriter::pre(hlfir::WhereOp whereOp) {
517 hlfir::Entity savedMask{maybeSaved->first};
518 mlir::Value shape = hlfir::genShape(loc, builder, savedMask);
519 whereLoopNest = hlfir::genLoopNest(loc, builder, shape);
522 mlir::Value cdt = hlfir::getElementAt(loc, builder, savedMask,
540 whereLoopNest = hlfir::genLoopNest(loc, builder, shape);
555 void OrderedAssignmentRewriter::post(hlfir::WhereOp whereOp) {
567 void OrderedAssignmentRewriter::enterElsewhere(hlfir::ElseWhereOp elseWhereOp) {
581 void OrderedAssignmentRewriter::pre(hlfir::ElseWhereOp elseWhereOp) {
590 void OrderedAssignmentRewriter::post(hlfir::ElseWhereOp elseWhereOp) {
599 /// Forall index are block arguments of hlfir.forall body, or the result
600 /// of hlfir.forall_index.
605 mlir::isa_and_nonnull<hlfir::ForallOp>(block->getParentOp());
608 return value.getDefiningOp<hlfir::ForallIndexOp>();
641 static hlfir::YieldOp getYield(mlir::Region ®ion) {
642 auto yield = mlir::dyn_cast_or_null<hlfir::YieldOp>(
688 // Clone all operations except the final hlfir.yield.
716 value = hlfir::loadTrivialScalar(loc, builder, hlfir::Entity{value});
725 std::optional<hlfir::Entity> loweredRhs) {
727 hlfir::ElementalAddrOp elementalAddrLhs =
728 mlir::dyn_cast<hlfir::ElementalAddrOp>(lhsRegion.back().back());
742 hlfir::genShape(loc, builder, *loweredRhs);
750 loweredLhs.vectorSubscriptLoopNest = hlfir::genLoopNest(
768 hlfir::genLoopNest(loc, builder, *loweredLhs.vectorSubscriptShape,
818 std::optional<hlfir::YieldOp> maybeYield) {
832 hlfir::OrderedAssignmentTreeOpInterface node,
833 llvm::SmallVectorImpl<hlfir::SaveEntity> &saveEntities) const {
835 if (hlfir::SaveEntity *savedEntity =
836 std::get_if<hlfir::SaveEntity>(&action))
843 hlfir::OrderedAssignmentTreeOpInterface node) const {
844 // hlfir.forall_index do not contain saved regions/assignments,
845 // but if their hlfir.forall parent was required, they are
847 if (mlir::isa<hlfir::ForallIndexOp>(node))
850 if (hlfir::SaveEntity *savedEntity =
851 std::get_if<hlfir::SaveEntity>(&action)) {
854 // contains it. For instance, an saving a bound in hlfir.forall B does not
860 auto assign = std::get<hlfir::RegionAssignOp>(action);
868 static bool isInOrderApply(hlfir::ApplyOp apply,
869 hlfir::ElementalOpInterface elemental) {
880 /// Gather the tree of hlfir::ElementalOpInterface use-def, if any, starting
883 gatherElementalTree(hlfir::ElementalOpInterface elemental,
888 // applying indices are in order. An hlfir::Elemental may have been created
897 if (auto apply = mlir::dyn_cast<hlfir::ApplyOp>(op)) {
901 mlir::dyn_cast_or_null<hlfir::ElementalOpInterface>(
913 mlir::dyn_cast<hlfir::ElementalOpInterface>(terminator)) {
914 // Vector subscripted designator (hlfir.elemental_addr terminator).
919 mlir::Value entity = mlir::cast<hlfir::YieldOp>(terminator).getEntity();
920 auto maybeElemental = mlir::dyn_cast_or_null<hlfir::ElementalOpInterface>(
932 // loops. But all of the operations outside of hlfir.elemental/
933 // hlfir.elemental_addr must be emitted now because their value may be
940 // even the scalar ones that are not encoded with hlfir.elemental, to be
945 // Clone only the bodies of all hlfir.exactly_once operations, which contain
950 if (auto exactlyOnce = mlir::dyn_cast<hlfir::ExactlyOnceOp>(op)) {
970 if (auto elementalAddrOp = mlir::dyn_cast<hlfir::ElementalAddrOp>(terminator))
972 mlir::Value entity = mlir::cast<hlfir::YieldOp>(terminator).getEntity();
973 if (auto elemental = entity.getDefiningOp<hlfir::ElementalOp>())
977 hlfir::Entity clonedEntity{mapper.lookupOrDefault(entity)};
978 return hlfir::genShape(loc, builder, hlfir::Entity{clonedEntity});
988 // Clone all operations that are not hlfir.exactly_once and that are not
989 // hlfir.elemental/hlfir.elemental_addr.
991 if (!mlir::isa<hlfir::ExactlyOnceOp>(op) && !elementalParts.contains(&op))
995 // Clone and "index" bodies of hlfir.elemental/hlfir.elemental_addr.
997 hlfir::ElementalOpInterface elemental =
998 mlir::dyn_cast<hlfir::ElementalAddrOp>(terminator);
1000 // If the terminator is not an hlfir.elemental_addr, try if the yielded
1001 // entity was produced by an hlfir.elemental.
1002 mlir::Value entity = mlir::cast<hlfir::YieldOp>(terminator).getEntity();
1003 elemental = entity.getDefiningOp<hlfir::ElementalOp>();
1007 hlfir::Entity clonedEntity{mapper.lookupOrDefault(entity)};
1008 return hlfir::getElementAt(loc, builder, clonedEntity, oneBasedIndices);
1013 [&](hlfir::ElementalOp appliedElemental) -> bool {
1024 // of the hlfir.elemental that have been inlined.
1027 if (auto elementalAddr = mlir::dyn_cast<hlfir::ElementalAddrOp>(terminator)) {
1030 auto yieldOp = mlir::cast<hlfir::YieldOp>(terminator);
1037 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(op))
1050 // Clone clean-ups of hlfir.exactly_once operations (in reverse order
1054 if (auto exactlyOnce = mlir::dyn_cast<hlfir::ExactlyOnceOp>(op)) {
1064 // outside of the loops since the mask non hlfir.elemental part
1070 static hlfir::RegionAssignOp
1072 auto assign = mlir::dyn_cast<hlfir::RegionAssignOp>(region.getParentOp());
1123 getTempName(hlfir::OrderedAssignmentTreeOpInterface root) {
1124 if (mlir::isa<hlfir::ForallOp>(root.getOperation()))
1126 if (mlir::isa<hlfir::WhereOp>(root.getOperation()))
1132 hlfir::SaveEntity savedEntity, bool willUseSavedEntityInSameRun) {
1135 if (hlfir::RegionAssignOp regionAssignOp =
1146 hlfir::Entity entity{clonedValue};
1147 entity = hlfir::loadTrivialScalar(loc, builder, entity);
1154 // WHERE construct, or an LHS/RHS temp of hlfir.region_assign outside of
1197 // this is not done for hlfir.expr because this use would prevent the
1198 // hlfir.expr storage from being moved when creating the temporary in
1202 !mlir::isa<hlfir::ExprType>(entity.getType()))) {
1209 mlir::isa<hlfir::RegionAssignOp>(region.getParentOp())) {
1227 static bool rhsIsArray(hlfir::RegionAssignOp regionAssignOp) {
1228 auto yieldOp = mlir::dyn_cast<hlfir::YieldOp>(
1230 return yieldOp && hlfir::Entity{yieldOp.getEntity()}.isArray();
1234 hlfir::SaveEntity savedEntity, hlfir::RegionAssignOp regionAssignOp) {
1292 /// Lower an ordered assignment tree to fir.do_loop and hlfir.assign given
1294 static void lower(hlfir::OrderedAssignmentTreeOpInterface root,
1295 mlir::PatternRewriter &rewriter, hlfir::Schedule &schedule) {
1306 static llvm::LogicalResult rewrite(hlfir::OrderedAssignmentTreeOpInterface root,
1309 hlfir::Schedule schedule =
1310 hlfir::buildEvaluationSchedule(root, tryFusingAssignments);
1329 class ForallOpConversion : public mlir::OpRewritePattern<hlfir::ForallOp> {
1335 matchAndRewrite(hlfir::ForallOp forallOp,
1337 auto root = mlir::cast<hlfir::OrderedAssignmentTreeOpInterface>(
1346 class WhereOpConversion : public mlir::OpRewritePattern<hlfir::WhereOp> {
1352 matchAndRewrite(hlfir::WhereOp whereOp,
1354 auto root = mlir::cast<hlfir::OrderedAssignmentTreeOpInterface>(
1362 : public mlir::OpRewritePattern<hlfir::RegionAssignOp> {
1368 matchAndRewrite(hlfir::RegionAssignOp regionAssignOp,
1370 auto root = mlir::cast<hlfir::OrderedAssignmentTreeOpInterface>(
1377 : public hlfir::impl::LowerHLFIROrderedAssignmentsBase<
1398 return !mlir::isa<hlfir::OrderedAssignmentTreeOpInterface>(op);