1 //===- BufferizeHLFIR.cpp - Bufferize HLFIR ------------------------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // This file defines a pass that bufferize hlfir.expr. It translates operations 9 // producing or consuming hlfir.expr into operations operating on memory. 10 // An hlfir.expr is translated to a tuple<variable address, cleanupflag> 11 // where cleanupflag is set to true if storage for the expression was allocated 12 // on the heap. 13 //===----------------------------------------------------------------------===// 14 15 #include "flang/Optimizer/Builder/Character.h" 16 #include "flang/Optimizer/Builder/FIRBuilder.h" 17 #include "flang/Optimizer/Builder/HLFIRTools.h" 18 #include "flang/Optimizer/Builder/MutableBox.h" 19 #include "flang/Optimizer/Builder/Runtime/Allocatable.h" 20 #include "flang/Optimizer/Builder/Runtime/Derived.h" 21 #include "flang/Optimizer/Builder/Todo.h" 22 #include "flang/Optimizer/Dialect/FIRDialect.h" 23 #include "flang/Optimizer/Dialect/FIROps.h" 24 #include "flang/Optimizer/Dialect/FIRType.h" 25 #include "flang/Optimizer/Dialect/Support/FIRContext.h" 26 #include "flang/Optimizer/HLFIR/HLFIRDialect.h" 27 #include "flang/Optimizer/HLFIR/HLFIROps.h" 28 #include "flang/Optimizer/HLFIR/Passes.h" 29 #include "flang/Optimizer/OpenMP/Passes.h" 30 #include "mlir/Dialect/OpenMP/OpenMPDialect.h" 31 #include "mlir/IR/Dominance.h" 32 #include "mlir/IR/PatternMatch.h" 33 #include "mlir/Pass/Pass.h" 34 #include "mlir/Pass/PassManager.h" 35 #include "mlir/Transforms/DialectConversion.h" 36 #include "llvm/ADT/TypeSwitch.h" 37 38 namespace hlfir { 39 #define GEN_PASS_DEF_BUFFERIZEHLFIR 40 #include "flang/Optimizer/HLFIR/Passes.h.inc" 41 } // namespace hlfir 42 43 namespace { 44 45 /// Helper to create tuple from a bufferized expr storage and clean up 46 /// instruction flag. The storage is an HLFIR variable so that it can 47 /// be manipulated as a variable later (all shape and length information 48 /// cam be retrieved from it). 49 static mlir::Value packageBufferizedExpr(mlir::Location loc, 50 fir::FirOpBuilder &builder, 51 hlfir::Entity storage, 52 mlir::Value mustFree) { 53 auto tupleType = mlir::TupleType::get( 54 builder.getContext(), 55 mlir::TypeRange{storage.getType(), mustFree.getType()}); 56 auto undef = builder.create<fir::UndefOp>(loc, tupleType); 57 auto insert = builder.create<fir::InsertValueOp>( 58 loc, tupleType, undef, mustFree, 59 builder.getArrayAttr( 60 {builder.getIntegerAttr(builder.getIndexType(), 1)})); 61 return builder.create<fir::InsertValueOp>( 62 loc, tupleType, insert, storage, 63 builder.getArrayAttr( 64 {builder.getIntegerAttr(builder.getIndexType(), 0)})); 65 } 66 67 /// Helper to create tuple from a bufferized expr storage and constant 68 /// boolean clean-up flag. 69 static mlir::Value packageBufferizedExpr(mlir::Location loc, 70 fir::FirOpBuilder &builder, 71 hlfir::Entity storage, bool mustFree) { 72 mlir::Value mustFreeValue = builder.createBool(loc, mustFree); 73 return packageBufferizedExpr(loc, builder, storage, mustFreeValue); 74 } 75 76 /// Helper to extract the storage from a tuple created by packageBufferizedExpr. 77 /// It assumes no tuples are used as HLFIR operation operands, which is 78 /// currently enforced by the verifiers that only accept HLFIR value or 79 /// variable types which do not include tuples. 80 static hlfir::Entity getBufferizedExprStorage(mlir::Value bufferizedExpr) { 81 auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType()); 82 if (!tupleType) 83 return hlfir::Entity{bufferizedExpr}; 84 assert(tupleType.size() == 2 && "unexpected tuple type"); 85 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>()) 86 if (insert.getVal().getType() == tupleType.getType(0)) 87 return hlfir::Entity{insert.getVal()}; 88 TODO(bufferizedExpr.getLoc(), "general extract storage case"); 89 } 90 91 /// Helper to extract the clean-up flag from a tuple created by 92 /// packageBufferizedExpr. 93 static mlir::Value getBufferizedExprMustFreeFlag(mlir::Value bufferizedExpr) { 94 auto tupleType = mlir::dyn_cast<mlir::TupleType>(bufferizedExpr.getType()); 95 if (!tupleType) 96 return bufferizedExpr; 97 assert(tupleType.size() == 2 && "unexpected tuple type"); 98 if (auto insert = bufferizedExpr.getDefiningOp<fir::InsertValueOp>()) 99 if (auto insert0 = insert.getAdt().getDefiningOp<fir::InsertValueOp>()) 100 if (insert0.getVal().getType() == tupleType.getType(1)) 101 return insert0.getVal(); 102 TODO(bufferizedExpr.getLoc(), "general extract storage case"); 103 } 104 105 static std::pair<hlfir::Entity, mlir::Value> 106 createArrayTemp(mlir::Location loc, fir::FirOpBuilder &builder, 107 mlir::Type exprType, mlir::Value shape, 108 mlir::ValueRange extents, mlir::ValueRange lenParams, 109 std::optional<hlfir::Entity> polymorphicMold) { 110 mlir::Type sequenceType = hlfir::getFortranElementOrSequenceType(exprType); 111 llvm::StringRef tmpName{".tmp.array"}; 112 113 if (polymorphicMold) { 114 // Create *allocated* polymorphic temporary using the dynamic type 115 // of the mold and the provided shape/extents. The created temporary 116 // array will be written element per element, that is why it has to be 117 // allocated. 118 mlir::Type boxHeapType = fir::HeapType::get(sequenceType); 119 mlir::Value alloc = fir::factory::genNullBoxStorage( 120 builder, loc, fir::ClassType::get(boxHeapType)); 121 mlir::Value isHeapAlloc = builder.createBool(loc, true); 122 fir::FortranVariableFlagsAttr declAttrs = 123 fir::FortranVariableFlagsAttr::get( 124 builder.getContext(), fir::FortranVariableFlagsEnum::allocatable); 125 126 auto declareOp = 127 builder.create<hlfir::DeclareOp>(loc, alloc, tmpName, 128 /*shape=*/nullptr, lenParams, 129 /*dummy_scope=*/nullptr, declAttrs); 130 131 int rank = extents.size(); 132 fir::runtime::genAllocatableApplyMold(builder, loc, alloc, 133 polymorphicMold->getFirBase(), rank); 134 if (!extents.empty()) { 135 mlir::Type idxTy = builder.getIndexType(); 136 mlir::Value one = builder.createIntegerConstant(loc, idxTy, 1); 137 unsigned dim = 0; 138 for (mlir::Value extent : extents) { 139 mlir::Value dimIndex = builder.createIntegerConstant(loc, idxTy, dim++); 140 fir::runtime::genAllocatableSetBounds(builder, loc, alloc, dimIndex, 141 one, extent); 142 } 143 } 144 if (!lenParams.empty()) { 145 // We should call AllocatableSetDerivedLength() here. 146 // TODO: does the mold provide the length parameters or 147 // the operation itself or should they be in sync? 148 TODO(loc, "polymorphic type with length parameters in HLFIR"); 149 } 150 fir::runtime::genAllocatableAllocate(builder, loc, alloc); 151 152 return {hlfir::Entity{declareOp.getBase()}, isHeapAlloc}; 153 } 154 155 mlir::Value allocmem = builder.createHeapTemporary(loc, sequenceType, tmpName, 156 extents, lenParams); 157 auto declareOp = builder.create<hlfir::DeclareOp>( 158 loc, allocmem, tmpName, shape, lenParams, 159 /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{}); 160 mlir::Value trueVal = builder.createBool(loc, true); 161 return {hlfir::Entity{declareOp.getBase()}, trueVal}; 162 } 163 164 /// Copy \p source into a new temporary and package the temporary into a 165 /// <temp,cleanup> tuple. The temporary may be heap or stack allocated. 166 static mlir::Value copyInTempAndPackage(mlir::Location loc, 167 fir::FirOpBuilder &builder, 168 hlfir::Entity source) { 169 auto [temp, cleanup] = hlfir::createTempFromMold(loc, builder, source); 170 builder.create<hlfir::AssignOp>(loc, source, temp, temp.isAllocatable(), 171 /*keep_lhs_length_if_realloc=*/false, 172 /*temporary_lhs=*/true); 173 // Dereference allocatable temporary directly to simplify processing 174 // of its uses. 175 if (temp.isAllocatable()) 176 temp = hlfir::derefPointersAndAllocatables(loc, builder, temp); 177 return packageBufferizedExpr(loc, builder, temp, cleanup); 178 } 179 180 struct AsExprOpConversion : public mlir::OpConversionPattern<hlfir::AsExprOp> { 181 using mlir::OpConversionPattern<hlfir::AsExprOp>::OpConversionPattern; 182 explicit AsExprOpConversion(mlir::MLIRContext *ctx) 183 : mlir::OpConversionPattern<hlfir::AsExprOp>{ctx} {} 184 llvm::LogicalResult 185 matchAndRewrite(hlfir::AsExprOp asExpr, OpAdaptor adaptor, 186 mlir::ConversionPatternRewriter &rewriter) const override { 187 mlir::Location loc = asExpr->getLoc(); 188 auto module = asExpr->getParentOfType<mlir::ModuleOp>(); 189 fir::FirOpBuilder builder(rewriter, module); 190 if (asExpr.isMove()) { 191 // Move variable storage for the hlfir.expr buffer. 192 mlir::Value bufferizedExpr = packageBufferizedExpr( 193 loc, builder, hlfir::Entity{adaptor.getVar()}, adaptor.getMustFree()); 194 rewriter.replaceOp(asExpr, bufferizedExpr); 195 return mlir::success(); 196 } 197 // Otherwise, create a copy in a new buffer. 198 hlfir::Entity source = hlfir::Entity{adaptor.getVar()}; 199 mlir::Value bufferizedExpr = copyInTempAndPackage(loc, builder, source); 200 rewriter.replaceOp(asExpr, bufferizedExpr); 201 return mlir::success(); 202 } 203 }; 204 205 struct ShapeOfOpConversion 206 : public mlir::OpConversionPattern<hlfir::ShapeOfOp> { 207 using mlir::OpConversionPattern<hlfir::ShapeOfOp>::OpConversionPattern; 208 209 llvm::LogicalResult 210 matchAndRewrite(hlfir::ShapeOfOp shapeOf, OpAdaptor adaptor, 211 mlir::ConversionPatternRewriter &rewriter) const override { 212 mlir::Location loc = shapeOf.getLoc(); 213 mlir::ModuleOp mod = shapeOf->getParentOfType<mlir::ModuleOp>(); 214 fir::FirOpBuilder builder(rewriter, mod); 215 216 mlir::Value shape; 217 hlfir::Entity bufferizedExpr{getBufferizedExprStorage(adaptor.getExpr())}; 218 if (bufferizedExpr.isVariable()) { 219 shape = hlfir::genShape(loc, builder, bufferizedExpr); 220 } else { 221 // everything else failed so try to create a shape from static type info 222 hlfir::ExprType exprTy = 223 mlir::dyn_cast_or_null<hlfir::ExprType>(adaptor.getExpr().getType()); 224 if (exprTy) 225 shape = hlfir::genExprShape(builder, loc, exprTy); 226 } 227 // expected to never happen 228 if (!shape) 229 return emitError(loc, 230 "Unresolvable hlfir.shape_of where extents are unknown"); 231 232 rewriter.replaceOp(shapeOf, shape); 233 return mlir::success(); 234 } 235 }; 236 237 struct ApplyOpConversion : public mlir::OpConversionPattern<hlfir::ApplyOp> { 238 using mlir::OpConversionPattern<hlfir::ApplyOp>::OpConversionPattern; 239 explicit ApplyOpConversion(mlir::MLIRContext *ctx) 240 : mlir::OpConversionPattern<hlfir::ApplyOp>{ctx} {} 241 llvm::LogicalResult 242 matchAndRewrite(hlfir::ApplyOp apply, OpAdaptor adaptor, 243 mlir::ConversionPatternRewriter &rewriter) const override { 244 mlir::Location loc = apply->getLoc(); 245 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr()); 246 mlir::Type resultType = hlfir::getVariableElementType(bufferizedExpr); 247 mlir::Value result = rewriter.create<hlfir::DesignateOp>( 248 loc, resultType, bufferizedExpr, adaptor.getIndices(), 249 adaptor.getTypeparams()); 250 if (fir::isa_trivial(apply.getType())) { 251 result = rewriter.create<fir::LoadOp>(loc, result); 252 } else { 253 fir::FirOpBuilder builder(rewriter, apply.getOperation()); 254 result = 255 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, false); 256 } 257 rewriter.replaceOp(apply, result); 258 return mlir::success(); 259 } 260 }; 261 262 struct AssignOpConversion : public mlir::OpConversionPattern<hlfir::AssignOp> { 263 using mlir::OpConversionPattern<hlfir::AssignOp>::OpConversionPattern; 264 explicit AssignOpConversion(mlir::MLIRContext *ctx) 265 : mlir::OpConversionPattern<hlfir::AssignOp>{ctx} {} 266 llvm::LogicalResult 267 matchAndRewrite(hlfir::AssignOp assign, OpAdaptor adaptor, 268 mlir::ConversionPatternRewriter &rewriter) const override { 269 llvm::SmallVector<mlir::Value> newOperands; 270 for (mlir::Value operand : adaptor.getOperands()) 271 newOperands.push_back(getBufferizedExprStorage(operand)); 272 rewriter.startOpModification(assign); 273 assign->setOperands(newOperands); 274 rewriter.finalizeOpModification(assign); 275 return mlir::success(); 276 } 277 }; 278 279 struct ConcatOpConversion : public mlir::OpConversionPattern<hlfir::ConcatOp> { 280 using mlir::OpConversionPattern<hlfir::ConcatOp>::OpConversionPattern; 281 explicit ConcatOpConversion(mlir::MLIRContext *ctx) 282 : mlir::OpConversionPattern<hlfir::ConcatOp>{ctx} {} 283 llvm::LogicalResult 284 matchAndRewrite(hlfir::ConcatOp concat, OpAdaptor adaptor, 285 mlir::ConversionPatternRewriter &rewriter) const override { 286 mlir::Location loc = concat->getLoc(); 287 fir::FirOpBuilder builder(rewriter, concat.getOperation()); 288 assert(adaptor.getStrings().size() >= 2 && 289 "must have at least two strings operands"); 290 if (adaptor.getStrings().size() > 2) 291 TODO(loc, "codegen of optimized chained concatenation of more than two " 292 "strings"); 293 hlfir::Entity lhs = getBufferizedExprStorage(adaptor.getStrings()[0]); 294 hlfir::Entity rhs = getBufferizedExprStorage(adaptor.getStrings()[1]); 295 auto [lhsExv, c1] = hlfir::translateToExtendedValue(loc, builder, lhs); 296 auto [rhsExv, c2] = hlfir::translateToExtendedValue(loc, builder, rhs); 297 assert(!c1 && !c2 && "expected variables"); 298 fir::ExtendedValue res = 299 fir::factory::CharacterExprHelper{builder, loc}.createConcatenate( 300 *lhsExv.getCharBox(), *rhsExv.getCharBox()); 301 // Ensure the memory type is the same as the result type. 302 mlir::Type addrType = fir::ReferenceType::get( 303 hlfir::getFortranElementType(concat.getResult().getType())); 304 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res)); 305 res = fir::substBase(res, cast); 306 hlfir::Entity hlfirTempRes = 307 hlfir::Entity{hlfir::genDeclare(loc, builder, res, "tmp", 308 fir::FortranVariableFlagsAttr{}) 309 .getBase()}; 310 mlir::Value bufferizedExpr = 311 packageBufferizedExpr(loc, builder, hlfirTempRes, false); 312 rewriter.replaceOp(concat, bufferizedExpr); 313 return mlir::success(); 314 } 315 }; 316 317 struct SetLengthOpConversion 318 : public mlir::OpConversionPattern<hlfir::SetLengthOp> { 319 using mlir::OpConversionPattern<hlfir::SetLengthOp>::OpConversionPattern; 320 explicit SetLengthOpConversion(mlir::MLIRContext *ctx) 321 : mlir::OpConversionPattern<hlfir::SetLengthOp>{ctx} {} 322 llvm::LogicalResult 323 matchAndRewrite(hlfir::SetLengthOp setLength, OpAdaptor adaptor, 324 mlir::ConversionPatternRewriter &rewriter) const override { 325 mlir::Location loc = setLength->getLoc(); 326 fir::FirOpBuilder builder(rewriter, setLength.getOperation()); 327 // Create a temp with the new length. 328 hlfir::Entity string = getBufferizedExprStorage(adaptor.getString()); 329 auto charType = hlfir::getFortranElementType(setLength.getType()); 330 llvm::StringRef tmpName{".tmp"}; 331 llvm::SmallVector<mlir::Value, 1> lenParams{adaptor.getLength()}; 332 auto alloca = builder.createTemporary(loc, charType, tmpName, 333 /*shape=*/std::nullopt, lenParams); 334 auto declareOp = builder.create<hlfir::DeclareOp>( 335 loc, alloca, tmpName, /*shape=*/mlir::Value{}, lenParams, 336 /*dummy_scope=*/nullptr, fir::FortranVariableFlagsAttr{}); 337 hlfir::Entity temp{declareOp.getBase()}; 338 // Assign string value to the created temp. 339 builder.create<hlfir::AssignOp>(loc, string, temp, 340 /*realloc=*/false, 341 /*keep_lhs_length_if_realloc=*/false, 342 /*temporary_lhs=*/true); 343 mlir::Value bufferizedExpr = 344 packageBufferizedExpr(loc, builder, temp, false); 345 rewriter.replaceOp(setLength, bufferizedExpr); 346 return mlir::success(); 347 } 348 }; 349 350 struct GetLengthOpConversion 351 : public mlir::OpConversionPattern<hlfir::GetLengthOp> { 352 using mlir::OpConversionPattern<hlfir::GetLengthOp>::OpConversionPattern; 353 explicit GetLengthOpConversion(mlir::MLIRContext *ctx) 354 : mlir::OpConversionPattern<hlfir::GetLengthOp>{ctx} {} 355 llvm::LogicalResult 356 matchAndRewrite(hlfir::GetLengthOp getLength, OpAdaptor adaptor, 357 mlir::ConversionPatternRewriter &rewriter) const override { 358 mlir::Location loc = getLength->getLoc(); 359 fir::FirOpBuilder builder(rewriter, getLength.getOperation()); 360 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr()); 361 mlir::Value length = hlfir::genCharLength(loc, builder, bufferizedExpr); 362 if (!length) 363 return rewriter.notifyMatchFailure( 364 getLength, "could not deduce length from GetLengthOp operand"); 365 length = builder.createConvert(loc, builder.getIndexType(), length); 366 rewriter.replaceOp(getLength, length); 367 return mlir::success(); 368 } 369 }; 370 371 /// The current hlfir.associate lowering does not handle multiple uses of a 372 /// non-trivial expression value because it generates the cleanup for the 373 /// expression bufferization at hlfir.end_associate. If there was more than one 374 /// hlfir.end_associate, it would be cleaned up multiple times, perhaps before 375 /// one of the other uses. 376 /// Note that we have to be careful about expressions used by a single 377 /// hlfir.end_associate that may be executed more times than the producer 378 /// of the expression value. This may also cause multiple clean-ups 379 /// for the same memory (e.g. cause double-free errors). For example, 380 /// hlfir.end_associate inside hlfir.elemental may cause such issues 381 /// for expressions produced outside of hlfir.elemental. 382 static bool allOtherUsesAreSafeForAssociate(mlir::Value value, 383 mlir::Operation *currentUse, 384 mlir::Operation *endAssociate) { 385 // If value producer is from a different region than 386 // hlfir.associate/end_associate, then conservatively assume 387 // that the hlfir.end_associate may execute more times than 388 // the value producer. 389 // TODO: this may be improved for operations that cannot 390 // result in multiple executions (e.g. ifOp). 391 if (value.getParentRegion() != currentUse->getParentRegion() || 392 (endAssociate && 393 value.getParentRegion() != endAssociate->getParentRegion())) 394 return false; 395 396 for (mlir::Operation *useOp : value.getUsers()) { 397 // Ignore DestroyOp's that do not imply finalization. 398 // If finalization is implied, then we must delegate 399 // the finalization to the correspoding EndAssociateOp, 400 // but we currently do not; so we disable the buffer 401 // reuse in this case. 402 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) { 403 if (destroy.mustFinalizeExpr()) 404 return false; 405 else 406 continue; 407 } 408 409 if (useOp != currentUse) { 410 // hlfir.shape_of and hlfir.get_length will not disrupt cleanup so it is 411 // safe for hlfir.associate. These operations might read from the box and 412 // so they need to come before the hflir.end_associate (which may 413 // deallocate). 414 if (mlir::isa<hlfir::ShapeOfOp>(useOp) || 415 mlir::isa<hlfir::GetLengthOp>(useOp)) { 416 if (!endAssociate) 417 continue; 418 // If useOp dominates the endAssociate, then it is definitely safe. 419 if (useOp->getBlock() != endAssociate->getBlock()) 420 if (mlir::DominanceInfo{}.dominates(useOp, endAssociate)) 421 continue; 422 if (useOp->isBeforeInBlock(endAssociate)) 423 continue; 424 } 425 return false; 426 } 427 } 428 return true; 429 } 430 431 static void eraseAllUsesInDestroys(mlir::Value value, 432 mlir::ConversionPatternRewriter &rewriter) { 433 for (mlir::Operation *useOp : value.getUsers()) 434 if (auto destroy = mlir::dyn_cast<hlfir::DestroyOp>(useOp)) { 435 assert(!destroy.mustFinalizeExpr() && 436 "deleting DestroyOp with finalize attribute"); 437 rewriter.eraseOp(destroy); 438 } 439 } 440 441 struct AssociateOpConversion 442 : public mlir::OpConversionPattern<hlfir::AssociateOp> { 443 using mlir::OpConversionPattern<hlfir::AssociateOp>::OpConversionPattern; 444 explicit AssociateOpConversion(mlir::MLIRContext *ctx) 445 : mlir::OpConversionPattern<hlfir::AssociateOp>{ctx} {} 446 llvm::LogicalResult 447 matchAndRewrite(hlfir::AssociateOp associate, OpAdaptor adaptor, 448 mlir::ConversionPatternRewriter &rewriter) const override { 449 mlir::Location loc = associate->getLoc(); 450 fir::FirOpBuilder builder(rewriter, associate.getOperation()); 451 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getSource()); 452 const bool isTrivialValue = fir::isa_trivial(bufferizedExpr.getType()); 453 454 auto getEndAssociate = 455 [](hlfir::AssociateOp associate) -> mlir::Operation * { 456 for (mlir::Operation *useOp : associate->getUsers()) 457 if (mlir::isa<hlfir::EndAssociateOp>(useOp)) 458 return useOp; 459 // happens in some hand coded mlir in tests 460 return nullptr; 461 }; 462 463 auto replaceWith = [&](mlir::Value hlfirVar, mlir::Value firVar, 464 mlir::Value flag) { 465 // 0-dim variables may need special handling: 466 // %0 = hlfir.as_expr %x move %true : 467 // (!fir.box<!fir.heap<!fir.type<_T{y:i32}>>>, i1) -> 468 // !hlfir.expr<!fir.type<_T{y:i32}>> 469 // %1:3 = hlfir.associate %0 {adapt.valuebyref} : 470 // (!hlfir.expr<!fir.type<_T{y:i32}>>) -> 471 // (!fir.ref<!fir.type<_T{y:i32}>>, 472 // !fir.ref<!fir.type<_T{y:i32}>>, 473 // i1) 474 // 475 // !fir.box<!fir.heap<!fir.type<_T{y:i32}>>> value must be 476 // propagated as the box address !fir.ref<!fir.type<_T{y:i32}>>. 477 auto adjustVar = [&](mlir::Value sourceVar, mlir::Type assocType) { 478 if (mlir::isa<fir::ReferenceType>(sourceVar.getType()) && 479 mlir::isa<fir::ClassType>( 480 fir::unwrapRefType(sourceVar.getType()))) { 481 // Association of a polymorphic value. 482 sourceVar = builder.create<fir::LoadOp>(loc, sourceVar); 483 assert(mlir::isa<fir::ClassType>(sourceVar.getType()) && 484 fir::isAllocatableType(sourceVar.getType())); 485 assert(sourceVar.getType() == assocType); 486 } else if ((mlir::isa<fir::BaseBoxType>(sourceVar.getType()) && 487 !mlir::isa<fir::BaseBoxType>(assocType)) || 488 ((mlir::isa<fir::BoxCharType>(sourceVar.getType()) && 489 !mlir::isa<fir::BoxCharType>(assocType)))) { 490 sourceVar = builder.create<fir::BoxAddrOp>(loc, assocType, sourceVar); 491 } else { 492 sourceVar = builder.createConvert(loc, assocType, sourceVar); 493 } 494 return sourceVar; 495 }; 496 497 mlir::Type associateHlfirVarType = associate.getResultTypes()[0]; 498 hlfirVar = adjustVar(hlfirVar, associateHlfirVarType); 499 associate.getResult(0).replaceAllUsesWith(hlfirVar); 500 501 mlir::Type associateFirVarType = associate.getResultTypes()[1]; 502 firVar = adjustVar(firVar, associateFirVarType); 503 associate.getResult(1).replaceAllUsesWith(firVar); 504 associate.getResult(2).replaceAllUsesWith(flag); 505 // FIXME: note that the AssociateOp that is being erased 506 // here will continue to be a user of the original Source 507 // operand (e.g. a result of hlfir.elemental), because 508 // the erasure is not immediate in the rewriter. 509 // In case there are multiple uses of the Source operand, 510 // the allOtherUsesAreSafeForAssociate() below will always 511 // see them, so there is no way to reuse the buffer. 512 // I think we have to run this analysis before doing 513 // the conversions, so that we can analyze HLFIR in its 514 // original form and decide which of the AssociateOp 515 // users of hlfir.expr can reuse the buffer (if it can). 516 rewriter.eraseOp(associate); 517 }; 518 519 // If this is the last use of the expression value and this is an hlfir.expr 520 // that was bufferized, re-use the storage. 521 // Otherwise, create a temp and assign the storage to it. 522 // 523 // WARNING: it is important to use the original Source operand 524 // of the AssociateOp to look for the users, because its replacement 525 // has zero materialized users at this point. 526 // So allOtherUsesAreSafeForAssociate() may incorrectly return 527 // true here. 528 if (!isTrivialValue && allOtherUsesAreSafeForAssociate( 529 associate.getSource(), associate.getOperation(), 530 getEndAssociate(associate))) { 531 // Re-use hlfir.expr buffer if this is the only use of the hlfir.expr 532 // outside of the hlfir.destroy. Take on the cleaning-up responsibility 533 // for the related hlfir.end_associate, and erase the hlfir.destroy (if 534 // any). 535 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getSource()); 536 mlir::Value firBase = hlfir::Entity{bufferizedExpr}.getFirBase(); 537 replaceWith(bufferizedExpr, firBase, mustFree); 538 eraseAllUsesInDestroys(associate.getSource(), rewriter); 539 // Make sure to erase the hlfir.destroy if there is an indirection through 540 // a hlfir.no_reassoc operation. 541 if (auto noReassoc = mlir::dyn_cast_or_null<hlfir::NoReassocOp>( 542 associate.getSource().getDefiningOp())) 543 eraseAllUsesInDestroys(noReassoc.getVal(), rewriter); 544 return mlir::success(); 545 } 546 if (isTrivialValue) { 547 llvm::SmallVector<mlir::NamedAttribute, 1> attrs; 548 if (associate->hasAttr(fir::getAdaptToByRefAttrName())) { 549 attrs.push_back(fir::getAdaptToByRefAttr(builder)); 550 } 551 llvm::StringRef name = ""; 552 if (associate.getUniqName()) 553 name = *associate.getUniqName(); 554 auto temp = 555 builder.createTemporary(loc, bufferizedExpr.getType(), name, attrs); 556 builder.create<fir::StoreOp>(loc, bufferizedExpr, temp); 557 mlir::Value mustFree = builder.createBool(loc, false); 558 replaceWith(temp, temp, mustFree); 559 return mlir::success(); 560 } 561 // non-trivial value with more than one use. We will have to make a copy and 562 // use that 563 hlfir::Entity source = hlfir::Entity{bufferizedExpr}; 564 mlir::Value bufferTuple = copyInTempAndPackage(loc, builder, source); 565 bufferizedExpr = getBufferizedExprStorage(bufferTuple); 566 replaceWith(bufferizedExpr, hlfir::Entity{bufferizedExpr}.getFirBase(), 567 getBufferizedExprMustFreeFlag(bufferTuple)); 568 return mlir::success(); 569 } 570 }; 571 572 static void genBufferDestruction(mlir::Location loc, fir::FirOpBuilder &builder, 573 mlir::Value var, mlir::Value mustFree, 574 bool mustFinalize) { 575 auto genFreeOrFinalize = [&](bool doFree, bool deallocComponents, 576 bool doFinalize) { 577 if (!doFree && !deallocComponents && !doFinalize) 578 return; 579 580 mlir::Value addr = var; 581 582 // fir::FreeMemOp operand type must be a fir::HeapType. 583 mlir::Type heapType = fir::HeapType::get( 584 hlfir::getFortranElementOrSequenceType(var.getType())); 585 if (mlir::isa<fir::ReferenceType>(var.getType()) && 586 mlir::isa<fir::ClassType>(fir::unwrapRefType(var.getType()))) { 587 // A temporary for a polymorphic expression is represented 588 // via an allocatable. Variable type in this case 589 // is !fir.ref<!fir.class<!fir.heap<!fir.type<>>>>. 590 // We need to free the allocatable data, not the box 591 // that is allocated on the stack. 592 var = builder.create<fir::LoadOp>(loc, var); 593 assert(mlir::isa<fir::ClassType>(var.getType()) && 594 fir::isAllocatableType(var.getType())); 595 addr = builder.create<fir::BoxAddrOp>(loc, heapType, var); 596 // Lowering currently does not produce DestroyOp with 'finalize' 597 // for polymorphic temporaries. It will have to do so, for example, 598 // for MERGE with polymorphic results. 599 if (mustFinalize) 600 TODO(loc, "finalizing polymorphic temporary in HLFIR"); 601 } else if (mlir::isa<fir::BaseBoxType, fir::BoxCharType>(var.getType())) { 602 if (mustFinalize && !mlir::isa<fir::BaseBoxType>(var.getType())) 603 fir::emitFatalError(loc, "non-finalizable variable"); 604 605 addr = builder.create<fir::BoxAddrOp>(loc, heapType, var); 606 } else { 607 if (!mlir::isa<fir::HeapType>(var.getType())) 608 addr = builder.create<fir::ConvertOp>(loc, heapType, var); 609 610 if (mustFinalize || deallocComponents) { 611 // Embox the raw pointer using proper shape and type params 612 // (note that the shape might be visible via the array finalization 613 // routines). 614 if (!hlfir::isFortranEntity(var)) 615 TODO(loc, "need a Fortran entity to create a box"); 616 617 hlfir::Entity entity{var}; 618 llvm::SmallVector<mlir::Value> lenParams; 619 hlfir::genLengthParameters(loc, builder, entity, lenParams); 620 mlir::Value shape; 621 if (entity.isArray()) 622 shape = hlfir::genShape(loc, builder, entity); 623 mlir::Type boxType = fir::BoxType::get(heapType); 624 var = builder.createBox(loc, boxType, addr, shape, /*slice=*/nullptr, 625 lenParams, /*tdesc=*/nullptr); 626 } 627 } 628 629 if (mustFinalize) 630 fir::runtime::genDerivedTypeFinalize(builder, loc, var); 631 632 // If there are allocatable components, they need to be deallocated 633 // (regardless of the mustFree and mustFinalize settings). 634 if (deallocComponents) 635 fir::runtime::genDerivedTypeDestroyWithoutFinalization(builder, loc, var); 636 637 if (doFree) 638 builder.create<fir::FreeMemOp>(loc, addr); 639 }; 640 bool deallocComponents = hlfir::mayHaveAllocatableComponent(var.getType()); 641 642 auto genFree = [&]() { 643 genFreeOrFinalize(/*doFree=*/true, /*deallocComponents=*/false, 644 /*doFinalize=*/false); 645 }; 646 if (auto cstMustFree = fir::getIntIfConstant(mustFree)) { 647 genFreeOrFinalize(*cstMustFree != 0 ? true : false, deallocComponents, 648 mustFinalize); 649 return; 650 } 651 652 // If mustFree is dynamic, first, deallocate any allocatable 653 // components and finalize. 654 genFreeOrFinalize(/*doFree=*/false, deallocComponents, 655 /*doFinalize=*/mustFinalize); 656 // Conditionally free the memory. 657 builder.genIfThen(loc, mustFree).genThen(genFree).end(); 658 } 659 660 struct EndAssociateOpConversion 661 : public mlir::OpConversionPattern<hlfir::EndAssociateOp> { 662 using mlir::OpConversionPattern<hlfir::EndAssociateOp>::OpConversionPattern; 663 explicit EndAssociateOpConversion(mlir::MLIRContext *ctx) 664 : mlir::OpConversionPattern<hlfir::EndAssociateOp>{ctx} {} 665 llvm::LogicalResult 666 matchAndRewrite(hlfir::EndAssociateOp endAssociate, OpAdaptor adaptor, 667 mlir::ConversionPatternRewriter &rewriter) const override { 668 mlir::Location loc = endAssociate->getLoc(); 669 fir::FirOpBuilder builder(rewriter, endAssociate.getOperation()); 670 genBufferDestruction(loc, builder, adaptor.getVar(), adaptor.getMustFree(), 671 /*mustFinalize=*/false); 672 rewriter.eraseOp(endAssociate); 673 return mlir::success(); 674 } 675 }; 676 677 struct DestroyOpConversion 678 : public mlir::OpConversionPattern<hlfir::DestroyOp> { 679 using mlir::OpConversionPattern<hlfir::DestroyOp>::OpConversionPattern; 680 explicit DestroyOpConversion(mlir::MLIRContext *ctx) 681 : mlir::OpConversionPattern<hlfir::DestroyOp>{ctx} {} 682 llvm::LogicalResult 683 matchAndRewrite(hlfir::DestroyOp destroy, OpAdaptor adaptor, 684 mlir::ConversionPatternRewriter &rewriter) const override { 685 // If expr was bufferized on the heap, now is time to deallocate the buffer. 686 mlir::Location loc = destroy->getLoc(); 687 hlfir::Entity bufferizedExpr = getBufferizedExprStorage(adaptor.getExpr()); 688 if (!fir::isa_trivial(bufferizedExpr.getType())) { 689 fir::FirOpBuilder builder(rewriter, destroy.getOperation()); 690 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getExpr()); 691 // Passing FIR base might be enough for cases when 692 // component deallocation and finalization are not required. 693 // If extra BoxAddr operations become a performance problem, 694 // we may pass both bases and let genBufferDestruction decide 695 // which one to use. 696 mlir::Value base = bufferizedExpr.getBase(); 697 genBufferDestruction(loc, builder, base, mustFree, 698 destroy.mustFinalizeExpr()); 699 } 700 701 rewriter.eraseOp(destroy); 702 return mlir::success(); 703 } 704 }; 705 706 struct NoReassocOpConversion 707 : public mlir::OpConversionPattern<hlfir::NoReassocOp> { 708 using mlir::OpConversionPattern<hlfir::NoReassocOp>::OpConversionPattern; 709 explicit NoReassocOpConversion(mlir::MLIRContext *ctx) 710 : mlir::OpConversionPattern<hlfir::NoReassocOp>{ctx} {} 711 llvm::LogicalResult 712 matchAndRewrite(hlfir::NoReassocOp noreassoc, OpAdaptor adaptor, 713 mlir::ConversionPatternRewriter &rewriter) const override { 714 mlir::Location loc = noreassoc->getLoc(); 715 fir::FirOpBuilder builder(rewriter, noreassoc.getOperation()); 716 mlir::Value bufferizedExpr = getBufferizedExprStorage(adaptor.getVal()); 717 mlir::Value result = 718 builder.create<hlfir::NoReassocOp>(loc, bufferizedExpr); 719 720 if (!fir::isa_trivial(bufferizedExpr.getType())) { 721 // NoReassocOp should not be needed on the mustFree path. 722 mlir::Value mustFree = getBufferizedExprMustFreeFlag(adaptor.getVal()); 723 result = 724 packageBufferizedExpr(loc, builder, hlfir::Entity{result}, mustFree); 725 } 726 rewriter.replaceOp(noreassoc, result); 727 return mlir::success(); 728 } 729 }; 730 731 /// Was \p value created in the mlir block where \p builder is currently set ? 732 static bool wasCreatedInCurrentBlock(mlir::Value value, 733 fir::FirOpBuilder &builder) { 734 if (mlir::Operation *op = value.getDefiningOp()) 735 return op->getBlock() == builder.getBlock(); 736 return false; 737 } 738 739 /// This Listener allows setting both the builder and the rewriter as 740 /// listeners. This is required when a pattern uses a firBuilder helper that 741 /// may create illegal operations that will need to be translated and requires 742 /// notifying the rewriter. 743 struct HLFIRListener : public mlir::OpBuilder::Listener { 744 HLFIRListener(fir::FirOpBuilder &builder, 745 mlir::ConversionPatternRewriter &rewriter) 746 : builder{builder}, rewriter{rewriter} {} 747 void notifyOperationInserted(mlir::Operation *op, 748 mlir::OpBuilder::InsertPoint previous) override { 749 builder.notifyOperationInserted(op, previous); 750 rewriter.getListener()->notifyOperationInserted(op, previous); 751 } 752 virtual void notifyBlockInserted(mlir::Block *block, mlir::Region *previous, 753 mlir::Region::iterator previousIt) override { 754 builder.notifyBlockInserted(block, previous, previousIt); 755 rewriter.getListener()->notifyBlockInserted(block, previous, previousIt); 756 } 757 fir::FirOpBuilder &builder; 758 mlir::ConversionPatternRewriter &rewriter; 759 }; 760 761 struct ElementalOpConversion 762 : public mlir::OpConversionPattern<hlfir::ElementalOp> { 763 using mlir::OpConversionPattern<hlfir::ElementalOp>::OpConversionPattern; 764 explicit ElementalOpConversion(mlir::MLIRContext *ctx, 765 bool optimizeEmptyElementals = false) 766 : mlir::OpConversionPattern<hlfir::ElementalOp>{ctx}, 767 optimizeEmptyElementals(optimizeEmptyElementals) { 768 // This pattern recursively converts nested ElementalOp's 769 // by cloning and then converting them, so we have to allow 770 // for recursive pattern application. The recursion is bounded 771 // by the nesting level of ElementalOp's. 772 setHasBoundedRewriteRecursion(); 773 } 774 llvm::LogicalResult 775 matchAndRewrite(hlfir::ElementalOp elemental, OpAdaptor adaptor, 776 mlir::ConversionPatternRewriter &rewriter) const override { 777 mlir::Location loc = elemental->getLoc(); 778 fir::FirOpBuilder builder(rewriter, elemental.getOperation()); 779 // The body of the elemental op may contain operation that will require 780 // to be translated. Notify the rewriter about the cloned operations. 781 HLFIRListener listener{builder, rewriter}; 782 builder.setListener(&listener); 783 784 mlir::Value shape = adaptor.getShape(); 785 std::optional<hlfir::Entity> mold; 786 if (adaptor.getMold()) 787 mold = getBufferizedExprStorage(adaptor.getMold()); 788 auto extents = hlfir::getIndexExtents(loc, builder, shape); 789 auto [temp, cleanup] = 790 createArrayTemp(loc, builder, elemental.getType(), shape, extents, 791 adaptor.getTypeparams(), mold); 792 // If the box load is needed, we'd better place it outside 793 // of the loop nest. 794 temp = derefPointersAndAllocatables(loc, builder, temp); 795 796 if (optimizeEmptyElementals) 797 extents = fir::factory::updateRuntimeExtentsForEmptyArrays(builder, loc, 798 extents); 799 800 // Generate a loop nest looping around the fir.elemental shape and clone 801 // fir.elemental region inside the inner loop. 802 hlfir::LoopNest loopNest = 803 hlfir::genLoopNest(loc, builder, extents, !elemental.isOrdered(), 804 flangomp::shouldUseWorkshareLowering(elemental)); 805 auto insPt = builder.saveInsertionPoint(); 806 builder.setInsertionPointToStart(loopNest.body); 807 auto yield = hlfir::inlineElementalOp(loc, builder, elemental, 808 loopNest.oneBasedIndices); 809 hlfir::Entity elementValue(yield.getElementValue()); 810 // Skip final AsExpr if any. It would create an element temporary, 811 // which is no needed since the element will be assigned right away in 812 // the array temporary. An hlfir.as_expr may have been added if the 813 // elemental is a "view" over a variable (e.g parentheses or transpose). 814 if (auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>()) { 815 if (asExpr->hasOneUse() && !asExpr.isMove()) { 816 // Check that the asExpr is the final operation before the yield, 817 // otherwise, clean-ups could impact the memory being re-used. 818 if (asExpr->getNextNode() == yield.getOperation()) { 819 elementValue = hlfir::Entity{asExpr.getVar()}; 820 rewriter.eraseOp(asExpr); 821 } 822 } 823 } 824 rewriter.eraseOp(yield); 825 // Assign the element value to the temp element for this iteration. 826 auto tempElement = 827 hlfir::getElementAt(loc, builder, temp, loopNest.oneBasedIndices); 828 // If the elemental result is a temporary of a derived type, 829 // we can avoid the deep copy implied by the AssignOp and just 830 // do the shallow copy with load/store. This helps avoiding the overhead 831 // of deallocating allocatable components of the temporary (if any) 832 // on each iteration of the elemental operation. 833 auto asExpr = elementValue.getDefiningOp<hlfir::AsExprOp>(); 834 auto elemType = hlfir::getFortranElementType(elementValue.getType()); 835 if (asExpr && asExpr.isMove() && mlir::isa<fir::RecordType>(elemType) && 836 hlfir::mayHaveAllocatableComponent(elemType) && 837 wasCreatedInCurrentBlock(elementValue, builder)) { 838 auto load = builder.create<fir::LoadOp>(loc, asExpr.getVar()); 839 builder.create<fir::StoreOp>(loc, load, tempElement); 840 } else { 841 builder.create<hlfir::AssignOp>(loc, elementValue, tempElement, 842 /*realloc=*/false, 843 /*keep_lhs_length_if_realloc=*/false, 844 /*temporary_lhs=*/true); 845 846 // hlfir.yield_element implicitly marks the end-of-life its operand if 847 // it is an expression created in the hlfir.elemental (since it is its 848 // last use and an hlfir.destroy could not be created afterwards) 849 // Now that this node has been removed and the expression has been used in 850 // the assign, insert an hlfir.destroy to mark the expression end-of-life. 851 // If the expression creation allocated a buffer on the heap inside the 852 // loop, this will ensure the buffer properly deallocated. 853 if (mlir::isa<hlfir::ExprType>(elementValue.getType()) && 854 wasCreatedInCurrentBlock(elementValue, builder)) 855 builder.create<hlfir::DestroyOp>(loc, elementValue); 856 } 857 builder.restoreInsertionPoint(insPt); 858 859 mlir::Value bufferizedExpr = 860 packageBufferizedExpr(loc, builder, temp, cleanup); 861 // Explicitly delete the body of the elemental to get rid 862 // of any users of hlfir.expr values inside the body as early 863 // as possible. 864 rewriter.startOpModification(elemental); 865 rewriter.eraseBlock(elemental.getBody()); 866 rewriter.finalizeOpModification(elemental); 867 rewriter.replaceOp(elemental, bufferizedExpr); 868 return mlir::success(); 869 } 870 871 private: 872 bool optimizeEmptyElementals = false; 873 }; 874 struct CharExtremumOpConversion 875 : public mlir::OpConversionPattern<hlfir::CharExtremumOp> { 876 using mlir::OpConversionPattern<hlfir::CharExtremumOp>::OpConversionPattern; 877 explicit CharExtremumOpConversion(mlir::MLIRContext *ctx) 878 : mlir::OpConversionPattern<hlfir::CharExtremumOp>{ctx} {} 879 llvm::LogicalResult 880 matchAndRewrite(hlfir::CharExtremumOp char_extremum, OpAdaptor adaptor, 881 mlir::ConversionPatternRewriter &rewriter) const override { 882 mlir::Location loc = char_extremum->getLoc(); 883 auto predicate = char_extremum.getPredicate(); 884 bool predIsMin = 885 predicate == hlfir::CharExtremumPredicate::min ? true : false; 886 fir::FirOpBuilder builder(rewriter, char_extremum.getOperation()); 887 assert(adaptor.getStrings().size() >= 2 && 888 "must have at least two strings operands"); 889 auto numOperands = adaptor.getStrings().size(); 890 891 std::vector<hlfir::Entity> chars; 892 std::vector< 893 std::pair<fir::ExtendedValue, std::optional<hlfir::CleanupFunction>>> 894 pairs; 895 llvm::SmallVector<fir::CharBoxValue> opCBVs; 896 for (size_t i = 0; i < numOperands; ++i) { 897 chars.emplace_back(getBufferizedExprStorage(adaptor.getStrings()[i])); 898 pairs.emplace_back( 899 hlfir::translateToExtendedValue(loc, builder, chars[i])); 900 assert(!pairs[i].second && "expected variables"); 901 opCBVs.emplace_back(*pairs[i].first.getCharBox()); 902 } 903 904 fir::ExtendedValue res = 905 fir::factory::CharacterExprHelper{builder, loc}.createCharExtremum( 906 predIsMin, opCBVs); 907 mlir::Type addrType = fir::ReferenceType::get( 908 hlfir::getFortranElementType(char_extremum.getResult().getType())); 909 mlir::Value cast = builder.createConvert(loc, addrType, fir::getBase(res)); 910 res = fir::substBase(res, cast); 911 hlfir::Entity hlfirTempRes = 912 hlfir::Entity{hlfir::genDeclare(loc, builder, res, ".tmp.char_extremum", 913 fir::FortranVariableFlagsAttr{}) 914 .getBase()}; 915 mlir::Value bufferizedExpr = 916 packageBufferizedExpr(loc, builder, hlfirTempRes, false); 917 rewriter.replaceOp(char_extremum, bufferizedExpr); 918 return mlir::success(); 919 } 920 }; 921 922 struct EvaluateInMemoryOpConversion 923 : public mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp> { 924 using mlir::OpConversionPattern< 925 hlfir::EvaluateInMemoryOp>::OpConversionPattern; 926 explicit EvaluateInMemoryOpConversion(mlir::MLIRContext *ctx) 927 : mlir::OpConversionPattern<hlfir::EvaluateInMemoryOp>{ctx} {} 928 llvm::LogicalResult 929 matchAndRewrite(hlfir::EvaluateInMemoryOp evalInMemOp, OpAdaptor adaptor, 930 mlir::ConversionPatternRewriter &rewriter) const override { 931 mlir::Location loc = evalInMemOp->getLoc(); 932 fir::FirOpBuilder builder(rewriter, evalInMemOp.getOperation()); 933 auto [temp, isHeapAlloc] = hlfir::computeEvaluateOpInNewTemp( 934 loc, builder, evalInMemOp, adaptor.getShape(), adaptor.getTypeparams()); 935 mlir::Value bufferizedExpr = 936 packageBufferizedExpr(loc, builder, temp, isHeapAlloc); 937 rewriter.replaceOp(evalInMemOp, bufferizedExpr); 938 return mlir::success(); 939 } 940 }; 941 942 class BufferizeHLFIR : public hlfir::impl::BufferizeHLFIRBase<BufferizeHLFIR> { 943 public: 944 using BufferizeHLFIRBase<BufferizeHLFIR>::BufferizeHLFIRBase; 945 946 void runOnOperation() override { 947 // TODO: make this a pass operating on FuncOp. The issue is that 948 // FirOpBuilder helpers may generate new FuncOp because of runtime/llvm 949 // intrinsics calls creation. This may create race conflict if the pass is 950 // scheduled on FuncOp. A solution could be to provide an optional mutex 951 // when building a FirOpBuilder and locking around FuncOp and GlobalOp 952 // creation, but this needs a bit more thinking, so at this point the pass 953 // is scheduled on the moduleOp. 954 auto module = this->getOperation(); 955 auto *context = &getContext(); 956 mlir::RewritePatternSet patterns(context); 957 patterns.insert<ApplyOpConversion, AsExprOpConversion, AssignOpConversion, 958 AssociateOpConversion, CharExtremumOpConversion, 959 ConcatOpConversion, DestroyOpConversion, 960 EndAssociateOpConversion, EvaluateInMemoryOpConversion, 961 NoReassocOpConversion, SetLengthOpConversion, 962 ShapeOfOpConversion, GetLengthOpConversion>(context); 963 patterns.insert<ElementalOpConversion>(context, optimizeEmptyElementals); 964 mlir::ConversionTarget target(*context); 965 // Note that YieldElementOp is not marked as an illegal operation. 966 // It must be erased by its parent converter and there is no explicit 967 // conversion pattern to YieldElementOp itself. If any YieldElementOp 968 // survives this pass, the verifier will detect it because it has to be 969 // a child of ElementalOp and ElementalOp's are explicitly illegal. 970 target.addIllegalOp<hlfir::ApplyOp, hlfir::AssociateOp, hlfir::ElementalOp, 971 hlfir::EndAssociateOp, hlfir::SetLengthOp>(); 972 973 target.markUnknownOpDynamicallyLegal([](mlir::Operation *op) { 974 return llvm::all_of(op->getResultTypes(), 975 [](mlir::Type ty) { 976 return !mlir::isa<hlfir::ExprType>(ty); 977 }) && 978 llvm::all_of(op->getOperandTypes(), [](mlir::Type ty) { 979 return !mlir::isa<hlfir::ExprType>(ty); 980 }); 981 }); 982 if (mlir::failed( 983 mlir::applyFullConversion(module, target, std::move(patterns)))) { 984 mlir::emitError(mlir::UnknownLoc::get(context), 985 "failure in HLFIR bufferization pass"); 986 signalPassFailure(); 987 } 988 } 989 }; 990 } // namespace 991