14d4af15cSKareem Ergawy //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===// 24d4af15cSKareem Ergawy // 34d4af15cSKareem Ergawy // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 44d4af15cSKareem Ergawy // See https://llvm.org/LICENSE.txt for license information. 54d4af15cSKareem Ergawy // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 64d4af15cSKareem Ergawy // 74d4af15cSKareem Ergawy //===----------------------------------------------------------------------===// 84d4af15cSKareem Ergawy // 94d4af15cSKareem Ergawy // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/ 104d4af15cSKareem Ergawy // 114d4af15cSKareem Ergawy //===----------------------------------------------------------------------===// 124d4af15cSKareem Ergawy 134d4af15cSKareem Ergawy #include "DataSharingProcessor.h" 144d4af15cSKareem Ergawy 154d4af15cSKareem Ergawy #include "Utils.h" 1698e733eaSTom Eccles #include "flang/Lower/ConvertVariable.h" 174d4af15cSKareem Ergawy #include "flang/Lower/PFTBuilder.h" 182703f7ecSKareem Ergawy #include "flang/Lower/SymbolMap.h" 1924f5fc77SKareem Ergawy #include "flang/Optimizer/Builder/HLFIRTools.h" 204d4af15cSKareem Ergawy #include "flang/Optimizer/Builder/Todo.h" 212e6558b8STom Eccles #include "flang/Optimizer/HLFIR/HLFIROps.h" 224d4af15cSKareem Ergawy #include "flang/Semantics/tools.h" 234d4af15cSKareem Ergawy 244d4af15cSKareem Ergawy namespace Fortran { 254d4af15cSKareem Ergawy namespace lower { 264d4af15cSKareem Ergawy namespace omp { 276af4118fSKareem Ergawy bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy( 286af4118fSKareem Ergawy const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const { 296af4118fSKareem Ergawy return eval.visit( 306af4118fSKareem Ergawy common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) { 316af4118fSKareem Ergawy return symDefMap.count(symbol) && 326af4118fSKareem Ergawy symDefMap.at(symbol) == &functionParserNode; 336af4118fSKareem Ergawy }, 346af4118fSKareem Ergawy [](const auto &functionParserNode) { return false; }}); 356af4118fSKareem Ergawy } 366af4118fSKareem Ergawy 376af4118fSKareem Ergawy DataSharingProcessor::DataSharingProcessor( 386af4118fSKareem Ergawy lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx, 396af4118fSKareem Ergawy const List<Clause> &clauses, lower::pft::Evaluation &eval, 406af4118fSKareem Ergawy bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization, 41db9856b5SLeandro Lupori lower::SymMap &symTable) 4212c4d26cSSergio Afonso : converter(converter), semaCtx(semaCtx), 436af4118fSKareem Ergawy firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval), 446af4118fSKareem Ergawy shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols), 456af4118fSKareem Ergawy useDelayedPrivatization(useDelayedPrivatization), symTable(symTable), 466af4118fSKareem Ergawy visitor() { 476af4118fSKareem Ergawy eval.visit([&](const auto &functionParserNode) { 486af4118fSKareem Ergawy parser::Walk(functionParserNode, visitor); 496af4118fSKareem Ergawy }); 506af4118fSKareem Ergawy } 514d4af15cSKareem Ergawy 5278eac466SSergio Afonso void DataSharingProcessor::processStep1( 53913a8244SKareem Ergawy mlir::omp::PrivateClauseOps *clauseOps) { 544d4af15cSKareem Ergawy collectSymbolsForPrivatization(); 554d4af15cSKareem Ergawy collectDefaultSymbols(); 561e9625e5SLeandro Lupori collectImplicitSymbols(); 576af4118fSKareem Ergawy collectPreDeterminedSymbols(); 586af4118fSKareem Ergawy 59913a8244SKareem Ergawy privatize(clauseOps); 606af4118fSKareem Ergawy 614d4af15cSKareem Ergawy insertBarrier(); 624d4af15cSKareem Ergawy } 634d4af15cSKareem Ergawy 644d4af15cSKareem Ergawy void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) { 656542e566SLeandro Lupori // 'sections' lastprivate is handled by genOMP() 666542e566SLeandro Lupori if (!mlir::isa<mlir::omp::SectionsOp>(op)) { 6712c4d26cSSergio Afonso mlir::OpBuilder::InsertionGuard guard(firOpBuilder); 684d4af15cSKareem Ergawy copyLastPrivatize(op); 696542e566SLeandro Lupori } 704d4af15cSKareem Ergawy 714d4af15cSKareem Ergawy if (isLoop) { 724d4af15cSKareem Ergawy // push deallocs out of the loop 734d4af15cSKareem Ergawy firOpBuilder.setInsertionPointAfter(op); 744d4af15cSKareem Ergawy insertDeallocs(); 754d4af15cSKareem Ergawy } else { 7699f6ff9cSSergio Afonso mlir::OpBuilder::InsertionGuard guard(firOpBuilder); 774d4af15cSKareem Ergawy insertDeallocs(); 784d4af15cSKareem Ergawy } 794d4af15cSKareem Ergawy } 804d4af15cSKareem Ergawy 814d4af15cSKareem Ergawy void DataSharingProcessor::insertDeallocs() { 826af4118fSKareem Ergawy for (const semantics::Symbol *sym : allPrivatizedSymbols) 837a66e420SKrzysztof Parzyszek if (semantics::IsAllocatable(sym->GetUltimate())) { 840632cb38SKareem Ergawy if (!useDelayedPrivatization) { 854d4af15cSKareem Ergawy converter.createHostAssociateVarCloneDealloc(*sym); 86e1aa8ad6SKareem Ergawy continue; 870632cb38SKareem Ergawy } 880632cb38SKareem Ergawy 897a66e420SKrzysztof Parzyszek lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym); 900632cb38SKareem Ergawy assert(hsb && "Host symbol box not found"); 910632cb38SKareem Ergawy mlir::Type symType = hsb.getAddr().getType(); 920632cb38SKareem Ergawy mlir::Location symLoc = hsb.getAddr().getLoc(); 930632cb38SKareem Ergawy fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym); 940632cb38SKareem Ergawy mlir::omp::PrivateClauseOp privatizer = symToPrivatizer.at(sym); 950632cb38SKareem Ergawy 96db9856b5SLeandro Lupori lower::SymMapScope scope(symTable); 970632cb38SKareem Ergawy mlir::OpBuilder::InsertionGuard guard(firOpBuilder); 980632cb38SKareem Ergawy 990632cb38SKareem Ergawy mlir::Region &deallocRegion = privatizer.getDeallocRegion(); 1000632cb38SKareem Ergawy fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); 1010632cb38SKareem Ergawy mlir::Block *deallocEntryBlock = firOpBuilder.createBlock( 1020632cb38SKareem Ergawy &deallocRegion, /*insertPt=*/{}, symType, symLoc); 1030632cb38SKareem Ergawy 1040632cb38SKareem Ergawy firOpBuilder.setInsertionPointToEnd(deallocEntryBlock); 105db9856b5SLeandro Lupori symTable.addSymbol(*sym, 1060632cb38SKareem Ergawy fir::substBase(symExV, deallocRegion.getArgument(0))); 1070632cb38SKareem Ergawy 1080632cb38SKareem Ergawy converter.createHostAssociateVarCloneDealloc(*sym); 1090632cb38SKareem Ergawy firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc()); 1104d4af15cSKareem Ergawy } 1114d4af15cSKareem Ergawy } 1124d4af15cSKareem Ergawy 1137a66e420SKrzysztof Parzyszek void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) { 114ff78cd5fSjeanPerier bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate); 115ff78cd5fSjeanPerier bool success = converter.createHostAssociateVarClone( 116ff78cd5fSjeanPerier *sym, /*skipDefaultInit=*/isFirstPrivate); 1174d4af15cSKareem Ergawy (void)success; 1184d4af15cSKareem Ergawy assert(success && "Privatization failed due to existing binding"); 1191fcb6a97SLeandro Lupori 1201fcb6a97SLeandro Lupori // Initialize clone from original object if it has any allocatable member. 1211fcb6a97SLeandro Lupori auto needInitClone = [&] { 1221fcb6a97SLeandro Lupori if (isFirstPrivate) 1231fcb6a97SLeandro Lupori return false; 1241fcb6a97SLeandro Lupori 1251fcb6a97SLeandro Lupori SymbolBox sb = symTable.lookupSymbol(sym); 1261fcb6a97SLeandro Lupori assert(sb); 1271fcb6a97SLeandro Lupori mlir::Value addr = sb.getAddr(); 1281fcb6a97SLeandro Lupori assert(addr); 1295130a4eaSLeandro Lupori return !fir::isPointerType(addr.getType()) && 1305130a4eaSLeandro Lupori hlfir::mayHaveAllocatableComponent(addr.getType()); 1311fcb6a97SLeandro Lupori }; 1321fcb6a97SLeandro Lupori 1331fcb6a97SLeandro Lupori if (needInitClone()) { 1341fcb6a97SLeandro Lupori Fortran::lower::initializeCloneAtRuntime(converter, *sym, symTable); 1351fcb6a97SLeandro Lupori callsInitClone = true; 1361fcb6a97SLeandro Lupori } 1374d4af15cSKareem Ergawy } 1384d4af15cSKareem Ergawy 1394d4af15cSKareem Ergawy void DataSharingProcessor::copyFirstPrivateSymbol( 1407a66e420SKrzysztof Parzyszek const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) { 1417a66e420SKrzysztof Parzyszek if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate)) 14226b8be20SKareem Ergawy converter.copyHostAssociateVar(*sym, copyAssignIP); 1434d4af15cSKareem Ergawy } 1444d4af15cSKareem Ergawy 1454d4af15cSKareem Ergawy void DataSharingProcessor::copyLastPrivateSymbol( 146797f0119SLeandro Lupori const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *lastPrivIP) { 147797f0119SLeandro Lupori if (sym->test(semantics::Symbol::Flag::OmpLastPrivate)) 148*a0406ce8SKareem Ergawy converter.copyHostAssociateVar(*sym, lastPrivIP, /*hostIsSource=*/false); 1494d4af15cSKareem Ergawy } 1504d4af15cSKareem Ergawy 1514d4af15cSKareem Ergawy void DataSharingProcessor::collectOmpObjectListSymbol( 152037a32a9SKrzysztof Parzyszek const omp::ObjectList &objects, 1537a66e420SKrzysztof Parzyszek llvm::SetVector<const semantics::Symbol *> &symbolSet) { 154037a32a9SKrzysztof Parzyszek for (const omp::Object &object : objects) 1558b18f2feSKrzysztof Parzyszek symbolSet.insert(object.sym()); 1564d4af15cSKareem Ergawy } 1574d4af15cSKareem Ergawy 1584d4af15cSKareem Ergawy void DataSharingProcessor::collectSymbolsForPrivatization() { 159037a32a9SKrzysztof Parzyszek for (const omp::Clause &clause : clauses) { 1604d4af15cSKareem Ergawy if (const auto &privateClause = 161037a32a9SKrzysztof Parzyszek std::get_if<omp::clause::Private>(&clause.u)) { 1626af4118fSKareem Ergawy collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols); 1634d4af15cSKareem Ergawy } else if (const auto &firstPrivateClause = 164037a32a9SKrzysztof Parzyszek std::get_if<omp::clause::Firstprivate>(&clause.u)) { 1656af4118fSKareem Ergawy collectOmpObjectListSymbol(firstPrivateClause->v, 1666af4118fSKareem Ergawy explicitlyPrivatizedSymbols); 1674d4af15cSKareem Ergawy } else if (const auto &lastPrivateClause = 168037a32a9SKrzysztof Parzyszek std::get_if<omp::clause::Lastprivate>(&clause.u)) { 169f9824439SKrzysztof Parzyszek lastprivateModifierNotSupported(*lastPrivateClause, 170f9824439SKrzysztof Parzyszek converter.getCurrentLocation()); 171148a5579SKrzysztof Parzyszek const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t); 1726af4118fSKareem Ergawy collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols); 1734d4af15cSKareem Ergawy } 1744d4af15cSKareem Ergawy } 1754d4af15cSKareem Ergawy 1766af4118fSKareem Ergawy for (auto *sym : explicitlyPrivatizedSymbols) 1776af4118fSKareem Ergawy allPrivatizedSymbols.insert(sym); 1784d4af15cSKareem Ergawy } 1794d4af15cSKareem Ergawy 1804d4af15cSKareem Ergawy bool DataSharingProcessor::needBarrier() { 1816542e566SLeandro Lupori // Emit implicit barrier to synchronize threads and avoid data races on 1826542e566SLeandro Lupori // initialization of firstprivate variables and post-update of lastprivate 1836542e566SLeandro Lupori // variables. 1846542e566SLeandro Lupori // Emit implicit barrier for linear clause. Maybe on somewhere else. 1856af4118fSKareem Ergawy for (const semantics::Symbol *sym : allPrivatizedSymbols) { 1861fcb6a97SLeandro Lupori if (sym->test(semantics::Symbol::Flag::OmpLastPrivate) && 1871fcb6a97SLeandro Lupori (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || callsInitClone)) 1884d4af15cSKareem Ergawy return true; 1894d4af15cSKareem Ergawy } 1904d4af15cSKareem Ergawy return false; 1914d4af15cSKareem Ergawy } 1924d4af15cSKareem Ergawy 1934d4af15cSKareem Ergawy void DataSharingProcessor::insertBarrier() { 1944d4af15cSKareem Ergawy if (needBarrier()) 1954d4af15cSKareem Ergawy firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation()); 1964d4af15cSKareem Ergawy } 1974d4af15cSKareem Ergawy 1984d4af15cSKareem Ergawy void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) { 199ca4dbc27SSergio Afonso mlir::omp::LoopNestOp loopOp; 200ca4dbc27SSergio Afonso if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op)) 2012784060cSSergio Afonso loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop()); 202ca4dbc27SSergio Afonso 2034d4af15cSKareem Ergawy bool cmpCreated = false; 2049dbf3e23SSergio Afonso mlir::OpBuilder::InsertionGuard guard(firOpBuilder); 205037a32a9SKrzysztof Parzyszek for (const omp::Clause &clause : clauses) { 206037a32a9SKrzysztof Parzyszek if (clause.id != llvm::omp::OMPC_lastprivate) 207037a32a9SKrzysztof Parzyszek continue; 208eafa1500SNimishMishra if (mlir::isa<mlir::omp::WsloopOp>(op) || 209eafa1500SNimishMishra mlir::isa<mlir::omp::SimdOp>(op)) { 2104d4af15cSKareem Ergawy // Update the original variable just before exiting the worksharing 2114d4af15cSKareem Ergawy // loop. Conversion as follows: 2124d4af15cSKareem Ergawy // 213eafa1500SNimishMishra // omp.wsloop / omp.simd { omp.wsloop / omp.simd { 214ca4dbc27SSergio Afonso // omp.loop_nest { omp.loop_nest { 215ca4dbc27SSergio Afonso // ... ... 216ca4dbc27SSergio Afonso // store ===> store 217ca4dbc27SSergio Afonso // omp.yield %v = arith.addi %iv, %step 218ca4dbc27SSergio Afonso // } %cmp = %step < 0 ? %v < %ub : %v > %ub 2190a17bdfcSSergio Afonso // } fir.if %cmp { 2200a17bdfcSSergio Afonso // fir.store %v to %loopIV 2214d4af15cSKareem Ergawy // ^%lpv_update_blk: 2224d4af15cSKareem Ergawy // } 2234d4af15cSKareem Ergawy // omp.yield 2244d4af15cSKareem Ergawy // } 225ca4dbc27SSergio Afonso // } 2264d4af15cSKareem Ergawy 2274d4af15cSKareem Ergawy // Only generate the compare once in presence of multiple LastPrivate 2284d4af15cSKareem Ergawy // clauses. 2294d4af15cSKareem Ergawy if (cmpCreated) 2304d4af15cSKareem Ergawy continue; 2314d4af15cSKareem Ergawy cmpCreated = true; 2324d4af15cSKareem Ergawy 233ca4dbc27SSergio Afonso mlir::Location loc = loopOp.getLoc(); 234ca4dbc27SSergio Afonso mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator(); 2354d4af15cSKareem Ergawy firOpBuilder.setInsertionPoint(lastOper); 2364d4af15cSKareem Ergawy 2374b9fab59SDavid Truby mlir::Value cmpOp; 2384b9fab59SDavid Truby llvm::SmallVector<mlir::Value> vs; 2394b9fab59SDavid Truby vs.reserve(loopOp.getIVs().size()); 240fdfeea5bSSergio Afonso for (auto [iv, ub, step] : 24146ecd7bbSSergio Afonso llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(), 24246ecd7bbSSergio Afonso loopOp.getLoopSteps())) { 2434d4af15cSKareem Ergawy // v = iv + step 2444d4af15cSKareem Ergawy // cmp = step < 0 ? v < ub : v > ub 2454d4af15cSKareem Ergawy mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step); 2464b9fab59SDavid Truby vs.push_back(v); 2474d4af15cSKareem Ergawy mlir::Value zero = 2484d4af15cSKareem Ergawy firOpBuilder.createIntegerConstant(loc, step.getType(), 0); 2494d4af15cSKareem Ergawy mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>( 2504d4af15cSKareem Ergawy loc, mlir::arith::CmpIPredicate::slt, step, zero); 2514d4af15cSKareem Ergawy mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>( 2524d4af15cSKareem Ergawy loc, mlir::arith::CmpIPredicate::slt, v, ub); 2534d4af15cSKareem Ergawy mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>( 2544d4af15cSKareem Ergawy loc, mlir::arith::CmpIPredicate::sgt, v, ub); 2554b9fab59SDavid Truby mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>( 2564d4af15cSKareem Ergawy loc, negativeStep, vLT, vGT); 2574d4af15cSKareem Ergawy 2584b9fab59SDavid Truby if (cmpOp) { 2594b9fab59SDavid Truby cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp); 2604b9fab59SDavid Truby } else { 2614b9fab59SDavid Truby cmpOp = icmpOp; 2624b9fab59SDavid Truby } 2634b9fab59SDavid Truby } 2644b9fab59SDavid Truby 2654d4af15cSKareem Ergawy auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false); 2664d4af15cSKareem Ergawy firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front()); 2674b9fab59SDavid Truby for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) { 2684d4af15cSKareem Ergawy assert(loopIV && "loopIV was not set"); 2697cee61c8SKiran Chandramohan firOpBuilder.createStoreWithConvert(loc, v, loopIV); 2704b9fab59SDavid Truby } 2714d4af15cSKareem Ergawy lastPrivIP = firOpBuilder.saveInsertionPoint(); 2726542e566SLeandro Lupori } else if (mlir::isa<mlir::omp::SectionsOp>(op)) { 2736542e566SLeandro Lupori // Already handled by genOMP() 2744d4af15cSKareem Ergawy } else { 2754d4af15cSKareem Ergawy TODO(converter.getCurrentLocation(), 2764d4af15cSKareem Ergawy "lastprivate clause in constructs other than " 2774d4af15cSKareem Ergawy "simd/worksharing-loop"); 2784d4af15cSKareem Ergawy } 2794d4af15cSKareem Ergawy } 2804d4af15cSKareem Ergawy } 2814d4af15cSKareem Ergawy 2827a66e420SKrzysztof Parzyszek static const parser::CharBlock * 2837a66e420SKrzysztof Parzyszek getSource(const semantics::SemanticsContext &semaCtx, 2847a66e420SKrzysztof Parzyszek const lower::pft::Evaluation &eval) { 2857a66e420SKrzysztof Parzyszek const parser::CharBlock *source = nullptr; 2861e9625e5SLeandro Lupori 2877a66e420SKrzysztof Parzyszek auto ompConsVisit = [&](const parser::OpenMPConstruct &x) { 2887a66e420SKrzysztof Parzyszek std::visit(common::visitors{ 2897a66e420SKrzysztof Parzyszek [&](const parser::OpenMPSectionsConstruct &x) { 2901e9625e5SLeandro Lupori source = &std::get<0>(x.t).source; 2911e9625e5SLeandro Lupori }, 2927a66e420SKrzysztof Parzyszek [&](const parser::OpenMPLoopConstruct &x) { 2931e9625e5SLeandro Lupori source = &std::get<0>(x.t).source; 2941e9625e5SLeandro Lupori }, 2957a66e420SKrzysztof Parzyszek [&](const parser::OpenMPBlockConstruct &x) { 2961e9625e5SLeandro Lupori source = &std::get<0>(x.t).source; 2971e9625e5SLeandro Lupori }, 2987a66e420SKrzysztof Parzyszek [&](const parser::OpenMPCriticalConstruct &x) { 2991e9625e5SLeandro Lupori source = &std::get<0>(x.t).source; 3001e9625e5SLeandro Lupori }, 3017a66e420SKrzysztof Parzyszek [&](const parser::OpenMPAtomicConstruct &x) { 3021e9625e5SLeandro Lupori std::visit([&](const auto &x) { source = &x.source; }, 3031e9625e5SLeandro Lupori x.u); 3041e9625e5SLeandro Lupori }, 3051e9625e5SLeandro Lupori [&](const auto &x) { source = &x.source; }, 3061e9625e5SLeandro Lupori }, 3071e9625e5SLeandro Lupori x.u); 3081e9625e5SLeandro Lupori }; 3091e9625e5SLeandro Lupori 3107a66e420SKrzysztof Parzyszek eval.visit(common::visitors{ 3117a66e420SKrzysztof Parzyszek [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); }, 3127a66e420SKrzysztof Parzyszek [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; }, 3137a66e420SKrzysztof Parzyszek [&](const parser::OmpEndLoopDirective &x) { source = &x.source; }, 3141e9625e5SLeandro Lupori [&](const auto &x) {}, 3151e9625e5SLeandro Lupori }); 3161e9625e5SLeandro Lupori 3171e9625e5SLeandro Lupori return source; 3181e9625e5SLeandro Lupori } 3191e9625e5SLeandro Lupori 32037f6ba4fSNimishMishra void DataSharingProcessor::collectSymbolsInNestedRegions( 3217a66e420SKrzysztof Parzyszek lower::pft::Evaluation &eval, semantics::Symbol::Flag flag, 3227a66e420SKrzysztof Parzyszek llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) { 3237a66e420SKrzysztof Parzyszek for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) { 32437f6ba4fSNimishMishra if (nestedEval.hasNestedEvaluations()) { 32537f6ba4fSNimishMishra if (nestedEval.isConstruct()) 32637f6ba4fSNimishMishra // Recursively look for OpenMP constructs within `nestedEval`'s region 32737f6ba4fSNimishMishra collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions); 3286af4118fSKareem Ergawy else { 32937f6ba4fSNimishMishra converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag, 33037f6ba4fSNimishMishra /*collectSymbols=*/true, 33137f6ba4fSNimishMishra /*collectHostAssociatedSymbols=*/false); 33237f6ba4fSNimishMishra } 33337f6ba4fSNimishMishra } 33437f6ba4fSNimishMishra } 3356af4118fSKareem Ergawy } 33637f6ba4fSNimishMishra 33737f6ba4fSNimishMishra // Collect symbols to be default privatized in two steps. 33837f6ba4fSNimishMishra // In step 1, collect all symbols in `eval` that match `flag` into 33937f6ba4fSNimishMishra // `defaultSymbols`. In step 2, for nested constructs (if any), if and only if 34037f6ba4fSNimishMishra // the nested construct is an OpenMP construct, collect those nested 34137f6ba4fSNimishMishra // symbols skipping host associated symbols into `symbolsInNestedRegions`. 34237f6ba4fSNimishMishra // Later, in current context, all symbols in the set 34337f6ba4fSNimishMishra // `defaultSymbols` - `symbolsInNestedRegions` will be privatized. 3444d4af15cSKareem Ergawy void DataSharingProcessor::collectSymbols( 3457a66e420SKrzysztof Parzyszek semantics::Symbol::Flag flag, 3467a66e420SKrzysztof Parzyszek llvm::SetVector<const semantics::Symbol *> &symbols) { 3471e9625e5SLeandro Lupori // Collect all scopes associated with 'eval'. 3487a66e420SKrzysztof Parzyszek llvm::SetVector<const semantics::Scope *> clauseScopes; 3497a66e420SKrzysztof Parzyszek std::function<void(const semantics::Scope *)> collectScopes = 3507a66e420SKrzysztof Parzyszek [&](const semantics::Scope *scope) { 3511e9625e5SLeandro Lupori clauseScopes.insert(scope); 3527a66e420SKrzysztof Parzyszek for (const semantics::Scope &child : scope->children()) 3531e9625e5SLeandro Lupori collectScopes(&child); 3541e9625e5SLeandro Lupori }; 3557a66e420SKrzysztof Parzyszek const parser::CharBlock *source = 3561e9625e5SLeandro Lupori clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source; 3577a66e420SKrzysztof Parzyszek const semantics::Scope *curScope = nullptr; 3581e9625e5SLeandro Lupori if (source && !source->empty()) { 3591e9625e5SLeandro Lupori curScope = &semaCtx.FindScope(*source); 3601e9625e5SLeandro Lupori collectScopes(curScope); 3611e9625e5SLeandro Lupori } 3621e9625e5SLeandro Lupori // Collect all symbols referenced in the evaluation being processed, 3631e9625e5SLeandro Lupori // that matches 'flag'. 3647a66e420SKrzysztof Parzyszek llvm::SetVector<const semantics::Symbol *> allSymbols; 3651e9625e5SLeandro Lupori converter.collectSymbolSet(eval, allSymbols, flag, 3664d4af15cSKareem Ergawy /*collectSymbols=*/true, 3674d4af15cSKareem Ergawy /*collectHostAssociatedSymbols=*/true); 3686af4118fSKareem Ergawy 3697a66e420SKrzysztof Parzyszek llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions; 37037f6ba4fSNimishMishra collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions); 3716af4118fSKareem Ergawy 3726af4118fSKareem Ergawy for (auto *symbol : allSymbols) 3736af4118fSKareem Ergawy if (visitor.isSymbolDefineBy(symbol, eval)) 3746af4118fSKareem Ergawy symbolsInNestedRegions.remove(symbol); 3756af4118fSKareem Ergawy 3761e9625e5SLeandro Lupori // Filter-out symbols that must not be privatized. 3777a66e420SKrzysztof Parzyszek bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit; 3786af4118fSKareem Ergawy bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined; 3796af4118fSKareem Ergawy 3807a66e420SKrzysztof Parzyszek auto isPrivatizable = [](const semantics::Symbol &sym) -> bool { 3817a66e420SKrzysztof Parzyszek return !semantics::IsProcedure(sym) && 3827a66e420SKrzysztof Parzyszek !sym.GetUltimate().has<semantics::DerivedTypeDetails>() && 3837a66e420SKrzysztof Parzyszek !sym.GetUltimate().has<semantics::NamelistDetails>() && 384c734d77bSTom Eccles !semantics::IsImpliedDoIndex(sym.GetUltimate()) && 385c734d77bSTom Eccles !semantics::IsStmtFunction(sym); 3861e9625e5SLeandro Lupori }; 3876af4118fSKareem Ergawy 3886af4118fSKareem Ergawy auto shouldCollectSymbol = [&](const semantics::Symbol *sym) { 3896af4118fSKareem Ergawy if (collectImplicit) 3906af4118fSKareem Ergawy return sym->test(semantics::Symbol::Flag::OmpImplicit); 3916af4118fSKareem Ergawy 3926af4118fSKareem Ergawy if (collectPreDetermined) 3936af4118fSKareem Ergawy return sym->test(semantics::Symbol::Flag::OmpPreDetermined); 3946af4118fSKareem Ergawy 3956af4118fSKareem Ergawy return !sym->test(semantics::Symbol::Flag::OmpImplicit) && 3966af4118fSKareem Ergawy !sym->test(semantics::Symbol::Flag::OmpPreDetermined); 3976af4118fSKareem Ergawy }; 3986af4118fSKareem Ergawy 3991e9625e5SLeandro Lupori for (const auto *sym : allSymbols) { 4001e9625e5SLeandro Lupori assert(curScope && "couldn't find current scope"); 4011e9625e5SLeandro Lupori if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) && 4026af4118fSKareem Ergawy !explicitlyPrivatizedSymbols.contains(sym) && 4036af4118fSKareem Ergawy shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) { 4046af4118fSKareem Ergawy allPrivatizedSymbols.insert(sym); 4051e9625e5SLeandro Lupori symbols.insert(sym); 4061e9625e5SLeandro Lupori } 4074d4af15cSKareem Ergawy } 4086af4118fSKareem Ergawy } 4094d4af15cSKareem Ergawy 4104d4af15cSKareem Ergawy void DataSharingProcessor::collectDefaultSymbols() { 411148a5579SKrzysztof Parzyszek using DataSharingAttribute = omp::clause::Default::DataSharingAttribute; 412037a32a9SKrzysztof Parzyszek for (const omp::Clause &clause : clauses) { 413037a32a9SKrzysztof Parzyszek if (const auto *defaultClause = 414037a32a9SKrzysztof Parzyszek std::get_if<omp::clause::Default>(&clause.u)) { 415148a5579SKrzysztof Parzyszek if (defaultClause->v == DataSharingAttribute::Private) 4167a66e420SKrzysztof Parzyszek collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols); 417148a5579SKrzysztof Parzyszek else if (defaultClause->v == DataSharingAttribute::Firstprivate) 4187a66e420SKrzysztof Parzyszek collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate, 4191e9625e5SLeandro Lupori defaultSymbols); 4204d4af15cSKareem Ergawy } 4214d4af15cSKareem Ergawy } 4224d4af15cSKareem Ergawy } 4234d4af15cSKareem Ergawy 4241e9625e5SLeandro Lupori void DataSharingProcessor::collectImplicitSymbols() { 4251e9625e5SLeandro Lupori // There will be no implicit symbols when a default clause is present. 4261e9625e5SLeandro Lupori if (defaultSymbols.empty()) 4277a66e420SKrzysztof Parzyszek collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols); 4281e9625e5SLeandro Lupori } 4291e9625e5SLeandro Lupori 4306af4118fSKareem Ergawy void DataSharingProcessor::collectPreDeterminedSymbols() { 4316af4118fSKareem Ergawy if (shouldCollectPreDeterminedSymbols) 4326af4118fSKareem Ergawy collectSymbols(semantics::Symbol::Flag::OmpPreDetermined, 4336af4118fSKareem Ergawy preDeterminedSymbols); 4346af4118fSKareem Ergawy } 4356af4118fSKareem Ergawy 436913a8244SKareem Ergawy void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) { 4376af4118fSKareem Ergawy for (const semantics::Symbol *sym : allPrivatizedSymbols) { 4384d4af15cSKareem Ergawy if (const auto *commonDet = 4397a66e420SKrzysztof Parzyszek sym->detailsIf<semantics::CommonBlockDetails>()) { 44026b8be20SKareem Ergawy for (const auto &mem : commonDet->objects()) 441913a8244SKareem Ergawy doPrivatize(&*mem, clauseOps); 44226b8be20SKareem Ergawy } else 443913a8244SKareem Ergawy doPrivatize(sym, clauseOps); 4444d4af15cSKareem Ergawy } 4454d4af15cSKareem Ergawy } 4464d4af15cSKareem Ergawy 4474d4af15cSKareem Ergawy void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) { 4484d4af15cSKareem Ergawy insertLastPrivateCompare(op); 4496af4118fSKareem Ergawy for (const semantics::Symbol *sym : allPrivatizedSymbols) 4504d4af15cSKareem Ergawy if (const auto *commonDet = 4517a66e420SKrzysztof Parzyszek sym->detailsIf<semantics::CommonBlockDetails>()) { 4524d4af15cSKareem Ergawy for (const auto &mem : commonDet->objects()) { 4534d4af15cSKareem Ergawy copyLastPrivateSymbol(&*mem, &lastPrivIP); 4544d4af15cSKareem Ergawy } 4554d4af15cSKareem Ergawy } else { 4564d4af15cSKareem Ergawy copyLastPrivateSymbol(sym, &lastPrivIP); 4574d4af15cSKareem Ergawy } 4584d4af15cSKareem Ergawy } 4594d4af15cSKareem Ergawy 460913a8244SKareem Ergawy void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym, 461913a8244SKareem Ergawy mlir::omp::PrivateClauseOps *clauseOps) { 46226b8be20SKareem Ergawy if (!useDelayedPrivatization) { 4634d4af15cSKareem Ergawy cloneSymbol(sym); 4644d4af15cSKareem Ergawy copyFirstPrivateSymbol(sym); 46526b8be20SKareem Ergawy return; 4664d4af15cSKareem Ergawy } 46726b8be20SKareem Ergawy 4687a66e420SKrzysztof Parzyszek lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym); 46926b8be20SKareem Ergawy assert(hsb && "Host symbol box not found"); 47026b8be20SKareem Ergawy 47126b8be20SKareem Ergawy mlir::Type symType = hsb.getAddr().getType(); 47226b8be20SKareem Ergawy mlir::Location symLoc = hsb.getAddr().getLoc(); 47326b8be20SKareem Ergawy std::string privatizerName = sym->name().ToString() + ".privatizer"; 4747a66e420SKrzysztof Parzyszek bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate); 47526b8be20SKareem Ergawy 47626b8be20SKareem Ergawy mlir::omp::PrivateClauseOp privatizerOp = [&]() { 47726b8be20SKareem Ergawy auto moduleOp = firOpBuilder.getModule(); 47826b8be20SKareem Ergawy auto uniquePrivatizerName = fir::getTypeAsString( 47926b8be20SKareem Ergawy symType, converter.getKindMap(), 48026b8be20SKareem Ergawy converter.mangleName(*sym) + 48126b8be20SKareem Ergawy (isFirstPrivate ? "_firstprivate" : "_private")); 48226b8be20SKareem Ergawy 48326b8be20SKareem Ergawy if (auto existingPrivatizer = 48426b8be20SKareem Ergawy moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>( 48526b8be20SKareem Ergawy uniquePrivatizerName)) 48626b8be20SKareem Ergawy return existingPrivatizer; 48726b8be20SKareem Ergawy 4889dbf3e23SSergio Afonso mlir::OpBuilder::InsertionGuard guard(firOpBuilder); 48949df12c0SMatthias Springer firOpBuilder.setInsertionPointToStart(moduleOp.getBody()); 49026b8be20SKareem Ergawy auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>( 49126b8be20SKareem Ergawy symLoc, uniquePrivatizerName, symType, 49226b8be20SKareem Ergawy isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate 49326b8be20SKareem Ergawy : mlir::omp::DataSharingClauseType::Private); 49487cee71bSKareem Ergawy fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym); 495db9856b5SLeandro Lupori lower::SymMapScope outerScope(symTable); 49626b8be20SKareem Ergawy 49726b8be20SKareem Ergawy // Populate the `alloc` region. 49826b8be20SKareem Ergawy { 49926b8be20SKareem Ergawy mlir::Region &allocRegion = result.getAllocRegion(); 50026b8be20SKareem Ergawy mlir::Block *allocEntryBlock = firOpBuilder.createBlock( 50126b8be20SKareem Ergawy &allocRegion, /*insertPt=*/{}, symType, symLoc); 50226b8be20SKareem Ergawy 50326b8be20SKareem Ergawy firOpBuilder.setInsertionPointToEnd(allocEntryBlock); 50424f5fc77SKareem Ergawy 50524f5fc77SKareem Ergawy fir::ExtendedValue localExV = 50624f5fc77SKareem Ergawy hlfir::translateToExtendedValue( 50724f5fc77SKareem Ergawy symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)}, 50824f5fc77SKareem Ergawy /*contiguousHint=*/ 5097a66e420SKrzysztof Parzyszek evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext())) 51024f5fc77SKareem Ergawy .first; 51124f5fc77SKareem Ergawy 512db9856b5SLeandro Lupori symTable.addSymbol(*sym, localExV); 513db9856b5SLeandro Lupori lower::SymMapScope innerScope(symTable); 51426b8be20SKareem Ergawy cloneSymbol(sym); 515db9856b5SLeandro Lupori mlir::Value cloneAddr = symTable.shallowLookupSymbol(*sym).getAddr(); 516bbadbf75SKareem Ergawy mlir::Type cloneType = cloneAddr.getType(); 517bbadbf75SKareem Ergawy 518bbadbf75SKareem Ergawy // A `convert` op is required for variables that are storage associated 519bbadbf75SKareem Ergawy // via `equivalence`. The problem is that these variables are declared as 520bbadbf75SKareem Ergawy // `fir.ptr`s while their privatized storage is declared as `fir.ref`, 521bbadbf75SKareem Ergawy // therefore we convert to proper symbol type. 522bbadbf75SKareem Ergawy mlir::Value yieldedValue = 523bbadbf75SKareem Ergawy (symType == cloneType) ? cloneAddr 524bbadbf75SKareem Ergawy : firOpBuilder.createConvert( 525bbadbf75SKareem Ergawy cloneAddr.getLoc(), symType, cloneAddr); 526bbadbf75SKareem Ergawy 527bbadbf75SKareem Ergawy firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc(), 528bbadbf75SKareem Ergawy yieldedValue); 5294d4af15cSKareem Ergawy } 53026b8be20SKareem Ergawy 53126b8be20SKareem Ergawy // Populate the `copy` region if this is a `firstprivate`. 53226b8be20SKareem Ergawy if (isFirstPrivate) { 53326b8be20SKareem Ergawy mlir::Region ©Region = result.getCopyRegion(); 53426b8be20SKareem Ergawy // First block argument corresponding to the original/host value while 53526b8be20SKareem Ergawy // second block argument corresponding to the privatized value. 53626b8be20SKareem Ergawy mlir::Block *copyEntryBlock = firOpBuilder.createBlock( 53726b8be20SKareem Ergawy ©Region, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc}); 53826b8be20SKareem Ergawy firOpBuilder.setInsertionPointToEnd(copyEntryBlock); 53924f5fc77SKareem Ergawy 54024f5fc77SKareem Ergawy auto addSymbol = [&](unsigned argIdx, bool force = false) { 54124f5fc77SKareem Ergawy symExV.match( 54224f5fc77SKareem Ergawy [&](const fir::MutableBoxValue &box) { 543db9856b5SLeandro Lupori symTable.addSymbol( 54424f5fc77SKareem Ergawy *sym, fir::substBase(box, copyRegion.getArgument(argIdx)), 54524f5fc77SKareem Ergawy force); 54624f5fc77SKareem Ergawy }, 54724f5fc77SKareem Ergawy [&](const auto &box) { 548db9856b5SLeandro Lupori symTable.addSymbol(*sym, copyRegion.getArgument(argIdx), force); 54924f5fc77SKareem Ergawy }); 55024f5fc77SKareem Ergawy }; 55124f5fc77SKareem Ergawy 55224f5fc77SKareem Ergawy addSymbol(0, true); 553db9856b5SLeandro Lupori lower::SymMapScope innerScope(symTable); 55424f5fc77SKareem Ergawy addSymbol(1); 55524f5fc77SKareem Ergawy 55626b8be20SKareem Ergawy auto ip = firOpBuilder.saveInsertionPoint(); 55726b8be20SKareem Ergawy copyFirstPrivateSymbol(sym, &ip); 55826b8be20SKareem Ergawy 55926b8be20SKareem Ergawy firOpBuilder.create<mlir::omp::YieldOp>( 560db9856b5SLeandro Lupori hsb.getAddr().getLoc(), symTable.shallowLookupSymbol(*sym).getAddr()); 56126b8be20SKareem Ergawy } 56226b8be20SKareem Ergawy 56326b8be20SKareem Ergawy return result; 56426b8be20SKareem Ergawy }(); 56526b8be20SKareem Ergawy 56678eac466SSergio Afonso if (clauseOps) { 567fdfeea5bSSergio Afonso clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp)); 56878eac466SSergio Afonso clauseOps->privateVars.push_back(hsb.getAddr()); 56978eac466SSergio Afonso } 57078eac466SSergio Afonso 5710632cb38SKareem Ergawy symToPrivatizer[sym] = privatizerOp; 5724d4af15cSKareem Ergawy } 5734d4af15cSKareem Ergawy 5744d4af15cSKareem Ergawy } // namespace omp 5754d4af15cSKareem Ergawy } // namespace lower 5764d4af15cSKareem Ergawy } // namespace Fortran 577