xref: /llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp (revision a0406ce823e8f1c1993b565d08b045c0104c3a5a)
14d4af15cSKareem Ergawy //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===//
24d4af15cSKareem Ergawy //
34d4af15cSKareem Ergawy // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
44d4af15cSKareem Ergawy // See https://llvm.org/LICENSE.txt for license information.
54d4af15cSKareem Ergawy // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
64d4af15cSKareem Ergawy //
74d4af15cSKareem Ergawy //===----------------------------------------------------------------------===//
84d4af15cSKareem Ergawy //
94d4af15cSKareem Ergawy // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
104d4af15cSKareem Ergawy //
114d4af15cSKareem Ergawy //===----------------------------------------------------------------------===//
124d4af15cSKareem Ergawy 
134d4af15cSKareem Ergawy #include "DataSharingProcessor.h"
144d4af15cSKareem Ergawy 
154d4af15cSKareem Ergawy #include "Utils.h"
1698e733eaSTom Eccles #include "flang/Lower/ConvertVariable.h"
174d4af15cSKareem Ergawy #include "flang/Lower/PFTBuilder.h"
182703f7ecSKareem Ergawy #include "flang/Lower/SymbolMap.h"
1924f5fc77SKareem Ergawy #include "flang/Optimizer/Builder/HLFIRTools.h"
204d4af15cSKareem Ergawy #include "flang/Optimizer/Builder/Todo.h"
212e6558b8STom Eccles #include "flang/Optimizer/HLFIR/HLFIROps.h"
224d4af15cSKareem Ergawy #include "flang/Semantics/tools.h"
234d4af15cSKareem Ergawy 
244d4af15cSKareem Ergawy namespace Fortran {
254d4af15cSKareem Ergawy namespace lower {
264d4af15cSKareem Ergawy namespace omp {
276af4118fSKareem Ergawy bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy(
286af4118fSKareem Ergawy     const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const {
296af4118fSKareem Ergawy   return eval.visit(
306af4118fSKareem Ergawy       common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) {
316af4118fSKareem Ergawy                          return symDefMap.count(symbol) &&
326af4118fSKareem Ergawy                                 symDefMap.at(symbol) == &functionParserNode;
336af4118fSKareem Ergawy                        },
346af4118fSKareem Ergawy                        [](const auto &functionParserNode) { return false; }});
356af4118fSKareem Ergawy }
366af4118fSKareem Ergawy 
376af4118fSKareem Ergawy DataSharingProcessor::DataSharingProcessor(
386af4118fSKareem Ergawy     lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
396af4118fSKareem Ergawy     const List<Clause> &clauses, lower::pft::Evaluation &eval,
406af4118fSKareem Ergawy     bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization,
41db9856b5SLeandro Lupori     lower::SymMap &symTable)
4212c4d26cSSergio Afonso     : converter(converter), semaCtx(semaCtx),
436af4118fSKareem Ergawy       firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
446af4118fSKareem Ergawy       shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
456af4118fSKareem Ergawy       useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
466af4118fSKareem Ergawy       visitor() {
476af4118fSKareem Ergawy   eval.visit([&](const auto &functionParserNode) {
486af4118fSKareem Ergawy     parser::Walk(functionParserNode, visitor);
496af4118fSKareem Ergawy   });
506af4118fSKareem Ergawy }
514d4af15cSKareem Ergawy 
5278eac466SSergio Afonso void DataSharingProcessor::processStep1(
53913a8244SKareem Ergawy     mlir::omp::PrivateClauseOps *clauseOps) {
544d4af15cSKareem Ergawy   collectSymbolsForPrivatization();
554d4af15cSKareem Ergawy   collectDefaultSymbols();
561e9625e5SLeandro Lupori   collectImplicitSymbols();
576af4118fSKareem Ergawy   collectPreDeterminedSymbols();
586af4118fSKareem Ergawy 
59913a8244SKareem Ergawy   privatize(clauseOps);
606af4118fSKareem Ergawy 
614d4af15cSKareem Ergawy   insertBarrier();
624d4af15cSKareem Ergawy }
634d4af15cSKareem Ergawy 
644d4af15cSKareem Ergawy void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
656542e566SLeandro Lupori   // 'sections' lastprivate is handled by genOMP()
666542e566SLeandro Lupori   if (!mlir::isa<mlir::omp::SectionsOp>(op)) {
6712c4d26cSSergio Afonso     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
684d4af15cSKareem Ergawy     copyLastPrivatize(op);
696542e566SLeandro Lupori   }
704d4af15cSKareem Ergawy 
714d4af15cSKareem Ergawy   if (isLoop) {
724d4af15cSKareem Ergawy     // push deallocs out of the loop
734d4af15cSKareem Ergawy     firOpBuilder.setInsertionPointAfter(op);
744d4af15cSKareem Ergawy     insertDeallocs();
754d4af15cSKareem Ergawy   } else {
7699f6ff9cSSergio Afonso     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
774d4af15cSKareem Ergawy     insertDeallocs();
784d4af15cSKareem Ergawy   }
794d4af15cSKareem Ergawy }
804d4af15cSKareem Ergawy 
814d4af15cSKareem Ergawy void DataSharingProcessor::insertDeallocs() {
826af4118fSKareem Ergawy   for (const semantics::Symbol *sym : allPrivatizedSymbols)
837a66e420SKrzysztof Parzyszek     if (semantics::IsAllocatable(sym->GetUltimate())) {
840632cb38SKareem Ergawy       if (!useDelayedPrivatization) {
854d4af15cSKareem Ergawy         converter.createHostAssociateVarCloneDealloc(*sym);
86e1aa8ad6SKareem Ergawy         continue;
870632cb38SKareem Ergawy       }
880632cb38SKareem Ergawy 
897a66e420SKrzysztof Parzyszek       lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
900632cb38SKareem Ergawy       assert(hsb && "Host symbol box not found");
910632cb38SKareem Ergawy       mlir::Type symType = hsb.getAddr().getType();
920632cb38SKareem Ergawy       mlir::Location symLoc = hsb.getAddr().getLoc();
930632cb38SKareem Ergawy       fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
940632cb38SKareem Ergawy       mlir::omp::PrivateClauseOp privatizer = symToPrivatizer.at(sym);
950632cb38SKareem Ergawy 
96db9856b5SLeandro Lupori       lower::SymMapScope scope(symTable);
970632cb38SKareem Ergawy       mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
980632cb38SKareem Ergawy 
990632cb38SKareem Ergawy       mlir::Region &deallocRegion = privatizer.getDeallocRegion();
1000632cb38SKareem Ergawy       fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
1010632cb38SKareem Ergawy       mlir::Block *deallocEntryBlock = firOpBuilder.createBlock(
1020632cb38SKareem Ergawy           &deallocRegion, /*insertPt=*/{}, symType, symLoc);
1030632cb38SKareem Ergawy 
1040632cb38SKareem Ergawy       firOpBuilder.setInsertionPointToEnd(deallocEntryBlock);
105db9856b5SLeandro Lupori       symTable.addSymbol(*sym,
1060632cb38SKareem Ergawy                          fir::substBase(symExV, deallocRegion.getArgument(0)));
1070632cb38SKareem Ergawy 
1080632cb38SKareem Ergawy       converter.createHostAssociateVarCloneDealloc(*sym);
1090632cb38SKareem Ergawy       firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc());
1104d4af15cSKareem Ergawy     }
1114d4af15cSKareem Ergawy }
1124d4af15cSKareem Ergawy 
1137a66e420SKrzysztof Parzyszek void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
114ff78cd5fSjeanPerier   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
115ff78cd5fSjeanPerier   bool success = converter.createHostAssociateVarClone(
116ff78cd5fSjeanPerier       *sym, /*skipDefaultInit=*/isFirstPrivate);
1174d4af15cSKareem Ergawy   (void)success;
1184d4af15cSKareem Ergawy   assert(success && "Privatization failed due to existing binding");
1191fcb6a97SLeandro Lupori 
1201fcb6a97SLeandro Lupori   // Initialize clone from original object if it has any allocatable member.
1211fcb6a97SLeandro Lupori   auto needInitClone = [&] {
1221fcb6a97SLeandro Lupori     if (isFirstPrivate)
1231fcb6a97SLeandro Lupori       return false;
1241fcb6a97SLeandro Lupori 
1251fcb6a97SLeandro Lupori     SymbolBox sb = symTable.lookupSymbol(sym);
1261fcb6a97SLeandro Lupori     assert(sb);
1271fcb6a97SLeandro Lupori     mlir::Value addr = sb.getAddr();
1281fcb6a97SLeandro Lupori     assert(addr);
1295130a4eaSLeandro Lupori     return !fir::isPointerType(addr.getType()) &&
1305130a4eaSLeandro Lupori            hlfir::mayHaveAllocatableComponent(addr.getType());
1311fcb6a97SLeandro Lupori   };
1321fcb6a97SLeandro Lupori 
1331fcb6a97SLeandro Lupori   if (needInitClone()) {
1341fcb6a97SLeandro Lupori     Fortran::lower::initializeCloneAtRuntime(converter, *sym, symTable);
1351fcb6a97SLeandro Lupori     callsInitClone = true;
1361fcb6a97SLeandro Lupori   }
1374d4af15cSKareem Ergawy }
1384d4af15cSKareem Ergawy 
1394d4af15cSKareem Ergawy void DataSharingProcessor::copyFirstPrivateSymbol(
1407a66e420SKrzysztof Parzyszek     const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) {
1417a66e420SKrzysztof Parzyszek   if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate))
14226b8be20SKareem Ergawy     converter.copyHostAssociateVar(*sym, copyAssignIP);
1434d4af15cSKareem Ergawy }
1444d4af15cSKareem Ergawy 
1454d4af15cSKareem Ergawy void DataSharingProcessor::copyLastPrivateSymbol(
146797f0119SLeandro Lupori     const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *lastPrivIP) {
147797f0119SLeandro Lupori   if (sym->test(semantics::Symbol::Flag::OmpLastPrivate))
148*a0406ce8SKareem Ergawy     converter.copyHostAssociateVar(*sym, lastPrivIP, /*hostIsSource=*/false);
1494d4af15cSKareem Ergawy }
1504d4af15cSKareem Ergawy 
1514d4af15cSKareem Ergawy void DataSharingProcessor::collectOmpObjectListSymbol(
152037a32a9SKrzysztof Parzyszek     const omp::ObjectList &objects,
1537a66e420SKrzysztof Parzyszek     llvm::SetVector<const semantics::Symbol *> &symbolSet) {
154037a32a9SKrzysztof Parzyszek   for (const omp::Object &object : objects)
1558b18f2feSKrzysztof Parzyszek     symbolSet.insert(object.sym());
1564d4af15cSKareem Ergawy }
1574d4af15cSKareem Ergawy 
1584d4af15cSKareem Ergawy void DataSharingProcessor::collectSymbolsForPrivatization() {
159037a32a9SKrzysztof Parzyszek   for (const omp::Clause &clause : clauses) {
1604d4af15cSKareem Ergawy     if (const auto &privateClause =
161037a32a9SKrzysztof Parzyszek             std::get_if<omp::clause::Private>(&clause.u)) {
1626af4118fSKareem Ergawy       collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols);
1634d4af15cSKareem Ergawy     } else if (const auto &firstPrivateClause =
164037a32a9SKrzysztof Parzyszek                    std::get_if<omp::clause::Firstprivate>(&clause.u)) {
1656af4118fSKareem Ergawy       collectOmpObjectListSymbol(firstPrivateClause->v,
1666af4118fSKareem Ergawy                                  explicitlyPrivatizedSymbols);
1674d4af15cSKareem Ergawy     } else if (const auto &lastPrivateClause =
168037a32a9SKrzysztof Parzyszek                    std::get_if<omp::clause::Lastprivate>(&clause.u)) {
169f9824439SKrzysztof Parzyszek       lastprivateModifierNotSupported(*lastPrivateClause,
170f9824439SKrzysztof Parzyszek                                       converter.getCurrentLocation());
171148a5579SKrzysztof Parzyszek       const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
1726af4118fSKareem Ergawy       collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
1734d4af15cSKareem Ergawy     }
1744d4af15cSKareem Ergawy   }
1754d4af15cSKareem Ergawy 
1766af4118fSKareem Ergawy   for (auto *sym : explicitlyPrivatizedSymbols)
1776af4118fSKareem Ergawy     allPrivatizedSymbols.insert(sym);
1784d4af15cSKareem Ergawy }
1794d4af15cSKareem Ergawy 
1804d4af15cSKareem Ergawy bool DataSharingProcessor::needBarrier() {
1816542e566SLeandro Lupori   // Emit implicit barrier to synchronize threads and avoid data races on
1826542e566SLeandro Lupori   // initialization of firstprivate variables and post-update of lastprivate
1836542e566SLeandro Lupori   // variables.
1846542e566SLeandro Lupori   // Emit implicit barrier for linear clause. Maybe on somewhere else.
1856af4118fSKareem Ergawy   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
1861fcb6a97SLeandro Lupori     if (sym->test(semantics::Symbol::Flag::OmpLastPrivate) &&
1871fcb6a97SLeandro Lupori         (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || callsInitClone))
1884d4af15cSKareem Ergawy       return true;
1894d4af15cSKareem Ergawy   }
1904d4af15cSKareem Ergawy   return false;
1914d4af15cSKareem Ergawy }
1924d4af15cSKareem Ergawy 
1934d4af15cSKareem Ergawy void DataSharingProcessor::insertBarrier() {
1944d4af15cSKareem Ergawy   if (needBarrier())
1954d4af15cSKareem Ergawy     firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
1964d4af15cSKareem Ergawy }
1974d4af15cSKareem Ergawy 
1984d4af15cSKareem Ergawy void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
199ca4dbc27SSergio Afonso   mlir::omp::LoopNestOp loopOp;
200ca4dbc27SSergio Afonso   if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op))
2012784060cSSergio Afonso     loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop());
202ca4dbc27SSergio Afonso 
2034d4af15cSKareem Ergawy   bool cmpCreated = false;
2049dbf3e23SSergio Afonso   mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
205037a32a9SKrzysztof Parzyszek   for (const omp::Clause &clause : clauses) {
206037a32a9SKrzysztof Parzyszek     if (clause.id != llvm::omp::OMPC_lastprivate)
207037a32a9SKrzysztof Parzyszek       continue;
208eafa1500SNimishMishra     if (mlir::isa<mlir::omp::WsloopOp>(op) ||
209eafa1500SNimishMishra         mlir::isa<mlir::omp::SimdOp>(op)) {
2104d4af15cSKareem Ergawy       // Update the original variable just before exiting the worksharing
2114d4af15cSKareem Ergawy       // loop. Conversion as follows:
2124d4af15cSKareem Ergawy       //
213eafa1500SNimishMishra       // omp.wsloop / omp.simd {    omp.wsloop / omp.simd {
214ca4dbc27SSergio Afonso       //   omp.loop_nest {            omp.loop_nest {
215ca4dbc27SSergio Afonso       //     ...                        ...
216ca4dbc27SSergio Afonso       //     store          ===>        store
217ca4dbc27SSergio Afonso       //     omp.yield                  %v = arith.addi %iv, %step
218ca4dbc27SSergio Afonso       //   }                            %cmp = %step < 0 ? %v < %ub : %v > %ub
2190a17bdfcSSergio Afonso       // }                              fir.if %cmp {
2200a17bdfcSSergio Afonso       //                                  fir.store %v to %loopIV
2214d4af15cSKareem Ergawy       //                                  ^%lpv_update_blk:
2224d4af15cSKareem Ergawy       //                                }
2234d4af15cSKareem Ergawy       //                                omp.yield
2244d4af15cSKareem Ergawy       //                              }
225ca4dbc27SSergio Afonso       //                            }
2264d4af15cSKareem Ergawy 
2274d4af15cSKareem Ergawy       // Only generate the compare once in presence of multiple LastPrivate
2284d4af15cSKareem Ergawy       // clauses.
2294d4af15cSKareem Ergawy       if (cmpCreated)
2304d4af15cSKareem Ergawy         continue;
2314d4af15cSKareem Ergawy       cmpCreated = true;
2324d4af15cSKareem Ergawy 
233ca4dbc27SSergio Afonso       mlir::Location loc = loopOp.getLoc();
234ca4dbc27SSergio Afonso       mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
2354d4af15cSKareem Ergawy       firOpBuilder.setInsertionPoint(lastOper);
2364d4af15cSKareem Ergawy 
2374b9fab59SDavid Truby       mlir::Value cmpOp;
2384b9fab59SDavid Truby       llvm::SmallVector<mlir::Value> vs;
2394b9fab59SDavid Truby       vs.reserve(loopOp.getIVs().size());
240fdfeea5bSSergio Afonso       for (auto [iv, ub, step] :
24146ecd7bbSSergio Afonso            llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(),
24246ecd7bbSSergio Afonso                            loopOp.getLoopSteps())) {
2434d4af15cSKareem Ergawy         // v = iv + step
2444d4af15cSKareem Ergawy         // cmp = step < 0 ? v < ub : v > ub
2454d4af15cSKareem Ergawy         mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
2464b9fab59SDavid Truby         vs.push_back(v);
2474d4af15cSKareem Ergawy         mlir::Value zero =
2484d4af15cSKareem Ergawy             firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
2494d4af15cSKareem Ergawy         mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
2504d4af15cSKareem Ergawy             loc, mlir::arith::CmpIPredicate::slt, step, zero);
2514d4af15cSKareem Ergawy         mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
2524d4af15cSKareem Ergawy             loc, mlir::arith::CmpIPredicate::slt, v, ub);
2534d4af15cSKareem Ergawy         mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
2544d4af15cSKareem Ergawy             loc, mlir::arith::CmpIPredicate::sgt, v, ub);
2554b9fab59SDavid Truby         mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
2564d4af15cSKareem Ergawy             loc, negativeStep, vLT, vGT);
2574d4af15cSKareem Ergawy 
2584b9fab59SDavid Truby         if (cmpOp) {
2594b9fab59SDavid Truby           cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
2604b9fab59SDavid Truby         } else {
2614b9fab59SDavid Truby           cmpOp = icmpOp;
2624b9fab59SDavid Truby         }
2634b9fab59SDavid Truby       }
2644b9fab59SDavid Truby 
2654d4af15cSKareem Ergawy       auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
2664d4af15cSKareem Ergawy       firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
2674b9fab59SDavid Truby       for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
2684d4af15cSKareem Ergawy         assert(loopIV && "loopIV was not set");
2697cee61c8SKiran Chandramohan         firOpBuilder.createStoreWithConvert(loc, v, loopIV);
2704b9fab59SDavid Truby       }
2714d4af15cSKareem Ergawy       lastPrivIP = firOpBuilder.saveInsertionPoint();
2726542e566SLeandro Lupori     } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
2736542e566SLeandro Lupori       // Already handled by genOMP()
2744d4af15cSKareem Ergawy     } else {
2754d4af15cSKareem Ergawy       TODO(converter.getCurrentLocation(),
2764d4af15cSKareem Ergawy            "lastprivate clause in constructs other than "
2774d4af15cSKareem Ergawy            "simd/worksharing-loop");
2784d4af15cSKareem Ergawy     }
2794d4af15cSKareem Ergawy   }
2804d4af15cSKareem Ergawy }
2814d4af15cSKareem Ergawy 
2827a66e420SKrzysztof Parzyszek static const parser::CharBlock *
2837a66e420SKrzysztof Parzyszek getSource(const semantics::SemanticsContext &semaCtx,
2847a66e420SKrzysztof Parzyszek           const lower::pft::Evaluation &eval) {
2857a66e420SKrzysztof Parzyszek   const parser::CharBlock *source = nullptr;
2861e9625e5SLeandro Lupori 
2877a66e420SKrzysztof Parzyszek   auto ompConsVisit = [&](const parser::OpenMPConstruct &x) {
2887a66e420SKrzysztof Parzyszek     std::visit(common::visitors{
2897a66e420SKrzysztof Parzyszek                    [&](const parser::OpenMPSectionsConstruct &x) {
2901e9625e5SLeandro Lupori                      source = &std::get<0>(x.t).source;
2911e9625e5SLeandro Lupori                    },
2927a66e420SKrzysztof Parzyszek                    [&](const parser::OpenMPLoopConstruct &x) {
2931e9625e5SLeandro Lupori                      source = &std::get<0>(x.t).source;
2941e9625e5SLeandro Lupori                    },
2957a66e420SKrzysztof Parzyszek                    [&](const parser::OpenMPBlockConstruct &x) {
2961e9625e5SLeandro Lupori                      source = &std::get<0>(x.t).source;
2971e9625e5SLeandro Lupori                    },
2987a66e420SKrzysztof Parzyszek                    [&](const parser::OpenMPCriticalConstruct &x) {
2991e9625e5SLeandro Lupori                      source = &std::get<0>(x.t).source;
3001e9625e5SLeandro Lupori                    },
3017a66e420SKrzysztof Parzyszek                    [&](const parser::OpenMPAtomicConstruct &x) {
3021e9625e5SLeandro Lupori                      std::visit([&](const auto &x) { source = &x.source; },
3031e9625e5SLeandro Lupori                                 x.u);
3041e9625e5SLeandro Lupori                    },
3051e9625e5SLeandro Lupori                    [&](const auto &x) { source = &x.source; },
3061e9625e5SLeandro Lupori                },
3071e9625e5SLeandro Lupori                x.u);
3081e9625e5SLeandro Lupori   };
3091e9625e5SLeandro Lupori 
3107a66e420SKrzysztof Parzyszek   eval.visit(common::visitors{
3117a66e420SKrzysztof Parzyszek       [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); },
3127a66e420SKrzysztof Parzyszek       [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; },
3137a66e420SKrzysztof Parzyszek       [&](const parser::OmpEndLoopDirective &x) { source = &x.source; },
3141e9625e5SLeandro Lupori       [&](const auto &x) {},
3151e9625e5SLeandro Lupori   });
3161e9625e5SLeandro Lupori 
3171e9625e5SLeandro Lupori   return source;
3181e9625e5SLeandro Lupori }
3191e9625e5SLeandro Lupori 
32037f6ba4fSNimishMishra void DataSharingProcessor::collectSymbolsInNestedRegions(
3217a66e420SKrzysztof Parzyszek     lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
3227a66e420SKrzysztof Parzyszek     llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) {
3237a66e420SKrzysztof Parzyszek   for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
32437f6ba4fSNimishMishra     if (nestedEval.hasNestedEvaluations()) {
32537f6ba4fSNimishMishra       if (nestedEval.isConstruct())
32637f6ba4fSNimishMishra         // Recursively look for OpenMP constructs within `nestedEval`'s region
32737f6ba4fSNimishMishra         collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
3286af4118fSKareem Ergawy       else {
32937f6ba4fSNimishMishra         converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
33037f6ba4fSNimishMishra                                    /*collectSymbols=*/true,
33137f6ba4fSNimishMishra                                    /*collectHostAssociatedSymbols=*/false);
33237f6ba4fSNimishMishra       }
33337f6ba4fSNimishMishra     }
33437f6ba4fSNimishMishra   }
3356af4118fSKareem Ergawy }
33637f6ba4fSNimishMishra 
33737f6ba4fSNimishMishra // Collect symbols to be default privatized in two steps.
33837f6ba4fSNimishMishra // In step 1, collect all symbols in `eval` that match `flag` into
33937f6ba4fSNimishMishra // `defaultSymbols`. In step 2, for nested constructs (if any), if and only if
34037f6ba4fSNimishMishra // the nested construct is an OpenMP construct, collect those nested
34137f6ba4fSNimishMishra // symbols skipping host associated symbols into `symbolsInNestedRegions`.
34237f6ba4fSNimishMishra // Later, in current context, all symbols in the set
34337f6ba4fSNimishMishra // `defaultSymbols` - `symbolsInNestedRegions` will be privatized.
3444d4af15cSKareem Ergawy void DataSharingProcessor::collectSymbols(
3457a66e420SKrzysztof Parzyszek     semantics::Symbol::Flag flag,
3467a66e420SKrzysztof Parzyszek     llvm::SetVector<const semantics::Symbol *> &symbols) {
3471e9625e5SLeandro Lupori   // Collect all scopes associated with 'eval'.
3487a66e420SKrzysztof Parzyszek   llvm::SetVector<const semantics::Scope *> clauseScopes;
3497a66e420SKrzysztof Parzyszek   std::function<void(const semantics::Scope *)> collectScopes =
3507a66e420SKrzysztof Parzyszek       [&](const semantics::Scope *scope) {
3511e9625e5SLeandro Lupori         clauseScopes.insert(scope);
3527a66e420SKrzysztof Parzyszek         for (const semantics::Scope &child : scope->children())
3531e9625e5SLeandro Lupori           collectScopes(&child);
3541e9625e5SLeandro Lupori       };
3557a66e420SKrzysztof Parzyszek   const parser::CharBlock *source =
3561e9625e5SLeandro Lupori       clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source;
3577a66e420SKrzysztof Parzyszek   const semantics::Scope *curScope = nullptr;
3581e9625e5SLeandro Lupori   if (source && !source->empty()) {
3591e9625e5SLeandro Lupori     curScope = &semaCtx.FindScope(*source);
3601e9625e5SLeandro Lupori     collectScopes(curScope);
3611e9625e5SLeandro Lupori   }
3621e9625e5SLeandro Lupori   // Collect all symbols referenced in the evaluation being processed,
3631e9625e5SLeandro Lupori   // that matches 'flag'.
3647a66e420SKrzysztof Parzyszek   llvm::SetVector<const semantics::Symbol *> allSymbols;
3651e9625e5SLeandro Lupori   converter.collectSymbolSet(eval, allSymbols, flag,
3664d4af15cSKareem Ergawy                              /*collectSymbols=*/true,
3674d4af15cSKareem Ergawy                              /*collectHostAssociatedSymbols=*/true);
3686af4118fSKareem Ergawy 
3697a66e420SKrzysztof Parzyszek   llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
37037f6ba4fSNimishMishra   collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
3716af4118fSKareem Ergawy 
3726af4118fSKareem Ergawy   for (auto *symbol : allSymbols)
3736af4118fSKareem Ergawy     if (visitor.isSymbolDefineBy(symbol, eval))
3746af4118fSKareem Ergawy       symbolsInNestedRegions.remove(symbol);
3756af4118fSKareem Ergawy 
3761e9625e5SLeandro Lupori   // Filter-out symbols that must not be privatized.
3777a66e420SKrzysztof Parzyszek   bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit;
3786af4118fSKareem Ergawy   bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined;
3796af4118fSKareem Ergawy 
3807a66e420SKrzysztof Parzyszek   auto isPrivatizable = [](const semantics::Symbol &sym) -> bool {
3817a66e420SKrzysztof Parzyszek     return !semantics::IsProcedure(sym) &&
3827a66e420SKrzysztof Parzyszek            !sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
3837a66e420SKrzysztof Parzyszek            !sym.GetUltimate().has<semantics::NamelistDetails>() &&
384c734d77bSTom Eccles            !semantics::IsImpliedDoIndex(sym.GetUltimate()) &&
385c734d77bSTom Eccles            !semantics::IsStmtFunction(sym);
3861e9625e5SLeandro Lupori   };
3876af4118fSKareem Ergawy 
3886af4118fSKareem Ergawy   auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
3896af4118fSKareem Ergawy     if (collectImplicit)
3906af4118fSKareem Ergawy       return sym->test(semantics::Symbol::Flag::OmpImplicit);
3916af4118fSKareem Ergawy 
3926af4118fSKareem Ergawy     if (collectPreDetermined)
3936af4118fSKareem Ergawy       return sym->test(semantics::Symbol::Flag::OmpPreDetermined);
3946af4118fSKareem Ergawy 
3956af4118fSKareem Ergawy     return !sym->test(semantics::Symbol::Flag::OmpImplicit) &&
3966af4118fSKareem Ergawy            !sym->test(semantics::Symbol::Flag::OmpPreDetermined);
3976af4118fSKareem Ergawy   };
3986af4118fSKareem Ergawy 
3991e9625e5SLeandro Lupori   for (const auto *sym : allSymbols) {
4001e9625e5SLeandro Lupori     assert(curScope && "couldn't find current scope");
4011e9625e5SLeandro Lupori     if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) &&
4026af4118fSKareem Ergawy         !explicitlyPrivatizedSymbols.contains(sym) &&
4036af4118fSKareem Ergawy         shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) {
4046af4118fSKareem Ergawy       allPrivatizedSymbols.insert(sym);
4051e9625e5SLeandro Lupori       symbols.insert(sym);
4061e9625e5SLeandro Lupori     }
4074d4af15cSKareem Ergawy   }
4086af4118fSKareem Ergawy }
4094d4af15cSKareem Ergawy 
4104d4af15cSKareem Ergawy void DataSharingProcessor::collectDefaultSymbols() {
411148a5579SKrzysztof Parzyszek   using DataSharingAttribute = omp::clause::Default::DataSharingAttribute;
412037a32a9SKrzysztof Parzyszek   for (const omp::Clause &clause : clauses) {
413037a32a9SKrzysztof Parzyszek     if (const auto *defaultClause =
414037a32a9SKrzysztof Parzyszek             std::get_if<omp::clause::Default>(&clause.u)) {
415148a5579SKrzysztof Parzyszek       if (defaultClause->v == DataSharingAttribute::Private)
4167a66e420SKrzysztof Parzyszek         collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols);
417148a5579SKrzysztof Parzyszek       else if (defaultClause->v == DataSharingAttribute::Firstprivate)
4187a66e420SKrzysztof Parzyszek         collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate,
4191e9625e5SLeandro Lupori                        defaultSymbols);
4204d4af15cSKareem Ergawy     }
4214d4af15cSKareem Ergawy   }
4224d4af15cSKareem Ergawy }
4234d4af15cSKareem Ergawy 
4241e9625e5SLeandro Lupori void DataSharingProcessor::collectImplicitSymbols() {
4251e9625e5SLeandro Lupori   // There will be no implicit symbols when a default clause is present.
4261e9625e5SLeandro Lupori   if (defaultSymbols.empty())
4277a66e420SKrzysztof Parzyszek     collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols);
4281e9625e5SLeandro Lupori }
4291e9625e5SLeandro Lupori 
4306af4118fSKareem Ergawy void DataSharingProcessor::collectPreDeterminedSymbols() {
4316af4118fSKareem Ergawy   if (shouldCollectPreDeterminedSymbols)
4326af4118fSKareem Ergawy     collectSymbols(semantics::Symbol::Flag::OmpPreDetermined,
4336af4118fSKareem Ergawy                    preDeterminedSymbols);
4346af4118fSKareem Ergawy }
4356af4118fSKareem Ergawy 
436913a8244SKareem Ergawy void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) {
4376af4118fSKareem Ergawy   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
4384d4af15cSKareem Ergawy     if (const auto *commonDet =
4397a66e420SKrzysztof Parzyszek             sym->detailsIf<semantics::CommonBlockDetails>()) {
44026b8be20SKareem Ergawy       for (const auto &mem : commonDet->objects())
441913a8244SKareem Ergawy         doPrivatize(&*mem, clauseOps);
44226b8be20SKareem Ergawy     } else
443913a8244SKareem Ergawy       doPrivatize(sym, clauseOps);
4444d4af15cSKareem Ergawy   }
4454d4af15cSKareem Ergawy }
4464d4af15cSKareem Ergawy 
4474d4af15cSKareem Ergawy void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
4484d4af15cSKareem Ergawy   insertLastPrivateCompare(op);
4496af4118fSKareem Ergawy   for (const semantics::Symbol *sym : allPrivatizedSymbols)
4504d4af15cSKareem Ergawy     if (const auto *commonDet =
4517a66e420SKrzysztof Parzyszek             sym->detailsIf<semantics::CommonBlockDetails>()) {
4524d4af15cSKareem Ergawy       for (const auto &mem : commonDet->objects()) {
4534d4af15cSKareem Ergawy         copyLastPrivateSymbol(&*mem, &lastPrivIP);
4544d4af15cSKareem Ergawy       }
4554d4af15cSKareem Ergawy     } else {
4564d4af15cSKareem Ergawy       copyLastPrivateSymbol(sym, &lastPrivIP);
4574d4af15cSKareem Ergawy     }
4584d4af15cSKareem Ergawy }
4594d4af15cSKareem Ergawy 
460913a8244SKareem Ergawy void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
461913a8244SKareem Ergawy                                        mlir::omp::PrivateClauseOps *clauseOps) {
46226b8be20SKareem Ergawy   if (!useDelayedPrivatization) {
4634d4af15cSKareem Ergawy     cloneSymbol(sym);
4644d4af15cSKareem Ergawy     copyFirstPrivateSymbol(sym);
46526b8be20SKareem Ergawy     return;
4664d4af15cSKareem Ergawy   }
46726b8be20SKareem Ergawy 
4687a66e420SKrzysztof Parzyszek   lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
46926b8be20SKareem Ergawy   assert(hsb && "Host symbol box not found");
47026b8be20SKareem Ergawy 
47126b8be20SKareem Ergawy   mlir::Type symType = hsb.getAddr().getType();
47226b8be20SKareem Ergawy   mlir::Location symLoc = hsb.getAddr().getLoc();
47326b8be20SKareem Ergawy   std::string privatizerName = sym->name().ToString() + ".privatizer";
4747a66e420SKrzysztof Parzyszek   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
47526b8be20SKareem Ergawy 
47626b8be20SKareem Ergawy   mlir::omp::PrivateClauseOp privatizerOp = [&]() {
47726b8be20SKareem Ergawy     auto moduleOp = firOpBuilder.getModule();
47826b8be20SKareem Ergawy     auto uniquePrivatizerName = fir::getTypeAsString(
47926b8be20SKareem Ergawy         symType, converter.getKindMap(),
48026b8be20SKareem Ergawy         converter.mangleName(*sym) +
48126b8be20SKareem Ergawy             (isFirstPrivate ? "_firstprivate" : "_private"));
48226b8be20SKareem Ergawy 
48326b8be20SKareem Ergawy     if (auto existingPrivatizer =
48426b8be20SKareem Ergawy             moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>(
48526b8be20SKareem Ergawy                 uniquePrivatizerName))
48626b8be20SKareem Ergawy       return existingPrivatizer;
48726b8be20SKareem Ergawy 
4889dbf3e23SSergio Afonso     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
48949df12c0SMatthias Springer     firOpBuilder.setInsertionPointToStart(moduleOp.getBody());
49026b8be20SKareem Ergawy     auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>(
49126b8be20SKareem Ergawy         symLoc, uniquePrivatizerName, symType,
49226b8be20SKareem Ergawy         isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate
49326b8be20SKareem Ergawy                        : mlir::omp::DataSharingClauseType::Private);
49487cee71bSKareem Ergawy     fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
495db9856b5SLeandro Lupori     lower::SymMapScope outerScope(symTable);
49626b8be20SKareem Ergawy 
49726b8be20SKareem Ergawy     // Populate the `alloc` region.
49826b8be20SKareem Ergawy     {
49926b8be20SKareem Ergawy       mlir::Region &allocRegion = result.getAllocRegion();
50026b8be20SKareem Ergawy       mlir::Block *allocEntryBlock = firOpBuilder.createBlock(
50126b8be20SKareem Ergawy           &allocRegion, /*insertPt=*/{}, symType, symLoc);
50226b8be20SKareem Ergawy 
50326b8be20SKareem Ergawy       firOpBuilder.setInsertionPointToEnd(allocEntryBlock);
50424f5fc77SKareem Ergawy 
50524f5fc77SKareem Ergawy       fir::ExtendedValue localExV =
50624f5fc77SKareem Ergawy           hlfir::translateToExtendedValue(
50724f5fc77SKareem Ergawy               symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)},
50824f5fc77SKareem Ergawy               /*contiguousHint=*/
5097a66e420SKrzysztof Parzyszek               evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
51024f5fc77SKareem Ergawy               .first;
51124f5fc77SKareem Ergawy 
512db9856b5SLeandro Lupori       symTable.addSymbol(*sym, localExV);
513db9856b5SLeandro Lupori       lower::SymMapScope innerScope(symTable);
51426b8be20SKareem Ergawy       cloneSymbol(sym);
515db9856b5SLeandro Lupori       mlir::Value cloneAddr = symTable.shallowLookupSymbol(*sym).getAddr();
516bbadbf75SKareem Ergawy       mlir::Type cloneType = cloneAddr.getType();
517bbadbf75SKareem Ergawy 
518bbadbf75SKareem Ergawy       // A `convert` op is required for variables that are storage associated
519bbadbf75SKareem Ergawy       // via `equivalence`. The problem is that these variables are declared as
520bbadbf75SKareem Ergawy       // `fir.ptr`s while their privatized storage is declared as `fir.ref`,
521bbadbf75SKareem Ergawy       // therefore we convert to proper symbol type.
522bbadbf75SKareem Ergawy       mlir::Value yieldedValue =
523bbadbf75SKareem Ergawy           (symType == cloneType) ? cloneAddr
524bbadbf75SKareem Ergawy                                  : firOpBuilder.createConvert(
525bbadbf75SKareem Ergawy                                        cloneAddr.getLoc(), symType, cloneAddr);
526bbadbf75SKareem Ergawy 
527bbadbf75SKareem Ergawy       firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc(),
528bbadbf75SKareem Ergawy                                               yieldedValue);
5294d4af15cSKareem Ergawy     }
53026b8be20SKareem Ergawy 
53126b8be20SKareem Ergawy     // Populate the `copy` region if this is a `firstprivate`.
53226b8be20SKareem Ergawy     if (isFirstPrivate) {
53326b8be20SKareem Ergawy       mlir::Region &copyRegion = result.getCopyRegion();
53426b8be20SKareem Ergawy       // First block argument corresponding to the original/host value while
53526b8be20SKareem Ergawy       // second block argument corresponding to the privatized value.
53626b8be20SKareem Ergawy       mlir::Block *copyEntryBlock = firOpBuilder.createBlock(
53726b8be20SKareem Ergawy           &copyRegion, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc});
53826b8be20SKareem Ergawy       firOpBuilder.setInsertionPointToEnd(copyEntryBlock);
53924f5fc77SKareem Ergawy 
54024f5fc77SKareem Ergawy       auto addSymbol = [&](unsigned argIdx, bool force = false) {
54124f5fc77SKareem Ergawy         symExV.match(
54224f5fc77SKareem Ergawy             [&](const fir::MutableBoxValue &box) {
543db9856b5SLeandro Lupori               symTable.addSymbol(
54424f5fc77SKareem Ergawy                   *sym, fir::substBase(box, copyRegion.getArgument(argIdx)),
54524f5fc77SKareem Ergawy                   force);
54624f5fc77SKareem Ergawy             },
54724f5fc77SKareem Ergawy             [&](const auto &box) {
548db9856b5SLeandro Lupori               symTable.addSymbol(*sym, copyRegion.getArgument(argIdx), force);
54924f5fc77SKareem Ergawy             });
55024f5fc77SKareem Ergawy       };
55124f5fc77SKareem Ergawy 
55224f5fc77SKareem Ergawy       addSymbol(0, true);
553db9856b5SLeandro Lupori       lower::SymMapScope innerScope(symTable);
55424f5fc77SKareem Ergawy       addSymbol(1);
55524f5fc77SKareem Ergawy 
55626b8be20SKareem Ergawy       auto ip = firOpBuilder.saveInsertionPoint();
55726b8be20SKareem Ergawy       copyFirstPrivateSymbol(sym, &ip);
55826b8be20SKareem Ergawy 
55926b8be20SKareem Ergawy       firOpBuilder.create<mlir::omp::YieldOp>(
560db9856b5SLeandro Lupori           hsb.getAddr().getLoc(), symTable.shallowLookupSymbol(*sym).getAddr());
56126b8be20SKareem Ergawy     }
56226b8be20SKareem Ergawy 
56326b8be20SKareem Ergawy     return result;
56426b8be20SKareem Ergawy   }();
56526b8be20SKareem Ergawy 
56678eac466SSergio Afonso   if (clauseOps) {
567fdfeea5bSSergio Afonso     clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp));
56878eac466SSergio Afonso     clauseOps->privateVars.push_back(hsb.getAddr());
56978eac466SSergio Afonso   }
57078eac466SSergio Afonso 
5710632cb38SKareem Ergawy   symToPrivatizer[sym] = privatizerOp;
5724d4af15cSKareem Ergawy }
5734d4af15cSKareem Ergawy 
5744d4af15cSKareem Ergawy } // namespace omp
5754d4af15cSKareem Ergawy } // namespace lower
5764d4af15cSKareem Ergawy } // namespace Fortran
577