xref: /llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp (revision 433ca3ebbef50002bec716ef2c6d6a82db71048d) 
1 //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "DataSharingProcessor.h"
14 
15 #include "Utils.h"
16 #include "flang/Lower/ConvertVariable.h"
17 #include "flang/Lower/PFTBuilder.h"
18 #include "flang/Lower/SymbolMap.h"
19 #include "flang/Optimizer/Builder/HLFIRTools.h"
20 #include "flang/Optimizer/Builder/Todo.h"
21 #include "flang/Optimizer/HLFIR/HLFIROps.h"
22 #include "flang/Semantics/tools.h"
23 
24 namespace Fortran {
25 namespace lower {
26 namespace omp {
27 bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy(
28     const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const {
29   return eval.visit(
30       common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) {
31                          return symDefMap.count(symbol) &&
32                                 symDefMap.at(symbol) == &functionParserNode;
33                        },
34                        [](const auto &functionParserNode) { return false; }});
35 }
36 
37 DataSharingProcessor::DataSharingProcessor(
38     lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
39     const List<Clause> &clauses, lower::pft::Evaluation &eval,
40     bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization,
41     lower::SymMap *symTable)
42     : converter(converter), semaCtx(semaCtx),
43       firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
44       shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
45       useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
46       visitor() {
47   eval.visit([&](const auto &functionParserNode) {
48     parser::Walk(functionParserNode, visitor);
49   });
50 }
51 
52 void DataSharingProcessor::processStep1(
53     mlir::omp::PrivateClauseOps *clauseOps) {
54   collectSymbolsForPrivatization();
55   collectDefaultSymbols();
56   collectImplicitSymbols();
57   collectPreDeterminedSymbols();
58 
59   privatize(clauseOps);
60 
61   insertBarrier();
62 }
63 
64 void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
65   // 'sections' lastprivate is handled by genOMP()
66   if (!mlir::isa<mlir::omp::SectionsOp>(op)) {
67     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
68     copyLastPrivatize(op);
69   }
70 
71   if (isLoop) {
72     // push deallocs out of the loop
73     firOpBuilder.setInsertionPointAfter(op);
74     insertDeallocs();
75   } else {
76     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
77     insertDeallocs();
78   }
79 }
80 
81 void DataSharingProcessor::insertDeallocs() {
82   for (const semantics::Symbol *sym : allPrivatizedSymbols)
83     if (semantics::IsAllocatable(sym->GetUltimate())) {
84       if (!useDelayedPrivatization) {
85         converter.createHostAssociateVarCloneDealloc(*sym);
86         continue;
87       }
88 
89       lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
90       assert(hsb && "Host symbol box not found");
91       mlir::Type symType = hsb.getAddr().getType();
92       mlir::Location symLoc = hsb.getAddr().getLoc();
93       fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
94       mlir::omp::PrivateClauseOp privatizer = symToPrivatizer.at(sym);
95 
96       lower::SymMapScope scope(*symTable);
97       mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
98 
99       mlir::Region &deallocRegion = privatizer.getDeallocRegion();
100       fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
101       mlir::Block *deallocEntryBlock = firOpBuilder.createBlock(
102           &deallocRegion, /*insertPt=*/{}, symType, symLoc);
103 
104       firOpBuilder.setInsertionPointToEnd(deallocEntryBlock);
105       symTable->addSymbol(*sym,
106                           fir::substBase(symExV, deallocRegion.getArgument(0)));
107 
108       converter.createHostAssociateVarCloneDealloc(*sym);
109       firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc());
110     }
111 }
112 
113 void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
114   bool success = converter.createHostAssociateVarClone(*sym);
115   (void)success;
116   assert(success && "Privatization failed due to existing binding");
117 
118   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
119   if (!isFirstPrivate &&
120       Fortran::lower::hasDefaultInitialization(sym->GetUltimate()))
121     Fortran::lower::defaultInitializeAtRuntime(converter, *sym, *symTable);
122 }
123 
124 void DataSharingProcessor::copyFirstPrivateSymbol(
125     const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) {
126   if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate))
127     converter.copyHostAssociateVar(*sym, copyAssignIP);
128 }
129 
130 void DataSharingProcessor::copyLastPrivateSymbol(
131     const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *lastPrivIP) {
132   if (sym->test(semantics::Symbol::Flag::OmpLastPrivate))
133     converter.copyHostAssociateVar(*sym, lastPrivIP);
134 }
135 
136 void DataSharingProcessor::collectOmpObjectListSymbol(
137     const omp::ObjectList &objects,
138     llvm::SetVector<const semantics::Symbol *> &symbolSet) {
139   for (const omp::Object &object : objects)
140     symbolSet.insert(object.sym());
141 }
142 
143 void DataSharingProcessor::collectSymbolsForPrivatization() {
144   for (const omp::Clause &clause : clauses) {
145     if (const auto &privateClause =
146             std::get_if<omp::clause::Private>(&clause.u)) {
147       collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols);
148     } else if (const auto &firstPrivateClause =
149                    std::get_if<omp::clause::Firstprivate>(&clause.u)) {
150       collectOmpObjectListSymbol(firstPrivateClause->v,
151                                  explicitlyPrivatizedSymbols);
152     } else if (const auto &lastPrivateClause =
153                    std::get_if<omp::clause::Lastprivate>(&clause.u)) {
154       const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
155       collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
156     }
157   }
158 
159   for (auto *sym : explicitlyPrivatizedSymbols)
160     allPrivatizedSymbols.insert(sym);
161 }
162 
163 bool DataSharingProcessor::needBarrier() {
164   // Emit implicit barrier to synchronize threads and avoid data races on
165   // initialization of firstprivate variables and post-update of lastprivate
166   // variables.
167   // Emit implicit barrier for linear clause. Maybe on somewhere else.
168   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
169     if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) &&
170         sym->test(semantics::Symbol::Flag::OmpLastPrivate))
171       return true;
172   }
173   return false;
174 }
175 
176 void DataSharingProcessor::insertBarrier() {
177   if (needBarrier())
178     firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
179 }
180 
181 void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
182   mlir::omp::LoopNestOp loopOp;
183   if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op))
184     loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop());
185 
186   bool cmpCreated = false;
187   mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
188   for (const omp::Clause &clause : clauses) {
189     if (clause.id != llvm::omp::OMPC_lastprivate)
190       continue;
191     // TODO: Add lastprivate support for simd construct
192     if (mlir::isa<mlir::omp::WsloopOp>(op)) {
193       // Update the original variable just before exiting the worksharing
194       // loop. Conversion as follows:
195       //
196       // omp.wsloop {             omp.wsloop {
197       //   omp.loop_nest {          omp.loop_nest {
198       //     ...                      ...
199       //     store          ===>      store
200       //     omp.yield                %v = arith.addi %iv, %step
201       //   }                          %cmp = %step < 0 ? %v < %ub : %v > %ub
202       //   omp.terminator             fir.if %cmp {
203       // }                              fir.store %v to %loopIV
204       //                                ^%lpv_update_blk:
205       //                              }
206       //                              omp.yield
207       //                            }
208       //                            omp.terminator
209       //                          }
210 
211       // Only generate the compare once in presence of multiple LastPrivate
212       // clauses.
213       if (cmpCreated)
214         continue;
215       cmpCreated = true;
216 
217       mlir::Location loc = loopOp.getLoc();
218       mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
219       firOpBuilder.setInsertionPoint(lastOper);
220 
221       mlir::Value cmpOp;
222       llvm::SmallVector<mlir::Value> vs;
223       vs.reserve(loopOp.getIVs().size());
224       for (auto [iv, ub, step] :
225            llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(),
226                            loopOp.getLoopSteps())) {
227         // v = iv + step
228         // cmp = step < 0 ? v < ub : v > ub
229         mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
230         vs.push_back(v);
231         mlir::Value zero =
232             firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
233         mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
234             loc, mlir::arith::CmpIPredicate::slt, step, zero);
235         mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
236             loc, mlir::arith::CmpIPredicate::slt, v, ub);
237         mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
238             loc, mlir::arith::CmpIPredicate::sgt, v, ub);
239         mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
240             loc, negativeStep, vLT, vGT);
241 
242         if (cmpOp) {
243           cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
244         } else {
245           cmpOp = icmpOp;
246         }
247       }
248 
249       auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
250       firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
251       for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
252         assert(loopIV && "loopIV was not set");
253         firOpBuilder.createStoreWithConvert(loc, v, loopIV);
254       }
255       lastPrivIP = firOpBuilder.saveInsertionPoint();
256     } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
257       // Already handled by genOMP()
258     } else {
259       TODO(converter.getCurrentLocation(),
260            "lastprivate clause in constructs other than "
261            "simd/worksharing-loop");
262     }
263   }
264 }
265 
266 static const parser::CharBlock *
267 getSource(const semantics::SemanticsContext &semaCtx,
268           const lower::pft::Evaluation &eval) {
269   const parser::CharBlock *source = nullptr;
270 
271   auto ompConsVisit = [&](const parser::OpenMPConstruct &x) {
272     std::visit(common::visitors{
273                    [&](const parser::OpenMPSectionsConstruct &x) {
274                      source = &std::get<0>(x.t).source;
275                    },
276                    [&](const parser::OpenMPLoopConstruct &x) {
277                      source = &std::get<0>(x.t).source;
278                    },
279                    [&](const parser::OpenMPBlockConstruct &x) {
280                      source = &std::get<0>(x.t).source;
281                    },
282                    [&](const parser::OpenMPCriticalConstruct &x) {
283                      source = &std::get<0>(x.t).source;
284                    },
285                    [&](const parser::OpenMPAtomicConstruct &x) {
286                      std::visit([&](const auto &x) { source = &x.source; },
287                                 x.u);
288                    },
289                    [&](const auto &x) { source = &x.source; },
290                },
291                x.u);
292   };
293 
294   eval.visit(common::visitors{
295       [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); },
296       [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; },
297       [&](const parser::OmpEndLoopDirective &x) { source = &x.source; },
298       [&](const auto &x) {},
299   });
300 
301   return source;
302 }
303 
304 void DataSharingProcessor::collectSymbolsInNestedRegions(
305     lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
306     llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) {
307   for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
308     if (nestedEval.hasNestedEvaluations()) {
309       if (nestedEval.isConstruct())
310         // Recursively look for OpenMP constructs within `nestedEval`'s region
311         collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
312       else {
313         converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
314                                    /*collectSymbols=*/true,
315                                    /*collectHostAssociatedSymbols=*/false);
316       }
317     }
318   }
319 }
320 
321 // Collect symbols to be default privatized in two steps.
322 // In step 1, collect all symbols in `eval` that match `flag` into
323 // `defaultSymbols`. In step 2, for nested constructs (if any), if and only if
324 // the nested construct is an OpenMP construct, collect those nested
325 // symbols skipping host associated symbols into `symbolsInNestedRegions`.
326 // Later, in current context, all symbols in the set
327 // `defaultSymbols` - `symbolsInNestedRegions` will be privatized.
328 void DataSharingProcessor::collectSymbols(
329     semantics::Symbol::Flag flag,
330     llvm::SetVector<const semantics::Symbol *> &symbols) {
331   // Collect all scopes associated with 'eval'.
332   llvm::SetVector<const semantics::Scope *> clauseScopes;
333   std::function<void(const semantics::Scope *)> collectScopes =
334       [&](const semantics::Scope *scope) {
335         clauseScopes.insert(scope);
336         for (const semantics::Scope &child : scope->children())
337           collectScopes(&child);
338       };
339   const parser::CharBlock *source =
340       clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source;
341   const semantics::Scope *curScope = nullptr;
342   if (source && !source->empty()) {
343     curScope = &semaCtx.FindScope(*source);
344     collectScopes(curScope);
345   }
346   // Collect all symbols referenced in the evaluation being processed,
347   // that matches 'flag'.
348   llvm::SetVector<const semantics::Symbol *> allSymbols;
349   converter.collectSymbolSet(eval, allSymbols, flag,
350                              /*collectSymbols=*/true,
351                              /*collectHostAssociatedSymbols=*/true);
352 
353   // Add implicitly referenced symbols from statement functions.
354   if (curScope) {
355     for (const auto &sym : curScope->GetSymbols()) {
356       if (sym->test(semantics::Symbol::Flag::OmpFromStmtFunction) &&
357           sym->test(flag))
358         allSymbols.insert(&*sym);
359     }
360   }
361 
362   llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
363   collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
364 
365   for (auto *symbol : allSymbols)
366     if (visitor.isSymbolDefineBy(symbol, eval))
367       symbolsInNestedRegions.remove(symbol);
368 
369   // Filter-out symbols that must not be privatized.
370   bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit;
371   bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined;
372 
373   auto isPrivatizable = [](const semantics::Symbol &sym) -> bool {
374     return !semantics::IsProcedure(sym) &&
375            !sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
376            !sym.GetUltimate().has<semantics::NamelistDetails>() &&
377            !semantics::IsImpliedDoIndex(sym.GetUltimate());
378   };
379 
380   auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
381     if (collectImplicit)
382       return sym->test(semantics::Symbol::Flag::OmpImplicit);
383 
384     if (collectPreDetermined)
385       return sym->test(semantics::Symbol::Flag::OmpPreDetermined);
386 
387     return !sym->test(semantics::Symbol::Flag::OmpImplicit) &&
388            !sym->test(semantics::Symbol::Flag::OmpPreDetermined);
389   };
390 
391   for (const auto *sym : allSymbols) {
392     assert(curScope && "couldn't find current scope");
393     if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) &&
394         !explicitlyPrivatizedSymbols.contains(sym) &&
395         shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) {
396       allPrivatizedSymbols.insert(sym);
397       symbols.insert(sym);
398     }
399   }
400 }
401 
402 void DataSharingProcessor::collectDefaultSymbols() {
403   using DataSharingAttribute = omp::clause::Default::DataSharingAttribute;
404   for (const omp::Clause &clause : clauses) {
405     if (const auto *defaultClause =
406             std::get_if<omp::clause::Default>(&clause.u)) {
407       if (defaultClause->v == DataSharingAttribute::Private)
408         collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols);
409       else if (defaultClause->v == DataSharingAttribute::Firstprivate)
410         collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate,
411                        defaultSymbols);
412     }
413   }
414 }
415 
416 void DataSharingProcessor::collectImplicitSymbols() {
417   // There will be no implicit symbols when a default clause is present.
418   if (defaultSymbols.empty())
419     collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols);
420 }
421 
422 void DataSharingProcessor::collectPreDeterminedSymbols() {
423   if (shouldCollectPreDeterminedSymbols)
424     collectSymbols(semantics::Symbol::Flag::OmpPreDetermined,
425                    preDeterminedSymbols);
426 }
427 
428 void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) {
429   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
430     if (const auto *commonDet =
431             sym->detailsIf<semantics::CommonBlockDetails>()) {
432       for (const auto &mem : commonDet->objects())
433         doPrivatize(&*mem, clauseOps);
434     } else
435       doPrivatize(sym, clauseOps);
436   }
437 }
438 
439 void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
440   insertLastPrivateCompare(op);
441   for (const semantics::Symbol *sym : allPrivatizedSymbols)
442     if (const auto *commonDet =
443             sym->detailsIf<semantics::CommonBlockDetails>()) {
444       for (const auto &mem : commonDet->objects()) {
445         copyLastPrivateSymbol(&*mem, &lastPrivIP);
446       }
447     } else {
448       copyLastPrivateSymbol(sym, &lastPrivIP);
449     }
450 }
451 
452 void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
453                                        mlir::omp::PrivateClauseOps *clauseOps) {
454   if (!useDelayedPrivatization) {
455     cloneSymbol(sym);
456     copyFirstPrivateSymbol(sym);
457     return;
458   }
459 
460   lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
461   assert(hsb && "Host symbol box not found");
462 
463   mlir::Type symType = hsb.getAddr().getType();
464   mlir::Location symLoc = hsb.getAddr().getLoc();
465   std::string privatizerName = sym->name().ToString() + ".privatizer";
466   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
467 
468   mlir::omp::PrivateClauseOp privatizerOp = [&]() {
469     auto moduleOp = firOpBuilder.getModule();
470     auto uniquePrivatizerName = fir::getTypeAsString(
471         symType, converter.getKindMap(),
472         converter.mangleName(*sym) +
473             (isFirstPrivate ? "_firstprivate" : "_private"));
474 
475     if (auto existingPrivatizer =
476             moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>(
477                 uniquePrivatizerName))
478       return existingPrivatizer;
479 
480     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
481     firOpBuilder.setInsertionPoint(&moduleOp.getBodyRegion().front(),
482                                    moduleOp.getBodyRegion().front().begin());
483     auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>(
484         symLoc, uniquePrivatizerName, symType,
485         isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate
486                        : mlir::omp::DataSharingClauseType::Private);
487     fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
488     lower::SymMapScope outerScope(*symTable);
489 
490     // Populate the `alloc` region.
491     {
492       mlir::Region &allocRegion = result.getAllocRegion();
493       mlir::Block *allocEntryBlock = firOpBuilder.createBlock(
494           &allocRegion, /*insertPt=*/{}, symType, symLoc);
495 
496       firOpBuilder.setInsertionPointToEnd(allocEntryBlock);
497 
498       fir::ExtendedValue localExV =
499           hlfir::translateToExtendedValue(
500               symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)},
501               /*contiguousHint=*/
502               evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
503               .first;
504 
505       symTable->addSymbol(*sym, localExV);
506       lower::SymMapScope innerScope(*symTable);
507       cloneSymbol(sym);
508       mlir::Value cloneAddr = symTable->shallowLookupSymbol(*sym).getAddr();
509       mlir::Type cloneType = cloneAddr.getType();
510 
511       // A `convert` op is required for variables that are storage associated
512       // via `equivalence`. The problem is that these variables are declared as
513       // `fir.ptr`s while their privatized storage is declared as `fir.ref`,
514       // therefore we convert to proper symbol type.
515       mlir::Value yieldedValue =
516           (symType == cloneType) ? cloneAddr
517                                  : firOpBuilder.createConvert(
518                                        cloneAddr.getLoc(), symType, cloneAddr);
519 
520       firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc(),
521                                               yieldedValue);
522     }
523 
524     // Populate the `copy` region if this is a `firstprivate`.
525     if (isFirstPrivate) {
526       mlir::Region &copyRegion = result.getCopyRegion();
527       // First block argument corresponding to the original/host value while
528       // second block argument corresponding to the privatized value.
529       mlir::Block *copyEntryBlock = firOpBuilder.createBlock(
530           &copyRegion, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc});
531       firOpBuilder.setInsertionPointToEnd(copyEntryBlock);
532 
533       auto addSymbol = [&](unsigned argIdx, bool force = false) {
534         symExV.match(
535             [&](const fir::MutableBoxValue &box) {
536               symTable->addSymbol(
537                   *sym, fir::substBase(box, copyRegion.getArgument(argIdx)),
538                   force);
539             },
540             [&](const auto &box) {
541               symTable->addSymbol(*sym, copyRegion.getArgument(argIdx), force);
542             });
543       };
544 
545       addSymbol(0, true);
546       lower::SymMapScope innerScope(*symTable);
547       addSymbol(1);
548 
549       auto ip = firOpBuilder.saveInsertionPoint();
550       copyFirstPrivateSymbol(sym, &ip);
551 
552       firOpBuilder.create<mlir::omp::YieldOp>(
553           hsb.getAddr().getLoc(),
554           symTable->shallowLookupSymbol(*sym).getAddr());
555     }
556 
557     return result;
558   }();
559 
560   if (clauseOps) {
561     clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp));
562     clauseOps->privateVars.push_back(hsb.getAddr());
563   }
564 
565   symToPrivatizer[sym] = privatizerOp;
566 }
567 
568 } // namespace omp
569 } // namespace lower
570 } // namespace Fortran
571