xref: /llvm-project/flang/lib/Lower/OpenMP/DataSharingProcessor.cpp (revision 1fcb6a9754a8db057e18f629cb90011b638901e7) 
1 //===-- DataSharingProcessor.cpp --------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Coding style: https://mlir.llvm.org/getting_started/DeveloperGuide/
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "DataSharingProcessor.h"
14 
15 #include "Utils.h"
16 #include "flang/Lower/ConvertVariable.h"
17 #include "flang/Lower/PFTBuilder.h"
18 #include "flang/Lower/SymbolMap.h"
19 #include "flang/Optimizer/Builder/HLFIRTools.h"
20 #include "flang/Optimizer/Builder/Todo.h"
21 #include "flang/Optimizer/HLFIR/HLFIROps.h"
22 #include "flang/Semantics/tools.h"
23 
24 namespace Fortran {
25 namespace lower {
26 namespace omp {
27 bool DataSharingProcessor::OMPConstructSymbolVisitor::isSymbolDefineBy(
28     const semantics::Symbol *symbol, lower::pft::Evaluation &eval) const {
29   return eval.visit(
30       common::visitors{[&](const parser::OpenMPConstruct &functionParserNode) {
31                          return symDefMap.count(symbol) &&
32                                 symDefMap.at(symbol) == &functionParserNode;
33                        },
34                        [](const auto &functionParserNode) { return false; }});
35 }
36 
37 DataSharingProcessor::DataSharingProcessor(
38     lower::AbstractConverter &converter, semantics::SemanticsContext &semaCtx,
39     const List<Clause> &clauses, lower::pft::Evaluation &eval,
40     bool shouldCollectPreDeterminedSymbols, bool useDelayedPrivatization,
41     lower::SymMap &symTable)
42     : converter(converter), semaCtx(semaCtx),
43       firOpBuilder(converter.getFirOpBuilder()), clauses(clauses), eval(eval),
44       shouldCollectPreDeterminedSymbols(shouldCollectPreDeterminedSymbols),
45       useDelayedPrivatization(useDelayedPrivatization), symTable(symTable),
46       visitor() {
47   eval.visit([&](const auto &functionParserNode) {
48     parser::Walk(functionParserNode, visitor);
49   });
50 }
51 
52 void DataSharingProcessor::processStep1(
53     mlir::omp::PrivateClauseOps *clauseOps) {
54   collectSymbolsForPrivatization();
55   collectDefaultSymbols();
56   collectImplicitSymbols();
57   collectPreDeterminedSymbols();
58 
59   privatize(clauseOps);
60 
61   insertBarrier();
62 }
63 
64 void DataSharingProcessor::processStep2(mlir::Operation *op, bool isLoop) {
65   // 'sections' lastprivate is handled by genOMP()
66   if (!mlir::isa<mlir::omp::SectionsOp>(op)) {
67     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
68     copyLastPrivatize(op);
69   }
70 
71   if (isLoop) {
72     // push deallocs out of the loop
73     firOpBuilder.setInsertionPointAfter(op);
74     insertDeallocs();
75   } else {
76     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
77     insertDeallocs();
78   }
79 }
80 
81 void DataSharingProcessor::insertDeallocs() {
82   for (const semantics::Symbol *sym : allPrivatizedSymbols)
83     if (semantics::IsAllocatable(sym->GetUltimate())) {
84       if (!useDelayedPrivatization) {
85         converter.createHostAssociateVarCloneDealloc(*sym);
86         continue;
87       }
88 
89       lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
90       assert(hsb && "Host symbol box not found");
91       mlir::Type symType = hsb.getAddr().getType();
92       mlir::Location symLoc = hsb.getAddr().getLoc();
93       fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
94       mlir::omp::PrivateClauseOp privatizer = symToPrivatizer.at(sym);
95 
96       lower::SymMapScope scope(symTable);
97       mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
98 
99       mlir::Region &deallocRegion = privatizer.getDeallocRegion();
100       fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder();
101       mlir::Block *deallocEntryBlock = firOpBuilder.createBlock(
102           &deallocRegion, /*insertPt=*/{}, symType, symLoc);
103 
104       firOpBuilder.setInsertionPointToEnd(deallocEntryBlock);
105       symTable.addSymbol(*sym,
106                          fir::substBase(symExV, deallocRegion.getArgument(0)));
107 
108       converter.createHostAssociateVarCloneDealloc(*sym);
109       firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc());
110     }
111 }
112 
113 void DataSharingProcessor::cloneSymbol(const semantics::Symbol *sym) {
114   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
115   bool success = converter.createHostAssociateVarClone(
116       *sym, /*skipDefaultInit=*/isFirstPrivate);
117   (void)success;
118   assert(success && "Privatization failed due to existing binding");
119 
120   // Initialize clone from original object if it has any allocatable member.
121   auto needInitClone = [&] {
122     if (isFirstPrivate)
123       return false;
124 
125     SymbolBox sb = symTable.lookupSymbol(sym);
126     assert(sb);
127     mlir::Value addr = sb.getAddr();
128     assert(addr);
129     return hlfir::mayHaveAllocatableComponent(addr.getType());
130   };
131 
132   if (needInitClone()) {
133     Fortran::lower::initializeCloneAtRuntime(converter, *sym, symTable);
134     callsInitClone = true;
135   }
136 }
137 
138 void DataSharingProcessor::copyFirstPrivateSymbol(
139     const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *copyAssignIP) {
140   if (sym->test(semantics::Symbol::Flag::OmpFirstPrivate))
141     converter.copyHostAssociateVar(*sym, copyAssignIP);
142 }
143 
144 void DataSharingProcessor::copyLastPrivateSymbol(
145     const semantics::Symbol *sym, mlir::OpBuilder::InsertPoint *lastPrivIP) {
146   if (sym->test(semantics::Symbol::Flag::OmpLastPrivate))
147     converter.copyHostAssociateVar(*sym, lastPrivIP);
148 }
149 
150 void DataSharingProcessor::collectOmpObjectListSymbol(
151     const omp::ObjectList &objects,
152     llvm::SetVector<const semantics::Symbol *> &symbolSet) {
153   for (const omp::Object &object : objects)
154     symbolSet.insert(object.sym());
155 }
156 
157 void DataSharingProcessor::collectSymbolsForPrivatization() {
158   for (const omp::Clause &clause : clauses) {
159     if (const auto &privateClause =
160             std::get_if<omp::clause::Private>(&clause.u)) {
161       collectOmpObjectListSymbol(privateClause->v, explicitlyPrivatizedSymbols);
162     } else if (const auto &firstPrivateClause =
163                    std::get_if<omp::clause::Firstprivate>(&clause.u)) {
164       collectOmpObjectListSymbol(firstPrivateClause->v,
165                                  explicitlyPrivatizedSymbols);
166     } else if (const auto &lastPrivateClause =
167                    std::get_if<omp::clause::Lastprivate>(&clause.u)) {
168       lastprivateModifierNotSupported(*lastPrivateClause,
169                                       converter.getCurrentLocation());
170       const ObjectList &objects = std::get<ObjectList>(lastPrivateClause->t);
171       collectOmpObjectListSymbol(objects, explicitlyPrivatizedSymbols);
172     }
173   }
174 
175   for (auto *sym : explicitlyPrivatizedSymbols)
176     allPrivatizedSymbols.insert(sym);
177 }
178 
179 bool DataSharingProcessor::needBarrier() {
180   // Emit implicit barrier to synchronize threads and avoid data races on
181   // initialization of firstprivate variables and post-update of lastprivate
182   // variables.
183   // Emit implicit barrier for linear clause. Maybe on somewhere else.
184   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
185     if (sym->test(semantics::Symbol::Flag::OmpLastPrivate) &&
186         (sym->test(semantics::Symbol::Flag::OmpFirstPrivate) || callsInitClone))
187       return true;
188   }
189   return false;
190 }
191 
192 void DataSharingProcessor::insertBarrier() {
193   if (needBarrier())
194     firOpBuilder.create<mlir::omp::BarrierOp>(converter.getCurrentLocation());
195 }
196 
197 void DataSharingProcessor::insertLastPrivateCompare(mlir::Operation *op) {
198   mlir::omp::LoopNestOp loopOp;
199   if (auto wrapper = mlir::dyn_cast<mlir::omp::LoopWrapperInterface>(op))
200     loopOp = mlir::cast<mlir::omp::LoopNestOp>(wrapper.getWrappedLoop());
201 
202   bool cmpCreated = false;
203   mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
204   for (const omp::Clause &clause : clauses) {
205     if (clause.id != llvm::omp::OMPC_lastprivate)
206       continue;
207     if (mlir::isa<mlir::omp::WsloopOp>(op) ||
208         mlir::isa<mlir::omp::SimdOp>(op)) {
209       // Update the original variable just before exiting the worksharing
210       // loop. Conversion as follows:
211       //
212       // omp.wsloop / omp.simd {    omp.wsloop / omp.simd {
213       //   omp.loop_nest {            omp.loop_nest {
214       //     ...                        ...
215       //     store          ===>        store
216       //     omp.yield                  %v = arith.addi %iv, %step
217       //   }                            %cmp = %step < 0 ? %v < %ub : %v > %ub
218       // }                              fir.if %cmp {
219       //                                  fir.store %v to %loopIV
220       //                                  ^%lpv_update_blk:
221       //                                }
222       //                                omp.yield
223       //                              }
224       //                            }
225 
226       // Only generate the compare once in presence of multiple LastPrivate
227       // clauses.
228       if (cmpCreated)
229         continue;
230       cmpCreated = true;
231 
232       mlir::Location loc = loopOp.getLoc();
233       mlir::Operation *lastOper = loopOp.getRegion().back().getTerminator();
234       firOpBuilder.setInsertionPoint(lastOper);
235 
236       mlir::Value cmpOp;
237       llvm::SmallVector<mlir::Value> vs;
238       vs.reserve(loopOp.getIVs().size());
239       for (auto [iv, ub, step] :
240            llvm::zip_equal(loopOp.getIVs(), loopOp.getLoopUpperBounds(),
241                            loopOp.getLoopSteps())) {
242         // v = iv + step
243         // cmp = step < 0 ? v < ub : v > ub
244         mlir::Value v = firOpBuilder.create<mlir::arith::AddIOp>(loc, iv, step);
245         vs.push_back(v);
246         mlir::Value zero =
247             firOpBuilder.createIntegerConstant(loc, step.getType(), 0);
248         mlir::Value negativeStep = firOpBuilder.create<mlir::arith::CmpIOp>(
249             loc, mlir::arith::CmpIPredicate::slt, step, zero);
250         mlir::Value vLT = firOpBuilder.create<mlir::arith::CmpIOp>(
251             loc, mlir::arith::CmpIPredicate::slt, v, ub);
252         mlir::Value vGT = firOpBuilder.create<mlir::arith::CmpIOp>(
253             loc, mlir::arith::CmpIPredicate::sgt, v, ub);
254         mlir::Value icmpOp = firOpBuilder.create<mlir::arith::SelectOp>(
255             loc, negativeStep, vLT, vGT);
256 
257         if (cmpOp) {
258           cmpOp = firOpBuilder.create<mlir::arith::AndIOp>(loc, cmpOp, icmpOp);
259         } else {
260           cmpOp = icmpOp;
261         }
262       }
263 
264       auto ifOp = firOpBuilder.create<fir::IfOp>(loc, cmpOp, /*else*/ false);
265       firOpBuilder.setInsertionPointToStart(&ifOp.getThenRegion().front());
266       for (auto [v, loopIV] : llvm::zip_equal(vs, loopIVs)) {
267         assert(loopIV && "loopIV was not set");
268         firOpBuilder.createStoreWithConvert(loc, v, loopIV);
269       }
270       lastPrivIP = firOpBuilder.saveInsertionPoint();
271     } else if (mlir::isa<mlir::omp::SectionsOp>(op)) {
272       // Already handled by genOMP()
273     } else {
274       TODO(converter.getCurrentLocation(),
275            "lastprivate clause in constructs other than "
276            "simd/worksharing-loop");
277     }
278   }
279 }
280 
281 static const parser::CharBlock *
282 getSource(const semantics::SemanticsContext &semaCtx,
283           const lower::pft::Evaluation &eval) {
284   const parser::CharBlock *source = nullptr;
285 
286   auto ompConsVisit = [&](const parser::OpenMPConstruct &x) {
287     std::visit(common::visitors{
288                    [&](const parser::OpenMPSectionsConstruct &x) {
289                      source = &std::get<0>(x.t).source;
290                    },
291                    [&](const parser::OpenMPLoopConstruct &x) {
292                      source = &std::get<0>(x.t).source;
293                    },
294                    [&](const parser::OpenMPBlockConstruct &x) {
295                      source = &std::get<0>(x.t).source;
296                    },
297                    [&](const parser::OpenMPCriticalConstruct &x) {
298                      source = &std::get<0>(x.t).source;
299                    },
300                    [&](const parser::OpenMPAtomicConstruct &x) {
301                      std::visit([&](const auto &x) { source = &x.source; },
302                                 x.u);
303                    },
304                    [&](const auto &x) { source = &x.source; },
305                },
306                x.u);
307   };
308 
309   eval.visit(common::visitors{
310       [&](const parser::OpenMPConstruct &x) { ompConsVisit(x); },
311       [&](const parser::OpenMPDeclarativeConstruct &x) { source = &x.source; },
312       [&](const parser::OmpEndLoopDirective &x) { source = &x.source; },
313       [&](const auto &x) {},
314   });
315 
316   return source;
317 }
318 
319 void DataSharingProcessor::collectSymbolsInNestedRegions(
320     lower::pft::Evaluation &eval, semantics::Symbol::Flag flag,
321     llvm::SetVector<const semantics::Symbol *> &symbolsInNestedRegions) {
322   for (lower::pft::Evaluation &nestedEval : eval.getNestedEvaluations()) {
323     if (nestedEval.hasNestedEvaluations()) {
324       if (nestedEval.isConstruct())
325         // Recursively look for OpenMP constructs within `nestedEval`'s region
326         collectSymbolsInNestedRegions(nestedEval, flag, symbolsInNestedRegions);
327       else {
328         converter.collectSymbolSet(nestedEval, symbolsInNestedRegions, flag,
329                                    /*collectSymbols=*/true,
330                                    /*collectHostAssociatedSymbols=*/false);
331       }
332     }
333   }
334 }
335 
336 // Collect symbols to be default privatized in two steps.
337 // In step 1, collect all symbols in `eval` that match `flag` into
338 // `defaultSymbols`. In step 2, for nested constructs (if any), if and only if
339 // the nested construct is an OpenMP construct, collect those nested
340 // symbols skipping host associated symbols into `symbolsInNestedRegions`.
341 // Later, in current context, all symbols in the set
342 // `defaultSymbols` - `symbolsInNestedRegions` will be privatized.
343 void DataSharingProcessor::collectSymbols(
344     semantics::Symbol::Flag flag,
345     llvm::SetVector<const semantics::Symbol *> &symbols) {
346   // Collect all scopes associated with 'eval'.
347   llvm::SetVector<const semantics::Scope *> clauseScopes;
348   std::function<void(const semantics::Scope *)> collectScopes =
349       [&](const semantics::Scope *scope) {
350         clauseScopes.insert(scope);
351         for (const semantics::Scope &child : scope->children())
352           collectScopes(&child);
353       };
354   const parser::CharBlock *source =
355       clauses.empty() ? getSource(semaCtx, eval) : &clauses.front().source;
356   const semantics::Scope *curScope = nullptr;
357   if (source && !source->empty()) {
358     curScope = &semaCtx.FindScope(*source);
359     collectScopes(curScope);
360   }
361   // Collect all symbols referenced in the evaluation being processed,
362   // that matches 'flag'.
363   llvm::SetVector<const semantics::Symbol *> allSymbols;
364   converter.collectSymbolSet(eval, allSymbols, flag,
365                              /*collectSymbols=*/true,
366                              /*collectHostAssociatedSymbols=*/true);
367 
368   llvm::SetVector<const semantics::Symbol *> symbolsInNestedRegions;
369   collectSymbolsInNestedRegions(eval, flag, symbolsInNestedRegions);
370 
371   for (auto *symbol : allSymbols)
372     if (visitor.isSymbolDefineBy(symbol, eval))
373       symbolsInNestedRegions.remove(symbol);
374 
375   // Filter-out symbols that must not be privatized.
376   bool collectImplicit = flag == semantics::Symbol::Flag::OmpImplicit;
377   bool collectPreDetermined = flag == semantics::Symbol::Flag::OmpPreDetermined;
378 
379   auto isPrivatizable = [](const semantics::Symbol &sym) -> bool {
380     return !semantics::IsProcedure(sym) &&
381            !sym.GetUltimate().has<semantics::DerivedTypeDetails>() &&
382            !sym.GetUltimate().has<semantics::NamelistDetails>() &&
383            !semantics::IsImpliedDoIndex(sym.GetUltimate()) &&
384            !semantics::IsStmtFunction(sym);
385   };
386 
387   auto shouldCollectSymbol = [&](const semantics::Symbol *sym) {
388     if (collectImplicit)
389       return sym->test(semantics::Symbol::Flag::OmpImplicit);
390 
391     if (collectPreDetermined)
392       return sym->test(semantics::Symbol::Flag::OmpPreDetermined);
393 
394     return !sym->test(semantics::Symbol::Flag::OmpImplicit) &&
395            !sym->test(semantics::Symbol::Flag::OmpPreDetermined);
396   };
397 
398   for (const auto *sym : allSymbols) {
399     assert(curScope && "couldn't find current scope");
400     if (isPrivatizable(*sym) && !symbolsInNestedRegions.contains(sym) &&
401         !explicitlyPrivatizedSymbols.contains(sym) &&
402         shouldCollectSymbol(sym) && clauseScopes.contains(&sym->owner())) {
403       allPrivatizedSymbols.insert(sym);
404       symbols.insert(sym);
405     }
406   }
407 }
408 
409 void DataSharingProcessor::collectDefaultSymbols() {
410   using DataSharingAttribute = omp::clause::Default::DataSharingAttribute;
411   for (const omp::Clause &clause : clauses) {
412     if (const auto *defaultClause =
413             std::get_if<omp::clause::Default>(&clause.u)) {
414       if (defaultClause->v == DataSharingAttribute::Private)
415         collectSymbols(semantics::Symbol::Flag::OmpPrivate, defaultSymbols);
416       else if (defaultClause->v == DataSharingAttribute::Firstprivate)
417         collectSymbols(semantics::Symbol::Flag::OmpFirstPrivate,
418                        defaultSymbols);
419     }
420   }
421 }
422 
423 void DataSharingProcessor::collectImplicitSymbols() {
424   // There will be no implicit symbols when a default clause is present.
425   if (defaultSymbols.empty())
426     collectSymbols(semantics::Symbol::Flag::OmpImplicit, implicitSymbols);
427 }
428 
429 void DataSharingProcessor::collectPreDeterminedSymbols() {
430   if (shouldCollectPreDeterminedSymbols)
431     collectSymbols(semantics::Symbol::Flag::OmpPreDetermined,
432                    preDeterminedSymbols);
433 }
434 
435 void DataSharingProcessor::privatize(mlir::omp::PrivateClauseOps *clauseOps) {
436   for (const semantics::Symbol *sym : allPrivatizedSymbols) {
437     if (const auto *commonDet =
438             sym->detailsIf<semantics::CommonBlockDetails>()) {
439       for (const auto &mem : commonDet->objects())
440         doPrivatize(&*mem, clauseOps);
441     } else
442       doPrivatize(sym, clauseOps);
443   }
444 }
445 
446 void DataSharingProcessor::copyLastPrivatize(mlir::Operation *op) {
447   insertLastPrivateCompare(op);
448   for (const semantics::Symbol *sym : allPrivatizedSymbols)
449     if (const auto *commonDet =
450             sym->detailsIf<semantics::CommonBlockDetails>()) {
451       for (const auto &mem : commonDet->objects()) {
452         copyLastPrivateSymbol(&*mem, &lastPrivIP);
453       }
454     } else {
455       copyLastPrivateSymbol(sym, &lastPrivIP);
456     }
457 }
458 
459 void DataSharingProcessor::doPrivatize(const semantics::Symbol *sym,
460                                        mlir::omp::PrivateClauseOps *clauseOps) {
461   if (!useDelayedPrivatization) {
462     cloneSymbol(sym);
463     copyFirstPrivateSymbol(sym);
464     return;
465   }
466 
467   lower::SymbolBox hsb = converter.lookupOneLevelUpSymbol(*sym);
468   assert(hsb && "Host symbol box not found");
469 
470   mlir::Type symType = hsb.getAddr().getType();
471   mlir::Location symLoc = hsb.getAddr().getLoc();
472   std::string privatizerName = sym->name().ToString() + ".privatizer";
473   bool isFirstPrivate = sym->test(semantics::Symbol::Flag::OmpFirstPrivate);
474 
475   mlir::omp::PrivateClauseOp privatizerOp = [&]() {
476     auto moduleOp = firOpBuilder.getModule();
477     auto uniquePrivatizerName = fir::getTypeAsString(
478         symType, converter.getKindMap(),
479         converter.mangleName(*sym) +
480             (isFirstPrivate ? "_firstprivate" : "_private"));
481 
482     if (auto existingPrivatizer =
483             moduleOp.lookupSymbol<mlir::omp::PrivateClauseOp>(
484                 uniquePrivatizerName))
485       return existingPrivatizer;
486 
487     mlir::OpBuilder::InsertionGuard guard(firOpBuilder);
488     firOpBuilder.setInsertionPointToStart(moduleOp.getBody());
489     auto result = firOpBuilder.create<mlir::omp::PrivateClauseOp>(
490         symLoc, uniquePrivatizerName, symType,
491         isFirstPrivate ? mlir::omp::DataSharingClauseType::FirstPrivate
492                        : mlir::omp::DataSharingClauseType::Private);
493     fir::ExtendedValue symExV = converter.getSymbolExtendedValue(*sym);
494     lower::SymMapScope outerScope(symTable);
495 
496     // Populate the `alloc` region.
497     {
498       mlir::Region &allocRegion = result.getAllocRegion();
499       mlir::Block *allocEntryBlock = firOpBuilder.createBlock(
500           &allocRegion, /*insertPt=*/{}, symType, symLoc);
501 
502       firOpBuilder.setInsertionPointToEnd(allocEntryBlock);
503 
504       fir::ExtendedValue localExV =
505           hlfir::translateToExtendedValue(
506               symLoc, firOpBuilder, hlfir::Entity{allocRegion.getArgument(0)},
507               /*contiguousHint=*/
508               evaluate::IsSimplyContiguous(*sym, converter.getFoldingContext()))
509               .first;
510 
511       symTable.addSymbol(*sym, localExV);
512       lower::SymMapScope innerScope(symTable);
513       cloneSymbol(sym);
514       mlir::Value cloneAddr = symTable.shallowLookupSymbol(*sym).getAddr();
515       mlir::Type cloneType = cloneAddr.getType();
516 
517       // A `convert` op is required for variables that are storage associated
518       // via `equivalence`. The problem is that these variables are declared as
519       // `fir.ptr`s while their privatized storage is declared as `fir.ref`,
520       // therefore we convert to proper symbol type.
521       mlir::Value yieldedValue =
522           (symType == cloneType) ? cloneAddr
523                                  : firOpBuilder.createConvert(
524                                        cloneAddr.getLoc(), symType, cloneAddr);
525 
526       firOpBuilder.create<mlir::omp::YieldOp>(hsb.getAddr().getLoc(),
527                                               yieldedValue);
528     }
529 
530     // Populate the `copy` region if this is a `firstprivate`.
531     if (isFirstPrivate) {
532       mlir::Region &copyRegion = result.getCopyRegion();
533       // First block argument corresponding to the original/host value while
534       // second block argument corresponding to the privatized value.
535       mlir::Block *copyEntryBlock = firOpBuilder.createBlock(
536           &copyRegion, /*insertPt=*/{}, {symType, symType}, {symLoc, symLoc});
537       firOpBuilder.setInsertionPointToEnd(copyEntryBlock);
538 
539       auto addSymbol = [&](unsigned argIdx, bool force = false) {
540         symExV.match(
541             [&](const fir::MutableBoxValue &box) {
542               symTable.addSymbol(
543                   *sym, fir::substBase(box, copyRegion.getArgument(argIdx)),
544                   force);
545             },
546             [&](const auto &box) {
547               symTable.addSymbol(*sym, copyRegion.getArgument(argIdx), force);
548             });
549       };
550 
551       addSymbol(0, true);
552       lower::SymMapScope innerScope(symTable);
553       addSymbol(1);
554 
555       auto ip = firOpBuilder.saveInsertionPoint();
556       copyFirstPrivateSymbol(sym, &ip);
557 
558       firOpBuilder.create<mlir::omp::YieldOp>(
559           hsb.getAddr().getLoc(), symTable.shallowLookupSymbol(*sym).getAddr());
560     }
561 
562     return result;
563   }();
564 
565   if (clauseOps) {
566     clauseOps->privateSyms.push_back(mlir::SymbolRefAttr::get(privatizerOp));
567     clauseOps->privateVars.push_back(hsb.getAddr());
568   }
569 
570   symToPrivatizer[sym] = privatizerOp;
571 }
572 
573 } // namespace omp
574 } // namespace lower
575 } // namespace Fortran
576