xref: /llvm-project/clang/lib/CodeGen/CGLoopInfo.cpp (revision d4c667c9af0335ac20015decce204c43f03a6cee) 
1 //===---- CGLoopInfo.cpp - LLVM CodeGen for loop metadata -*- C++ -*-------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "CGLoopInfo.h"
10 #include "clang/AST/ASTContext.h"
11 #include "clang/AST/Attr.h"
12 #include "clang/AST/Expr.h"
13 #include "clang/Basic/CodeGenOptions.h"
14 #include "llvm/IR/BasicBlock.h"
15 #include "llvm/IR/CFG.h"
16 #include "llvm/IR/Constants.h"
17 #include "llvm/IR/InstrTypes.h"
18 #include "llvm/IR/Instructions.h"
19 #include "llvm/IR/Metadata.h"
20 using namespace clang::CodeGen;
21 using namespace llvm;
22 
23 MDNode *
24 LoopInfo::createLoopPropertiesMetadata(ArrayRef<Metadata *> LoopProperties) {
25   LLVMContext &Ctx = Header->getContext();
26   SmallVector<Metadata *, 4> NewLoopProperties;
27   NewLoopProperties.push_back(nullptr);
28   NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
29 
30   MDNode *LoopID = MDNode::getDistinct(Ctx, NewLoopProperties);
31   LoopID->replaceOperandWith(0, LoopID);
32   return LoopID;
33 }
34 
35 MDNode *LoopInfo::createPipeliningMetadata(const LoopAttributes &Attrs,
36                                            ArrayRef<Metadata *> LoopProperties,
37                                            bool &HasUserTransforms) {
38   LLVMContext &Ctx = Header->getContext();
39 
40   Optional<bool> Enabled;
41   if (Attrs.PipelineDisabled)
42     Enabled = false;
43   else if (Attrs.PipelineInitiationInterval != 0)
44     Enabled = true;
45 
46   if (Enabled != true) {
47     SmallVector<Metadata *, 4> NewLoopProperties;
48     if (Enabled == false) {
49       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
50       NewLoopProperties.push_back(
51           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.pipeline.disable"),
52                             ConstantAsMetadata::get(ConstantInt::get(
53                                 llvm::Type::getInt1Ty(Ctx), 1))}));
54       LoopProperties = NewLoopProperties;
55     }
56     return createLoopPropertiesMetadata(LoopProperties);
57   }
58 
59   SmallVector<Metadata *, 4> Args;
60   Args.push_back(nullptr);
61   Args.append(LoopProperties.begin(), LoopProperties.end());
62 
63   if (Attrs.PipelineInitiationInterval > 0) {
64     Metadata *Vals[] = {
65         MDString::get(Ctx, "llvm.loop.pipeline.initiationinterval"),
66         ConstantAsMetadata::get(ConstantInt::get(
67             llvm::Type::getInt32Ty(Ctx), Attrs.PipelineInitiationInterval))};
68     Args.push_back(MDNode::get(Ctx, Vals));
69   }
70 
71   // No follow-up: This is the last transformation.
72 
73   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
74   LoopID->replaceOperandWith(0, LoopID);
75   HasUserTransforms = true;
76   return LoopID;
77 }
78 
79 MDNode *
80 LoopInfo::createPartialUnrollMetadata(const LoopAttributes &Attrs,
81                                       ArrayRef<Metadata *> LoopProperties,
82                                       bool &HasUserTransforms) {
83   LLVMContext &Ctx = Header->getContext();
84 
85   Optional<bool> Enabled;
86   if (Attrs.UnrollEnable == LoopAttributes::Disable)
87     Enabled = false;
88   else if (Attrs.UnrollEnable == LoopAttributes::Full)
89     Enabled = None;
90   else if (Attrs.UnrollEnable != LoopAttributes::Unspecified ||
91            Attrs.UnrollCount != 0)
92     Enabled = true;
93 
94   if (Enabled != true) {
95     // createFullUnrollMetadata will already have added llvm.loop.unroll.disable
96     // if unrolling is disabled.
97     return createPipeliningMetadata(Attrs, LoopProperties, HasUserTransforms);
98   }
99 
100   SmallVector<Metadata *, 4> FollowupLoopProperties;
101 
102   // Apply all loop properties to the unrolled loop.
103   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
104 
105   // Don't unroll an already unrolled loop.
106   FollowupLoopProperties.push_back(
107       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
108 
109   bool FollowupHasTransforms = false;
110   MDNode *Followup = createPipeliningMetadata(Attrs, FollowupLoopProperties,
111                                               FollowupHasTransforms);
112 
113   SmallVector<Metadata *, 4> Args;
114   Args.push_back(nullptr);
115   Args.append(LoopProperties.begin(), LoopProperties.end());
116 
117   // Setting unroll.count
118   if (Attrs.UnrollCount > 0) {
119     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.count"),
120                         ConstantAsMetadata::get(ConstantInt::get(
121                             llvm::Type::getInt32Ty(Ctx), Attrs.UnrollCount))};
122     Args.push_back(MDNode::get(Ctx, Vals));
123   }
124 
125   // Setting unroll.full or unroll.disable
126   if (Attrs.UnrollEnable == LoopAttributes::Enable) {
127     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll.enable")};
128     Args.push_back(MDNode::get(Ctx, Vals));
129   }
130 
131   if (FollowupHasTransforms)
132     Args.push_back(MDNode::get(
133         Ctx, {MDString::get(Ctx, "llvm.loop.unroll.followup_all"), Followup}));
134 
135   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
136   LoopID->replaceOperandWith(0, LoopID);
137   HasUserTransforms = true;
138   return LoopID;
139 }
140 
141 MDNode *
142 LoopInfo::createUnrollAndJamMetadata(const LoopAttributes &Attrs,
143                                      ArrayRef<Metadata *> LoopProperties,
144                                      bool &HasUserTransforms) {
145   LLVMContext &Ctx = Header->getContext();
146 
147   Optional<bool> Enabled;
148   if (Attrs.UnrollAndJamEnable == LoopAttributes::Disable)
149     Enabled = false;
150   else if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable ||
151            Attrs.UnrollAndJamCount != 0)
152     Enabled = true;
153 
154   if (Enabled != true) {
155     SmallVector<Metadata *, 4> NewLoopProperties;
156     if (Enabled == false) {
157       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
158       NewLoopProperties.push_back(MDNode::get(
159           Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
160       LoopProperties = NewLoopProperties;
161     }
162     return createPartialUnrollMetadata(Attrs, LoopProperties,
163                                        HasUserTransforms);
164   }
165 
166   SmallVector<Metadata *, 4> FollowupLoopProperties;
167   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
168   FollowupLoopProperties.push_back(
169       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll_and_jam.disable")));
170 
171   bool FollowupHasTransforms = false;
172   MDNode *Followup = createPartialUnrollMetadata(Attrs, FollowupLoopProperties,
173                                                  FollowupHasTransforms);
174 
175   SmallVector<Metadata *, 4> Args;
176   Args.push_back(nullptr);
177   Args.append(LoopProperties.begin(), LoopProperties.end());
178 
179   // Setting unroll_and_jam.count
180   if (Attrs.UnrollAndJamCount > 0) {
181     Metadata *Vals[] = {
182         MDString::get(Ctx, "llvm.loop.unroll_and_jam.count"),
183         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
184                                                  Attrs.UnrollAndJamCount))};
185     Args.push_back(MDNode::get(Ctx, Vals));
186   }
187 
188   if (Attrs.UnrollAndJamEnable == LoopAttributes::Enable) {
189     Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.unroll_and_jam.enable")};
190     Args.push_back(MDNode::get(Ctx, Vals));
191   }
192 
193   if (FollowupHasTransforms)
194     Args.push_back(MDNode::get(
195         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_outer"),
196               Followup}));
197 
198   if (UnrollAndJamInnerFollowup)
199     Args.push_back(MDNode::get(
200         Ctx, {MDString::get(Ctx, "llvm.loop.unroll_and_jam.followup_inner"),
201               UnrollAndJamInnerFollowup}));
202 
203   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
204   LoopID->replaceOperandWith(0, LoopID);
205   HasUserTransforms = true;
206   return LoopID;
207 }
208 
209 MDNode *
210 LoopInfo::createLoopVectorizeMetadata(const LoopAttributes &Attrs,
211                                       ArrayRef<Metadata *> LoopProperties,
212                                       bool &HasUserTransforms) {
213   LLVMContext &Ctx = Header->getContext();
214 
215   Optional<bool> Enabled;
216   if (Attrs.VectorizeEnable == LoopAttributes::Disable)
217     Enabled = false;
218   else if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
219            Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified ||
220            Attrs.InterleaveCount != 0 || Attrs.VectorizeWidth != 0)
221     Enabled = true;
222 
223   if (Enabled != true) {
224     SmallVector<Metadata *, 4> NewLoopProperties;
225     if (Enabled == false) {
226       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
227       NewLoopProperties.push_back(
228           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
229                             ConstantAsMetadata::get(ConstantInt::get(
230                                 llvm::Type::getInt1Ty(Ctx), 0))}));
231       LoopProperties = NewLoopProperties;
232     }
233     return createUnrollAndJamMetadata(Attrs, LoopProperties, HasUserTransforms);
234   }
235 
236   // Apply all loop properties to the vectorized loop.
237   SmallVector<Metadata *, 4> FollowupLoopProperties;
238   FollowupLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
239 
240   // Don't vectorize an already vectorized loop.
241   FollowupLoopProperties.push_back(
242       MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
243 
244   bool FollowupHasTransforms = false;
245   MDNode *Followup = createUnrollAndJamMetadata(Attrs, FollowupLoopProperties,
246                                                 FollowupHasTransforms);
247 
248   SmallVector<Metadata *, 4> Args;
249   Args.push_back(nullptr);
250   Args.append(LoopProperties.begin(), LoopProperties.end());
251 
252   // Setting vectorize.predicate
253   bool IsVectorPredicateEnabled = false;
254   if (Attrs.VectorizePredicateEnable != LoopAttributes::Unspecified &&
255       Attrs.VectorizeEnable != LoopAttributes::Disable &&
256       Attrs.VectorizeWidth < 1) {
257 
258     IsVectorPredicateEnabled =
259         (Attrs.VectorizePredicateEnable == LoopAttributes::Enable);
260 
261     Metadata *Vals[] = {
262         MDString::get(Ctx, "llvm.loop.vectorize.predicate.enable"),
263         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt1Ty(Ctx),
264                                                  IsVectorPredicateEnabled))};
265     Args.push_back(MDNode::get(Ctx, Vals));
266   }
267 
268   // Setting vectorize.width
269   if (Attrs.VectorizeWidth > 0) {
270     Metadata *Vals[] = {
271         MDString::get(Ctx, "llvm.loop.vectorize.width"),
272         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
273                                                  Attrs.VectorizeWidth))};
274     Args.push_back(MDNode::get(Ctx, Vals));
275   }
276 
277   // Setting interleave.count
278   if (Attrs.InterleaveCount > 0) {
279     Metadata *Vals[] = {
280         MDString::get(Ctx, "llvm.loop.interleave.count"),
281         ConstantAsMetadata::get(ConstantInt::get(llvm::Type::getInt32Ty(Ctx),
282                                                  Attrs.InterleaveCount))};
283     Args.push_back(MDNode::get(Ctx, Vals));
284   }
285 
286   // vectorize.enable is set if:
287   // 1) loop hint vectorize.enable is set, or
288   // 2) it is implied when vectorize.predicate is set, or
289   // 3) it is implied when vectorize.width is set.
290   if (Attrs.VectorizeEnable != LoopAttributes::Unspecified ||
291       IsVectorPredicateEnabled ||
292       Attrs.VectorizeWidth > 1 ) {
293     bool AttrVal = Attrs.VectorizeEnable != LoopAttributes::Disable;
294     Args.push_back(
295         MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.vectorize.enable"),
296                           ConstantAsMetadata::get(ConstantInt::get(
297                               llvm::Type::getInt1Ty(Ctx), AttrVal))}));
298   }
299 
300   if (FollowupHasTransforms)
301     Args.push_back(MDNode::get(
302         Ctx,
303         {MDString::get(Ctx, "llvm.loop.vectorize.followup_all"), Followup}));
304 
305   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
306   LoopID->replaceOperandWith(0, LoopID);
307   HasUserTransforms = true;
308   return LoopID;
309 }
310 
311 MDNode *
312 LoopInfo::createLoopDistributeMetadata(const LoopAttributes &Attrs,
313                                        ArrayRef<Metadata *> LoopProperties,
314                                        bool &HasUserTransforms) {
315   LLVMContext &Ctx = Header->getContext();
316 
317   Optional<bool> Enabled;
318   if (Attrs.DistributeEnable == LoopAttributes::Disable)
319     Enabled = false;
320   if (Attrs.DistributeEnable == LoopAttributes::Enable)
321     Enabled = true;
322 
323   if (Enabled != true) {
324     SmallVector<Metadata *, 4> NewLoopProperties;
325     if (Enabled == false) {
326       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
327       NewLoopProperties.push_back(
328           MDNode::get(Ctx, {MDString::get(Ctx, "llvm.loop.distribute.enable"),
329                             ConstantAsMetadata::get(ConstantInt::get(
330                                 llvm::Type::getInt1Ty(Ctx), 0))}));
331       LoopProperties = NewLoopProperties;
332     }
333     return createLoopVectorizeMetadata(Attrs, LoopProperties,
334                                        HasUserTransforms);
335   }
336 
337   bool FollowupHasTransforms = false;
338   MDNode *Followup =
339       createLoopVectorizeMetadata(Attrs, LoopProperties, FollowupHasTransforms);
340 
341   SmallVector<Metadata *, 4> Args;
342   Args.push_back(nullptr);
343   Args.append(LoopProperties.begin(), LoopProperties.end());
344 
345   Metadata *Vals[] = {MDString::get(Ctx, "llvm.loop.distribute.enable"),
346                       ConstantAsMetadata::get(ConstantInt::get(
347                           llvm::Type::getInt1Ty(Ctx),
348                           (Attrs.DistributeEnable == LoopAttributes::Enable)))};
349   Args.push_back(MDNode::get(Ctx, Vals));
350 
351   if (FollowupHasTransforms)
352     Args.push_back(MDNode::get(
353         Ctx,
354         {MDString::get(Ctx, "llvm.loop.distribute.followup_all"), Followup}));
355 
356   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
357   LoopID->replaceOperandWith(0, LoopID);
358   HasUserTransforms = true;
359   return LoopID;
360 }
361 
362 MDNode *LoopInfo::createFullUnrollMetadata(const LoopAttributes &Attrs,
363                                            ArrayRef<Metadata *> LoopProperties,
364                                            bool &HasUserTransforms) {
365   LLVMContext &Ctx = Header->getContext();
366 
367   Optional<bool> Enabled;
368   if (Attrs.UnrollEnable == LoopAttributes::Disable)
369     Enabled = false;
370   else if (Attrs.UnrollEnable == LoopAttributes::Full)
371     Enabled = true;
372 
373   if (Enabled != true) {
374     SmallVector<Metadata *, 4> NewLoopProperties;
375     if (Enabled == false) {
376       NewLoopProperties.append(LoopProperties.begin(), LoopProperties.end());
377       NewLoopProperties.push_back(
378           MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.disable")));
379       LoopProperties = NewLoopProperties;
380     }
381     return createLoopDistributeMetadata(Attrs, LoopProperties,
382                                         HasUserTransforms);
383   }
384 
385   SmallVector<Metadata *, 4> Args;
386   Args.push_back(nullptr);
387   Args.append(LoopProperties.begin(), LoopProperties.end());
388   Args.push_back(MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.unroll.full")));
389 
390   // No follow-up: there is no loop after full unrolling.
391   // TODO: Warn if there are transformations after full unrolling.
392 
393   MDNode *LoopID = MDNode::getDistinct(Ctx, Args);
394   LoopID->replaceOperandWith(0, LoopID);
395   HasUserTransforms = true;
396   return LoopID;
397 }
398 
399 MDNode *LoopInfo::createMetadata(
400     const LoopAttributes &Attrs,
401     llvm::ArrayRef<llvm::Metadata *> AdditionalLoopProperties,
402     bool &HasUserTransforms) {
403   SmallVector<Metadata *, 3> LoopProperties;
404 
405   // If we have a valid start debug location for the loop, add it.
406   if (StartLoc) {
407     LoopProperties.push_back(StartLoc.getAsMDNode());
408 
409     // If we also have a valid end debug location for the loop, add it.
410     if (EndLoc)
411       LoopProperties.push_back(EndLoc.getAsMDNode());
412   }
413 
414   assert(!!AccGroup == Attrs.IsParallel &&
415          "There must be an access group iff the loop is parallel");
416   if (Attrs.IsParallel) {
417     LLVMContext &Ctx = Header->getContext();
418     LoopProperties.push_back(MDNode::get(
419         Ctx, {MDString::get(Ctx, "llvm.loop.parallel_accesses"), AccGroup}));
420   }
421 
422   LoopProperties.insert(LoopProperties.end(), AdditionalLoopProperties.begin(),
423                         AdditionalLoopProperties.end());
424   return createFullUnrollMetadata(Attrs, LoopProperties, HasUserTransforms);
425 }
426 
427 LoopAttributes::LoopAttributes(bool IsParallel)
428     : IsParallel(IsParallel), VectorizeEnable(LoopAttributes::Unspecified),
429       UnrollEnable(LoopAttributes::Unspecified),
430       UnrollAndJamEnable(LoopAttributes::Unspecified),
431       VectorizePredicateEnable(LoopAttributes::Unspecified), VectorizeWidth(0),
432       InterleaveCount(0), UnrollCount(0), UnrollAndJamCount(0),
433       DistributeEnable(LoopAttributes::Unspecified), PipelineDisabled(false),
434       PipelineInitiationInterval(0) {}
435 
436 void LoopAttributes::clear() {
437   IsParallel = false;
438   VectorizeWidth = 0;
439   InterleaveCount = 0;
440   UnrollCount = 0;
441   UnrollAndJamCount = 0;
442   VectorizeEnable = LoopAttributes::Unspecified;
443   UnrollEnable = LoopAttributes::Unspecified;
444   UnrollAndJamEnable = LoopAttributes::Unspecified;
445   VectorizePredicateEnable = LoopAttributes::Unspecified;
446   DistributeEnable = LoopAttributes::Unspecified;
447   PipelineDisabled = false;
448   PipelineInitiationInterval = 0;
449 }
450 
451 LoopInfo::LoopInfo(BasicBlock *Header, const LoopAttributes &Attrs,
452                    const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
453                    LoopInfo *Parent)
454     : Header(Header), Attrs(Attrs), StartLoc(StartLoc), EndLoc(EndLoc),
455       Parent(Parent) {
456 
457   if (Attrs.IsParallel) {
458     // Create an access group for this loop.
459     LLVMContext &Ctx = Header->getContext();
460     AccGroup = MDNode::getDistinct(Ctx, {});
461   }
462 
463   if (!Attrs.IsParallel && Attrs.VectorizeWidth == 0 &&
464       Attrs.InterleaveCount == 0 && Attrs.UnrollCount == 0 &&
465       Attrs.UnrollAndJamCount == 0 && !Attrs.PipelineDisabled &&
466       Attrs.PipelineInitiationInterval == 0 &&
467       Attrs.VectorizePredicateEnable == LoopAttributes::Unspecified &&
468       Attrs.VectorizeEnable == LoopAttributes::Unspecified &&
469       Attrs.UnrollEnable == LoopAttributes::Unspecified &&
470       Attrs.UnrollAndJamEnable == LoopAttributes::Unspecified &&
471       Attrs.DistributeEnable == LoopAttributes::Unspecified && !StartLoc &&
472       !EndLoc)
473     return;
474 
475   TempLoopID = MDNode::getTemporary(Header->getContext(), None);
476 }
477 
478 void LoopInfo::finish() {
479   // We did not annotate the loop body instructions because there are no
480   // attributes for this loop.
481   if (!TempLoopID)
482     return;
483 
484   MDNode *LoopID;
485   LoopAttributes CurLoopAttr = Attrs;
486   LLVMContext &Ctx = Header->getContext();
487 
488   if (Parent && (Parent->Attrs.UnrollAndJamEnable ||
489                  Parent->Attrs.UnrollAndJamCount != 0)) {
490     // Parent unroll-and-jams this loop.
491     // Split the transformations in those that happens before the unroll-and-jam
492     // and those after.
493 
494     LoopAttributes BeforeJam, AfterJam;
495 
496     BeforeJam.IsParallel = AfterJam.IsParallel = Attrs.IsParallel;
497 
498     BeforeJam.VectorizeWidth = Attrs.VectorizeWidth;
499     BeforeJam.InterleaveCount = Attrs.InterleaveCount;
500     BeforeJam.VectorizeEnable = Attrs.VectorizeEnable;
501     BeforeJam.DistributeEnable = Attrs.DistributeEnable;
502     BeforeJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
503 
504     switch (Attrs.UnrollEnable) {
505     case LoopAttributes::Unspecified:
506     case LoopAttributes::Disable:
507       BeforeJam.UnrollEnable = Attrs.UnrollEnable;
508       AfterJam.UnrollEnable = Attrs.UnrollEnable;
509       break;
510     case LoopAttributes::Full:
511       BeforeJam.UnrollEnable = LoopAttributes::Full;
512       break;
513     case LoopAttributes::Enable:
514       AfterJam.UnrollEnable = LoopAttributes::Enable;
515       break;
516     }
517 
518     AfterJam.VectorizePredicateEnable = Attrs.VectorizePredicateEnable;
519     AfterJam.UnrollCount = Attrs.UnrollCount;
520     AfterJam.PipelineDisabled = Attrs.PipelineDisabled;
521     AfterJam.PipelineInitiationInterval = Attrs.PipelineInitiationInterval;
522 
523     // If this loop is subject of an unroll-and-jam by the parent loop, and has
524     // an unroll-and-jam annotation itself, we have to decide whether to first
525     // apply the parent's unroll-and-jam or this loop's unroll-and-jam. The
526     // UnrollAndJam pass processes loops from inner to outer, so we apply the
527     // inner first.
528     BeforeJam.UnrollAndJamCount = Attrs.UnrollAndJamCount;
529     BeforeJam.UnrollAndJamEnable = Attrs.UnrollAndJamEnable;
530 
531     // Set the inner followup metadata to process by the outer loop. Only
532     // consider the first inner loop.
533     if (!Parent->UnrollAndJamInnerFollowup) {
534       // Splitting the attributes into a BeforeJam and an AfterJam part will
535       // stop 'llvm.loop.isvectorized' (generated by vectorization in BeforeJam)
536       // to be forwarded to the AfterJam part. We detect the situation here and
537       // add it manually.
538       SmallVector<Metadata *, 1> BeforeLoopProperties;
539       if (BeforeJam.VectorizeEnable != LoopAttributes::Unspecified ||
540           BeforeJam.VectorizePredicateEnable != LoopAttributes::Unspecified ||
541           BeforeJam.InterleaveCount != 0 || BeforeJam.VectorizeWidth != 0)
542         BeforeLoopProperties.push_back(
543             MDNode::get(Ctx, MDString::get(Ctx, "llvm.loop.isvectorized")));
544 
545       bool InnerFollowupHasTransform = false;
546       MDNode *InnerFollowup = createMetadata(AfterJam, BeforeLoopProperties,
547                                              InnerFollowupHasTransform);
548       if (InnerFollowupHasTransform)
549         Parent->UnrollAndJamInnerFollowup = InnerFollowup;
550     }
551 
552     CurLoopAttr = BeforeJam;
553   }
554 
555   bool HasUserTransforms = false;
556   LoopID = createMetadata(CurLoopAttr, {}, HasUserTransforms);
557   TempLoopID->replaceAllUsesWith(LoopID);
558 }
559 
560 void LoopInfoStack::push(BasicBlock *Header, const llvm::DebugLoc &StartLoc,
561                          const llvm::DebugLoc &EndLoc) {
562   Active.emplace_back(
563       new LoopInfo(Header, StagedAttrs, StartLoc, EndLoc,
564                    Active.empty() ? nullptr : Active.back().get()));
565   // Clear the attributes so nested loops do not inherit them.
566   StagedAttrs.clear();
567 }
568 
569 void LoopInfoStack::push(BasicBlock *Header, clang::ASTContext &Ctx,
570                          const clang::CodeGenOptions &CGOpts,
571                          ArrayRef<const clang::Attr *> Attrs,
572                          const llvm::DebugLoc &StartLoc,
573                          const llvm::DebugLoc &EndLoc) {
574 
575   // Identify loop hint attributes from Attrs.
576   for (const auto *Attr : Attrs) {
577     const LoopHintAttr *LH = dyn_cast<LoopHintAttr>(Attr);
578     const OpenCLUnrollHintAttr *OpenCLHint =
579         dyn_cast<OpenCLUnrollHintAttr>(Attr);
580 
581     // Skip non loop hint attributes
582     if (!LH && !OpenCLHint) {
583       continue;
584     }
585 
586     LoopHintAttr::OptionType Option = LoopHintAttr::Unroll;
587     LoopHintAttr::LoopHintState State = LoopHintAttr::Disable;
588     unsigned ValueInt = 1;
589     // Translate opencl_unroll_hint attribute argument to
590     // equivalent LoopHintAttr enums.
591     // OpenCL v2.0 s6.11.5:
592     // 0 - enable unroll (no argument).
593     // 1 - disable unroll.
594     // other positive integer n - unroll by n.
595     if (OpenCLHint) {
596       ValueInt = OpenCLHint->getUnrollHint();
597       if (ValueInt == 0) {
598         State = LoopHintAttr::Enable;
599       } else if (ValueInt != 1) {
600         Option = LoopHintAttr::UnrollCount;
601         State = LoopHintAttr::Numeric;
602       }
603     } else if (LH) {
604       auto *ValueExpr = LH->getValue();
605       if (ValueExpr) {
606         llvm::APSInt ValueAPS = ValueExpr->EvaluateKnownConstInt(Ctx);
607         ValueInt = ValueAPS.getSExtValue();
608       }
609 
610       Option = LH->getOption();
611       State = LH->getState();
612     }
613     switch (State) {
614     case LoopHintAttr::Disable:
615       switch (Option) {
616       case LoopHintAttr::Vectorize:
617         // Disable vectorization by specifying a width of 1.
618         setVectorizeWidth(1);
619         break;
620       case LoopHintAttr::Interleave:
621         // Disable interleaving by speciyfing a count of 1.
622         setInterleaveCount(1);
623         break;
624       case LoopHintAttr::Unroll:
625         setUnrollState(LoopAttributes::Disable);
626         break;
627       case LoopHintAttr::UnrollAndJam:
628         setUnrollAndJamState(LoopAttributes::Disable);
629         break;
630       case LoopHintAttr::VectorizePredicate:
631         setVectorizePredicateState(LoopAttributes::Disable);
632         break;
633       case LoopHintAttr::Distribute:
634         setDistributeState(false);
635         break;
636       case LoopHintAttr::PipelineDisabled:
637         setPipelineDisabled(true);
638         break;
639       case LoopHintAttr::UnrollCount:
640       case LoopHintAttr::UnrollAndJamCount:
641       case LoopHintAttr::VectorizeWidth:
642       case LoopHintAttr::InterleaveCount:
643       case LoopHintAttr::PipelineInitiationInterval:
644         llvm_unreachable("Options cannot be disabled.");
645         break;
646       }
647       break;
648     case LoopHintAttr::Enable:
649       switch (Option) {
650       case LoopHintAttr::Vectorize:
651       case LoopHintAttr::Interleave:
652         setVectorizeEnable(true);
653         break;
654       case LoopHintAttr::Unroll:
655         setUnrollState(LoopAttributes::Enable);
656         break;
657       case LoopHintAttr::UnrollAndJam:
658         setUnrollAndJamState(LoopAttributes::Enable);
659         break;
660       case LoopHintAttr::VectorizePredicate:
661         setVectorizePredicateState(LoopAttributes::Enable);
662         break;
663       case LoopHintAttr::Distribute:
664         setDistributeState(true);
665         break;
666       case LoopHintAttr::UnrollCount:
667       case LoopHintAttr::UnrollAndJamCount:
668       case LoopHintAttr::VectorizeWidth:
669       case LoopHintAttr::InterleaveCount:
670       case LoopHintAttr::PipelineDisabled:
671       case LoopHintAttr::PipelineInitiationInterval:
672         llvm_unreachable("Options cannot enabled.");
673         break;
674       }
675       break;
676     case LoopHintAttr::AssumeSafety:
677       switch (Option) {
678       case LoopHintAttr::Vectorize:
679       case LoopHintAttr::Interleave:
680         // Apply "llvm.mem.parallel_loop_access" metadata to load/stores.
681         setParallel(true);
682         setVectorizeEnable(true);
683         break;
684       case LoopHintAttr::Unroll:
685       case LoopHintAttr::UnrollAndJam:
686       case LoopHintAttr::VectorizePredicate:
687       case LoopHintAttr::UnrollCount:
688       case LoopHintAttr::UnrollAndJamCount:
689       case LoopHintAttr::VectorizeWidth:
690       case LoopHintAttr::InterleaveCount:
691       case LoopHintAttr::Distribute:
692       case LoopHintAttr::PipelineDisabled:
693       case LoopHintAttr::PipelineInitiationInterval:
694         llvm_unreachable("Options cannot be used to assume mem safety.");
695         break;
696       }
697       break;
698     case LoopHintAttr::Full:
699       switch (Option) {
700       case LoopHintAttr::Unroll:
701         setUnrollState(LoopAttributes::Full);
702         break;
703       case LoopHintAttr::UnrollAndJam:
704         setUnrollAndJamState(LoopAttributes::Full);
705         break;
706       case LoopHintAttr::Vectorize:
707       case LoopHintAttr::Interleave:
708       case LoopHintAttr::UnrollCount:
709       case LoopHintAttr::UnrollAndJamCount:
710       case LoopHintAttr::VectorizeWidth:
711       case LoopHintAttr::InterleaveCount:
712       case LoopHintAttr::Distribute:
713       case LoopHintAttr::PipelineDisabled:
714       case LoopHintAttr::PipelineInitiationInterval:
715       case LoopHintAttr::VectorizePredicate:
716         llvm_unreachable("Options cannot be used with 'full' hint.");
717         break;
718       }
719       break;
720     case LoopHintAttr::Numeric:
721       switch (Option) {
722       case LoopHintAttr::VectorizeWidth:
723         setVectorizeWidth(ValueInt);
724         break;
725       case LoopHintAttr::InterleaveCount:
726         setInterleaveCount(ValueInt);
727         break;
728       case LoopHintAttr::UnrollCount:
729         setUnrollCount(ValueInt);
730         break;
731       case LoopHintAttr::UnrollAndJamCount:
732         setUnrollAndJamCount(ValueInt);
733         break;
734       case LoopHintAttr::PipelineInitiationInterval:
735         setPipelineInitiationInterval(ValueInt);
736         break;
737       case LoopHintAttr::Unroll:
738       case LoopHintAttr::UnrollAndJam:
739       case LoopHintAttr::VectorizePredicate:
740       case LoopHintAttr::Vectorize:
741       case LoopHintAttr::Interleave:
742       case LoopHintAttr::Distribute:
743       case LoopHintAttr::PipelineDisabled:
744         llvm_unreachable("Options cannot be assigned a value.");
745         break;
746       }
747       break;
748     }
749   }
750 
751   if (CGOpts.OptimizationLevel > 0)
752     // Disable unrolling for the loop, if unrolling is disabled (via
753     // -fno-unroll-loops) and no pragmas override the decision.
754     if (!CGOpts.UnrollLoops &&
755         (StagedAttrs.UnrollEnable == LoopAttributes::Unspecified &&
756          StagedAttrs.UnrollCount == 0))
757       setUnrollState(LoopAttributes::Disable);
758 
759   /// Stage the attributes.
760   push(Header, StartLoc, EndLoc);
761 }
762 
763 void LoopInfoStack::pop() {
764   assert(!Active.empty() && "No active loops to pop");
765   Active.back()->finish();
766   Active.pop_back();
767 }
768 
769 void LoopInfoStack::InsertHelper(Instruction *I) const {
770   if (I->mayReadOrWriteMemory()) {
771     SmallVector<Metadata *, 4> AccessGroups;
772     for (const auto &AL : Active) {
773       // Here we assume that every loop that has an access group is parallel.
774       if (MDNode *Group = AL->getAccessGroup())
775         AccessGroups.push_back(Group);
776     }
777     MDNode *UnionMD = nullptr;
778     if (AccessGroups.size() == 1)
779       UnionMD = cast<MDNode>(AccessGroups[0]);
780     else if (AccessGroups.size() >= 2)
781       UnionMD = MDNode::get(I->getContext(), AccessGroups);
782     I->setMetadata("llvm.access.group", UnionMD);
783   }
784 
785   if (!hasInfo())
786     return;
787 
788   const LoopInfo &L = getInfo();
789   if (!L.getLoopID())
790     return;
791 
792   if (I->isTerminator()) {
793     for (BasicBlock *Succ : successors(I))
794       if (Succ == L.getHeader()) {
795         I->setMetadata(llvm::LLVMContext::MD_loop, L.getLoopID());
796         break;
797       }
798     return;
799   }
800 }
801