xref: /llvm-project/mlir/tools/mlir-tblgen/OpDocGen.cpp (revision 2754f88a09b19c572aedb45b5333d25da04e4dde) 
1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // OpDocGen uses the description of operations to generate documentation for the
10 // operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "DialectGenUtilities.h"
15 #include "DocGenUtilities.h"
16 #include "OpGenHelpers.h"
17 #include "mlir/Support/IndentedOstream.h"
18 #include "mlir/TableGen/AttrOrTypeDef.h"
19 #include "mlir/TableGen/GenInfo.h"
20 #include "mlir/TableGen/Operator.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/SetVector.h"
23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/FormatVariadic.h"
27 #include "llvm/Support/Regex.h"
28 #include "llvm/Support/Signals.h"
29 #include "llvm/TableGen/Error.h"
30 #include "llvm/TableGen/Record.h"
31 #include "llvm/TableGen/TableGenBackend.h"
32 
33 #include <set>
34 #include <string>
35 
36 //===----------------------------------------------------------------------===//
37 // Commandline Options
38 //===----------------------------------------------------------------------===//
39 static llvm::cl::OptionCategory
40     docCat("Options for -gen-(attrdef|typedef|op|dialect)-doc");
41 llvm::cl::opt<std::string>
42     stripPrefix("strip-prefix",
43                 llvm::cl::desc("Strip prefix of the fully qualified names"),
44                 llvm::cl::init("::mlir::"), llvm::cl::cat(docCat));
45 llvm::cl::opt<bool> allowHugoSpecificFeatures(
46     "allow-hugo-specific-features",
47     llvm::cl::desc("Allows using features specific to Hugo"),
48     llvm::cl::init(false), llvm::cl::cat(docCat));
49 
50 using namespace llvm;
51 using namespace mlir;
52 using namespace mlir::tblgen;
53 using mlir::tblgen::Operator;
54 
55 void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
56   if (!summary.empty()) {
57     char first = std::toupper(summary.front());
58     llvm::StringRef rest = summary.drop_front();
59     os << "\n_" << first << rest << "_\n\n";
60   }
61 }
62 
63 // Emit the description by aligning the text to the left per line (e.g.,
64 // removing the minimum indentation across the block).
65 //
66 // This expects that the description in the tablegen file is already formatted
67 // in a way the user wanted but has some additional indenting due to being
68 // nested in the op definition.
69 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
70   raw_indented_ostream ros(os);
71   ros.printReindented(description.rtrim(" \t"));
72 }
73 
74 void mlir::tblgen::emitDescriptionComment(StringRef description,
75                                           raw_ostream &os, StringRef prefix) {
76   if (description.empty())
77     return;
78   raw_indented_ostream ros(os);
79   StringRef trimmed = description.rtrim(" \t");
80   ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());
81   if (!trimmed.endswith("\n"))
82     ros << "\n";
83 }
84 
85 // Emits `str` with trailing newline if not empty.
86 static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
87   if (!str.empty()) {
88     emitDescription(str, os);
89     os << "\n";
90   }
91 }
92 
93 /// Emit the given named constraint.
94 template <typename T>
95 static void emitNamedConstraint(const T &it, raw_ostream &os) {
96   if (!it.name.empty())
97     os << "| `" << it.name << "`";
98   else
99     os << "&laquo;unnamed&raquo;";
100   os << " | " << it.constraint.getSummary() << "\n";
101 }
102 
103 //===----------------------------------------------------------------------===//
104 // Operation Documentation
105 //===----------------------------------------------------------------------===//
106 
107 /// Emit the assembly format of an operation.
108 static void emitAssemblyFormat(StringRef opName, StringRef format,
109                                raw_ostream &os) {
110   os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
111 
112   // Print the assembly format aligned.
113   unsigned indent = strlen("operation ::= ");
114   std::pair<StringRef, StringRef> split = format.split('\n');
115   os << split.first.trim() << "\n";
116   do {
117     split = split.second.split('\n');
118     StringRef formatChunk = split.first.trim();
119     if (!formatChunk.empty())
120       os.indent(indent) << formatChunk << "\n";
121   } while (!split.second.empty());
122   os << "```\n\n";
123 }
124 
125 static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
126   // TODO: We should link to the trait/documentation of it. That also means we
127   // should add descriptions to traits that can be queried.
128   // Collect using set to sort effects, interfaces & traits.
129   std::set<std::string> effects, interfaces, traits;
130   for (auto &trait : op.getTraits()) {
131     if (isa<PredTrait>(&trait))
132       continue;
133 
134     std::string name = trait.getDef().getName().str();
135     StringRef ref = name;
136     StringRef traitName = trait.getDef().getValueAsString("trait");
137     traitName.consume_back("::Trait");
138     traitName.consume_back("::Impl");
139     if (ref.startswith("anonymous_"))
140       name = traitName.str();
141     if (isa<InterfaceTrait>(&trait)) {
142       if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {
143         auto effectName = trait.getDef().getValueAsString("baseEffectName");
144         effectName.consume_front("::");
145         effectName.consume_front("mlir::");
146         std::string effectStr;
147         llvm::raw_string_ostream os(effectStr);
148         os << effectName << "{";
149         auto list = trait.getDef().getValueAsListOfDefs("effects");
150         llvm::interleaveComma(list, os, [&](Record *rec) {
151           StringRef effect = rec->getValueAsString("effect");
152           effect.consume_front("::");
153           effect.consume_front("mlir::");
154           os << effect << " on " << rec->getValueAsString("resource");
155         });
156         os << "}";
157         effects.insert(os.str());
158         name.append(llvm::formatv(" ({0})", traitName).str());
159       }
160       interfaces.insert(name);
161       continue;
162     }
163 
164     traits.insert(name);
165   }
166   if (!traits.empty()) {
167     llvm::interleaveComma(traits, os << "\nTraits: ");
168     os << "\n";
169   }
170   if (!interfaces.empty()) {
171     llvm::interleaveComma(interfaces, os << "\nInterfaces: ");
172     os << "\n";
173   }
174   if (!effects.empty()) {
175     llvm::interleaveComma(effects, os << "\nEffects: ");
176     os << "\n";
177   }
178 }
179 
180 static StringRef resolveAttrDescription(const Attribute &attr) {
181   StringRef description = attr.getDescription();
182   if (description.empty())
183     return attr.getBaseAttr().getDescription();
184   return description;
185 }
186 
187 static void emitOpDoc(const Operator &op, raw_ostream &os) {
188   std::string classNameStr = op.getQualCppClassName();
189   StringRef className = classNameStr;
190   (void)className.consume_front(stripPrefix);
191   os << llvm::formatv("### `{0}` ({1})\n", op.getOperationName(), className);
192 
193   // Emit the summary, syntax, and description if present.
194   if (op.hasSummary())
195     emitSummary(op.getSummary(), os);
196   if (op.hasAssemblyFormat())
197     emitAssemblyFormat(op.getOperationName(), op.getAssemblyFormat().trim(),
198                        os);
199   if (op.hasDescription())
200     mlir::tblgen::emitDescription(op.getDescription(), os);
201 
202   emitOpTraitsDoc(op, os);
203 
204   // Emit attributes.
205   if (op.getNumAttributes() != 0) {
206     os << "\n#### Attributes:\n\n";
207     // Note: This table is HTML rather than markdown so the attribute's
208     // description can appear in an expandable region. The description may be
209     // multiple lines, which is not supported in a markdown table cell.
210     os << "<table>\n";
211     // Header.
212     os << "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";
213     for (const auto &it : op.getAttributes()) {
214       StringRef storageType = it.attr.getStorageType();
215       // Name and storage type.
216       os << "<tr>";
217       os << "<td><code>" << it.name << "</code></td><td>" << storageType
218          << "</td><td>";
219       StringRef description = resolveAttrDescription(it.attr);
220       if (allowHugoSpecificFeatures && !description.empty()) {
221         // Expandable description.
222         // This appears as just the summary, but when clicked shows the full
223         // description.
224         os << "<details>"
225            << "<summary>" << it.attr.getSummary() << "</summary>"
226            << "{{% markdown %}}" << description << "{{% /markdown %}}"
227            << "</details>";
228       } else {
229         // Fallback: Single-line summary.
230         os << it.attr.getSummary();
231       }
232       os << "</td></tr>\n";
233     }
234     os << "</table>\n";
235   }
236 
237   // Emit each of the operands.
238   if (op.getNumOperands() != 0) {
239     os << "\n#### Operands:\n\n";
240     os << "| Operand | Description |\n"
241        << "| :-----: | ----------- |\n";
242     for (const auto &it : op.getOperands())
243       emitNamedConstraint(it, os);
244   }
245 
246   // Emit results.
247   if (op.getNumResults() != 0) {
248     os << "\n#### Results:\n\n";
249     os << "| Result | Description |\n"
250        << "| :----: | ----------- |\n";
251     for (const auto &it : op.getResults())
252       emitNamedConstraint(it, os);
253   }
254 
255   // Emit successors.
256   if (op.getNumSuccessors() != 0) {
257     os << "\n#### Successors:\n\n";
258     os << "| Successor | Description |\n"
259        << "| :-------: | ----------- |\n";
260     for (const auto &it : op.getSuccessors())
261       emitNamedConstraint(it, os);
262   }
263 
264   os << "\n";
265 }
266 
267 static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
268   auto opDefs = getRequestedOpDefinitions(recordKeeper);
269 
270   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
271   for (const llvm::Record *opDef : opDefs)
272     emitOpDoc(Operator(opDef), os);
273 }
274 
275 //===----------------------------------------------------------------------===//
276 // Attribute Documentation
277 //===----------------------------------------------------------------------===//
278 
279 static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
280   os << "### " << attr.getSummary() << "\n\n";
281   emitDescription(attr.getDescription(), os);
282   os << "\n\n";
283 }
284 
285 //===----------------------------------------------------------------------===//
286 // Type Documentation
287 //===----------------------------------------------------------------------===//
288 
289 static void emitTypeDoc(const Type &type, raw_ostream &os) {
290   os << "### " << type.getSummary() << "\n\n";
291   emitDescription(type.getDescription(), os);
292   os << "\n\n";
293 }
294 
295 //===----------------------------------------------------------------------===//
296 // TypeDef Documentation
297 //===----------------------------------------------------------------------===//
298 
299 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
300                                             raw_ostream &os) {
301   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
302   char prefix = isa<AttrDef>(def) ? '#' : '!';
303   if (parameters.empty()) {
304     os << "\nSyntax: `" << prefix << def.getDialect().getName() << "."
305        << def.getMnemonic() << "`\n";
306     return;
307   }
308 
309   os << "\nSyntax:\n\n```\n"
310      << prefix << def.getDialect().getName() << "." << def.getMnemonic()
311      << "<\n";
312   for (const auto &it : llvm::enumerate(parameters)) {
313     const AttrOrTypeParameter &param = it.value();
314     os << "  " << param.getSyntax();
315     if (it.index() < (parameters.size() - 1))
316       os << ",";
317     os << "   # " << param.getName() << "\n";
318   }
319   os << ">\n```\n";
320 }
321 
322 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
323   os << llvm::formatv("### {0}\n", def.getCppClassName());
324 
325   // Emit the summary if present.
326   if (def.hasSummary())
327     os << "\n" << def.getSummary() << "\n";
328 
329   // Emit the syntax if present.
330   if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
331     emitAttrOrTypeDefAssemblyFormat(def, os);
332 
333   // Emit the description if present.
334   if (def.hasDescription()) {
335     os << "\n";
336     mlir::tblgen::emitDescription(def.getDescription(), os);
337   }
338 
339   // Emit parameter documentation.
340   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
341   if (!parameters.empty()) {
342     os << "\n#### Parameters:\n\n";
343     os << "| Parameter | C++ type | Description |\n"
344        << "| :-------: | :-------: | ----------- |\n";
345     for (const auto &it : parameters) {
346       auto desc = it.getSummary();
347       os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
348          << (desc ? *desc : "") << " |\n";
349     }
350   }
351 
352   os << "\n";
353 }
354 
355 static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
356                                  raw_ostream &os, StringRef recordTypeName) {
357   std::vector<llvm::Record *> defs =
358       recordKeeper.getAllDerivedDefinitions(recordTypeName);
359 
360   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
361   for (const llvm::Record *def : defs)
362     emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
363 }
364 
365 //===----------------------------------------------------------------------===//
366 // Dialect Documentation
367 //===----------------------------------------------------------------------===//
368 
369 struct OpDocGroup {
370   const Dialect &getDialect() const { return ops.front().getDialect(); }
371 
372   // Returns the summary description of the section.
373   std::string summary = "";
374 
375   // Returns the description of the section.
376   StringRef description = "";
377 
378   // Instances inside the section.
379   std::vector<Operator> ops;
380 };
381 
382 static void maybeNest(bool nest, llvm::function_ref<void(raw_ostream &os)> fn,
383                       raw_ostream &os) {
384   std::string str;
385   llvm::raw_string_ostream ss(str);
386   fn(ss);
387   for (StringRef x : llvm::split(ss.str(), "\n")) {
388     if (nest && x.starts_with("#"))
389       os << "#";
390     os << x << "\n";
391   }
392 }
393 
394 static void emitBlock(ArrayRef<Attribute> attributes,
395                       ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
396                       ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
397                       raw_ostream &os) {
398   if (!ops.empty()) {
399     os << "## Operation definition\n\n";
400     for (const OpDocGroup &grouping : ops) {
401       bool nested = !grouping.summary.empty();
402       maybeNest(
403           nested,
404           [&](raw_ostream &os) {
405             if (nested) {
406               os << "## " << StringRef(grouping.summary).trim() << "\n\n";
407               emitDescription(grouping.description, os);
408               os << "\n\n";
409             }
410             for (const Operator &op : grouping.ops) {
411               emitOpDoc(op, os);
412             }
413           },
414           os);
415     }
416   }
417 
418   if (!attributes.empty()) {
419     os << "## Attribute constraint definition\n\n";
420     for (const Attribute &attr : attributes)
421       emitAttrDoc(attr, os);
422   }
423 
424   if (!attrDefs.empty()) {
425     os << "## Attribute definition\n\n";
426     for (const AttrDef &def : attrDefs)
427       emitAttrOrTypeDefDoc(def, os);
428   }
429 
430   // TODO: Add link between use and def for types
431   if (!types.empty()) {
432     os << "## Type constraint definition\n\n";
433     for (const Type &type : types)
434       emitTypeDoc(type, os);
435   }
436 
437   if (!typeDefs.empty()) {
438     os << "## Type definition\n\n";
439     for (const TypeDef &def : typeDefs)
440       emitAttrOrTypeDefDoc(def, os);
441   }
442 }
443 
444 static void emitDialectDoc(const Dialect &dialect,
445                            ArrayRef<Attribute> attributes,
446                            ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
447                            ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
448                            raw_ostream &os) {
449   os << "# '" << dialect.getName() << "' Dialect\n\n";
450   emitIfNotEmpty(dialect.getSummary(), os);
451   emitIfNotEmpty(dialect.getDescription(), os);
452 
453   // Generate a TOC marker except if description already contains one.
454   llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
455   if (!r.match(dialect.getDescription()))
456     os << "[TOC]\n\n";
457 
458   emitBlock(attributes, attrDefs, ops, types, typeDefs, os);
459 }
460 
461 static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
462   std::vector<Record *> dialectDefs =
463       recordKeeper.getAllDerivedDefinitionsIfDefined("Dialect");
464   SmallVector<Dialect> dialects(dialectDefs.begin(), dialectDefs.end());
465   std::optional<Dialect> dialect = findDialectToGenerate(dialects);
466   if (!dialect)
467     return true;
468 
469   std::vector<Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
470   std::vector<Record *> attrDefs =
471       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectAttr");
472   std::vector<Record *> typeDefs =
473       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectType");
474   std::vector<Record *> typeDefDefs =
475       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef");
476   std::vector<Record *> attrDefDefs =
477       recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef");
478 
479   std::vector<Attribute> dialectAttrs;
480   std::vector<AttrDef> dialectAttrDefs;
481   std::vector<OpDocGroup> dialectOps;
482   std::vector<Type> dialectTypes;
483   std::vector<TypeDef> dialectTypeDefs;
484 
485   llvm::SmallDenseSet<Record *> seen;
486   auto addIfInDialect = [&](llvm::Record *record, const auto &def, auto &vec) {
487     if (seen.insert(record).second && def.getDialect() == *dialect) {
488       vec.push_back(def);
489       return true;
490     }
491     return false;
492   };
493 
494   SmallDenseMap<Record *, OpDocGroup> opDocGroup;
495 
496   for (Record *def : attrDefDefs)
497     addIfInDialect(def, AttrDef(def), dialectAttrDefs);
498   for (Record *def : attrDefs)
499     addIfInDialect(def, Attribute(def), dialectAttrs);
500   for (Record *def : opDefs) {
501     if (Record *group = def->getValueAsOptionalDef("opDocGroup")) {
502       OpDocGroup &op = opDocGroup[group];
503       addIfInDialect(def, Operator(def), op.ops);
504     } else {
505       OpDocGroup op;
506       op.ops.emplace_back(def);
507       addIfInDialect(def, op, dialectOps);
508     }
509   }
510   for (Record *rec :
511        recordKeeper.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
512     if (opDocGroup[rec].ops.empty())
513       continue;
514     opDocGroup[rec].summary = rec->getValueAsString("summary");
515     opDocGroup[rec].description = rec->getValueAsString("description");
516     dialectOps.push_back(opDocGroup[rec]);
517   }
518   for (Record *def : typeDefDefs)
519     addIfInDialect(def, TypeDef(def), dialectTypeDefs);
520   for (Record *def : typeDefs)
521     addIfInDialect(def, Type(def), dialectTypes);
522 
523   // Sort alphabetically ignorning dialect for ops and section name for
524   // sections.
525   // TODO: The sorting order could be revised, currently attempting to sort of
526   // keep in alphabetical order.
527   std::sort(dialectOps.begin(), dialectOps.end(),
528             [](const OpDocGroup &lhs, const OpDocGroup &rhs) {
529               auto getDesc = [](const OpDocGroup &arg) -> StringRef {
530                 if (!arg.summary.empty())
531                   return arg.summary;
532                 return arg.ops.front().getDef().getValueAsString("opName");
533               };
534               return getDesc(lhs).compare_insensitive(getDesc(rhs)) < 0;
535             });
536 
537   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
538   emitDialectDoc(*dialect, dialectAttrs, dialectAttrDefs, dialectOps,
539                  dialectTypes, dialectTypeDefs, os);
540   return false;
541 }
542 
543 //===----------------------------------------------------------------------===//
544 // Gen Registration
545 //===----------------------------------------------------------------------===//
546 
547 static mlir::GenRegistration
548     genAttrRegister("gen-attrdef-doc",
549                     "Generate dialect attribute documentation",
550                     [](const RecordKeeper &records, raw_ostream &os) {
551                       emitAttrOrTypeDefDoc(records, os, "AttrDef");
552                       return false;
553                     });
554 
555 static mlir::GenRegistration
556     genOpRegister("gen-op-doc", "Generate dialect documentation",
557                   [](const RecordKeeper &records, raw_ostream &os) {
558                     emitOpDoc(records, os);
559                     return false;
560                   });
561 
562 static mlir::GenRegistration
563     genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
564                     [](const RecordKeeper &records, raw_ostream &os) {
565                       emitAttrOrTypeDefDoc(records, os, "TypeDef");
566                       return false;
567                     });
568 
569 static mlir::GenRegistration
570     genRegister("gen-dialect-doc", "Generate dialect documentation",
571                 [](const RecordKeeper &records, raw_ostream &os) {
572                   return emitDialectDoc(records, os);
573                 });
574