xref: /llvm-project/mlir/tools/mlir-tblgen/OpDocGen.cpp (revision d339d8fede356f0acfad6d6c1bb960269f2467a9) 
1 //===- OpDocGen.cpp - MLIR operation documentation generator --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // OpDocGen uses the description of operations to generate documentation for the
10 // operations.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "DialectGenUtilities.h"
15 #include "DocGenUtilities.h"
16 #include "OpGenHelpers.h"
17 #include "mlir/Support/IndentedOstream.h"
18 #include "mlir/TableGen/AttrOrTypeDef.h"
19 #include "mlir/TableGen/GenInfo.h"
20 #include "mlir/TableGen/Operator.h"
21 #include "llvm/ADT/DenseMap.h"
22 #include "llvm/ADT/SetVector.h"
23 #include "llvm/ADT/StringExtras.h"
24 #include "llvm/ADT/StringRef.h"
25 #include "llvm/Support/CommandLine.h"
26 #include "llvm/Support/FormatVariadic.h"
27 #include "llvm/Support/Regex.h"
28 #include "llvm/Support/Signals.h"
29 #include "llvm/TableGen/Error.h"
30 #include "llvm/TableGen/Record.h"
31 #include "llvm/TableGen/TableGenBackend.h"
32 
33 #include <set>
34 #include <string>
35 
36 //===----------------------------------------------------------------------===//
37 // Commandline Options
38 //===----------------------------------------------------------------------===//
39 static llvm::cl::OptionCategory
40     docCat("Options for -gen-(attrdef|typedef|op|dialect)-doc");
41 llvm::cl::opt<std::string>
42     stripPrefix("strip-prefix",
43                 llvm::cl::desc("Strip prefix of the fully qualified names"),
44                 llvm::cl::init("::mlir::"), llvm::cl::cat(docCat));
45 
46 using namespace llvm;
47 using namespace mlir;
48 using namespace mlir::tblgen;
49 using mlir::tblgen::Operator;
50 
51 void mlir::tblgen::emitSummary(StringRef summary, raw_ostream &os) {
52   if (!summary.empty()) {
53     char first = std::toupper(summary.front());
54     llvm::StringRef rest = summary.drop_front();
55     os << "\n_" << first << rest << "_\n\n";
56   }
57 }
58 
59 // Emit the description by aligning the text to the left per line (e.g.,
60 // removing the minimum indentation across the block).
61 //
62 // This expects that the description in the tablegen file is already formatted
63 // in a way the user wanted but has some additional indenting due to being
64 // nested in the op definition.
65 void mlir::tblgen::emitDescription(StringRef description, raw_ostream &os) {
66   raw_indented_ostream ros(os);
67   ros.printReindented(description.rtrim(" \t"));
68 }
69 
70 void mlir::tblgen::emitDescriptionComment(StringRef description,
71                                           raw_ostream &os, StringRef prefix) {
72   if (description.empty())
73     return;
74   raw_indented_ostream ros(os);
75   StringRef trimmed = description.rtrim(" \t");
76   ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());
77   if (!trimmed.endswith("\n"))
78     ros << "\n";
79 }
80 
81 // Emits `str` with trailing newline if not empty.
82 static void emitIfNotEmpty(StringRef str, raw_ostream &os) {
83   if (!str.empty()) {
84     emitDescription(str, os);
85     os << "\n";
86   }
87 }
88 
89 /// Emit the given named constraint.
90 template <typename T>
91 static void emitNamedConstraint(const T &it, raw_ostream &os) {
92   if (!it.name.empty())
93     os << "| `" << it.name << "`";
94   else
95     os << "&laquo;unnamed&raquo;";
96   os << " | " << it.constraint.getSummary() << "\n";
97 }
98 
99 //===----------------------------------------------------------------------===//
100 // Operation Documentation
101 //===----------------------------------------------------------------------===//
102 
103 /// Emit the assembly format of an operation.
104 static void emitAssemblyFormat(StringRef opName, StringRef format,
105                                raw_ostream &os) {
106   os << "\nSyntax:\n\n```\noperation ::= `" << opName << "` ";
107 
108   // Print the assembly format aligned.
109   unsigned indent = strlen("operation ::= ");
110   std::pair<StringRef, StringRef> split = format.split('\n');
111   os << split.first.trim() << "\n";
112   do {
113     split = split.second.split('\n');
114     StringRef formatChunk = split.first.trim();
115     if (!formatChunk.empty())
116       os.indent(indent) << formatChunk << "\n";
117   } while (!split.second.empty());
118   os << "```\n\n";
119 }
120 
121 static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
122   // TODO: We should link to the trait/documentation of it. That also means we
123   // should add descriptions to traits that can be queried.
124   // Collect using set to sort effects, interfaces & traits.
125   std::set<std::string> effects, interfaces, traits;
126   for (auto &trait : op.getTraits()) {
127     if (isa<PredTrait>(&trait))
128       continue;
129 
130     std::string name = trait.getDef().getName().str();
131     StringRef ref = name;
132     StringRef traitName = trait.getDef().getValueAsString("trait");
133     traitName.consume_back("::Trait");
134     traitName.consume_back("::Impl");
135     if (ref.startswith("anonymous_"))
136       name = traitName.str();
137     if (isa<InterfaceTrait>(&trait)) {
138       if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {
139         auto effectName = trait.getDef().getValueAsString("baseEffectName");
140         effectName.consume_front("::");
141         effectName.consume_front("mlir::");
142         std::string effectStr;
143         llvm::raw_string_ostream os(effectStr);
144         os << effectName << "{";
145         auto list = trait.getDef().getValueAsListOfDefs("effects");
146         llvm::interleaveComma(list, os, [&](Record *rec) {
147           StringRef effect = rec->getValueAsString("effect");
148           effect.consume_front("::");
149           effect.consume_front("mlir::");
150           os << effect << " on " << rec->getValueAsString("resource");
151         });
152         os << "}";
153         effects.insert(os.str());
154         name.append(llvm::formatv(" ({0})", traitName).str());
155       }
156       interfaces.insert(name);
157       continue;
158     }
159 
160     traits.insert(name);
161   }
162   if (!traits.empty()) {
163     llvm::interleaveComma(traits, os << "\nTraits: ");
164     os << "\n";
165   }
166   if (!interfaces.empty()) {
167     llvm::interleaveComma(interfaces, os << "\nInterfaces: ");
168     os << "\n";
169   }
170   if (!effects.empty()) {
171     llvm::interleaveComma(effects, os << "\nEffects: ");
172     os << "\n";
173   }
174 }
175 
176 static StringRef resolveAttrDescription(const Attribute &attr) {
177   StringRef description = attr.getDescription();
178   if (description.empty())
179     return attr.getBaseAttr().getDescription();
180   return description;
181 }
182 
183 static void emitOpDoc(const Operator &op, raw_ostream &os) {
184   std::string classNameStr = op.getQualCppClassName();
185   StringRef className = classNameStr;
186   (void)className.consume_front(stripPrefix);
187   os << llvm::formatv("### `{0}` ({1})\n", op.getOperationName(), className);
188 
189   // Emit the summary, syntax, and description if present.
190   if (op.hasSummary())
191     emitSummary(op.getSummary(), os);
192   if (op.hasAssemblyFormat())
193     emitAssemblyFormat(op.getOperationName(), op.getAssemblyFormat().trim(),
194                        os);
195   if (op.hasDescription())
196     mlir::tblgen::emitDescription(op.getDescription(), os);
197 
198   emitOpTraitsDoc(op, os);
199 
200   // Emit attributes.
201   if (op.getNumAttributes() != 0) {
202     os << "\n#### Attributes:\n\n";
203     // Note: This table is HTML rather than markdown so the attribute's
204     // description can appear in an expandable region. The description may be
205     // multiple lines, which is not supported in a markdown table cell.
206     os << "<table>\n";
207     // Header.
208     os << "<tr><th>Attribute</th><th>MLIR Type</th><th>Description</th></tr>\n";
209     for (const auto &it : op.getAttributes()) {
210       StringRef storageType = it.attr.getStorageType();
211       // Name and storage type.
212       os << "<tr>";
213       os << "<td><code>" << it.name << "</code></td><td>" << storageType
214          << "</td><td>";
215       StringRef description = resolveAttrDescription(it.attr);
216       if (!description.empty()) {
217         // Expandable description.
218         // This appears as just the summary, but when clicked shows the full
219         // description.
220         os << "<details>"
221            << "<summary>" << it.attr.getSummary() << "</summary>"
222            << "{{% markdown %}}" << description << "{{% /markdown %}}"
223            << "</details>";
224       } else {
225         // Fallback: Single-line summary.
226         os << it.attr.getSummary();
227       }
228       os << "</td></tr>\n";
229     }
230     os << "<table>\n";
231   }
232 
233   // Emit each of the operands.
234   if (op.getNumOperands() != 0) {
235     os << "\n#### Operands:\n\n";
236     os << "| Operand | Description |\n"
237        << "| :-----: | ----------- |\n";
238     for (const auto &it : op.getOperands())
239       emitNamedConstraint(it, os);
240   }
241 
242   // Emit results.
243   if (op.getNumResults() != 0) {
244     os << "\n#### Results:\n\n";
245     os << "| Result | Description |\n"
246        << "| :----: | ----------- |\n";
247     for (const auto &it : op.getResults())
248       emitNamedConstraint(it, os);
249   }
250 
251   // Emit successors.
252   if (op.getNumSuccessors() != 0) {
253     os << "\n#### Successors:\n\n";
254     os << "| Successor | Description |\n"
255        << "| :-------: | ----------- |\n";
256     for (const auto &it : op.getSuccessors())
257       emitNamedConstraint(it, os);
258   }
259 
260   os << "\n";
261 }
262 
263 static void emitOpDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
264   auto opDefs = getRequestedOpDefinitions(recordKeeper);
265 
266   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
267   for (const llvm::Record *opDef : opDefs)
268     emitOpDoc(Operator(opDef), os);
269 }
270 
271 //===----------------------------------------------------------------------===//
272 // Attribute Documentation
273 //===----------------------------------------------------------------------===//
274 
275 static void emitAttrDoc(const Attribute &attr, raw_ostream &os) {
276   os << "### " << attr.getSummary() << "\n\n";
277   emitDescription(attr.getDescription(), os);
278   os << "\n\n";
279 }
280 
281 //===----------------------------------------------------------------------===//
282 // Type Documentation
283 //===----------------------------------------------------------------------===//
284 
285 static void emitTypeDoc(const Type &type, raw_ostream &os) {
286   os << "### " << type.getSummary() << "\n\n";
287   emitDescription(type.getDescription(), os);
288   os << "\n\n";
289 }
290 
291 //===----------------------------------------------------------------------===//
292 // TypeDef Documentation
293 //===----------------------------------------------------------------------===//
294 
295 static void emitAttrOrTypeDefAssemblyFormat(const AttrOrTypeDef &def,
296                                             raw_ostream &os) {
297   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
298   char prefix = isa<AttrDef>(def) ? '#' : '!';
299   if (parameters.empty()) {
300     os << "\nSyntax: `" << prefix << def.getDialect().getName() << "."
301        << def.getMnemonic() << "`\n";
302     return;
303   }
304 
305   os << "\nSyntax:\n\n```\n"
306      << prefix << def.getDialect().getName() << "." << def.getMnemonic()
307      << "<\n";
308   for (const auto &it : llvm::enumerate(parameters)) {
309     const AttrOrTypeParameter &param = it.value();
310     os << "  " << param.getSyntax();
311     if (it.index() < (parameters.size() - 1))
312       os << ",";
313     os << "   # " << param.getName() << "\n";
314   }
315   os << ">\n```\n";
316 }
317 
318 static void emitAttrOrTypeDefDoc(const AttrOrTypeDef &def, raw_ostream &os) {
319   os << llvm::formatv("### {0}\n", def.getCppClassName());
320 
321   // Emit the summary if present.
322   if (def.hasSummary())
323     os << "\n" << def.getSummary() << "\n";
324 
325   // Emit the syntax if present.
326   if (def.getMnemonic() && !def.hasCustomAssemblyFormat())
327     emitAttrOrTypeDefAssemblyFormat(def, os);
328 
329   // Emit the description if present.
330   if (def.hasDescription()) {
331     os << "\n";
332     mlir::tblgen::emitDescription(def.getDescription(), os);
333   }
334 
335   // Emit parameter documentation.
336   ArrayRef<AttrOrTypeParameter> parameters = def.getParameters();
337   if (!parameters.empty()) {
338     os << "\n#### Parameters:\n\n";
339     os << "| Parameter | C++ type | Description |\n"
340        << "| :-------: | :-------: | ----------- |\n";
341     for (const auto &it : parameters) {
342       auto desc = it.getSummary();
343       os << "| " << it.getName() << " | `" << it.getCppType() << "` | "
344          << (desc ? *desc : "") << " |\n";
345     }
346   }
347 
348   os << "\n";
349 }
350 
351 static void emitAttrOrTypeDefDoc(const RecordKeeper &recordKeeper,
352                                  raw_ostream &os, StringRef recordTypeName) {
353   std::vector<llvm::Record *> defs =
354       recordKeeper.getAllDerivedDefinitions(recordTypeName);
355 
356   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
357   for (const llvm::Record *def : defs)
358     emitAttrOrTypeDefDoc(AttrOrTypeDef(def), os);
359 }
360 
361 //===----------------------------------------------------------------------===//
362 // Dialect Documentation
363 //===----------------------------------------------------------------------===//
364 
365 struct OpDocGroup {
366   const Dialect &getDialect() const { return ops.front().getDialect(); }
367 
368   // Returns the summary description of the section.
369   std::string summary = "";
370 
371   // Returns the description of the section.
372   StringRef description = "";
373 
374   // Instances inside the section.
375   std::vector<Operator> ops;
376 };
377 
378 static void maybeNest(bool nest, llvm::function_ref<void(raw_ostream &os)> fn,
379                       raw_ostream &os) {
380   std::string str;
381   llvm::raw_string_ostream ss(str);
382   fn(ss);
383   for (StringRef x : llvm::split(ss.str(), "\n")) {
384     if (nest && x.starts_with("#"))
385       os << "#";
386     os << x << "\n";
387   }
388 }
389 
390 static void emitBlock(ArrayRef<Attribute> attributes,
391                       ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
392                       ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
393                       raw_ostream &os) {
394   if (!ops.empty()) {
395     os << "## Operation definition\n\n";
396     for (const OpDocGroup &grouping : ops) {
397       bool nested = !grouping.summary.empty();
398       maybeNest(
399           nested,
400           [&](raw_ostream &os) {
401             if (nested) {
402               os << "## " << StringRef(grouping.summary).trim() << "\n\n";
403               emitDescription(grouping.description, os);
404               os << "\n\n";
405             }
406             for (const Operator &op : grouping.ops) {
407               emitOpDoc(op, os);
408             }
409           },
410           os);
411     }
412   }
413 
414   if (!attributes.empty()) {
415     os << "## Attribute constraint definition\n\n";
416     for (const Attribute &attr : attributes)
417       emitAttrDoc(attr, os);
418   }
419 
420   if (!attrDefs.empty()) {
421     os << "## Attribute definition\n\n";
422     for (const AttrDef &def : attrDefs)
423       emitAttrOrTypeDefDoc(def, os);
424   }
425 
426   // TODO: Add link between use and def for types
427   if (!types.empty()) {
428     os << "## Type constraint definition\n\n";
429     for (const Type &type : types)
430       emitTypeDoc(type, os);
431   }
432 
433   if (!typeDefs.empty()) {
434     os << "## Type definition\n\n";
435     for (const TypeDef &def : typeDefs)
436       emitAttrOrTypeDefDoc(def, os);
437   }
438 }
439 
440 static void emitDialectDoc(const Dialect &dialect,
441                            ArrayRef<Attribute> attributes,
442                            ArrayRef<AttrDef> attrDefs, ArrayRef<OpDocGroup> ops,
443                            ArrayRef<Type> types, ArrayRef<TypeDef> typeDefs,
444                            raw_ostream &os) {
445   os << "# '" << dialect.getName() << "' Dialect\n\n";
446   emitIfNotEmpty(dialect.getSummary(), os);
447   emitIfNotEmpty(dialect.getDescription(), os);
448 
449   // Generate a TOC marker except if description already contains one.
450   llvm::Regex r("^[[:space:]]*\\[TOC\\]$", llvm::Regex::RegexFlags::Newline);
451   if (!r.match(dialect.getDescription()))
452     os << "[TOC]\n\n";
453 
454   emitBlock(attributes, attrDefs, ops, types, typeDefs, os);
455 }
456 
457 static bool emitDialectDoc(const RecordKeeper &recordKeeper, raw_ostream &os) {
458   std::vector<Record *> dialectDefs =
459       recordKeeper.getAllDerivedDefinitionsIfDefined("Dialect");
460   SmallVector<Dialect> dialects(dialectDefs.begin(), dialectDefs.end());
461   std::optional<Dialect> dialect = findDialectToGenerate(dialects);
462   if (!dialect)
463     return true;
464 
465   std::vector<Record *> opDefs = getRequestedOpDefinitions(recordKeeper);
466   std::vector<Record *> attrDefs =
467       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectAttr");
468   std::vector<Record *> typeDefs =
469       recordKeeper.getAllDerivedDefinitionsIfDefined("DialectType");
470   std::vector<Record *> typeDefDefs =
471       recordKeeper.getAllDerivedDefinitionsIfDefined("TypeDef");
472   std::vector<Record *> attrDefDefs =
473       recordKeeper.getAllDerivedDefinitionsIfDefined("AttrDef");
474 
475   std::vector<Attribute> dialectAttrs;
476   std::vector<AttrDef> dialectAttrDefs;
477   std::vector<OpDocGroup> dialectOps;
478   std::vector<Type> dialectTypes;
479   std::vector<TypeDef> dialectTypeDefs;
480 
481   llvm::SmallDenseSet<Record *> seen;
482   auto addIfInDialect = [&](llvm::Record *record, const auto &def, auto &vec) {
483     if (seen.insert(record).second && def.getDialect() == *dialect) {
484       vec.push_back(def);
485       return true;
486     }
487     return false;
488   };
489 
490   SmallDenseMap<Record *, OpDocGroup> opDocGroup;
491 
492   for (Record *def : attrDefDefs)
493     addIfInDialect(def, AttrDef(def), dialectAttrDefs);
494   for (Record *def : attrDefs)
495     addIfInDialect(def, Attribute(def), dialectAttrs);
496   for (Record *def : opDefs) {
497     if (Record *group = def->getValueAsOptionalDef("opDocGroup")) {
498       OpDocGroup &op = opDocGroup[group];
499       addIfInDialect(def, Operator(def), op.ops);
500     } else {
501       OpDocGroup op;
502       op.ops.emplace_back(def);
503       addIfInDialect(def, op, dialectOps);
504     }
505   }
506   for (Record *rec :
507        recordKeeper.getAllDerivedDefinitionsIfDefined("OpDocGroup")) {
508     if (opDocGroup[rec].ops.empty())
509       continue;
510     opDocGroup[rec].summary = rec->getValueAsString("summary");
511     opDocGroup[rec].description = rec->getValueAsString("description");
512     dialectOps.push_back(opDocGroup[rec]);
513   }
514   for (Record *def : typeDefDefs)
515     addIfInDialect(def, TypeDef(def), dialectTypeDefs);
516   for (Record *def : typeDefs)
517     addIfInDialect(def, Type(def), dialectTypes);
518 
519   // Sort alphabetically ignorning dialect for ops and section name for
520   // sections.
521   // TODO: The sorting order could be revised, currently attempting to sort of
522   // keep in alphabetical order.
523   std::sort(dialectOps.begin(), dialectOps.end(),
524             [](const OpDocGroup &lhs, const OpDocGroup &rhs) {
525               auto getDesc = [](const OpDocGroup &arg) -> StringRef {
526                 if (!arg.summary.empty())
527                   return arg.summary;
528                 return arg.ops.front().getDef().getValueAsString("opName");
529               };
530               return getDesc(lhs).compare_insensitive(getDesc(rhs)) < 0;
531             });
532 
533   os << "<!-- Autogenerated by mlir-tblgen; don't manually edit -->\n";
534   emitDialectDoc(*dialect, dialectAttrs, dialectAttrDefs, dialectOps,
535                  dialectTypes, dialectTypeDefs, os);
536   return false;
537 }
538 
539 //===----------------------------------------------------------------------===//
540 // Gen Registration
541 //===----------------------------------------------------------------------===//
542 
543 static mlir::GenRegistration
544     genAttrRegister("gen-attrdef-doc",
545                     "Generate dialect attribute documentation",
546                     [](const RecordKeeper &records, raw_ostream &os) {
547                       emitAttrOrTypeDefDoc(records, os, "AttrDef");
548                       return false;
549                     });
550 
551 static mlir::GenRegistration
552     genOpRegister("gen-op-doc", "Generate dialect documentation",
553                   [](const RecordKeeper &records, raw_ostream &os) {
554                     emitOpDoc(records, os);
555                     return false;
556                   });
557 
558 static mlir::GenRegistration
559     genTypeRegister("gen-typedef-doc", "Generate dialect type documentation",
560                     [](const RecordKeeper &records, raw_ostream &os) {
561                       emitAttrOrTypeDefDoc(records, os, "TypeDef");
562                       return false;
563                     });
564 
565 static mlir::GenRegistration
566     genRegister("gen-dialect-doc", "Generate dialect documentation",
567                 [](const RecordKeeper &records, raw_ostream &os) {
568                   return emitDialectDoc(records, os);
569                 });
570