1 //===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This tablegen backend emits information about intrinsic functions.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "CodeGenIntrinsics.h"
14 #include "CodeGenTarget.h"
15 #include "SequenceToOffsetTable.h"
16 #include "TableGenBackends.h"
17 #include "llvm/ADT/StringExtras.h"
18 #include "llvm/Support/CommandLine.h"
19 #include "llvm/TableGen/Error.h"
20 #include "llvm/TableGen/Record.h"
21 #include "llvm/TableGen/StringToOffsetTable.h"
22 #include "llvm/TableGen/TableGenBackend.h"
23 #include <algorithm>
24 using namespace llvm;
25
26 cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
27 cl::opt<std::string>
28 IntrinsicPrefix("intrinsic-prefix",
29 cl::desc("Generate intrinsics with this target prefix"),
30 cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat));
31
32 namespace {
33 class IntrinsicEmitter {
34 RecordKeeper &Records;
35
36 public:
IntrinsicEmitter(RecordKeeper & R)37 IntrinsicEmitter(RecordKeeper &R) : Records(R) {}
38
39 void run(raw_ostream &OS, bool Enums);
40
41 void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
42 void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
43 void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints,
44 raw_ostream &OS);
45 void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints,
46 raw_ostream &OS);
47 void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
48 void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS);
49 void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints, bool IsClang,
50 raw_ostream &OS);
51 };
52 } // End anonymous namespace
53
54 //===----------------------------------------------------------------------===//
55 // IntrinsicEmitter Implementation
56 //===----------------------------------------------------------------------===//
57
run(raw_ostream & OS,bool Enums)58 void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
59 emitSourceFileHeader("Intrinsic Function Source Fragment", OS);
60
61 CodeGenIntrinsicTable Ints(Records);
62
63 if (Enums) {
64 // Emit the enum information.
65 EmitEnumInfo(Ints, OS);
66 } else {
67 // Emit the target metadata.
68 EmitTargetInfo(Ints, OS);
69
70 // Emit the intrinsic ID -> name table.
71 EmitIntrinsicToNameTable(Ints, OS);
72
73 // Emit the intrinsic ID -> overload table.
74 EmitIntrinsicToOverloadTable(Ints, OS);
75
76 // Emit the intrinsic declaration generator.
77 EmitGenerator(Ints, OS);
78
79 // Emit the intrinsic parameter attributes.
80 EmitAttributes(Ints, OS);
81
82 // Emit code to translate GCC builtins into LLVM intrinsics.
83 EmitIntrinsicToBuiltinMap(Ints, true, OS);
84
85 // Emit code to translate MS builtins into LLVM intrinsics.
86 EmitIntrinsicToBuiltinMap(Ints, false, OS);
87 }
88 }
89
EmitEnumInfo(const CodeGenIntrinsicTable & Ints,raw_ostream & OS)90 void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
91 raw_ostream &OS) {
92 // Find the TargetSet for which to generate enums. There will be an initial
93 // set with an empty target prefix which will include target independent
94 // intrinsics like dbg.value.
95 const CodeGenIntrinsicTable::TargetSet *Set = nullptr;
96 for (const auto &Target : Ints.Targets) {
97 if (Target.Name == IntrinsicPrefix) {
98 Set = &Target;
99 break;
100 }
101 }
102 if (!Set) {
103 std::vector<std::string> KnownTargets;
104 for (const auto &Target : Ints.Targets)
105 if (!Target.Name.empty())
106 KnownTargets.push_back(Target.Name);
107 PrintFatalError("tried to generate intrinsics for unknown target " +
108 IntrinsicPrefix +
109 "\nKnown targets are: " + join(KnownTargets, ", ") + "\n");
110 }
111
112 // Generate a complete header for target specific intrinsics.
113 if (!IntrinsicPrefix.empty()) {
114 std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
115 OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n";
116 OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n";
117 OS << "namespace llvm {\n";
118 OS << "namespace Intrinsic {\n";
119 OS << "enum " << UpperPrefix << "Intrinsics : unsigned {\n";
120 }
121
122 OS << "// Enum values for intrinsics\n";
123 for (unsigned i = Set->Offset, e = Set->Offset + Set->Count; i != e; ++i) {
124 OS << " " << Ints[i].EnumName;
125
126 // Assign a value to the first intrinsic in this target set so that all
127 // intrinsic ids are distinct.
128 if (i == Set->Offset)
129 OS << " = " << (Set->Offset + 1);
130
131 OS << ", ";
132 if (Ints[i].EnumName.size() < 40)
133 OS.indent(40 - Ints[i].EnumName.size());
134 OS << " // " << Ints[i].Name << "\n";
135 }
136
137 // Emit num_intrinsics into the target neutral enum.
138 if (IntrinsicPrefix.empty()) {
139 OS << " num_intrinsics = " << (Ints.size() + 1) << "\n";
140 } else {
141 OS << "}; // enum\n";
142 OS << "} // namespace Intrinsic\n";
143 OS << "} // namespace llvm\n\n";
144 OS << "#endif\n";
145 }
146 }
147
EmitTargetInfo(const CodeGenIntrinsicTable & Ints,raw_ostream & OS)148 void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
149 raw_ostream &OS) {
150 OS << "// Target mapping\n";
151 OS << "#ifdef GET_INTRINSIC_TARGET_DATA\n";
152 OS << "struct IntrinsicTargetInfo {\n"
153 << " llvm::StringLiteral Name;\n"
154 << " size_t Offset;\n"
155 << " size_t Count;\n"
156 << "};\n";
157 OS << "static constexpr IntrinsicTargetInfo TargetInfos[] = {\n";
158 for (auto Target : Ints.Targets)
159 OS << " {llvm::StringLiteral(\"" << Target.Name << "\"), " << Target.Offset
160 << ", " << Target.Count << "},\n";
161 OS << "};\n";
162 OS << "#endif\n\n";
163 }
164
EmitIntrinsicToNameTable(const CodeGenIntrinsicTable & Ints,raw_ostream & OS)165 void IntrinsicEmitter::EmitIntrinsicToNameTable(
166 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
167 OS << "// Intrinsic ID to name table\n";
168 OS << "#ifdef GET_INTRINSIC_NAME_TABLE\n";
169 OS << " // Note that entry #0 is the invalid intrinsic!\n";
170 for (unsigned i = 0, e = Ints.size(); i != e; ++i)
171 OS << " \"" << Ints[i].Name << "\",\n";
172 OS << "#endif\n\n";
173 }
174
EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable & Ints,raw_ostream & OS)175 void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
176 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
177 OS << "// Intrinsic ID to overload bitset\n";
178 OS << "#ifdef GET_INTRINSIC_OVERLOAD_TABLE\n";
179 OS << "static const uint8_t OTable[] = {\n";
180 OS << " 0";
181 for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
182 // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
183 if ((i+1)%8 == 0)
184 OS << ",\n 0";
185 if (Ints[i].isOverloaded)
186 OS << " | (1<<" << (i+1)%8 << ')';
187 }
188 OS << "\n};\n\n";
189 // OTable contains a true bit at the position if the intrinsic is overloaded.
190 OS << "return (OTable[id/8] & (1 << (id%8))) != 0;\n";
191 OS << "#endif\n\n";
192 }
193
194
195 // NOTE: This must be kept in synch with the copy in lib/IR/Function.cpp!
196 enum IIT_Info {
197 // Common values should be encoded with 0-15.
198 IIT_Done = 0,
199 IIT_I1 = 1,
200 IIT_I8 = 2,
201 IIT_I16 = 3,
202 IIT_I32 = 4,
203 IIT_I64 = 5,
204 IIT_F16 = 6,
205 IIT_F32 = 7,
206 IIT_F64 = 8,
207 IIT_V2 = 9,
208 IIT_V4 = 10,
209 IIT_V8 = 11,
210 IIT_V16 = 12,
211 IIT_V32 = 13,
212 IIT_PTR = 14,
213 IIT_ARG = 15,
214
215 // Values from 16+ are only encodable with the inefficient encoding.
216 IIT_V64 = 16,
217 IIT_MMX = 17,
218 IIT_TOKEN = 18,
219 IIT_METADATA = 19,
220 IIT_EMPTYSTRUCT = 20,
221 IIT_STRUCT2 = 21,
222 IIT_STRUCT3 = 22,
223 IIT_STRUCT4 = 23,
224 IIT_STRUCT5 = 24,
225 IIT_EXTEND_ARG = 25,
226 IIT_TRUNC_ARG = 26,
227 IIT_ANYPTR = 27,
228 IIT_V1 = 28,
229 IIT_VARARG = 29,
230 IIT_HALF_VEC_ARG = 30,
231 IIT_SAME_VEC_WIDTH_ARG = 31,
232 IIT_PTR_TO_ARG = 32,
233 IIT_PTR_TO_ELT = 33,
234 IIT_VEC_OF_ANYPTRS_TO_ELT = 34,
235 IIT_I128 = 35,
236 IIT_V512 = 36,
237 IIT_V1024 = 37,
238 IIT_STRUCT6 = 38,
239 IIT_STRUCT7 = 39,
240 IIT_STRUCT8 = 40,
241 IIT_F128 = 41,
242 IIT_VEC_ELEMENT = 42,
243 IIT_SCALABLE_VEC = 43,
244 IIT_SUBDIVIDE2_ARG = 44,
245 IIT_SUBDIVIDE4_ARG = 45,
246 IIT_VEC_OF_BITCASTS_TO_INT = 46,
247 IIT_V128 = 47,
248 IIT_BF16 = 48,
249 IIT_STRUCT9 = 49,
250 IIT_V256 = 50,
251 IIT_AMX = 51,
252 IIT_PPCF128 = 52,
253 IIT_V3 = 53,
254 IIT_EXTERNREF = 54,
255 IIT_FUNCREF = 55,
256 IIT_ANYPTR_TO_ELT = 56,
257 IIT_I2 = 57,
258 IIT_I4 = 58,
259 };
260
EncodeFixedValueType(MVT::SimpleValueType VT,std::vector<unsigned char> & Sig)261 static void EncodeFixedValueType(MVT::SimpleValueType VT,
262 std::vector<unsigned char> &Sig) {
263 // clang-format off
264 if (MVT(VT).isInteger()) {
265 unsigned BitWidth = MVT(VT).getFixedSizeInBits();
266 switch (BitWidth) {
267 default: PrintFatalError("unhandled integer type width in intrinsic!");
268 case 1: return Sig.push_back(IIT_I1);
269 case 2: return Sig.push_back(IIT_I2);
270 case 4: return Sig.push_back(IIT_I4);
271 case 8: return Sig.push_back(IIT_I8);
272 case 16: return Sig.push_back(IIT_I16);
273 case 32: return Sig.push_back(IIT_I32);
274 case 64: return Sig.push_back(IIT_I64);
275 case 128: return Sig.push_back(IIT_I128);
276 }
277 }
278
279 switch (VT) {
280 default: PrintFatalError("unhandled MVT in intrinsic!");
281 case MVT::f16: return Sig.push_back(IIT_F16);
282 case MVT::bf16: return Sig.push_back(IIT_BF16);
283 case MVT::f32: return Sig.push_back(IIT_F32);
284 case MVT::f64: return Sig.push_back(IIT_F64);
285 case MVT::f128: return Sig.push_back(IIT_F128);
286 case MVT::ppcf128: return Sig.push_back(IIT_PPCF128);
287 case MVT::token: return Sig.push_back(IIT_TOKEN);
288 case MVT::Metadata: return Sig.push_back(IIT_METADATA);
289 case MVT::x86mmx: return Sig.push_back(IIT_MMX);
290 case MVT::x86amx: return Sig.push_back(IIT_AMX);
291 // MVT::OtherVT is used to mean the empty struct type here.
292 case MVT::Other: return Sig.push_back(IIT_EMPTYSTRUCT);
293 // MVT::isVoid is used to represent varargs here.
294 case MVT::isVoid: return Sig.push_back(IIT_VARARG);
295 case MVT::externref:
296 return Sig.push_back(IIT_EXTERNREF);
297 case MVT::funcref:
298 return Sig.push_back(IIT_FUNCREF);
299 }
300 // clang-format on
301 }
302
303 #if defined(_MSC_VER) && !defined(__clang__)
304 #pragma optimize("",off) // MSVC 2015 optimizer can't deal with this function.
305 #endif
306
EncodeFixedType(Record * R,std::vector<unsigned char> & ArgCodes,unsigned & NextArgCode,std::vector<unsigned char> & Sig,ArrayRef<unsigned char> Mapping)307 static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes,
308 unsigned &NextArgCode,
309 std::vector<unsigned char> &Sig,
310 ArrayRef<unsigned char> Mapping) {
311
312 if (R->isSubClassOf("LLVMMatchType")) {
313 unsigned Number = Mapping[R->getValueAsInt("Number")];
314 assert(Number < ArgCodes.size() && "Invalid matching number!");
315 if (R->isSubClassOf("LLVMExtendedType"))
316 Sig.push_back(IIT_EXTEND_ARG);
317 else if (R->isSubClassOf("LLVMTruncatedType"))
318 Sig.push_back(IIT_TRUNC_ARG);
319 else if (R->isSubClassOf("LLVMHalfElementsVectorType"))
320 Sig.push_back(IIT_HALF_VEC_ARG);
321 else if (R->isSubClassOf("LLVMScalarOrSameVectorWidth")) {
322 Sig.push_back(IIT_SAME_VEC_WIDTH_ARG);
323 Sig.push_back((Number << 3) | ArgCodes[Number]);
324 MVT::SimpleValueType VT = getValueType(R->getValueAsDef("ElTy"));
325 EncodeFixedValueType(VT, Sig);
326 return;
327 }
328 else if (R->isSubClassOf("LLVMPointerTo"))
329 Sig.push_back(IIT_PTR_TO_ARG);
330 else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
331 Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT);
332 // Encode overloaded ArgNo
333 Sig.push_back(NextArgCode++);
334 // Encode LLVMMatchType<Number> ArgNo
335 Sig.push_back(Number);
336 return;
337 } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
338 Sig.push_back(IIT_ANYPTR_TO_ELT);
339 // Encode overloaded ArgNo
340 Sig.push_back(NextArgCode++);
341 // Encode LLVMMatchType<Number> ArgNo
342 Sig.push_back(Number);
343 return;
344 } else if (R->isSubClassOf("LLVMPointerToElt"))
345 Sig.push_back(IIT_PTR_TO_ELT);
346 else if (R->isSubClassOf("LLVMVectorElementType"))
347 Sig.push_back(IIT_VEC_ELEMENT);
348 else if (R->isSubClassOf("LLVMSubdivide2VectorType"))
349 Sig.push_back(IIT_SUBDIVIDE2_ARG);
350 else if (R->isSubClassOf("LLVMSubdivide4VectorType"))
351 Sig.push_back(IIT_SUBDIVIDE4_ARG);
352 else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt"))
353 Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT);
354 else
355 Sig.push_back(IIT_ARG);
356 return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/);
357 }
358
359 MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT"));
360
361 unsigned Tmp = 0;
362 switch (VT) {
363 default: break;
364 case MVT::iPTRAny: ++Tmp; [[fallthrough]];
365 case MVT::vAny: ++Tmp; [[fallthrough]];
366 case MVT::fAny: ++Tmp; [[fallthrough]];
367 case MVT::iAny: ++Tmp; [[fallthrough]];
368 case MVT::Any: {
369 // If this is an "any" valuetype, then the type is the type of the next
370 // type in the list specified to getIntrinsic().
371 Sig.push_back(IIT_ARG);
372
373 // Figure out what arg # this is consuming, and remember what kind it was.
374 assert(NextArgCode < ArgCodes.size() && ArgCodes[NextArgCode] == Tmp &&
375 "Invalid or no ArgCode associated with overloaded VT!");
376 unsigned ArgNo = NextArgCode++;
377
378 // Encode what sort of argument it must be in the low 3 bits of the ArgNo.
379 return Sig.push_back((ArgNo << 3) | Tmp);
380 }
381
382 case MVT::iPTR: {
383 unsigned AddrSpace = 0;
384 if (R->isSubClassOf("LLVMQualPointerType")) {
385 AddrSpace = R->getValueAsInt("AddrSpace");
386 assert(AddrSpace < 256 && "Address space exceeds 255");
387 }
388 if (AddrSpace) {
389 Sig.push_back(IIT_ANYPTR);
390 Sig.push_back(AddrSpace);
391 } else {
392 Sig.push_back(IIT_PTR);
393 }
394 return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig,
395 Mapping);
396 }
397 }
398
399 if (MVT(VT).isVector()) {
400 MVT VVT = VT;
401 if (VVT.isScalableVector())
402 Sig.push_back(IIT_SCALABLE_VEC);
403 switch (VVT.getVectorMinNumElements()) {
404 default: PrintFatalError("unhandled vector type width in intrinsic!");
405 case 1: Sig.push_back(IIT_V1); break;
406 case 2: Sig.push_back(IIT_V2); break;
407 case 3: Sig.push_back(IIT_V3); break;
408 case 4: Sig.push_back(IIT_V4); break;
409 case 8: Sig.push_back(IIT_V8); break;
410 case 16: Sig.push_back(IIT_V16); break;
411 case 32: Sig.push_back(IIT_V32); break;
412 case 64: Sig.push_back(IIT_V64); break;
413 case 128: Sig.push_back(IIT_V128); break;
414 case 256: Sig.push_back(IIT_V256); break;
415 case 512: Sig.push_back(IIT_V512); break;
416 case 1024: Sig.push_back(IIT_V1024); break;
417 }
418
419 return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig);
420 }
421
422 EncodeFixedValueType(VT, Sig);
423 }
424
UpdateArgCodes(Record * R,std::vector<unsigned char> & ArgCodes,unsigned int & NumInserted,SmallVectorImpl<unsigned char> & Mapping)425 static void UpdateArgCodes(Record *R, std::vector<unsigned char> &ArgCodes,
426 unsigned int &NumInserted,
427 SmallVectorImpl<unsigned char> &Mapping) {
428 if (R->isSubClassOf("LLVMMatchType")) {
429 if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) {
430 ArgCodes.push_back(3 /*vAny*/);
431 ++NumInserted;
432 } else if (R->isSubClassOf("LLVMAnyPointerToElt")) {
433 ArgCodes.push_back(4 /*iPTRAny*/);
434 ++NumInserted;
435 }
436 return;
437 }
438
439 unsigned Tmp = 0;
440 switch (getValueType(R->getValueAsDef("VT"))) {
441 default: break;
442 case MVT::iPTR:
443 UpdateArgCodes(R->getValueAsDef("ElTy"), ArgCodes, NumInserted, Mapping);
444 break;
445 case MVT::iPTRAny:
446 ++Tmp;
447 [[fallthrough]];
448 case MVT::vAny:
449 ++Tmp;
450 [[fallthrough]];
451 case MVT::fAny:
452 ++Tmp;
453 [[fallthrough]];
454 case MVT::iAny:
455 ++Tmp;
456 [[fallthrough]];
457 case MVT::Any:
458 unsigned OriginalIdx = ArgCodes.size() - NumInserted;
459 assert(OriginalIdx >= Mapping.size());
460 Mapping.resize(OriginalIdx+1);
461 Mapping[OriginalIdx] = ArgCodes.size();
462 ArgCodes.push_back(Tmp);
463 break;
464 }
465 }
466
467 #if defined(_MSC_VER) && !defined(__clang__)
468 #pragma optimize("",on)
469 #endif
470
471 /// ComputeFixedEncoding - If we can encode the type signature for this
472 /// intrinsic into 32 bits, return it. If not, return ~0U.
ComputeFixedEncoding(const CodeGenIntrinsic & Int,std::vector<unsigned char> & TypeSig)473 static void ComputeFixedEncoding(const CodeGenIntrinsic &Int,
474 std::vector<unsigned char> &TypeSig) {
475 std::vector<unsigned char> ArgCodes;
476
477 // Add codes for any overloaded result VTs.
478 unsigned int NumInserted = 0;
479 SmallVector<unsigned char, 8> ArgMapping;
480 for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i)
481 UpdateArgCodes(Int.IS.RetTypeDefs[i], ArgCodes, NumInserted, ArgMapping);
482
483 // Add codes for any overloaded operand VTs.
484 for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i)
485 UpdateArgCodes(Int.IS.ParamTypeDefs[i], ArgCodes, NumInserted, ArgMapping);
486
487 unsigned NextArgCode = 0;
488 if (Int.IS.RetVTs.empty())
489 TypeSig.push_back(IIT_Done);
490 else if (Int.IS.RetVTs.size() == 1 &&
491 Int.IS.RetVTs[0] == MVT::isVoid)
492 TypeSig.push_back(IIT_Done);
493 else {
494 switch (Int.IS.RetVTs.size()) {
495 case 1: break;
496 case 2: TypeSig.push_back(IIT_STRUCT2); break;
497 case 3: TypeSig.push_back(IIT_STRUCT3); break;
498 case 4: TypeSig.push_back(IIT_STRUCT4); break;
499 case 5: TypeSig.push_back(IIT_STRUCT5); break;
500 case 6: TypeSig.push_back(IIT_STRUCT6); break;
501 case 7: TypeSig.push_back(IIT_STRUCT7); break;
502 case 8: TypeSig.push_back(IIT_STRUCT8); break;
503 case 9: TypeSig.push_back(IIT_STRUCT9); break;
504 default: llvm_unreachable("Unhandled case in struct");
505 }
506
507 for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i)
508 EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, NextArgCode, TypeSig,
509 ArgMapping);
510 }
511
512 for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i)
513 EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, NextArgCode, TypeSig,
514 ArgMapping);
515 }
516
printIITEntry(raw_ostream & OS,unsigned char X)517 static void printIITEntry(raw_ostream &OS, unsigned char X) {
518 OS << (unsigned)X;
519 }
520
EmitGenerator(const CodeGenIntrinsicTable & Ints,raw_ostream & OS)521 void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
522 raw_ostream &OS) {
523 // If we can compute a 32-bit fixed encoding for this intrinsic, do so and
524 // capture it in this vector, otherwise store a ~0U.
525 std::vector<unsigned> FixedEncodings;
526
527 SequenceToOffsetTable<std::vector<unsigned char> > LongEncodingTable;
528
529 std::vector<unsigned char> TypeSig;
530
531 // Compute the unique argument type info.
532 for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
533 // Get the signature for the intrinsic.
534 TypeSig.clear();
535 ComputeFixedEncoding(Ints[i], TypeSig);
536
537 // Check to see if we can encode it into a 32-bit word. We can only encode
538 // 8 nibbles into a 32-bit word.
539 if (TypeSig.size() <= 8) {
540 bool Failed = false;
541 unsigned Result = 0;
542 for (unsigned i = 0, e = TypeSig.size(); i != e; ++i) {
543 // If we had an unencodable argument, bail out.
544 if (TypeSig[i] > 15) {
545 Failed = true;
546 break;
547 }
548 Result = (Result << 4) | TypeSig[e-i-1];
549 }
550
551 // If this could be encoded into a 31-bit word, return it.
552 if (!Failed && (Result >> 31) == 0) {
553 FixedEncodings.push_back(Result);
554 continue;
555 }
556 }
557
558 // Otherwise, we're going to unique the sequence into the
559 // LongEncodingTable, and use its offset in the 32-bit table instead.
560 LongEncodingTable.add(TypeSig);
561
562 // This is a placehold that we'll replace after the table is laid out.
563 FixedEncodings.push_back(~0U);
564 }
565
566 LongEncodingTable.layout();
567
568 OS << "// Global intrinsic function declaration type table.\n";
569 OS << "#ifdef GET_INTRINSIC_GENERATOR_GLOBAL\n";
570
571 OS << "static const unsigned IIT_Table[] = {\n ";
572
573 for (unsigned i = 0, e = FixedEncodings.size(); i != e; ++i) {
574 if ((i & 7) == 7)
575 OS << "\n ";
576
577 // If the entry fit in the table, just emit it.
578 if (FixedEncodings[i] != ~0U) {
579 OS << "0x" << Twine::utohexstr(FixedEncodings[i]) << ", ";
580 continue;
581 }
582
583 TypeSig.clear();
584 ComputeFixedEncoding(Ints[i], TypeSig);
585
586
587 // Otherwise, emit the offset into the long encoding table. We emit it this
588 // way so that it is easier to read the offset in the .def file.
589 OS << "(1U<<31) | " << LongEncodingTable.get(TypeSig) << ", ";
590 }
591
592 OS << "0\n};\n\n";
593
594 // Emit the shared table of register lists.
595 OS << "static const unsigned char IIT_LongEncodingTable[] = {\n";
596 if (!LongEncodingTable.empty())
597 LongEncodingTable.emit(OS, printIITEntry);
598 OS << " 255\n};\n\n";
599
600 OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL
601 }
602
603 namespace {
compareFnAttributes(const CodeGenIntrinsic * L,const CodeGenIntrinsic * R)604 std::optional<bool> compareFnAttributes(const CodeGenIntrinsic *L,
605 const CodeGenIntrinsic *R) {
606 // Sort throwing intrinsics after non-throwing intrinsics.
607 if (L->canThrow != R->canThrow)
608 return R->canThrow;
609
610 if (L->isNoDuplicate != R->isNoDuplicate)
611 return R->isNoDuplicate;
612
613 if (L->isNoMerge != R->isNoMerge)
614 return R->isNoMerge;
615
616 if (L->isNoReturn != R->isNoReturn)
617 return R->isNoReturn;
618
619 if (L->isNoCallback != R->isNoCallback)
620 return R->isNoCallback;
621
622 if (L->isNoSync != R->isNoSync)
623 return R->isNoSync;
624
625 if (L->isNoFree != R->isNoFree)
626 return R->isNoFree;
627
628 if (L->isWillReturn != R->isWillReturn)
629 return R->isWillReturn;
630
631 if (L->isCold != R->isCold)
632 return R->isCold;
633
634 if (L->isConvergent != R->isConvergent)
635 return R->isConvergent;
636
637 if (L->isSpeculatable != R->isSpeculatable)
638 return R->isSpeculatable;
639
640 if (L->hasSideEffects != R->hasSideEffects)
641 return R->hasSideEffects;
642
643 // Try to order by readonly/readnone attribute.
644 uint32_t LK = L->ME.toIntValue();
645 uint32_t RK = R->ME.toIntValue();
646 if (LK != RK) return (LK > RK);
647
648 return std::nullopt;
649 }
650
651 struct FnAttributeComparator {
operator ()__anonffb27fdf0211::FnAttributeComparator652 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
653 return compareFnAttributes(L, R).value_or(false);
654 }
655 };
656
657 struct AttributeComparator {
operator ()__anonffb27fdf0211::AttributeComparator658 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const {
659 if (std::optional<bool> Res = compareFnAttributes(L, R))
660 return *Res;
661
662 // Order by argument attributes.
663 // This is reliable because each side is already sorted internally.
664 return (L->ArgumentAttributes < R->ArgumentAttributes);
665 }
666 };
667 } // End anonymous namespace
668
669 /// EmitAttributes - This emits the Intrinsic::getAttributes method.
EmitAttributes(const CodeGenIntrinsicTable & Ints,raw_ostream & OS)670 void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints,
671 raw_ostream &OS) {
672 OS << "// Add parameter attributes that are not common to all intrinsics.\n";
673 OS << "#ifdef GET_INTRINSIC_ATTRIBUTES\n";
674
675 // Compute unique argument attribute sets.
676 std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned>
677 UniqArgAttributes;
678 OS << "static AttributeSet getIntrinsicArgAttributeSet("
679 << "LLVMContext &C, unsigned ID) {\n"
680 << " switch (ID) {\n"
681 << " default: llvm_unreachable(\"Invalid attribute set number\");\n";
682 for (const CodeGenIntrinsic &Int : Ints) {
683 for (auto &Attrs : Int.ArgumentAttributes) {
684 if (Attrs.empty())
685 continue;
686
687 unsigned ID = UniqArgAttributes.size();
688 if (!UniqArgAttributes.try_emplace(Attrs, ID).second)
689 continue;
690
691 assert(is_sorted(Attrs) &&
692 "Argument attributes are not sorted");
693
694 OS << " case " << ID << ":\n";
695 OS << " return AttributeSet::get(C, {\n";
696 for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) {
697 switch (Attr.Kind) {
698 case CodeGenIntrinsic::NoCapture:
699 OS << " Attribute::get(C, Attribute::NoCapture),\n";
700 break;
701 case CodeGenIntrinsic::NoAlias:
702 OS << " Attribute::get(C, Attribute::NoAlias),\n";
703 break;
704 case CodeGenIntrinsic::NoUndef:
705 OS << " Attribute::get(C, Attribute::NoUndef),\n";
706 break;
707 case CodeGenIntrinsic::NonNull:
708 OS << " Attribute::get(C, Attribute::NonNull),\n";
709 break;
710 case CodeGenIntrinsic::Returned:
711 OS << " Attribute::get(C, Attribute::Returned),\n";
712 break;
713 case CodeGenIntrinsic::ReadOnly:
714 OS << " Attribute::get(C, Attribute::ReadOnly),\n";
715 break;
716 case CodeGenIntrinsic::WriteOnly:
717 OS << " Attribute::get(C, Attribute::WriteOnly),\n";
718 break;
719 case CodeGenIntrinsic::ReadNone:
720 OS << " Attribute::get(C, Attribute::ReadNone),\n";
721 break;
722 case CodeGenIntrinsic::ImmArg:
723 OS << " Attribute::get(C, Attribute::ImmArg),\n";
724 break;
725 case CodeGenIntrinsic::Alignment:
726 OS << " Attribute::get(C, Attribute::Alignment, "
727 << Attr.Value << "),\n";
728 break;
729 }
730 }
731 OS << " });\n";
732 }
733 }
734 OS << " }\n";
735 OS << "}\n\n";
736
737 // Compute unique function attribute sets.
738 std::map<const CodeGenIntrinsic*, unsigned, FnAttributeComparator>
739 UniqFnAttributes;
740 OS << "static AttributeSet getIntrinsicFnAttributeSet("
741 << "LLVMContext &C, unsigned ID) {\n"
742 << " switch (ID) {\n"
743 << " default: llvm_unreachable(\"Invalid attribute set number\");\n";
744 for (const CodeGenIntrinsic &Intrinsic : Ints) {
745 unsigned ID = UniqFnAttributes.size();
746 if (!UniqFnAttributes.try_emplace(&Intrinsic, ID).second)
747 continue;
748
749 OS << " case " << ID << ":\n"
750 << " return AttributeSet::get(C, {\n";
751 if (!Intrinsic.canThrow)
752 OS << " Attribute::get(C, Attribute::NoUnwind),\n";
753 if (Intrinsic.isNoReturn)
754 OS << " Attribute::get(C, Attribute::NoReturn),\n";
755 if (Intrinsic.isNoCallback)
756 OS << " Attribute::get(C, Attribute::NoCallback),\n";
757 if (Intrinsic.isNoSync)
758 OS << " Attribute::get(C, Attribute::NoSync),\n";
759 if (Intrinsic.isNoFree)
760 OS << " Attribute::get(C, Attribute::NoFree),\n";
761 if (Intrinsic.isWillReturn)
762 OS << " Attribute::get(C, Attribute::WillReturn),\n";
763 if (Intrinsic.isCold)
764 OS << " Attribute::get(C, Attribute::Cold),\n";
765 if (Intrinsic.isNoDuplicate)
766 OS << " Attribute::get(C, Attribute::NoDuplicate),\n";
767 if (Intrinsic.isNoMerge)
768 OS << " Attribute::get(C, Attribute::NoMerge),\n";
769 if (Intrinsic.isConvergent)
770 OS << " Attribute::get(C, Attribute::Convergent),\n";
771 if (Intrinsic.isSpeculatable)
772 OS << " Attribute::get(C, Attribute::Speculatable),\n";
773
774 MemoryEffects ME = Intrinsic.ME;
775 // TODO: IntrHasSideEffects should affect not only readnone intrinsics.
776 if (ME.doesNotAccessMemory() && Intrinsic.hasSideEffects)
777 ME = MemoryEffects::unknown();
778 if (ME != MemoryEffects::unknown()) {
779 OS << " Attribute::getWithMemoryEffects(C, "
780 << "MemoryEffects::createFromIntValue(" << ME.toIntValue() << ")),\n";
781 }
782 OS << " });\n";
783 }
784 OS << " }\n";
785 OS << "}\n\n";
786 OS << "AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {\n";
787
788 // Compute the maximum number of attribute arguments and the map
789 typedef std::map<const CodeGenIntrinsic*, unsigned,
790 AttributeComparator> UniqAttrMapTy;
791 UniqAttrMapTy UniqAttributes;
792 unsigned maxArgAttrs = 0;
793 unsigned AttrNum = 0;
794 for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
795 const CodeGenIntrinsic &intrinsic = Ints[i];
796 maxArgAttrs =
797 std::max(maxArgAttrs, unsigned(intrinsic.ArgumentAttributes.size()));
798 unsigned &N = UniqAttributes[&intrinsic];
799 if (N) continue;
800 N = ++AttrNum;
801 assert(N < 65536 && "Too many unique attributes for table!");
802 }
803
804 // Emit an array of AttributeList. Most intrinsics will have at least one
805 // entry, for the function itself (index ~1), which is usually nounwind.
806 OS << " static const uint16_t IntrinsicsToAttributesMap[] = {\n";
807
808 for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
809 const CodeGenIntrinsic &intrinsic = Ints[i];
810
811 OS << " " << UniqAttributes[&intrinsic] << ", // "
812 << intrinsic.Name << "\n";
813 }
814 OS << " };\n\n";
815
816 OS << " std::pair<unsigned, AttributeSet> AS[" << maxArgAttrs + 1 << "];\n";
817 OS << " unsigned NumAttrs = 0;\n";
818 OS << " if (id != 0) {\n";
819 OS << " switch(IntrinsicsToAttributesMap[id - 1]) {\n";
820 OS << " default: llvm_unreachable(\"Invalid attribute number\");\n";
821 for (auto UniqAttribute : UniqAttributes) {
822 OS << " case " << UniqAttribute.second << ": {\n";
823
824 const CodeGenIntrinsic &Intrinsic = *(UniqAttribute.first);
825
826 // Keep track of the number of attributes we're writing out.
827 unsigned numAttrs = 0;
828
829 for (const auto &[AttrIdx, Attrs] :
830 enumerate(Intrinsic.ArgumentAttributes)) {
831 if (Attrs.empty())
832 continue;
833
834 unsigned ID = UniqArgAttributes.find(Attrs)->second;
835 OS << " AS[" << numAttrs++ << "] = {" << AttrIdx
836 << ", getIntrinsicArgAttributeSet(C, " << ID << ")};\n";
837 }
838
839 if (!Intrinsic.canThrow ||
840 (Intrinsic.ME != MemoryEffects::unknown() &&
841 !Intrinsic.hasSideEffects) ||
842 Intrinsic.isNoReturn || Intrinsic.isNoCallback || Intrinsic.isNoSync ||
843 Intrinsic.isNoFree || Intrinsic.isWillReturn || Intrinsic.isCold ||
844 Intrinsic.isNoDuplicate || Intrinsic.isNoMerge ||
845 Intrinsic.isConvergent || Intrinsic.isSpeculatable) {
846 unsigned ID = UniqFnAttributes.find(&Intrinsic)->second;
847 OS << " AS[" << numAttrs++ << "] = {AttributeList::FunctionIndex, "
848 << "getIntrinsicFnAttributeSet(C, " << ID << ")};\n";
849 }
850
851 if (numAttrs) {
852 OS << " NumAttrs = " << numAttrs << ";\n";
853 OS << " break;\n";
854 OS << " }\n";
855 } else {
856 OS << " return AttributeList();\n";
857 OS << " }\n";
858 }
859 }
860
861 OS << " }\n";
862 OS << " }\n";
863 OS << " return AttributeList::get(C, ArrayRef(AS, NumAttrs));\n";
864 OS << "}\n";
865 OS << "#endif // GET_INTRINSIC_ATTRIBUTES\n\n";
866 }
867
EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable & Ints,bool IsClang,raw_ostream & OS)868 void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
869 const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) {
870 StringRef CompilerName = (IsClang ? "Clang" : "MS");
871 StringRef UpperCompilerName = (IsClang ? "CLANG" : "MS");
872 typedef std::map<std::string, std::map<std::string, std::string>> BIMTy;
873 BIMTy BuiltinMap;
874 StringToOffsetTable Table;
875 for (unsigned i = 0, e = Ints.size(); i != e; ++i) {
876 const std::string &BuiltinName =
877 IsClang ? Ints[i].ClangBuiltinName : Ints[i].MSBuiltinName;
878 if (!BuiltinName.empty()) {
879 // Get the map for this target prefix.
880 std::map<std::string, std::string> &BIM =
881 BuiltinMap[Ints[i].TargetPrefix];
882
883 if (!BIM.insert(std::make_pair(BuiltinName, Ints[i].EnumName)).second)
884 PrintFatalError(Ints[i].TheDef->getLoc(),
885 "Intrinsic '" + Ints[i].TheDef->getName() +
886 "': duplicate " + CompilerName + " builtin name!");
887 Table.GetOrAddStringOffset(BuiltinName);
888 }
889 }
890
891 OS << "// Get the LLVM intrinsic that corresponds to a builtin.\n";
892 OS << "// This is used by the C front-end. The builtin name is passed\n";
893 OS << "// in as BuiltinName, and a target prefix (e.g. 'ppc') is passed\n";
894 OS << "// in as TargetPrefix. The result is assigned to 'IntrinsicID'.\n";
895 OS << "#ifdef GET_LLVM_INTRINSIC_FOR_" << UpperCompilerName << "_BUILTIN\n";
896
897 OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName
898 << "Builtin(const char "
899 << "*TargetPrefixStr, StringRef BuiltinNameStr) {\n";
900
901 if (Table.Empty()) {
902 OS << " return Intrinsic::not_intrinsic;\n";
903 OS << "}\n";
904 OS << "#endif\n\n";
905 return;
906 }
907
908 OS << " static const char BuiltinNames[] = {\n";
909 Table.EmitCharArray(OS);
910 OS << " };\n\n";
911
912 OS << " struct BuiltinEntry {\n";
913 OS << " Intrinsic::ID IntrinID;\n";
914 OS << " unsigned StrTabOffset;\n";
915 OS << " const char *getName() const {\n";
916 OS << " return &BuiltinNames[StrTabOffset];\n";
917 OS << " }\n";
918 OS << " bool operator<(StringRef RHS) const {\n";
919 OS << " return strncmp(getName(), RHS.data(), RHS.size()) < 0;\n";
920 OS << " }\n";
921 OS << " };\n";
922
923 OS << " StringRef TargetPrefix(TargetPrefixStr);\n\n";
924
925 // Note: this could emit significantly better code if we cared.
926 for (auto &I : BuiltinMap) {
927 OS << " ";
928 if (!I.first.empty())
929 OS << "if (TargetPrefix == \"" << I.first << "\") ";
930 else
931 OS << "/* Target Independent Builtins */ ";
932 OS << "{\n";
933
934 // Emit the comparisons for this target prefix.
935 OS << " static const BuiltinEntry " << I.first << "Names[] = {\n";
936 for (const auto &P : I.second) {
937 OS << " {Intrinsic::" << P.second << ", "
938 << Table.GetOrAddStringOffset(P.first) << "}, // " << P.first << "\n";
939 }
940 OS << " };\n";
941 OS << " auto I = std::lower_bound(std::begin(" << I.first << "Names),\n";
942 OS << " std::end(" << I.first << "Names),\n";
943 OS << " BuiltinNameStr);\n";
944 OS << " if (I != std::end(" << I.first << "Names) &&\n";
945 OS << " I->getName() == BuiltinNameStr)\n";
946 OS << " return I->IntrinID;\n";
947 OS << " }\n";
948 }
949 OS << " return ";
950 OS << "Intrinsic::not_intrinsic;\n";
951 OS << "}\n";
952 OS << "#endif\n\n";
953 }
954
EmitIntrinsicEnums(RecordKeeper & RK,raw_ostream & OS)955 void llvm::EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS) {
956 IntrinsicEmitter(RK).run(OS, /*Enums=*/true);
957 }
958
EmitIntrinsicImpl(RecordKeeper & RK,raw_ostream & OS)959 void llvm::EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) {
960 IntrinsicEmitter(RK).run(OS, /*Enums=*/false);
961 }
962