1a7dea167SDimitry Andric //===--- ByteCodeEmitter.cpp - Instruction emitter for the VM ---*- C++ -*-===// 2a7dea167SDimitry Andric // 3a7dea167SDimitry Andric // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4a7dea167SDimitry Andric // See https://llvm.org/LICENSE.txt for license information. 5a7dea167SDimitry Andric // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6a7dea167SDimitry Andric // 7a7dea167SDimitry Andric //===----------------------------------------------------------------------===// 8a7dea167SDimitry Andric 9a7dea167SDimitry Andric #include "ByteCodeEmitter.h" 10a7dea167SDimitry Andric #include "Context.h" 1106c3fb27SDimitry Andric #include "Floating.h" 12*0fca6ea1SDimitry Andric #include "IntegralAP.h" 13a7dea167SDimitry Andric #include "Opcode.h" 14a7dea167SDimitry Andric #include "Program.h" 1506c3fb27SDimitry Andric #include "clang/AST/ASTLambda.h" 16*0fca6ea1SDimitry Andric #include "clang/AST/Attr.h" 17a7dea167SDimitry Andric #include "clang/AST/DeclCXX.h" 185f757f3fSDimitry Andric #include "clang/Basic/Builtins.h" 19349cc55cSDimitry Andric #include <type_traits> 20a7dea167SDimitry Andric 21a7dea167SDimitry Andric using namespace clang; 22a7dea167SDimitry Andric using namespace clang::interp; 23a7dea167SDimitry Andric 24*0fca6ea1SDimitry Andric /// Unevaluated builtins don't get their arguments put on the stack 25*0fca6ea1SDimitry Andric /// automatically. They instead operate on the AST of their Call 26*0fca6ea1SDimitry Andric /// Expression. 27*0fca6ea1SDimitry Andric /// Similar information is available via ASTContext::BuiltinInfo, 28*0fca6ea1SDimitry Andric /// but that is not correct for our use cases. 29*0fca6ea1SDimitry Andric static bool isUnevaluatedBuiltin(unsigned BuiltinID) { 30*0fca6ea1SDimitry Andric return BuiltinID == Builtin::BI__builtin_classify_type || 31*0fca6ea1SDimitry Andric BuiltinID == Builtin::BI__builtin_os_log_format_buffer_size; 32*0fca6ea1SDimitry Andric } 33*0fca6ea1SDimitry Andric 347a6dacacSDimitry Andric Function *ByteCodeEmitter::compileFunc(const FunctionDecl *FuncDecl) { 35*0fca6ea1SDimitry Andric 36*0fca6ea1SDimitry Andric // Manually created functions that haven't been assigned proper 37*0fca6ea1SDimitry Andric // parameters yet. 38*0fca6ea1SDimitry Andric if (!FuncDecl->param_empty() && !FuncDecl->param_begin()) 39*0fca6ea1SDimitry Andric return nullptr; 40*0fca6ea1SDimitry Andric 41*0fca6ea1SDimitry Andric bool IsLambdaStaticInvoker = false; 42*0fca6ea1SDimitry Andric if (const auto *MD = dyn_cast<CXXMethodDecl>(FuncDecl); 43*0fca6ea1SDimitry Andric MD && MD->isLambdaStaticInvoker()) { 44*0fca6ea1SDimitry Andric // For a lambda static invoker, we might have to pick a specialized 45*0fca6ea1SDimitry Andric // version if the lambda is generic. In that case, the picked function 46*0fca6ea1SDimitry Andric // will *NOT* be a static invoker anymore. However, it will still 47*0fca6ea1SDimitry Andric // be a non-static member function, this (usually) requiring an 48*0fca6ea1SDimitry Andric // instance pointer. We suppress that later in this function. 49*0fca6ea1SDimitry Andric IsLambdaStaticInvoker = true; 50*0fca6ea1SDimitry Andric 51*0fca6ea1SDimitry Andric const CXXRecordDecl *ClosureClass = MD->getParent(); 52*0fca6ea1SDimitry Andric assert(ClosureClass->captures_begin() == ClosureClass->captures_end()); 53*0fca6ea1SDimitry Andric if (ClosureClass->isGenericLambda()) { 54*0fca6ea1SDimitry Andric const CXXMethodDecl *LambdaCallOp = ClosureClass->getLambdaCallOperator(); 55*0fca6ea1SDimitry Andric assert(MD->isFunctionTemplateSpecialization() && 56*0fca6ea1SDimitry Andric "A generic lambda's static-invoker function must be a " 57*0fca6ea1SDimitry Andric "template specialization"); 58*0fca6ea1SDimitry Andric const TemplateArgumentList *TAL = MD->getTemplateSpecializationArgs(); 59*0fca6ea1SDimitry Andric FunctionTemplateDecl *CallOpTemplate = 60*0fca6ea1SDimitry Andric LambdaCallOp->getDescribedFunctionTemplate(); 61*0fca6ea1SDimitry Andric void *InsertPos = nullptr; 62*0fca6ea1SDimitry Andric const FunctionDecl *CorrespondingCallOpSpecialization = 63*0fca6ea1SDimitry Andric CallOpTemplate->findSpecialization(TAL->asArray(), InsertPos); 64*0fca6ea1SDimitry Andric assert(CorrespondingCallOpSpecialization); 65*0fca6ea1SDimitry Andric FuncDecl = cast<CXXMethodDecl>(CorrespondingCallOpSpecialization); 66*0fca6ea1SDimitry Andric } 67*0fca6ea1SDimitry Andric } 68*0fca6ea1SDimitry Andric 69a7dea167SDimitry Andric // Set up argument indices. 70a7dea167SDimitry Andric unsigned ParamOffset = 0; 71a7dea167SDimitry Andric SmallVector<PrimType, 8> ParamTypes; 7206c3fb27SDimitry Andric SmallVector<unsigned, 8> ParamOffsets; 73a7dea167SDimitry Andric llvm::DenseMap<unsigned, Function::ParamDescriptor> ParamDescriptors; 74a7dea167SDimitry Andric 75bdd1243dSDimitry Andric // If the return is not a primitive, a pointer to the storage where the 76bdd1243dSDimitry Andric // value is initialized in is passed as the first argument. See 'RVO' 77bdd1243dSDimitry Andric // elsewhere in the code. 78bdd1243dSDimitry Andric QualType Ty = FuncDecl->getReturnType(); 79bdd1243dSDimitry Andric bool HasRVO = false; 80a7dea167SDimitry Andric if (!Ty->isVoidType() && !Ctx.classify(Ty)) { 81bdd1243dSDimitry Andric HasRVO = true; 82bdd1243dSDimitry Andric ParamTypes.push_back(PT_Ptr); 8306c3fb27SDimitry Andric ParamOffsets.push_back(ParamOffset); 84bdd1243dSDimitry Andric ParamOffset += align(primSize(PT_Ptr)); 85bdd1243dSDimitry Andric } 86bdd1243dSDimitry Andric 87bdd1243dSDimitry Andric // If the function decl is a member decl, the next parameter is 88bdd1243dSDimitry Andric // the 'this' pointer. This parameter is pop()ed from the 89bdd1243dSDimitry Andric // InterpStack when calling the function. 90bdd1243dSDimitry Andric bool HasThisPointer = false; 9106c3fb27SDimitry Andric if (const auto *MD = dyn_cast<CXXMethodDecl>(FuncDecl)) { 92*0fca6ea1SDimitry Andric if (!IsLambdaStaticInvoker) { 93*0fca6ea1SDimitry Andric HasThisPointer = MD->isInstance(); 945f757f3fSDimitry Andric if (MD->isImplicitObjectMemberFunction()) { 95a7dea167SDimitry Andric ParamTypes.push_back(PT_Ptr); 9606c3fb27SDimitry Andric ParamOffsets.push_back(ParamOffset); 97a7dea167SDimitry Andric ParamOffset += align(primSize(PT_Ptr)); 98a7dea167SDimitry Andric } 99*0fca6ea1SDimitry Andric } 100a7dea167SDimitry Andric 10106c3fb27SDimitry Andric // Set up lambda capture to closure record field mapping. 10206c3fb27SDimitry Andric if (isLambdaCallOperator(MD)) { 103*0fca6ea1SDimitry Andric // The parent record needs to be complete, we need to know about all 104*0fca6ea1SDimitry Andric // the lambda captures. 105*0fca6ea1SDimitry Andric if (!MD->getParent()->isCompleteDefinition()) 106*0fca6ea1SDimitry Andric return nullptr; 107*0fca6ea1SDimitry Andric 10806c3fb27SDimitry Andric const Record *R = P.getOrCreateRecord(MD->getParent()); 10906c3fb27SDimitry Andric llvm::DenseMap<const ValueDecl *, FieldDecl *> LC; 11006c3fb27SDimitry Andric FieldDecl *LTC; 11106c3fb27SDimitry Andric 11206c3fb27SDimitry Andric MD->getParent()->getCaptureFields(LC, LTC); 11306c3fb27SDimitry Andric 11406c3fb27SDimitry Andric for (auto Cap : LC) { 1155f757f3fSDimitry Andric // Static lambdas cannot have any captures. If this one does, 1165f757f3fSDimitry Andric // it has already been diagnosed and we can only ignore it. 1175f757f3fSDimitry Andric if (MD->isStatic()) 1185f757f3fSDimitry Andric return nullptr; 1195f757f3fSDimitry Andric 12006c3fb27SDimitry Andric unsigned Offset = R->getField(Cap.second)->Offset; 12106c3fb27SDimitry Andric this->LambdaCaptures[Cap.first] = { 12206c3fb27SDimitry Andric Offset, Cap.second->getType()->isReferenceType()}; 12306c3fb27SDimitry Andric } 124*0fca6ea1SDimitry Andric if (LTC) { 125*0fca6ea1SDimitry Andric QualType CaptureType = R->getField(LTC)->Decl->getType(); 126*0fca6ea1SDimitry Andric this->LambdaThisCapture = {R->getField(LTC)->Offset, 127*0fca6ea1SDimitry Andric CaptureType->isReferenceType() || 128*0fca6ea1SDimitry Andric CaptureType->isPointerType()}; 129*0fca6ea1SDimitry Andric } 13006c3fb27SDimitry Andric } 13106c3fb27SDimitry Andric } 13206c3fb27SDimitry Andric 133a7dea167SDimitry Andric // Assign descriptors to all parameters. 134a7dea167SDimitry Andric // Composite objects are lowered to pointers. 135bdd1243dSDimitry Andric for (const ParmVarDecl *PD : FuncDecl->parameters()) { 1365f757f3fSDimitry Andric std::optional<PrimType> T = Ctx.classify(PD->getType()); 1375f757f3fSDimitry Andric PrimType PT = T.value_or(PT_Ptr); 1385f757f3fSDimitry Andric Descriptor *Desc = P.createDescriptor(PD, PT); 1395f757f3fSDimitry Andric ParamDescriptors.insert({ParamOffset, {PT, Desc}}); 1405f757f3fSDimitry Andric Params.insert({PD, {ParamOffset, T != std::nullopt}}); 14106c3fb27SDimitry Andric ParamOffsets.push_back(ParamOffset); 1425f757f3fSDimitry Andric ParamOffset += align(primSize(PT)); 1435f757f3fSDimitry Andric ParamTypes.push_back(PT); 144a7dea167SDimitry Andric } 145a7dea167SDimitry Andric 14606c3fb27SDimitry Andric // Create a handle over the emitted code. 14706c3fb27SDimitry Andric Function *Func = P.getFunction(FuncDecl); 1485f757f3fSDimitry Andric if (!Func) { 1495f757f3fSDimitry Andric bool IsUnevaluatedBuiltin = false; 1505f757f3fSDimitry Andric if (unsigned BI = FuncDecl->getBuiltinID()) 151*0fca6ea1SDimitry Andric IsUnevaluatedBuiltin = isUnevaluatedBuiltin(BI); 1525f757f3fSDimitry Andric 1535f757f3fSDimitry Andric Func = 1545f757f3fSDimitry Andric P.createFunction(FuncDecl, ParamOffset, std::move(ParamTypes), 1555f757f3fSDimitry Andric std::move(ParamDescriptors), std::move(ParamOffsets), 1565f757f3fSDimitry Andric HasThisPointer, HasRVO, IsUnevaluatedBuiltin); 1575f757f3fSDimitry Andric } 158bdd1243dSDimitry Andric 159bdd1243dSDimitry Andric assert(Func); 16006c3fb27SDimitry Andric // For not-yet-defined functions, we only create a Function instance and 16106c3fb27SDimitry Andric // compile their body later. 162*0fca6ea1SDimitry Andric if (!FuncDecl->isDefined() || 163*0fca6ea1SDimitry Andric (FuncDecl->willHaveBody() && !FuncDecl->hasBody())) { 1645f757f3fSDimitry Andric Func->setDefined(false); 165bdd1243dSDimitry Andric return Func; 1665f757f3fSDimitry Andric } 1675f757f3fSDimitry Andric 1685f757f3fSDimitry Andric Func->setDefined(true); 1695f757f3fSDimitry Andric 1705f757f3fSDimitry Andric // Lambda static invokers are a special case that we emit custom code for. 1715f757f3fSDimitry Andric bool IsEligibleForCompilation = false; 1725f757f3fSDimitry Andric if (const auto *MD = dyn_cast<CXXMethodDecl>(FuncDecl)) 1735f757f3fSDimitry Andric IsEligibleForCompilation = MD->isLambdaStaticInvoker(); 1745f757f3fSDimitry Andric if (!IsEligibleForCompilation) 175*0fca6ea1SDimitry Andric IsEligibleForCompilation = 176*0fca6ea1SDimitry Andric FuncDecl->isConstexpr() || FuncDecl->hasAttr<MSConstexprAttr>(); 177bdd1243dSDimitry Andric 178a7dea167SDimitry Andric // Compile the function body. 1795f757f3fSDimitry Andric if (!IsEligibleForCompilation || !visitFunc(FuncDecl)) { 180bdd1243dSDimitry Andric Func->setIsFullyCompiled(true); 181a7dea167SDimitry Andric return Func; 182bdd1243dSDimitry Andric } 1835f757f3fSDimitry Andric 184a7dea167SDimitry Andric // Create scopes from descriptors. 185a7dea167SDimitry Andric llvm::SmallVector<Scope, 2> Scopes; 186a7dea167SDimitry Andric for (auto &DS : Descriptors) { 187a7dea167SDimitry Andric Scopes.emplace_back(std::move(DS)); 188a7dea167SDimitry Andric } 189a7dea167SDimitry Andric 190a7dea167SDimitry Andric // Set the function's code. 191a7dea167SDimitry Andric Func->setCode(NextLocalOffset, std::move(Code), std::move(SrcMap), 19206c3fb27SDimitry Andric std::move(Scopes), FuncDecl->hasBody()); 193bdd1243dSDimitry Andric Func->setIsFullyCompiled(true); 194a7dea167SDimitry Andric return Func; 195a7dea167SDimitry Andric } 196a7dea167SDimitry Andric 197a7dea167SDimitry Andric Scope::Local ByteCodeEmitter::createLocal(Descriptor *D) { 198a7dea167SDimitry Andric NextLocalOffset += sizeof(Block); 199a7dea167SDimitry Andric unsigned Location = NextLocalOffset; 200a7dea167SDimitry Andric NextLocalOffset += align(D->getAllocSize()); 201a7dea167SDimitry Andric return {Location, D}; 202a7dea167SDimitry Andric } 203a7dea167SDimitry Andric 204a7dea167SDimitry Andric void ByteCodeEmitter::emitLabel(LabelTy Label) { 205a7dea167SDimitry Andric const size_t Target = Code.size(); 206a7dea167SDimitry Andric LabelOffsets.insert({Label, Target}); 20706c3fb27SDimitry Andric 20806c3fb27SDimitry Andric if (auto It = LabelRelocs.find(Label); 20906c3fb27SDimitry Andric It != LabelRelocs.end()) { 210a7dea167SDimitry Andric for (unsigned Reloc : It->second) { 211a7dea167SDimitry Andric using namespace llvm::support; 212a7dea167SDimitry Andric 21306c3fb27SDimitry Andric // Rewrite the operand of all jumps to this label. 214bdd1243dSDimitry Andric void *Location = Code.data() + Reloc - align(sizeof(int32_t)); 215bdd1243dSDimitry Andric assert(aligned(Location)); 216a7dea167SDimitry Andric const int32_t Offset = Target - static_cast<int64_t>(Reloc); 2175f757f3fSDimitry Andric endian::write<int32_t, llvm::endianness::native>(Location, Offset); 218a7dea167SDimitry Andric } 219a7dea167SDimitry Andric LabelRelocs.erase(It); 220a7dea167SDimitry Andric } 221a7dea167SDimitry Andric } 222a7dea167SDimitry Andric 223a7dea167SDimitry Andric int32_t ByteCodeEmitter::getOffset(LabelTy Label) { 224a7dea167SDimitry Andric // Compute the PC offset which the jump is relative to. 225bdd1243dSDimitry Andric const int64_t Position = 226bdd1243dSDimitry Andric Code.size() + align(sizeof(Opcode)) + align(sizeof(int32_t)); 227bdd1243dSDimitry Andric assert(aligned(Position)); 228a7dea167SDimitry Andric 229a7dea167SDimitry Andric // If target is known, compute jump offset. 23006c3fb27SDimitry Andric if (auto It = LabelOffsets.find(Label); 23106c3fb27SDimitry Andric It != LabelOffsets.end()) 232a7dea167SDimitry Andric return It->second - Position; 233a7dea167SDimitry Andric 234a7dea167SDimitry Andric // Otherwise, record relocation and return dummy offset. 235a7dea167SDimitry Andric LabelRelocs[Label].push_back(Position); 236a7dea167SDimitry Andric return 0ull; 237a7dea167SDimitry Andric } 238a7dea167SDimitry Andric 239a7dea167SDimitry Andric /// Helper to write bytecode and bail out if 32-bit offsets become invalid. 240349cc55cSDimitry Andric /// Pointers will be automatically marshalled as 32-bit IDs. 241349cc55cSDimitry Andric template <typename T> 24206c3fb27SDimitry Andric static void emit(Program &P, std::vector<std::byte> &Code, const T &Val, 243bdd1243dSDimitry Andric bool &Success) { 244bdd1243dSDimitry Andric size_t Size; 245bdd1243dSDimitry Andric 246bdd1243dSDimitry Andric if constexpr (std::is_pointer_v<T>) 247bdd1243dSDimitry Andric Size = sizeof(uint32_t); 248bdd1243dSDimitry Andric else 249bdd1243dSDimitry Andric Size = sizeof(T); 250bdd1243dSDimitry Andric 251a7dea167SDimitry Andric if (Code.size() + Size > std::numeric_limits<unsigned>::max()) { 252a7dea167SDimitry Andric Success = false; 253a7dea167SDimitry Andric return; 254a7dea167SDimitry Andric } 255349cc55cSDimitry Andric 256bdd1243dSDimitry Andric // Access must be aligned! 257bdd1243dSDimitry Andric size_t ValPos = align(Code.size()); 258bdd1243dSDimitry Andric Size = align(Size); 259bdd1243dSDimitry Andric assert(aligned(ValPos + Size)); 260bdd1243dSDimitry Andric Code.resize(ValPos + Size); 261349cc55cSDimitry Andric 262bdd1243dSDimitry Andric if constexpr (!std::is_pointer_v<T>) { 263bdd1243dSDimitry Andric new (Code.data() + ValPos) T(Val); 264bdd1243dSDimitry Andric } else { 265349cc55cSDimitry Andric uint32_t ID = P.getOrCreateNativePointer(Val); 266bdd1243dSDimitry Andric new (Code.data() + ValPos) uint32_t(ID); 267bdd1243dSDimitry Andric } 268349cc55cSDimitry Andric } 269349cc55cSDimitry Andric 270*0fca6ea1SDimitry Andric /// Emits a serializable value. These usually (potentially) contain 271*0fca6ea1SDimitry Andric /// heap-allocated memory and aren't trivially copyable. 272*0fca6ea1SDimitry Andric template <typename T> 273*0fca6ea1SDimitry Andric static void emitSerialized(std::vector<std::byte> &Code, const T &Val, 2745f757f3fSDimitry Andric bool &Success) { 2755f757f3fSDimitry Andric size_t Size = Val.bytesToSerialize(); 2765f757f3fSDimitry Andric 2775f757f3fSDimitry Andric if (Code.size() + Size > std::numeric_limits<unsigned>::max()) { 2785f757f3fSDimitry Andric Success = false; 2795f757f3fSDimitry Andric return; 2805f757f3fSDimitry Andric } 2815f757f3fSDimitry Andric 2825f757f3fSDimitry Andric // Access must be aligned! 2835f757f3fSDimitry Andric size_t ValPos = align(Code.size()); 2845f757f3fSDimitry Andric Size = align(Size); 2855f757f3fSDimitry Andric assert(aligned(ValPos + Size)); 2865f757f3fSDimitry Andric Code.resize(ValPos + Size); 2875f757f3fSDimitry Andric 2885f757f3fSDimitry Andric Val.serialize(Code.data() + ValPos); 2895f757f3fSDimitry Andric } 2905f757f3fSDimitry Andric 291*0fca6ea1SDimitry Andric template <> 292*0fca6ea1SDimitry Andric void emit(Program &P, std::vector<std::byte> &Code, const Floating &Val, 293*0fca6ea1SDimitry Andric bool &Success) { 294*0fca6ea1SDimitry Andric emitSerialized(Code, Val, Success); 295*0fca6ea1SDimitry Andric } 296*0fca6ea1SDimitry Andric 297*0fca6ea1SDimitry Andric template <> 298*0fca6ea1SDimitry Andric void emit(Program &P, std::vector<std::byte> &Code, 299*0fca6ea1SDimitry Andric const IntegralAP<false> &Val, bool &Success) { 300*0fca6ea1SDimitry Andric emitSerialized(Code, Val, Success); 301*0fca6ea1SDimitry Andric } 302*0fca6ea1SDimitry Andric 303*0fca6ea1SDimitry Andric template <> 304*0fca6ea1SDimitry Andric void emit(Program &P, std::vector<std::byte> &Code, const IntegralAP<true> &Val, 305*0fca6ea1SDimitry Andric bool &Success) { 306*0fca6ea1SDimitry Andric emitSerialized(Code, Val, Success); 307*0fca6ea1SDimitry Andric } 308*0fca6ea1SDimitry Andric 309349cc55cSDimitry Andric template <typename... Tys> 310349cc55cSDimitry Andric bool ByteCodeEmitter::emitOp(Opcode Op, const Tys &... Args, const SourceInfo &SI) { 311349cc55cSDimitry Andric bool Success = true; 312a7dea167SDimitry Andric 31306c3fb27SDimitry Andric // The opcode is followed by arguments. The source info is 31406c3fb27SDimitry Andric // attached to the address after the opcode. 315349cc55cSDimitry Andric emit(P, Code, Op, Success); 316a7dea167SDimitry Andric if (SI) 317a7dea167SDimitry Andric SrcMap.emplace_back(Code.size(), SI); 318a7dea167SDimitry Andric 319*0fca6ea1SDimitry Andric (..., emit(P, Code, Args, Success)); 320a7dea167SDimitry Andric return Success; 321a7dea167SDimitry Andric } 322a7dea167SDimitry Andric 323a7dea167SDimitry Andric bool ByteCodeEmitter::jumpTrue(const LabelTy &Label) { 324a7dea167SDimitry Andric return emitJt(getOffset(Label), SourceInfo{}); 325a7dea167SDimitry Andric } 326a7dea167SDimitry Andric 327a7dea167SDimitry Andric bool ByteCodeEmitter::jumpFalse(const LabelTy &Label) { 328a7dea167SDimitry Andric return emitJf(getOffset(Label), SourceInfo{}); 329a7dea167SDimitry Andric } 330a7dea167SDimitry Andric 331a7dea167SDimitry Andric bool ByteCodeEmitter::jump(const LabelTy &Label) { 332a7dea167SDimitry Andric return emitJmp(getOffset(Label), SourceInfo{}); 333a7dea167SDimitry Andric } 334a7dea167SDimitry Andric 335a7dea167SDimitry Andric bool ByteCodeEmitter::fallthrough(const LabelTy &Label) { 336a7dea167SDimitry Andric emitLabel(Label); 337a7dea167SDimitry Andric return true; 338a7dea167SDimitry Andric } 339a7dea167SDimitry Andric 340a7dea167SDimitry Andric //===----------------------------------------------------------------------===// 341a7dea167SDimitry Andric // Opcode emitters 342a7dea167SDimitry Andric //===----------------------------------------------------------------------===// 343a7dea167SDimitry Andric 344a7dea167SDimitry Andric #define GET_LINK_IMPL 345a7dea167SDimitry Andric #include "Opcodes.inc" 346a7dea167SDimitry Andric #undef GET_LINK_IMPL 347