1 //===---------------------------- StackMaps.cpp ---------------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 10 #define DEBUG_TYPE "stackmaps" 11 12 #include "llvm/CodeGen/StackMaps.h" 13 #include "llvm/CodeGen/AsmPrinter.h" 14 #include "llvm/CodeGen/MachineInstr.h" 15 #include "llvm/IR/DataLayout.h" 16 #include "llvm/MC/MCContext.h" 17 #include "llvm/MC/MCExpr.h" 18 #include "llvm/MC/MCObjectFileInfo.h" 19 #include "llvm/MC/MCSectionMachO.h" 20 #include "llvm/MC/MCStreamer.h" 21 #include "llvm/Support/Debug.h" 22 #include "llvm/Support/raw_ostream.h" 23 #include "llvm/Target/TargetMachine.h" 24 #include "llvm/Target/TargetOpcodes.h" 25 #include "llvm/Target/TargetRegisterInfo.h" 26 #include <iterator> 27 28 using namespace llvm; 29 30 PatchPointOpers::PatchPointOpers(const MachineInstr *MI) 31 : MI(MI), 32 HasDef(MI->getOperand(0).isReg() && MI->getOperand(0).isDef() && 33 !MI->getOperand(0).isImplicit()), 34 IsAnyReg(MI->getOperand(getMetaIdx(CCPos)).getImm() == CallingConv::AnyReg) 35 { 36 #ifndef NDEBUG 37 unsigned CheckStartIdx = 0, e = MI->getNumOperands(); 38 while (CheckStartIdx < e && MI->getOperand(CheckStartIdx).isReg() && 39 MI->getOperand(CheckStartIdx).isDef() && 40 !MI->getOperand(CheckStartIdx).isImplicit()) 41 ++CheckStartIdx; 42 43 assert(getMetaIdx() == CheckStartIdx && 44 "Unexpected additonal definition in Patchpoint intrinsic."); 45 #endif 46 } 47 48 unsigned PatchPointOpers::getNextScratchIdx(unsigned StartIdx) const { 49 if (!StartIdx) 50 StartIdx = getVarIdx(); 51 52 // Find the next scratch register (implicit def and early clobber) 53 unsigned ScratchIdx = StartIdx, e = MI->getNumOperands(); 54 while (ScratchIdx < e && 55 !(MI->getOperand(ScratchIdx).isReg() && 56 MI->getOperand(ScratchIdx).isDef() && 57 MI->getOperand(ScratchIdx).isImplicit() && 58 MI->getOperand(ScratchIdx).isEarlyClobber())) 59 ++ScratchIdx; 60 61 assert(ScratchIdx != e && "No scratch register available"); 62 return ScratchIdx; 63 } 64 65 MachineInstr::const_mop_iterator 66 StackMaps::parseOperand(MachineInstr::const_mop_iterator MOI, 67 MachineInstr::const_mop_iterator MOE, 68 LocationVec &Locs, LiveOutVec &LiveOuts) const { 69 if (MOI->isImm()) { 70 switch (MOI->getImm()) { 71 default: llvm_unreachable("Unrecognized operand type."); 72 case StackMaps::DirectMemRefOp: { 73 unsigned Size = AP.TM.getDataLayout()->getPointerSizeInBits(); 74 assert((Size % 8) == 0 && "Need pointer size in bytes."); 75 Size /= 8; 76 unsigned Reg = (++MOI)->getReg(); 77 int64_t Imm = (++MOI)->getImm(); 78 Locs.push_back(Location(StackMaps::Location::Direct, Size, Reg, Imm)); 79 break; 80 } 81 case StackMaps::IndirectMemRefOp: { 82 int64_t Size = (++MOI)->getImm(); 83 assert(Size > 0 && "Need a valid size for indirect memory locations."); 84 unsigned Reg = (++MOI)->getReg(); 85 int64_t Imm = (++MOI)->getImm(); 86 Locs.push_back(Location(StackMaps::Location::Indirect, Size, Reg, Imm)); 87 break; 88 } 89 case StackMaps::ConstantOp: { 90 ++MOI; 91 assert(MOI->isImm() && "Expected constant operand."); 92 int64_t Imm = MOI->getImm(); 93 Locs.push_back(Location(Location::Constant, sizeof(int64_t), 0, Imm)); 94 break; 95 } 96 } 97 return ++MOI; 98 } 99 100 // The physical register number will ultimately be encoded as a DWARF regno. 101 // The stack map also records the size of a spill slot that can hold the 102 // register content. (The runtime can track the actual size of the data type 103 // if it needs to.) 104 if (MOI->isReg()) { 105 // Skip implicit registers (this includes our scratch registers) 106 if (MOI->isImplicit()) 107 return ++MOI; 108 109 assert(TargetRegisterInfo::isPhysicalRegister(MOI->getReg()) && 110 "Virtreg operands should have been rewritten before now."); 111 const TargetRegisterClass *RC = 112 AP.TM.getRegisterInfo()->getMinimalPhysRegClass(MOI->getReg()); 113 assert(!MOI->getSubReg() && "Physical subreg still around."); 114 Locs.push_back( 115 Location(Location::Register, RC->getSize(), MOI->getReg(), 0)); 116 return ++MOI; 117 } 118 119 if (MOI->isRegLiveOut()) 120 LiveOuts = parseRegisterLiveOutMask(MOI->getRegLiveOut()); 121 122 return ++MOI; 123 } 124 125 /// Go up the super-register chain until we hit a valid dwarf register number. 126 static unsigned short getDwarfRegNum(unsigned Reg, const MCRegisterInfo &MCRI, 127 const TargetRegisterInfo *TRI) { 128 int RegNo = MCRI.getDwarfRegNum(Reg, false); 129 for (MCSuperRegIterator SR(Reg, TRI); 130 SR.isValid() && RegNo < 0; ++SR) 131 RegNo = TRI->getDwarfRegNum(*SR, false); 132 133 assert(RegNo >= 0 && "Invalid Dwarf register number."); 134 return (unsigned short) RegNo; 135 } 136 137 /// Create a live-out register record for the given register Reg. 138 StackMaps::LiveOutReg 139 StackMaps::createLiveOutReg(unsigned Reg, const MCRegisterInfo &MCRI, 140 const TargetRegisterInfo *TRI) const { 141 unsigned RegNo = getDwarfRegNum(Reg, MCRI, TRI); 142 unsigned Size = TRI->getMinimalPhysRegClass(Reg)->getSize(); 143 return LiveOutReg(Reg, RegNo, Size); 144 } 145 146 /// Parse the register live-out mask and return a vector of live-out registers 147 /// that need to be recorded in the stackmap. 148 StackMaps::LiveOutVec 149 StackMaps::parseRegisterLiveOutMask(const uint32_t *Mask) const { 150 assert(Mask && "No register mask specified"); 151 const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo(); 152 MCContext &OutContext = AP.OutStreamer.getContext(); 153 const MCRegisterInfo &MCRI = *OutContext.getRegisterInfo(); 154 LiveOutVec LiveOuts; 155 156 // Create a LiveOutReg for each bit that is set in the register mask. 157 for (unsigned Reg = 0, NumRegs = TRI->getNumRegs(); Reg != NumRegs; ++Reg) 158 if ((Mask[Reg / 32] >> Reg % 32) & 1) 159 LiveOuts.push_back(createLiveOutReg(Reg, MCRI, TRI)); 160 161 // We don't need to keep track of a register if its super-register is already 162 // in the list. Merge entries that refer to the same dwarf register and use 163 // the maximum size that needs to be spilled. 164 std::sort(LiveOuts.begin(), LiveOuts.end()); 165 for (LiveOutVec::iterator I = LiveOuts.begin(), E = LiveOuts.end(); 166 I != E; ++I) { 167 for (LiveOutVec::iterator II = next(I); II != E; ++II) { 168 if (I->RegNo != II->RegNo) { 169 // Skip all the now invalid entries. 170 I = --II; 171 break; 172 } 173 I->Size = std::max(I->Size, II->Size); 174 if (TRI->isSuperRegister(I->Reg, II->Reg)) 175 I->Reg = II->Reg; 176 II->MarkInvalid(); 177 } 178 } 179 LiveOuts.erase(std::remove_if(LiveOuts.begin(), LiveOuts.end(), 180 LiveOutReg::IsInvalid), LiveOuts.end()); 181 return LiveOuts; 182 } 183 184 void StackMaps::recordStackMapOpers(const MachineInstr &MI, uint64_t ID, 185 MachineInstr::const_mop_iterator MOI, 186 MachineInstr::const_mop_iterator MOE, 187 bool recordResult) { 188 189 MCContext &OutContext = AP.OutStreamer.getContext(); 190 MCSymbol *MILabel = OutContext.CreateTempSymbol(); 191 AP.OutStreamer.EmitLabel(MILabel); 192 193 LocationVec Locations; 194 LiveOutVec LiveOuts; 195 196 if (recordResult) { 197 assert(PatchPointOpers(&MI).hasDef() && "Stackmap has no return value."); 198 parseOperand(MI.operands_begin(), llvm::next(MI.operands_begin()), 199 Locations, LiveOuts); 200 } 201 202 // Parse operands. 203 while (MOI != MOE) { 204 MOI = parseOperand(MOI, MOE, Locations, LiveOuts); 205 } 206 207 // Move large constants into the constant pool. 208 for (LocationVec::iterator I = Locations.begin(), E = Locations.end(); 209 I != E; ++I) { 210 // Constants are encoded as sign-extended integers. 211 // -1 is directly encoded as .long 0xFFFFFFFF with no constant pool. 212 if (I->LocType == Location::Constant && 213 ((I->Offset + (int64_t(1)<<31)) >> 32) != 0) { 214 I->LocType = Location::ConstantIndex; 215 I->Offset = ConstPool.getConstantIndex(I->Offset); 216 } 217 } 218 219 const MCExpr *CSOffsetExpr = MCBinaryExpr::CreateSub( 220 MCSymbolRefExpr::Create(MILabel, OutContext), 221 MCSymbolRefExpr::Create(AP.CurrentFnSym, OutContext), 222 OutContext); 223 224 CSInfos.push_back(CallsiteInfo(CSOffsetExpr, ID, Locations, LiveOuts)); 225 } 226 227 void StackMaps::recordStackMap(const MachineInstr &MI) { 228 assert(MI.getOpcode() == TargetOpcode::STACKMAP && "expected stackmap"); 229 230 int64_t ID = MI.getOperand(0).getImm(); 231 recordStackMapOpers(MI, ID, llvm::next(MI.operands_begin(), 2), 232 MI.operands_end()); 233 } 234 235 void StackMaps::recordPatchPoint(const MachineInstr &MI) { 236 assert(MI.getOpcode() == TargetOpcode::PATCHPOINT && "expected patchpoint"); 237 238 PatchPointOpers opers(&MI); 239 int64_t ID = opers.getMetaOper(PatchPointOpers::IDPos).getImm(); 240 241 MachineInstr::const_mop_iterator MOI = 242 llvm::next(MI.operands_begin(), opers.getStackMapStartIdx()); 243 recordStackMapOpers(MI, ID, MOI, MI.operands_end(), 244 opers.isAnyReg() && opers.hasDef()); 245 246 #ifndef NDEBUG 247 // verify anyregcc 248 LocationVec &Locations = CSInfos.back().Locations; 249 if (opers.isAnyReg()) { 250 unsigned NArgs = opers.getMetaOper(PatchPointOpers::NArgPos).getImm(); 251 for (unsigned i = 0, e = (opers.hasDef() ? NArgs+1 : NArgs); i != e; ++i) 252 assert(Locations[i].LocType == Location::Register && 253 "anyreg arg must be in reg."); 254 } 255 #endif 256 } 257 258 /// serializeToStackMapSection conceptually populates the following fields: 259 /// 260 /// uint32 : Reserved (header) 261 /// uint32 : NumConstants 262 /// int64 : Constants[NumConstants] 263 /// uint32 : NumRecords 264 /// StkMapRecord[NumRecords] { 265 /// uint64 : PatchPoint ID 266 /// uint32 : Instruction Offset 267 /// uint16 : Reserved (record flags) 268 /// uint16 : NumLocations 269 /// Location[NumLocations] { 270 /// uint8 : Register | Direct | Indirect | Constant | ConstantIndex 271 /// uint8 : Size in Bytes 272 /// uint16 : Dwarf RegNum 273 /// int32 : Offset 274 /// } 275 /// uint16 : NumLiveOuts 276 /// LiveOuts[NumLiveOuts] 277 /// uint16 : Dwarf RegNum 278 /// uint8 : Reserved 279 /// uint8 : Size in Bytes 280 /// } 281 /// 282 /// Location Encoding, Type, Value: 283 /// 0x1, Register, Reg (value in register) 284 /// 0x2, Direct, Reg + Offset (frame index) 285 /// 0x3, Indirect, [Reg + Offset] (spilled value) 286 /// 0x4, Constant, Offset (small constant) 287 /// 0x5, ConstIndex, Constants[Offset] (large constant) 288 /// 289 void StackMaps::serializeToStackMapSection() { 290 // Bail out if there's no stack map data. 291 if (CSInfos.empty()) 292 return; 293 294 MCContext &OutContext = AP.OutStreamer.getContext(); 295 const TargetRegisterInfo *TRI = AP.TM.getRegisterInfo(); 296 297 // Create the section. 298 const MCSection *StackMapSection = 299 OutContext.getObjectFileInfo()->getStackMapSection(); 300 AP.OutStreamer.SwitchSection(StackMapSection); 301 302 // Emit a dummy symbol to force section inclusion. 303 AP.OutStreamer.EmitLabel( 304 OutContext.GetOrCreateSymbol(Twine("__LLVM_StackMaps"))); 305 306 // Serialize data. 307 const char *WSMP = "Stack Maps: "; 308 (void)WSMP; 309 const MCRegisterInfo &MCRI = *OutContext.getRegisterInfo(); 310 311 DEBUG(dbgs() << "********** Stack Map Output **********\n"); 312 313 // Header. 314 AP.OutStreamer.EmitIntValue(0, 4); 315 316 // Num constants. 317 AP.OutStreamer.EmitIntValue(ConstPool.getNumConstants(), 4); 318 319 // Constant pool entries. 320 for (unsigned i = 0; i < ConstPool.getNumConstants(); ++i) 321 AP.OutStreamer.EmitIntValue(ConstPool.getConstant(i), 8); 322 323 DEBUG(dbgs() << WSMP << "#callsites = " << CSInfos.size() << "\n"); 324 AP.OutStreamer.EmitIntValue(CSInfos.size(), 4); 325 326 for (CallsiteInfoList::const_iterator CSII = CSInfos.begin(), 327 CSIE = CSInfos.end(); 328 CSII != CSIE; ++CSII) { 329 330 uint64_t CallsiteID = CSII->ID; 331 const LocationVec &CSLocs = CSII->Locations; 332 const LiveOutVec &LiveOuts = CSII->LiveOuts; 333 334 DEBUG(dbgs() << WSMP << "callsite " << CallsiteID << "\n"); 335 336 // Verify stack map entry. It's better to communicate a problem to the 337 // runtime than crash in case of in-process compilation. Currently, we do 338 // simple overflow checks, but we may eventually communicate other 339 // compilation errors this way. 340 if (CSLocs.size() > UINT16_MAX || LiveOuts.size() > UINT16_MAX) { 341 AP.OutStreamer.EmitIntValue(UINT64_MAX, 8); // Invalid ID. 342 AP.OutStreamer.EmitValue(CSII->CSOffsetExpr, 4); 343 AP.OutStreamer.EmitIntValue(0, 2); // Reserved. 344 AP.OutStreamer.EmitIntValue(0, 2); // 0 locations. 345 AP.OutStreamer.EmitIntValue(0, 2); // 0 live-out registers. 346 continue; 347 } 348 349 AP.OutStreamer.EmitIntValue(CallsiteID, 8); 350 AP.OutStreamer.EmitValue(CSII->CSOffsetExpr, 4); 351 352 // Reserved for flags. 353 AP.OutStreamer.EmitIntValue(0, 2); 354 355 DEBUG(dbgs() << WSMP << " has " << CSLocs.size() << " locations\n"); 356 357 AP.OutStreamer.EmitIntValue(CSLocs.size(), 2); 358 359 unsigned operIdx = 0; 360 for (LocationVec::const_iterator LocI = CSLocs.begin(), LocE = CSLocs.end(); 361 LocI != LocE; ++LocI, ++operIdx) { 362 const Location &Loc = *LocI; 363 unsigned RegNo = 0; 364 int Offset = Loc.Offset; 365 if(Loc.Reg) { 366 RegNo = MCRI.getDwarfRegNum(Loc.Reg, false); 367 for (MCSuperRegIterator SR(Loc.Reg, TRI); 368 SR.isValid() && (int)RegNo < 0; ++SR) { 369 RegNo = TRI->getDwarfRegNum(*SR, false); 370 } 371 // If this is a register location, put the subregister byte offset in 372 // the location offset. 373 if (Loc.LocType == Location::Register) { 374 assert(!Loc.Offset && "Register location should have zero offset"); 375 unsigned LLVMRegNo = MCRI.getLLVMRegNum(RegNo, false); 376 unsigned SubRegIdx = MCRI.getSubRegIndex(LLVMRegNo, Loc.Reg); 377 if (SubRegIdx) 378 Offset = MCRI.getSubRegIdxOffset(SubRegIdx); 379 } 380 } 381 else { 382 assert(Loc.LocType != Location::Register && 383 "Missing location register"); 384 } 385 386 DEBUG( 387 dbgs() << WSMP << " Loc " << operIdx << ": "; 388 switch (Loc.LocType) { 389 case Location::Unprocessed: 390 dbgs() << "<Unprocessed operand>"; 391 break; 392 case Location::Register: 393 dbgs() << "Register " << MCRI.getName(Loc.Reg); 394 break; 395 case Location::Direct: 396 dbgs() << "Direct " << MCRI.getName(Loc.Reg); 397 if (Loc.Offset) 398 dbgs() << " + " << Loc.Offset; 399 break; 400 case Location::Indirect: 401 dbgs() << "Indirect " << MCRI.getName(Loc.Reg) 402 << " + " << Loc.Offset; 403 break; 404 case Location::Constant: 405 dbgs() << "Constant " << Loc.Offset; 406 break; 407 case Location::ConstantIndex: 408 dbgs() << "Constant Index " << Loc.Offset; 409 break; 410 } 411 dbgs() << " [encoding: .byte " << Loc.LocType 412 << ", .byte " << Loc.Size 413 << ", .short " << RegNo 414 << ", .int " << Offset << "]\n"; 415 ); 416 417 AP.OutStreamer.EmitIntValue(Loc.LocType, 1); 418 AP.OutStreamer.EmitIntValue(Loc.Size, 1); 419 AP.OutStreamer.EmitIntValue(RegNo, 2); 420 AP.OutStreamer.EmitIntValue(Offset, 4); 421 } 422 423 DEBUG(dbgs() << WSMP << " has " << LiveOuts.size() 424 << " live-out registers\n"); 425 426 AP.OutStreamer.EmitIntValue(LiveOuts.size(), 2); 427 428 operIdx = 0; 429 for (LiveOutVec::const_iterator LI = LiveOuts.begin(), LE = LiveOuts.end(); 430 LI != LE; ++LI, ++operIdx) { 431 DEBUG(dbgs() << WSMP << " LO " << operIdx << ": " 432 << MCRI.getName(LI->Reg) 433 << " [encoding: .short " << LI->RegNo 434 << ", .byte 0, .byte " << LI->Size << "]\n"); 435 436 AP.OutStreamer.EmitIntValue(LI->RegNo, 2); 437 AP.OutStreamer.EmitIntValue(0, 1); 438 AP.OutStreamer.EmitIntValue(LI->Size, 1); 439 } 440 } 441 442 AP.OutStreamer.AddBlankLine(); 443 444 CSInfos.clear(); 445 } 446