1 //===- llvm/ModuleSummaryIndex.h - Module Summary Index ---------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// @file 10 /// ModuleSummaryIndex.h This file contains the declarations the classes that 11 /// hold the module index and summary for function importing. 12 // 13 //===----------------------------------------------------------------------===// 14 15 #ifndef LLVM_IR_MODULESUMMARYINDEX_H 16 #define LLVM_IR_MODULESUMMARYINDEX_H 17 18 #include "llvm/ADT/ArrayRef.h" 19 #include "llvm/ADT/DenseMap.h" 20 #include "llvm/ADT/STLExtras.h" 21 #include "llvm/ADT/SmallString.h" 22 #include "llvm/ADT/SmallVector.h" 23 #include "llvm/ADT/StringExtras.h" 24 #include "llvm/ADT/StringMap.h" 25 #include "llvm/ADT/StringRef.h" 26 #include "llvm/IR/ConstantRange.h" 27 #include "llvm/IR/GlobalValue.h" 28 #include "llvm/IR/Module.h" 29 #include "llvm/Support/Allocator.h" 30 #include "llvm/Support/MathExtras.h" 31 #include "llvm/Support/ScaledNumber.h" 32 #include "llvm/Support/StringSaver.h" 33 #include "llvm/Support/raw_ostream.h" 34 #include <algorithm> 35 #include <array> 36 #include <cassert> 37 #include <cstddef> 38 #include <cstdint> 39 #include <map> 40 #include <memory> 41 #include <optional> 42 #include <set> 43 #include <string> 44 #include <unordered_set> 45 #include <utility> 46 #include <vector> 47 48 namespace llvm { 49 50 template <class GraphType> struct GraphTraits; 51 52 namespace yaml { 53 54 template <typename T> struct MappingTraits; 55 56 } // end namespace yaml 57 58 /// Class to accumulate and hold information about a callee. 59 struct CalleeInfo { 60 enum class HotnessType : uint8_t { 61 Unknown = 0, 62 Cold = 1, 63 None = 2, 64 Hot = 3, 65 Critical = 4 66 }; 67 68 // The size of the bit-field might need to be adjusted if more values are 69 // added to HotnessType enum. 70 uint32_t Hotness : 3; 71 72 // True if at least one of the calls to the callee is a tail call. 73 bool HasTailCall : 1; 74 75 /// The value stored in RelBlockFreq has to be interpreted as the digits of 76 /// a scaled number with a scale of \p -ScaleShift. 77 static constexpr unsigned RelBlockFreqBits = 28; 78 uint32_t RelBlockFreq : RelBlockFreqBits; 79 static constexpr int32_t ScaleShift = 8; 80 static constexpr uint64_t MaxRelBlockFreq = (1 << RelBlockFreqBits) - 1; 81 82 CalleeInfo() 83 : Hotness(static_cast<uint32_t>(HotnessType::Unknown)), 84 HasTailCall(false), RelBlockFreq(0) {} 85 explicit CalleeInfo(HotnessType Hotness, bool HasTC, uint64_t RelBF) 86 : Hotness(static_cast<uint32_t>(Hotness)), HasTailCall(HasTC), 87 RelBlockFreq(RelBF) {} 88 89 void updateHotness(const HotnessType OtherHotness) { 90 Hotness = std::max(Hotness, static_cast<uint32_t>(OtherHotness)); 91 } 92 93 bool hasTailCall() const { return HasTailCall; } 94 95 void setHasTailCall(const bool HasTC) { HasTailCall = HasTC; } 96 97 HotnessType getHotness() const { return HotnessType(Hotness); } 98 99 /// Update \p RelBlockFreq from \p BlockFreq and \p EntryFreq 100 /// 101 /// BlockFreq is divided by EntryFreq and added to RelBlockFreq. To represent 102 /// fractional values, the result is represented as a fixed point number with 103 /// scale of -ScaleShift. 104 void updateRelBlockFreq(uint64_t BlockFreq, uint64_t EntryFreq) { 105 if (EntryFreq == 0) 106 return; 107 using Scaled64 = ScaledNumber<uint64_t>; 108 Scaled64 Temp(BlockFreq, ScaleShift); 109 Temp /= Scaled64::get(EntryFreq); 110 111 uint64_t Sum = 112 SaturatingAdd<uint64_t>(Temp.toInt<uint64_t>(), RelBlockFreq); 113 Sum = std::min(Sum, uint64_t(MaxRelBlockFreq)); 114 RelBlockFreq = static_cast<uint32_t>(Sum); 115 } 116 }; 117 118 inline const char *getHotnessName(CalleeInfo::HotnessType HT) { 119 switch (HT) { 120 case CalleeInfo::HotnessType::Unknown: 121 return "unknown"; 122 case CalleeInfo::HotnessType::Cold: 123 return "cold"; 124 case CalleeInfo::HotnessType::None: 125 return "none"; 126 case CalleeInfo::HotnessType::Hot: 127 return "hot"; 128 case CalleeInfo::HotnessType::Critical: 129 return "critical"; 130 } 131 llvm_unreachable("invalid hotness"); 132 } 133 134 class GlobalValueSummary; 135 136 using GlobalValueSummaryList = std::vector<std::unique_ptr<GlobalValueSummary>>; 137 138 struct alignas(8) GlobalValueSummaryInfo { 139 union NameOrGV { 140 NameOrGV(bool HaveGVs) { 141 if (HaveGVs) 142 GV = nullptr; 143 else 144 Name = ""; 145 } 146 147 /// The GlobalValue corresponding to this summary. This is only used in 148 /// per-module summaries and when the IR is available. E.g. when module 149 /// analysis is being run, or when parsing both the IR and the summary 150 /// from assembly. 151 const GlobalValue *GV; 152 153 /// Summary string representation. This StringRef points to BC module 154 /// string table and is valid until module data is stored in memory. 155 /// This is guaranteed to happen until runThinLTOBackend function is 156 /// called, so it is safe to use this field during thin link. This field 157 /// is only valid if summary index was loaded from BC file. 158 StringRef Name; 159 } U; 160 161 inline GlobalValueSummaryInfo(bool HaveGVs); 162 163 /// List of global value summary structures for a particular value held 164 /// in the GlobalValueMap. Requires a vector in the case of multiple 165 /// COMDAT values of the same name. 166 GlobalValueSummaryList SummaryList; 167 }; 168 169 /// Map from global value GUID to corresponding summary structures. Use a 170 /// std::map rather than a DenseMap so that pointers to the map's value_type 171 /// (which are used by ValueInfo) are not invalidated by insertion. Also it will 172 /// likely incur less overhead, as the value type is not very small and the size 173 /// of the map is unknown, resulting in inefficiencies due to repeated 174 /// insertions and resizing. 175 using GlobalValueSummaryMapTy = 176 std::map<GlobalValue::GUID, GlobalValueSummaryInfo>; 177 178 /// Struct that holds a reference to a particular GUID in a global value 179 /// summary. 180 struct ValueInfo { 181 enum Flags { HaveGV = 1, ReadOnly = 2, WriteOnly = 4 }; 182 PointerIntPair<const GlobalValueSummaryMapTy::value_type *, 3, int> 183 RefAndFlags; 184 185 ValueInfo() = default; 186 ValueInfo(bool HaveGVs, const GlobalValueSummaryMapTy::value_type *R) { 187 RefAndFlags.setPointer(R); 188 RefAndFlags.setInt(HaveGVs); 189 } 190 191 explicit operator bool() const { return getRef(); } 192 193 GlobalValue::GUID getGUID() const { return getRef()->first; } 194 const GlobalValue *getValue() const { 195 assert(haveGVs()); 196 return getRef()->second.U.GV; 197 } 198 199 ArrayRef<std::unique_ptr<GlobalValueSummary>> getSummaryList() const { 200 return getRef()->second.SummaryList; 201 } 202 203 // Even if the index is built with GVs available, we may not have one for 204 // summary entries synthesized for profiled indirect call targets. 205 bool hasName() const { return !haveGVs() || getValue(); } 206 207 StringRef name() const { 208 assert(!haveGVs() || getRef()->second.U.GV); 209 return haveGVs() ? getRef()->second.U.GV->getName() 210 : getRef()->second.U.Name; 211 } 212 213 bool haveGVs() const { return RefAndFlags.getInt() & HaveGV; } 214 bool isReadOnly() const { 215 assert(isValidAccessSpecifier()); 216 return RefAndFlags.getInt() & ReadOnly; 217 } 218 bool isWriteOnly() const { 219 assert(isValidAccessSpecifier()); 220 return RefAndFlags.getInt() & WriteOnly; 221 } 222 unsigned getAccessSpecifier() const { 223 assert(isValidAccessSpecifier()); 224 return RefAndFlags.getInt() & (ReadOnly | WriteOnly); 225 } 226 bool isValidAccessSpecifier() const { 227 unsigned BadAccessMask = ReadOnly | WriteOnly; 228 return (RefAndFlags.getInt() & BadAccessMask) != BadAccessMask; 229 } 230 void setReadOnly() { 231 // We expect ro/wo attribute to set only once during 232 // ValueInfo lifetime. 233 assert(getAccessSpecifier() == 0); 234 RefAndFlags.setInt(RefAndFlags.getInt() | ReadOnly); 235 } 236 void setWriteOnly() { 237 assert(getAccessSpecifier() == 0); 238 RefAndFlags.setInt(RefAndFlags.getInt() | WriteOnly); 239 } 240 241 const GlobalValueSummaryMapTy::value_type *getRef() const { 242 return RefAndFlags.getPointer(); 243 } 244 245 /// Returns the most constraining visibility among summaries. The 246 /// visibilities, ordered from least to most constraining, are: default, 247 /// protected and hidden. 248 GlobalValue::VisibilityTypes getELFVisibility() const; 249 250 /// Checks if all summaries are DSO local (have the flag set). When DSOLocal 251 /// propagation has been done, set the parameter to enable fast check. 252 bool isDSOLocal(bool WithDSOLocalPropagation = false) const; 253 254 /// Checks if all copies are eligible for auto-hiding (have flag set). 255 bool canAutoHide() const; 256 }; 257 258 inline raw_ostream &operator<<(raw_ostream &OS, const ValueInfo &VI) { 259 OS << VI.getGUID(); 260 if (!VI.name().empty()) 261 OS << " (" << VI.name() << ")"; 262 return OS; 263 } 264 265 inline bool operator==(const ValueInfo &A, const ValueInfo &B) { 266 assert(A.getRef() && B.getRef() && 267 "Need ValueInfo with non-null Ref for comparison"); 268 return A.getRef() == B.getRef(); 269 } 270 271 inline bool operator!=(const ValueInfo &A, const ValueInfo &B) { 272 assert(A.getRef() && B.getRef() && 273 "Need ValueInfo with non-null Ref for comparison"); 274 return A.getRef() != B.getRef(); 275 } 276 277 inline bool operator<(const ValueInfo &A, const ValueInfo &B) { 278 assert(A.getRef() && B.getRef() && 279 "Need ValueInfo with non-null Ref to compare GUIDs"); 280 return A.getGUID() < B.getGUID(); 281 } 282 283 template <> struct DenseMapInfo<ValueInfo> { 284 static inline ValueInfo getEmptyKey() { 285 return ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-8); 286 } 287 288 static inline ValueInfo getTombstoneKey() { 289 return ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-16); 290 } 291 292 static inline bool isSpecialKey(ValueInfo V) { 293 return V == getTombstoneKey() || V == getEmptyKey(); 294 } 295 296 static bool isEqual(ValueInfo L, ValueInfo R) { 297 // We are not supposed to mix ValueInfo(s) with different HaveGVs flag 298 // in a same container. 299 assert(isSpecialKey(L) || isSpecialKey(R) || (L.haveGVs() == R.haveGVs())); 300 return L.getRef() == R.getRef(); 301 } 302 static unsigned getHashValue(ValueInfo I) { return (uintptr_t)I.getRef(); } 303 }; 304 305 // For optional hinted size reporting, holds a pair of the full stack id 306 // (pre-trimming, from the full context in the profile), and the associated 307 // total profiled size. 308 struct ContextTotalSize { 309 uint64_t FullStackId; 310 uint64_t TotalSize; 311 }; 312 313 /// Summary of memprof callsite metadata. 314 struct CallsiteInfo { 315 // Actual callee function. 316 ValueInfo Callee; 317 318 // Used to record whole program analysis cloning decisions. 319 // The ThinLTO backend will need to create as many clones as there are entries 320 // in the vector (it is expected and should be confirmed that all such 321 // summaries in the same FunctionSummary have the same number of entries). 322 // Each index records version info for the corresponding clone of this 323 // function. The value is the callee clone it calls (becomes the appended 324 // suffix id). Index 0 is the original version, and a value of 0 calls the 325 // original callee. 326 SmallVector<unsigned> Clones{0}; 327 328 // Represents stack ids in this context, recorded as indices into the 329 // StackIds vector in the summary index, which in turn holds the full 64-bit 330 // stack ids. This reduces memory as there are in practice far fewer unique 331 // stack ids than stack id references. 332 SmallVector<unsigned> StackIdIndices; 333 334 CallsiteInfo(ValueInfo Callee, SmallVector<unsigned> StackIdIndices) 335 : Callee(Callee), StackIdIndices(std::move(StackIdIndices)) {} 336 CallsiteInfo(ValueInfo Callee, SmallVector<unsigned> Clones, 337 SmallVector<unsigned> StackIdIndices) 338 : Callee(Callee), Clones(std::move(Clones)), 339 StackIdIndices(std::move(StackIdIndices)) {} 340 }; 341 342 inline raw_ostream &operator<<(raw_ostream &OS, const CallsiteInfo &SNI) { 343 OS << "Callee: " << SNI.Callee; 344 bool First = true; 345 OS << " Clones: "; 346 for (auto V : SNI.Clones) { 347 if (!First) 348 OS << ", "; 349 First = false; 350 OS << V; 351 } 352 First = true; 353 OS << " StackIds: "; 354 for (auto Id : SNI.StackIdIndices) { 355 if (!First) 356 OS << ", "; 357 First = false; 358 OS << Id; 359 } 360 return OS; 361 } 362 363 // Allocation type assigned to an allocation reached by a given context. 364 // More can be added, now this is cold, notcold and hot. 365 // Values should be powers of two so that they can be ORed, in particular to 366 // track allocations that have different behavior with different calling 367 // contexts. 368 enum class AllocationType : uint8_t { 369 None = 0, 370 NotCold = 1, 371 Cold = 2, 372 Hot = 4, 373 All = 7 // This should always be set to the OR of all values. 374 }; 375 376 /// Summary of a single MIB in a memprof metadata on allocations. 377 struct MIBInfo { 378 // The allocation type for this profiled context. 379 AllocationType AllocType; 380 381 // Represents stack ids in this context, recorded as indices into the 382 // StackIds vector in the summary index, which in turn holds the full 64-bit 383 // stack ids. This reduces memory as there are in practice far fewer unique 384 // stack ids than stack id references. 385 SmallVector<unsigned> StackIdIndices; 386 387 MIBInfo(AllocationType AllocType, SmallVector<unsigned> StackIdIndices) 388 : AllocType(AllocType), StackIdIndices(std::move(StackIdIndices)) {} 389 }; 390 391 inline raw_ostream &operator<<(raw_ostream &OS, const MIBInfo &MIB) { 392 OS << "AllocType " << (unsigned)MIB.AllocType; 393 bool First = true; 394 OS << " StackIds: "; 395 for (auto Id : MIB.StackIdIndices) { 396 if (!First) 397 OS << ", "; 398 First = false; 399 OS << Id; 400 } 401 return OS; 402 } 403 404 /// Summary of memprof metadata on allocations. 405 struct AllocInfo { 406 // Used to record whole program analysis cloning decisions. 407 // The ThinLTO backend will need to create as many clones as there are entries 408 // in the vector (it is expected and should be confirmed that all such 409 // summaries in the same FunctionSummary have the same number of entries). 410 // Each index records version info for the corresponding clone of this 411 // function. The value is the allocation type of the corresponding allocation. 412 // Index 0 is the original version. Before cloning, index 0 may have more than 413 // one allocation type. 414 SmallVector<uint8_t> Versions; 415 416 // Vector of MIBs in this memprof metadata. 417 std::vector<MIBInfo> MIBs; 418 419 // If requested, keep track of full stack contexts and total profiled sizes 420 // for each MIB. This will be a vector of the same length and order as the 421 // MIBs vector, if non-empty. Note that each MIB in the summary can have 422 // multiple of these as we trim the contexts when possible during matching. 423 // For hinted size reporting we, however, want the original pre-trimmed full 424 // stack context id for better correlation with the profile. 425 std::vector<std::vector<ContextTotalSize>> ContextSizeInfos; 426 427 AllocInfo(std::vector<MIBInfo> MIBs) : MIBs(std::move(MIBs)) { 428 Versions.push_back(0); 429 } 430 AllocInfo(SmallVector<uint8_t> Versions, std::vector<MIBInfo> MIBs) 431 : Versions(std::move(Versions)), MIBs(std::move(MIBs)) {} 432 }; 433 434 inline raw_ostream &operator<<(raw_ostream &OS, const AllocInfo &AE) { 435 bool First = true; 436 OS << "Versions: "; 437 for (auto V : AE.Versions) { 438 if (!First) 439 OS << ", "; 440 First = false; 441 OS << (unsigned)V; 442 } 443 OS << " MIB:\n"; 444 for (auto &M : AE.MIBs) { 445 OS << "\t\t" << M << "\n"; 446 } 447 if (!AE.ContextSizeInfos.empty()) { 448 OS << "\tContextSizeInfo per MIB:\n"; 449 for (auto Infos : AE.ContextSizeInfos) { 450 OS << "\t\t"; 451 bool FirstInfo = true; 452 for (auto [FullStackId, TotalSize] : Infos) { 453 if (!FirstInfo) 454 OS << ", "; 455 FirstInfo = false; 456 OS << "{ " << FullStackId << ", " << TotalSize << " }"; 457 } 458 OS << "\n"; 459 } 460 } 461 return OS; 462 } 463 464 /// Function and variable summary information to aid decisions and 465 /// implementation of importing. 466 class GlobalValueSummary { 467 public: 468 /// Sububclass discriminator (for dyn_cast<> et al.) 469 enum SummaryKind : unsigned { AliasKind, FunctionKind, GlobalVarKind }; 470 471 enum ImportKind : unsigned { 472 // The global value definition corresponding to the summary should be 473 // imported from source module 474 Definition = 0, 475 476 // When its definition doesn't exist in the destination module and not 477 // imported (e.g., function is too large to be inlined), the global value 478 // declaration corresponding to the summary should be imported, or the 479 // attributes from summary should be annotated on the function declaration. 480 Declaration = 1, 481 }; 482 483 /// Group flags (Linkage, NotEligibleToImport, etc.) as a bitfield. 484 struct GVFlags { 485 /// The linkage type of the associated global value. 486 /// 487 /// One use is to flag values that have local linkage types and need to 488 /// have module identifier appended before placing into the combined 489 /// index, to disambiguate from other values with the same name. 490 /// In the future this will be used to update and optimize linkage 491 /// types based on global summary-based analysis. 492 unsigned Linkage : 4; 493 494 /// Indicates the visibility. 495 unsigned Visibility : 2; 496 497 /// Indicate if the global value cannot be imported (e.g. it cannot 498 /// be renamed or references something that can't be renamed). 499 unsigned NotEligibleToImport : 1; 500 501 /// In per-module summary, indicate that the global value must be considered 502 /// a live root for index-based liveness analysis. Used for special LLVM 503 /// values such as llvm.global_ctors that the linker does not know about. 504 /// 505 /// In combined summary, indicate that the global value is live. 506 unsigned Live : 1; 507 508 /// Indicates that the linker resolved the symbol to a definition from 509 /// within the same linkage unit. 510 unsigned DSOLocal : 1; 511 512 /// In the per-module summary, indicates that the global value is 513 /// linkonce_odr and global unnamed addr (so eligible for auto-hiding 514 /// via hidden visibility). In the combined summary, indicates that the 515 /// prevailing linkonce_odr copy can be auto-hidden via hidden visibility 516 /// when it is upgraded to weak_odr in the backend. This is legal when 517 /// all copies are eligible for auto-hiding (i.e. all copies were 518 /// linkonce_odr global unnamed addr. If any copy is not (e.g. it was 519 /// originally weak_odr, we cannot auto-hide the prevailing copy as it 520 /// means the symbol was externally visible. 521 unsigned CanAutoHide : 1; 522 523 /// This field is written by the ThinLTO indexing step to postlink combined 524 /// summary. The value is interpreted as 'ImportKind' enum defined above. 525 unsigned ImportType : 1; 526 527 /// Convenience Constructors 528 explicit GVFlags(GlobalValue::LinkageTypes Linkage, 529 GlobalValue::VisibilityTypes Visibility, 530 bool NotEligibleToImport, bool Live, bool IsLocal, 531 bool CanAutoHide, ImportKind ImportType) 532 : Linkage(Linkage), Visibility(Visibility), 533 NotEligibleToImport(NotEligibleToImport), Live(Live), 534 DSOLocal(IsLocal), CanAutoHide(CanAutoHide), 535 ImportType(static_cast<unsigned>(ImportType)) {} 536 }; 537 538 private: 539 /// Kind of summary for use in dyn_cast<> et al. 540 SummaryKind Kind; 541 542 GVFlags Flags; 543 544 /// This is the hash of the name of the symbol in the original file. It is 545 /// identical to the GUID for global symbols, but differs for local since the 546 /// GUID includes the module level id in the hash. 547 GlobalValue::GUID OriginalName = 0; 548 549 /// Path of module IR containing value's definition, used to locate 550 /// module during importing. 551 /// 552 /// This is only used during parsing of the combined index, or when 553 /// parsing the per-module index for creation of the combined summary index, 554 /// not during writing of the per-module index which doesn't contain a 555 /// module path string table. 556 StringRef ModulePath; 557 558 /// List of values referenced by this global value's definition 559 /// (either by the initializer of a global variable, or referenced 560 /// from within a function). This does not include functions called, which 561 /// are listed in the derived FunctionSummary object. 562 /// We use SmallVector<ValueInfo, 0> instead of std::vector<ValueInfo> for its 563 /// smaller memory footprint. 564 SmallVector<ValueInfo, 0> RefEdgeList; 565 566 protected: 567 GlobalValueSummary(SummaryKind K, GVFlags Flags, 568 SmallVectorImpl<ValueInfo> &&Refs) 569 : Kind(K), Flags(Flags), RefEdgeList(std::move(Refs)) { 570 assert((K != AliasKind || Refs.empty()) && 571 "Expect no references for AliasSummary"); 572 } 573 574 public: 575 virtual ~GlobalValueSummary() = default; 576 577 /// Returns the hash of the original name, it is identical to the GUID for 578 /// externally visible symbols, but not for local ones. 579 GlobalValue::GUID getOriginalName() const { return OriginalName; } 580 581 /// Initialize the original name hash in this summary. 582 void setOriginalName(GlobalValue::GUID Name) { OriginalName = Name; } 583 584 /// Which kind of summary subclass this is. 585 SummaryKind getSummaryKind() const { return Kind; } 586 587 /// Set the path to the module containing this function, for use in 588 /// the combined index. 589 void setModulePath(StringRef ModPath) { ModulePath = ModPath; } 590 591 /// Get the path to the module containing this function. 592 StringRef modulePath() const { return ModulePath; } 593 594 /// Get the flags for this GlobalValue (see \p struct GVFlags). 595 GVFlags flags() const { return Flags; } 596 597 /// Return linkage type recorded for this global value. 598 GlobalValue::LinkageTypes linkage() const { 599 return static_cast<GlobalValue::LinkageTypes>(Flags.Linkage); 600 } 601 602 /// Sets the linkage to the value determined by global summary-based 603 /// optimization. Will be applied in the ThinLTO backends. 604 void setLinkage(GlobalValue::LinkageTypes Linkage) { 605 Flags.Linkage = Linkage; 606 } 607 608 /// Return true if this global value can't be imported. 609 bool notEligibleToImport() const { return Flags.NotEligibleToImport; } 610 611 bool isLive() const { return Flags.Live; } 612 613 void setLive(bool Live) { Flags.Live = Live; } 614 615 void setDSOLocal(bool Local) { Flags.DSOLocal = Local; } 616 617 bool isDSOLocal() const { return Flags.DSOLocal; } 618 619 void setCanAutoHide(bool CanAutoHide) { Flags.CanAutoHide = CanAutoHide; } 620 621 bool canAutoHide() const { return Flags.CanAutoHide; } 622 623 bool shouldImportAsDecl() const { 624 return Flags.ImportType == GlobalValueSummary::ImportKind::Declaration; 625 } 626 627 void setImportKind(ImportKind IK) { Flags.ImportType = IK; } 628 629 GlobalValueSummary::ImportKind importType() const { 630 return static_cast<ImportKind>(Flags.ImportType); 631 } 632 633 GlobalValue::VisibilityTypes getVisibility() const { 634 return (GlobalValue::VisibilityTypes)Flags.Visibility; 635 } 636 void setVisibility(GlobalValue::VisibilityTypes Vis) { 637 Flags.Visibility = (unsigned)Vis; 638 } 639 640 /// Flag that this global value cannot be imported. 641 void setNotEligibleToImport() { Flags.NotEligibleToImport = true; } 642 643 /// Return the list of values referenced by this global value definition. 644 ArrayRef<ValueInfo> refs() const { return RefEdgeList; } 645 646 /// If this is an alias summary, returns the summary of the aliased object (a 647 /// global variable or function), otherwise returns itself. 648 GlobalValueSummary *getBaseObject(); 649 const GlobalValueSummary *getBaseObject() const; 650 651 friend class ModuleSummaryIndex; 652 }; 653 654 GlobalValueSummaryInfo::GlobalValueSummaryInfo(bool HaveGVs) : U(HaveGVs) {} 655 656 /// Alias summary information. 657 class AliasSummary : public GlobalValueSummary { 658 ValueInfo AliaseeValueInfo; 659 660 /// This is the Aliasee in the same module as alias (could get from VI, trades 661 /// memory for time). Note that this pointer may be null (and the value info 662 /// empty) when we have a distributed index where the alias is being imported 663 /// (as a copy of the aliasee), but the aliasee is not. 664 GlobalValueSummary *AliaseeSummary = nullptr; 665 666 public: 667 AliasSummary(GVFlags Flags) 668 : GlobalValueSummary(AliasKind, Flags, SmallVector<ValueInfo, 0>{}) {} 669 670 /// Check if this is an alias summary. 671 static bool classof(const GlobalValueSummary *GVS) { 672 return GVS->getSummaryKind() == AliasKind; 673 } 674 675 void setAliasee(ValueInfo &AliaseeVI, GlobalValueSummary *Aliasee) { 676 AliaseeValueInfo = AliaseeVI; 677 AliaseeSummary = Aliasee; 678 } 679 680 bool hasAliasee() const { 681 assert(!!AliaseeSummary == (AliaseeValueInfo && 682 !AliaseeValueInfo.getSummaryList().empty()) && 683 "Expect to have both aliasee summary and summary list or neither"); 684 return !!AliaseeSummary; 685 } 686 687 const GlobalValueSummary &getAliasee() const { 688 assert(AliaseeSummary && "Unexpected missing aliasee summary"); 689 return *AliaseeSummary; 690 } 691 692 GlobalValueSummary &getAliasee() { 693 return const_cast<GlobalValueSummary &>( 694 static_cast<const AliasSummary *>(this)->getAliasee()); 695 } 696 ValueInfo getAliaseeVI() const { 697 assert(AliaseeValueInfo && "Unexpected missing aliasee"); 698 return AliaseeValueInfo; 699 } 700 GlobalValue::GUID getAliaseeGUID() const { 701 assert(AliaseeValueInfo && "Unexpected missing aliasee"); 702 return AliaseeValueInfo.getGUID(); 703 } 704 }; 705 706 const inline GlobalValueSummary *GlobalValueSummary::getBaseObject() const { 707 if (auto *AS = dyn_cast<AliasSummary>(this)) 708 return &AS->getAliasee(); 709 return this; 710 } 711 712 inline GlobalValueSummary *GlobalValueSummary::getBaseObject() { 713 if (auto *AS = dyn_cast<AliasSummary>(this)) 714 return &AS->getAliasee(); 715 return this; 716 } 717 718 /// Function summary information to aid decisions and implementation of 719 /// importing. 720 class FunctionSummary : public GlobalValueSummary { 721 public: 722 /// <CalleeValueInfo, CalleeInfo> call edge pair. 723 using EdgeTy = std::pair<ValueInfo, CalleeInfo>; 724 725 /// Types for -force-summary-edges-cold debugging option. 726 enum ForceSummaryHotnessType : unsigned { 727 FSHT_None, 728 FSHT_AllNonCritical, 729 FSHT_All 730 }; 731 732 /// An "identifier" for a virtual function. This contains the type identifier 733 /// represented as a GUID and the offset from the address point to the virtual 734 /// function pointer, where "address point" is as defined in the Itanium ABI: 735 /// https://itanium-cxx-abi.github.io/cxx-abi/abi.html#vtable-general 736 struct VFuncId { 737 GlobalValue::GUID GUID; 738 uint64_t Offset; 739 }; 740 741 /// A specification for a virtual function call with all constant integer 742 /// arguments. This is used to perform virtual constant propagation on the 743 /// summary. 744 struct ConstVCall { 745 VFuncId VFunc; 746 std::vector<uint64_t> Args; 747 }; 748 749 /// All type identifier related information. Because these fields are 750 /// relatively uncommon we only allocate space for them if necessary. 751 struct TypeIdInfo { 752 /// List of type identifiers used by this function in llvm.type.test 753 /// intrinsics referenced by something other than an llvm.assume intrinsic, 754 /// represented as GUIDs. 755 std::vector<GlobalValue::GUID> TypeTests; 756 757 /// List of virtual calls made by this function using (respectively) 758 /// llvm.assume(llvm.type.test) or llvm.type.checked.load intrinsics that do 759 /// not have all constant integer arguments. 760 std::vector<VFuncId> TypeTestAssumeVCalls, TypeCheckedLoadVCalls; 761 762 /// List of virtual calls made by this function using (respectively) 763 /// llvm.assume(llvm.type.test) or llvm.type.checked.load intrinsics with 764 /// all constant integer arguments. 765 std::vector<ConstVCall> TypeTestAssumeConstVCalls, 766 TypeCheckedLoadConstVCalls; 767 }; 768 769 /// Flags specific to function summaries. 770 struct FFlags { 771 // Function attribute flags. Used to track if a function accesses memory, 772 // recurses or aliases. 773 unsigned ReadNone : 1; 774 unsigned ReadOnly : 1; 775 unsigned NoRecurse : 1; 776 unsigned ReturnDoesNotAlias : 1; 777 778 // Indicate if the global value cannot be inlined. 779 unsigned NoInline : 1; 780 // Indicate if function should be always inlined. 781 unsigned AlwaysInline : 1; 782 // Indicate if function never raises an exception. Can be modified during 783 // thinlink function attribute propagation 784 unsigned NoUnwind : 1; 785 // Indicate if function contains instructions that mayThrow 786 unsigned MayThrow : 1; 787 788 // If there are calls to unknown targets (e.g. indirect) 789 unsigned HasUnknownCall : 1; 790 791 // Indicate if a function must be an unreachable function. 792 // 793 // This bit is sufficient but not necessary; 794 // if this bit is on, the function must be regarded as unreachable; 795 // if this bit is off, the function might be reachable or unreachable. 796 unsigned MustBeUnreachable : 1; 797 798 FFlags &operator&=(const FFlags &RHS) { 799 this->ReadNone &= RHS.ReadNone; 800 this->ReadOnly &= RHS.ReadOnly; 801 this->NoRecurse &= RHS.NoRecurse; 802 this->ReturnDoesNotAlias &= RHS.ReturnDoesNotAlias; 803 this->NoInline &= RHS.NoInline; 804 this->AlwaysInline &= RHS.AlwaysInline; 805 this->NoUnwind &= RHS.NoUnwind; 806 this->MayThrow &= RHS.MayThrow; 807 this->HasUnknownCall &= RHS.HasUnknownCall; 808 this->MustBeUnreachable &= RHS.MustBeUnreachable; 809 return *this; 810 } 811 812 bool anyFlagSet() { 813 return this->ReadNone | this->ReadOnly | this->NoRecurse | 814 this->ReturnDoesNotAlias | this->NoInline | this->AlwaysInline | 815 this->NoUnwind | this->MayThrow | this->HasUnknownCall | 816 this->MustBeUnreachable; 817 } 818 819 operator std::string() { 820 std::string Output; 821 raw_string_ostream OS(Output); 822 OS << "funcFlags: ("; 823 OS << "readNone: " << this->ReadNone; 824 OS << ", readOnly: " << this->ReadOnly; 825 OS << ", noRecurse: " << this->NoRecurse; 826 OS << ", returnDoesNotAlias: " << this->ReturnDoesNotAlias; 827 OS << ", noInline: " << this->NoInline; 828 OS << ", alwaysInline: " << this->AlwaysInline; 829 OS << ", noUnwind: " << this->NoUnwind; 830 OS << ", mayThrow: " << this->MayThrow; 831 OS << ", hasUnknownCall: " << this->HasUnknownCall; 832 OS << ", mustBeUnreachable: " << this->MustBeUnreachable; 833 OS << ")"; 834 return Output; 835 } 836 }; 837 838 /// Describes the uses of a parameter by the function. 839 struct ParamAccess { 840 static constexpr uint32_t RangeWidth = 64; 841 842 /// Describes the use of a value in a call instruction, specifying the 843 /// call's target, the value's parameter number, and the possible range of 844 /// offsets from the beginning of the value that are passed. 845 struct Call { 846 uint64_t ParamNo = 0; 847 ValueInfo Callee; 848 ConstantRange Offsets{/*BitWidth=*/RangeWidth, /*isFullSet=*/true}; 849 850 Call() = default; 851 Call(uint64_t ParamNo, ValueInfo Callee, const ConstantRange &Offsets) 852 : ParamNo(ParamNo), Callee(Callee), Offsets(Offsets) {} 853 }; 854 855 uint64_t ParamNo = 0; 856 /// The range contains byte offsets from the parameter pointer which 857 /// accessed by the function. In the per-module summary, it only includes 858 /// accesses made by the function instructions. In the combined summary, it 859 /// also includes accesses by nested function calls. 860 ConstantRange Use{/*BitWidth=*/RangeWidth, /*isFullSet=*/true}; 861 /// In the per-module summary, it summarizes the byte offset applied to each 862 /// pointer parameter before passing to each corresponding callee. 863 /// In the combined summary, it's empty and information is propagated by 864 /// inter-procedural analysis and applied to the Use field. 865 std::vector<Call> Calls; 866 867 ParamAccess() = default; 868 ParamAccess(uint64_t ParamNo, const ConstantRange &Use) 869 : ParamNo(ParamNo), Use(Use) {} 870 }; 871 872 /// Create an empty FunctionSummary (with specified call edges). 873 /// Used to represent external nodes and the dummy root node. 874 static FunctionSummary 875 makeDummyFunctionSummary(SmallVectorImpl<FunctionSummary::EdgeTy> &&Edges) { 876 return FunctionSummary( 877 FunctionSummary::GVFlags( 878 GlobalValue::LinkageTypes::AvailableExternallyLinkage, 879 GlobalValue::DefaultVisibility, 880 /*NotEligibleToImport=*/true, /*Live=*/true, /*IsLocal=*/false, 881 /*CanAutoHide=*/false, GlobalValueSummary::ImportKind::Definition), 882 /*NumInsts=*/0, FunctionSummary::FFlags{}, SmallVector<ValueInfo, 0>(), 883 std::move(Edges), std::vector<GlobalValue::GUID>(), 884 std::vector<FunctionSummary::VFuncId>(), 885 std::vector<FunctionSummary::VFuncId>(), 886 std::vector<FunctionSummary::ConstVCall>(), 887 std::vector<FunctionSummary::ConstVCall>(), 888 std::vector<FunctionSummary::ParamAccess>(), 889 std::vector<CallsiteInfo>(), std::vector<AllocInfo>()); 890 } 891 892 /// A dummy node to reference external functions that aren't in the index 893 static FunctionSummary ExternalNode; 894 895 private: 896 /// Number of instructions (ignoring debug instructions, e.g.) computed 897 /// during the initial compile step when the summary index is first built. 898 unsigned InstCount; 899 900 /// Function summary specific flags. 901 FFlags FunFlags; 902 903 /// List of <CalleeValueInfo, CalleeInfo> call edge pairs from this function. 904 /// We use SmallVector<ValueInfo, 0> instead of std::vector<ValueInfo> for its 905 /// smaller memory footprint. 906 SmallVector<EdgeTy, 0> CallGraphEdgeList; 907 908 std::unique_ptr<TypeIdInfo> TIdInfo; 909 910 /// Uses for every parameter to this function. 911 using ParamAccessesTy = std::vector<ParamAccess>; 912 std::unique_ptr<ParamAccessesTy> ParamAccesses; 913 914 /// Optional list of memprof callsite metadata summaries. The correspondence 915 /// between the callsite summary and the callsites in the function is implied 916 /// by the order in the vector (and can be validated by comparing the stack 917 /// ids in the CallsiteInfo to those in the instruction callsite metadata). 918 /// As a memory savings optimization, we only create these for the prevailing 919 /// copy of a symbol when creating the combined index during LTO. 920 using CallsitesTy = std::vector<CallsiteInfo>; 921 std::unique_ptr<CallsitesTy> Callsites; 922 923 /// Optional list of allocation memprof metadata summaries. The correspondence 924 /// between the alloc memprof summary and the allocation callsites in the 925 /// function is implied by the order in the vector (and can be validated by 926 /// comparing the stack ids in the AllocInfo to those in the instruction 927 /// memprof metadata). 928 /// As a memory savings optimization, we only create these for the prevailing 929 /// copy of a symbol when creating the combined index during LTO. 930 using AllocsTy = std::vector<AllocInfo>; 931 std::unique_ptr<AllocsTy> Allocs; 932 933 public: 934 FunctionSummary(GVFlags Flags, unsigned NumInsts, FFlags FunFlags, 935 SmallVectorImpl<ValueInfo> &&Refs, 936 SmallVectorImpl<EdgeTy> &&CGEdges, 937 std::vector<GlobalValue::GUID> TypeTests, 938 std::vector<VFuncId> TypeTestAssumeVCalls, 939 std::vector<VFuncId> TypeCheckedLoadVCalls, 940 std::vector<ConstVCall> TypeTestAssumeConstVCalls, 941 std::vector<ConstVCall> TypeCheckedLoadConstVCalls, 942 std::vector<ParamAccess> Params, CallsitesTy CallsiteList, 943 AllocsTy AllocList) 944 : GlobalValueSummary(FunctionKind, Flags, std::move(Refs)), 945 InstCount(NumInsts), FunFlags(FunFlags), 946 CallGraphEdgeList(std::move(CGEdges)) { 947 if (!TypeTests.empty() || !TypeTestAssumeVCalls.empty() || 948 !TypeCheckedLoadVCalls.empty() || !TypeTestAssumeConstVCalls.empty() || 949 !TypeCheckedLoadConstVCalls.empty()) 950 TIdInfo = std::make_unique<TypeIdInfo>( 951 TypeIdInfo{std::move(TypeTests), std::move(TypeTestAssumeVCalls), 952 std::move(TypeCheckedLoadVCalls), 953 std::move(TypeTestAssumeConstVCalls), 954 std::move(TypeCheckedLoadConstVCalls)}); 955 if (!Params.empty()) 956 ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(Params)); 957 if (!CallsiteList.empty()) 958 Callsites = std::make_unique<CallsitesTy>(std::move(CallsiteList)); 959 if (!AllocList.empty()) 960 Allocs = std::make_unique<AllocsTy>(std::move(AllocList)); 961 } 962 // Gets the number of readonly and writeonly refs in RefEdgeList 963 std::pair<unsigned, unsigned> specialRefCounts() const; 964 965 /// Check if this is a function summary. 966 static bool classof(const GlobalValueSummary *GVS) { 967 return GVS->getSummaryKind() == FunctionKind; 968 } 969 970 /// Get function summary flags. 971 FFlags fflags() const { return FunFlags; } 972 973 void setNoRecurse() { FunFlags.NoRecurse = true; } 974 975 void setNoUnwind() { FunFlags.NoUnwind = true; } 976 977 /// Get the instruction count recorded for this function. 978 unsigned instCount() const { return InstCount; } 979 980 /// Return the list of <CalleeValueInfo, CalleeInfo> pairs. 981 ArrayRef<EdgeTy> calls() const { return CallGraphEdgeList; } 982 983 SmallVector<EdgeTy, 0> &mutableCalls() { return CallGraphEdgeList; } 984 985 void addCall(EdgeTy E) { CallGraphEdgeList.push_back(E); } 986 987 /// Returns the list of type identifiers used by this function in 988 /// llvm.type.test intrinsics other than by an llvm.assume intrinsic, 989 /// represented as GUIDs. 990 ArrayRef<GlobalValue::GUID> type_tests() const { 991 if (TIdInfo) 992 return TIdInfo->TypeTests; 993 return {}; 994 } 995 996 /// Returns the list of virtual calls made by this function using 997 /// llvm.assume(llvm.type.test) intrinsics that do not have all constant 998 /// integer arguments. 999 ArrayRef<VFuncId> type_test_assume_vcalls() const { 1000 if (TIdInfo) 1001 return TIdInfo->TypeTestAssumeVCalls; 1002 return {}; 1003 } 1004 1005 /// Returns the list of virtual calls made by this function using 1006 /// llvm.type.checked.load intrinsics that do not have all constant integer 1007 /// arguments. 1008 ArrayRef<VFuncId> type_checked_load_vcalls() const { 1009 if (TIdInfo) 1010 return TIdInfo->TypeCheckedLoadVCalls; 1011 return {}; 1012 } 1013 1014 /// Returns the list of virtual calls made by this function using 1015 /// llvm.assume(llvm.type.test) intrinsics with all constant integer 1016 /// arguments. 1017 ArrayRef<ConstVCall> type_test_assume_const_vcalls() const { 1018 if (TIdInfo) 1019 return TIdInfo->TypeTestAssumeConstVCalls; 1020 return {}; 1021 } 1022 1023 /// Returns the list of virtual calls made by this function using 1024 /// llvm.type.checked.load intrinsics with all constant integer arguments. 1025 ArrayRef<ConstVCall> type_checked_load_const_vcalls() const { 1026 if (TIdInfo) 1027 return TIdInfo->TypeCheckedLoadConstVCalls; 1028 return {}; 1029 } 1030 1031 /// Returns the list of known uses of pointer parameters. 1032 ArrayRef<ParamAccess> paramAccesses() const { 1033 if (ParamAccesses) 1034 return *ParamAccesses; 1035 return {}; 1036 } 1037 1038 /// Sets the list of known uses of pointer parameters. 1039 void setParamAccesses(std::vector<ParamAccess> NewParams) { 1040 if (NewParams.empty()) 1041 ParamAccesses.reset(); 1042 else if (ParamAccesses) 1043 *ParamAccesses = std::move(NewParams); 1044 else 1045 ParamAccesses = std::make_unique<ParamAccessesTy>(std::move(NewParams)); 1046 } 1047 1048 /// Add a type test to the summary. This is used by WholeProgramDevirt if we 1049 /// were unable to devirtualize a checked call. 1050 void addTypeTest(GlobalValue::GUID Guid) { 1051 if (!TIdInfo) 1052 TIdInfo = std::make_unique<TypeIdInfo>(); 1053 TIdInfo->TypeTests.push_back(Guid); 1054 } 1055 1056 const TypeIdInfo *getTypeIdInfo() const { return TIdInfo.get(); }; 1057 1058 ArrayRef<CallsiteInfo> callsites() const { 1059 if (Callsites) 1060 return *Callsites; 1061 return {}; 1062 } 1063 1064 CallsitesTy &mutableCallsites() { 1065 assert(Callsites); 1066 return *Callsites; 1067 } 1068 1069 void addCallsite(CallsiteInfo &Callsite) { 1070 if (!Callsites) 1071 Callsites = std::make_unique<CallsitesTy>(); 1072 Callsites->push_back(Callsite); 1073 } 1074 1075 ArrayRef<AllocInfo> allocs() const { 1076 if (Allocs) 1077 return *Allocs; 1078 return {}; 1079 } 1080 1081 AllocsTy &mutableAllocs() { 1082 assert(Allocs); 1083 return *Allocs; 1084 } 1085 1086 friend struct GraphTraits<ValueInfo>; 1087 }; 1088 1089 template <> struct DenseMapInfo<FunctionSummary::VFuncId> { 1090 static FunctionSummary::VFuncId getEmptyKey() { return {0, uint64_t(-1)}; } 1091 1092 static FunctionSummary::VFuncId getTombstoneKey() { 1093 return {0, uint64_t(-2)}; 1094 } 1095 1096 static bool isEqual(FunctionSummary::VFuncId L, FunctionSummary::VFuncId R) { 1097 return L.GUID == R.GUID && L.Offset == R.Offset; 1098 } 1099 1100 static unsigned getHashValue(FunctionSummary::VFuncId I) { return I.GUID; } 1101 }; 1102 1103 template <> struct DenseMapInfo<FunctionSummary::ConstVCall> { 1104 static FunctionSummary::ConstVCall getEmptyKey() { 1105 return {{0, uint64_t(-1)}, {}}; 1106 } 1107 1108 static FunctionSummary::ConstVCall getTombstoneKey() { 1109 return {{0, uint64_t(-2)}, {}}; 1110 } 1111 1112 static bool isEqual(FunctionSummary::ConstVCall L, 1113 FunctionSummary::ConstVCall R) { 1114 return DenseMapInfo<FunctionSummary::VFuncId>::isEqual(L.VFunc, R.VFunc) && 1115 L.Args == R.Args; 1116 } 1117 1118 static unsigned getHashValue(FunctionSummary::ConstVCall I) { 1119 return I.VFunc.GUID; 1120 } 1121 }; 1122 1123 /// The ValueInfo and offset for a function within a vtable definition 1124 /// initializer array. 1125 struct VirtFuncOffset { 1126 VirtFuncOffset(ValueInfo VI, uint64_t Offset) 1127 : FuncVI(VI), VTableOffset(Offset) {} 1128 1129 ValueInfo FuncVI; 1130 uint64_t VTableOffset; 1131 }; 1132 /// List of functions referenced by a particular vtable definition. 1133 using VTableFuncList = std::vector<VirtFuncOffset>; 1134 1135 /// Global variable summary information to aid decisions and 1136 /// implementation of importing. 1137 /// 1138 /// Global variable summary has two extra flag, telling if it is 1139 /// readonly or writeonly. Both readonly and writeonly variables 1140 /// can be optimized in the backed: readonly variables can be 1141 /// const-folded, while writeonly vars can be completely eliminated 1142 /// together with corresponding stores. We let both things happen 1143 /// by means of internalizing such variables after ThinLTO import. 1144 class GlobalVarSummary : public GlobalValueSummary { 1145 private: 1146 /// For vtable definitions this holds the list of functions and 1147 /// their corresponding offsets within the initializer array. 1148 std::unique_ptr<VTableFuncList> VTableFuncs; 1149 1150 public: 1151 struct GVarFlags { 1152 GVarFlags(bool ReadOnly, bool WriteOnly, bool Constant, 1153 GlobalObject::VCallVisibility Vis) 1154 : MaybeReadOnly(ReadOnly), MaybeWriteOnly(WriteOnly), 1155 Constant(Constant), VCallVisibility(Vis) {} 1156 1157 // If true indicates that this global variable might be accessed 1158 // purely by non-volatile load instructions. This in turn means 1159 // it can be internalized in source and destination modules during 1160 // thin LTO import because it neither modified nor its address 1161 // is taken. 1162 unsigned MaybeReadOnly : 1; 1163 // If true indicates that variable is possibly only written to, so 1164 // its value isn't loaded and its address isn't taken anywhere. 1165 // False, when 'Constant' attribute is set. 1166 unsigned MaybeWriteOnly : 1; 1167 // Indicates that value is a compile-time constant. Global variable 1168 // can be 'Constant' while not being 'ReadOnly' on several occasions: 1169 // - it is volatile, (e.g mapped device address) 1170 // - its address is taken, meaning that unlike 'ReadOnly' vars we can't 1171 // internalize it. 1172 // Constant variables are always imported thus giving compiler an 1173 // opportunity to make some extra optimizations. Readonly constants 1174 // are also internalized. 1175 unsigned Constant : 1; 1176 // Set from metadata on vtable definitions during the module summary 1177 // analysis. 1178 unsigned VCallVisibility : 2; 1179 } VarFlags; 1180 1181 GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags, 1182 SmallVectorImpl<ValueInfo> &&Refs) 1183 : GlobalValueSummary(GlobalVarKind, Flags, std::move(Refs)), 1184 VarFlags(VarFlags) {} 1185 1186 /// Check if this is a global variable summary. 1187 static bool classof(const GlobalValueSummary *GVS) { 1188 return GVS->getSummaryKind() == GlobalVarKind; 1189 } 1190 1191 GVarFlags varflags() const { return VarFlags; } 1192 void setReadOnly(bool RO) { VarFlags.MaybeReadOnly = RO; } 1193 void setWriteOnly(bool WO) { VarFlags.MaybeWriteOnly = WO; } 1194 bool maybeReadOnly() const { return VarFlags.MaybeReadOnly; } 1195 bool maybeWriteOnly() const { return VarFlags.MaybeWriteOnly; } 1196 bool isConstant() const { return VarFlags.Constant; } 1197 void setVCallVisibility(GlobalObject::VCallVisibility Vis) { 1198 VarFlags.VCallVisibility = Vis; 1199 } 1200 GlobalObject::VCallVisibility getVCallVisibility() const { 1201 return (GlobalObject::VCallVisibility)VarFlags.VCallVisibility; 1202 } 1203 1204 void setVTableFuncs(VTableFuncList Funcs) { 1205 assert(!VTableFuncs); 1206 VTableFuncs = std::make_unique<VTableFuncList>(std::move(Funcs)); 1207 } 1208 1209 ArrayRef<VirtFuncOffset> vTableFuncs() const { 1210 if (VTableFuncs) 1211 return *VTableFuncs; 1212 return {}; 1213 } 1214 }; 1215 1216 struct TypeTestResolution { 1217 /// Specifies which kind of type check we should emit for this byte array. 1218 /// See http://clang.llvm.org/docs/ControlFlowIntegrityDesign.html for full 1219 /// details on each kind of check; the enumerators are described with 1220 /// reference to that document. 1221 enum Kind { 1222 Unsat, ///< Unsatisfiable type (i.e. no global has this type metadata) 1223 ByteArray, ///< Test a byte array (first example) 1224 Inline, ///< Inlined bit vector ("Short Inline Bit Vectors") 1225 Single, ///< Single element (last example in "Short Inline Bit Vectors") 1226 AllOnes, ///< All-ones bit vector ("Eliminating Bit Vector Checks for 1227 /// All-Ones Bit Vectors") 1228 Unknown, ///< Unknown (analysis not performed, don't lower) 1229 } TheKind = Unknown; 1230 1231 /// Range of size-1 expressed as a bit width. For example, if the size is in 1232 /// range [1,256], this number will be 8. This helps generate the most compact 1233 /// instruction sequences. 1234 unsigned SizeM1BitWidth = 0; 1235 1236 // The following fields are only used if the target does not support the use 1237 // of absolute symbols to store constants. Their meanings are the same as the 1238 // corresponding fields in LowerTypeTestsModule::TypeIdLowering in 1239 // LowerTypeTests.cpp. 1240 1241 uint64_t AlignLog2 = 0; 1242 uint64_t SizeM1 = 0; 1243 uint8_t BitMask = 0; 1244 uint64_t InlineBits = 0; 1245 }; 1246 1247 struct WholeProgramDevirtResolution { 1248 enum Kind { 1249 Indir, ///< Just do a regular virtual call 1250 SingleImpl, ///< Single implementation devirtualization 1251 BranchFunnel, ///< When retpoline mitigation is enabled, use a branch funnel 1252 ///< that is defined in the merged module. Otherwise same as 1253 ///< Indir. 1254 } TheKind = Indir; 1255 1256 std::string SingleImplName; 1257 1258 struct ByArg { 1259 enum Kind { 1260 Indir, ///< Just do a regular virtual call 1261 UniformRetVal, ///< Uniform return value optimization 1262 UniqueRetVal, ///< Unique return value optimization 1263 VirtualConstProp, ///< Virtual constant propagation 1264 } TheKind = Indir; 1265 1266 /// Additional information for the resolution: 1267 /// - UniformRetVal: the uniform return value. 1268 /// - UniqueRetVal: the return value associated with the unique vtable (0 or 1269 /// 1). 1270 uint64_t Info = 0; 1271 1272 // The following fields are only used if the target does not support the use 1273 // of absolute symbols to store constants. 1274 1275 uint32_t Byte = 0; 1276 uint32_t Bit = 0; 1277 }; 1278 1279 /// Resolutions for calls with all constant integer arguments (excluding the 1280 /// first argument, "this"), where the key is the argument vector. 1281 std::map<std::vector<uint64_t>, ByArg> ResByArg; 1282 }; 1283 1284 struct TypeIdSummary { 1285 TypeTestResolution TTRes; 1286 1287 /// Mapping from byte offset to whole-program devirt resolution for that 1288 /// (typeid, byte offset) pair. 1289 std::map<uint64_t, WholeProgramDevirtResolution> WPDRes; 1290 }; 1291 1292 /// 160 bits SHA1 1293 using ModuleHash = std::array<uint32_t, 5>; 1294 1295 /// Type used for iterating through the global value summary map. 1296 using const_gvsummary_iterator = GlobalValueSummaryMapTy::const_iterator; 1297 using gvsummary_iterator = GlobalValueSummaryMapTy::iterator; 1298 1299 /// String table to hold/own module path strings, as well as a hash 1300 /// of the module. The StringMap makes a copy of and owns inserted strings. 1301 using ModulePathStringTableTy = StringMap<ModuleHash>; 1302 1303 /// Map of global value GUID to its summary, used to identify values defined in 1304 /// a particular module, and provide efficient access to their summary. 1305 using GVSummaryMapTy = DenseMap<GlobalValue::GUID, GlobalValueSummary *>; 1306 1307 /// Map of a module name to the GUIDs and summaries we will import from that 1308 /// module. 1309 using ModuleToSummariesForIndexTy = 1310 std::map<std::string, GVSummaryMapTy, std::less<>>; 1311 1312 /// A set of global value summary pointers. 1313 using GVSummaryPtrSet = std::unordered_set<GlobalValueSummary *>; 1314 1315 /// Map of a type GUID to type id string and summary (multimap used 1316 /// in case of GUID conflicts). 1317 using TypeIdSummaryMapTy = 1318 std::multimap<GlobalValue::GUID, std::pair<StringRef, TypeIdSummary>>; 1319 1320 /// The following data structures summarize type metadata information. 1321 /// For type metadata overview see https://llvm.org/docs/TypeMetadata.html. 1322 /// Each type metadata includes both the type identifier and the offset of 1323 /// the address point of the type (the address held by objects of that type 1324 /// which may not be the beginning of the virtual table). Vtable definitions 1325 /// are decorated with type metadata for the types they are compatible with. 1326 /// 1327 /// Holds information about vtable definitions decorated with type metadata: 1328 /// the vtable definition value and its address point offset in a type 1329 /// identifier metadata it is decorated (compatible) with. 1330 struct TypeIdOffsetVtableInfo { 1331 TypeIdOffsetVtableInfo(uint64_t Offset, ValueInfo VI) 1332 : AddressPointOffset(Offset), VTableVI(VI) {} 1333 1334 uint64_t AddressPointOffset; 1335 ValueInfo VTableVI; 1336 }; 1337 /// List of vtable definitions decorated by a particular type identifier, 1338 /// and their corresponding offsets in that type identifier's metadata. 1339 /// Note that each type identifier may be compatible with multiple vtables, due 1340 /// to inheritance, which is why this is a vector. 1341 using TypeIdCompatibleVtableInfo = std::vector<TypeIdOffsetVtableInfo>; 1342 1343 /// Class to hold module path string table and global value map, 1344 /// and encapsulate methods for operating on them. 1345 class ModuleSummaryIndex { 1346 private: 1347 /// Map from value name to list of summary instances for values of that 1348 /// name (may be duplicates in the COMDAT case, e.g.). 1349 GlobalValueSummaryMapTy GlobalValueMap; 1350 1351 /// Holds strings for combined index, mapping to the corresponding module ID. 1352 ModulePathStringTableTy ModulePathStringTable; 1353 1354 BumpPtrAllocator TypeIdSaverAlloc; 1355 UniqueStringSaver TypeIdSaver; 1356 1357 /// Mapping from type identifier GUIDs to type identifier and its summary 1358 /// information. Produced by thin link. 1359 TypeIdSummaryMapTy TypeIdMap; 1360 1361 /// Mapping from type identifier to information about vtables decorated 1362 /// with that type identifier's metadata. Produced by per module summary 1363 /// analysis and consumed by thin link. For more information, see description 1364 /// above where TypeIdCompatibleVtableInfo is defined. 1365 std::map<StringRef, TypeIdCompatibleVtableInfo, std::less<>> 1366 TypeIdCompatibleVtableMap; 1367 1368 /// Mapping from original ID to GUID. If original ID can map to multiple 1369 /// GUIDs, it will be mapped to 0. 1370 DenseMap<GlobalValue::GUID, GlobalValue::GUID> OidGuidMap; 1371 1372 /// Indicates that summary-based GlobalValue GC has run, and values with 1373 /// GVFlags::Live==false are really dead. Otherwise, all values must be 1374 /// considered live. 1375 bool WithGlobalValueDeadStripping = false; 1376 1377 /// Indicates that summary-based attribute propagation has run and 1378 /// GVarFlags::MaybeReadonly / GVarFlags::MaybeWriteonly are really 1379 /// read/write only. 1380 bool WithAttributePropagation = false; 1381 1382 /// Indicates that summary-based DSOLocal propagation has run and the flag in 1383 /// every summary of a GV is synchronized. 1384 bool WithDSOLocalPropagation = false; 1385 1386 /// Indicates that we have whole program visibility. 1387 bool WithWholeProgramVisibility = false; 1388 1389 /// Indicates that summary-based synthetic entry count propagation has run 1390 bool HasSyntheticEntryCounts = false; 1391 1392 /// Indicates that we linked with allocator supporting hot/cold new operators. 1393 bool WithSupportsHotColdNew = false; 1394 1395 /// Indicates that distributed backend should skip compilation of the 1396 /// module. Flag is suppose to be set by distributed ThinLTO indexing 1397 /// when it detected that the module is not needed during the final 1398 /// linking. As result distributed backend should just output a minimal 1399 /// valid object file. 1400 bool SkipModuleByDistributedBackend = false; 1401 1402 /// If true then we're performing analysis of IR module, or parsing along with 1403 /// the IR from assembly. The value of 'false' means we're reading summary 1404 /// from BC or YAML source. Affects the type of value stored in NameOrGV 1405 /// union. 1406 bool HaveGVs; 1407 1408 // True if the index was created for a module compiled with -fsplit-lto-unit. 1409 bool EnableSplitLTOUnit; 1410 1411 // True if the index was created for a module compiled with -funified-lto 1412 bool UnifiedLTO; 1413 1414 // True if some of the modules were compiled with -fsplit-lto-unit and 1415 // some were not. Set when the combined index is created during the thin link. 1416 bool PartiallySplitLTOUnits = false; 1417 1418 /// True if some of the FunctionSummary contains a ParamAccess. 1419 bool HasParamAccess = false; 1420 1421 std::set<std::string, std::less<>> CfiFunctionDefs; 1422 std::set<std::string, std::less<>> CfiFunctionDecls; 1423 1424 // Used in cases where we want to record the name of a global, but 1425 // don't have the string owned elsewhere (e.g. the Strtab on a module). 1426 BumpPtrAllocator Alloc; 1427 StringSaver Saver; 1428 1429 // The total number of basic blocks in the module in the per-module summary or 1430 // the total number of basic blocks in the LTO unit in the combined index. 1431 // FIXME: Putting this in the distributed ThinLTO index files breaks LTO 1432 // backend caching on any BB change to any linked file. It is currently not 1433 // used except in the case of a SamplePGO partial profile, and should be 1434 // reevaluated/redesigned to allow more effective incremental builds in that 1435 // case. 1436 uint64_t BlockCount = 0; 1437 1438 // List of unique stack ids (hashes). We use a 4B index of the id in the 1439 // stack id lists on the alloc and callsite summaries for memory savings, 1440 // since the number of unique ids is in practice much smaller than the 1441 // number of stack id references in the summaries. 1442 std::vector<uint64_t> StackIds; 1443 1444 // Temporary map while building StackIds list. Clear when index is completely 1445 // built via releaseTemporaryMemory. 1446 DenseMap<uint64_t, unsigned> StackIdToIndex; 1447 1448 // YAML I/O support. 1449 friend yaml::MappingTraits<ModuleSummaryIndex>; 1450 1451 GlobalValueSummaryMapTy::value_type * 1452 getOrInsertValuePtr(GlobalValue::GUID GUID) { 1453 return &*GlobalValueMap.emplace(GUID, GlobalValueSummaryInfo(HaveGVs)) 1454 .first; 1455 } 1456 1457 public: 1458 // See HaveGVs variable comment. 1459 ModuleSummaryIndex(bool HaveGVs, bool EnableSplitLTOUnit = false, 1460 bool UnifiedLTO = false) 1461 : TypeIdSaver(TypeIdSaverAlloc), HaveGVs(HaveGVs), 1462 EnableSplitLTOUnit(EnableSplitLTOUnit), UnifiedLTO(UnifiedLTO), 1463 Saver(Alloc) {} 1464 1465 // Current version for the module summary in bitcode files. 1466 // The BitcodeSummaryVersion should be bumped whenever we introduce changes 1467 // in the way some record are interpreted, like flags for instance. 1468 // Note that incrementing this may require changes in both BitcodeReader.cpp 1469 // and BitcodeWriter.cpp. 1470 static constexpr uint64_t BitcodeSummaryVersion = 12; 1471 1472 // Regular LTO module name for ASM writer 1473 static constexpr const char *getRegularLTOModuleName() { 1474 return "[Regular LTO]"; 1475 } 1476 1477 bool haveGVs() const { return HaveGVs; } 1478 1479 uint64_t getFlags() const; 1480 void setFlags(uint64_t Flags); 1481 1482 uint64_t getBlockCount() const { return BlockCount; } 1483 void addBlockCount(uint64_t C) { BlockCount += C; } 1484 void setBlockCount(uint64_t C) { BlockCount = C; } 1485 1486 gvsummary_iterator begin() { return GlobalValueMap.begin(); } 1487 const_gvsummary_iterator begin() const { return GlobalValueMap.begin(); } 1488 gvsummary_iterator end() { return GlobalValueMap.end(); } 1489 const_gvsummary_iterator end() const { return GlobalValueMap.end(); } 1490 size_t size() const { return GlobalValueMap.size(); } 1491 1492 const std::vector<uint64_t> &stackIds() const { return StackIds; } 1493 1494 unsigned addOrGetStackIdIndex(uint64_t StackId) { 1495 auto Inserted = StackIdToIndex.insert({StackId, StackIds.size()}); 1496 if (Inserted.second) 1497 StackIds.push_back(StackId); 1498 return Inserted.first->second; 1499 } 1500 1501 uint64_t getStackIdAtIndex(unsigned Index) const { 1502 assert(StackIds.size() > Index); 1503 return StackIds[Index]; 1504 } 1505 1506 // Facility to release memory from data structures only needed during index 1507 // construction (including while building combined index). Currently this only 1508 // releases the temporary map used while constructing a correspondence between 1509 // stack ids and their index in the StackIds vector. Mostly impactful when 1510 // building a large combined index. 1511 void releaseTemporaryMemory() { 1512 assert(StackIdToIndex.size() == StackIds.size()); 1513 StackIdToIndex.clear(); 1514 StackIds.shrink_to_fit(); 1515 } 1516 1517 /// Convenience function for doing a DFS on a ValueInfo. Marks the function in 1518 /// the FunctionHasParent map. 1519 static void discoverNodes(ValueInfo V, 1520 std::map<ValueInfo, bool> &FunctionHasParent) { 1521 if (!V.getSummaryList().size()) 1522 return; // skip external functions that don't have summaries 1523 1524 // Mark discovered if we haven't yet 1525 auto S = FunctionHasParent.emplace(V, false); 1526 1527 // Stop if we've already discovered this node 1528 if (!S.second) 1529 return; 1530 1531 FunctionSummary *F = 1532 dyn_cast<FunctionSummary>(V.getSummaryList().front().get()); 1533 assert(F != nullptr && "Expected FunctionSummary node"); 1534 1535 for (const auto &C : F->calls()) { 1536 // Insert node if necessary 1537 auto S = FunctionHasParent.emplace(C.first, true); 1538 1539 // Skip nodes that we're sure have parents 1540 if (!S.second && S.first->second) 1541 continue; 1542 1543 if (S.second) 1544 discoverNodes(C.first, FunctionHasParent); 1545 else 1546 S.first->second = true; 1547 } 1548 } 1549 1550 // Calculate the callgraph root 1551 FunctionSummary calculateCallGraphRoot() { 1552 // Functions that have a parent will be marked in FunctionHasParent pair. 1553 // Once we've marked all functions, the functions in the map that are false 1554 // have no parent (so they're the roots) 1555 std::map<ValueInfo, bool> FunctionHasParent; 1556 1557 for (auto &S : *this) { 1558 // Skip external functions 1559 if (!S.second.SummaryList.size() || 1560 !isa<FunctionSummary>(S.second.SummaryList.front().get())) 1561 continue; 1562 discoverNodes(ValueInfo(HaveGVs, &S), FunctionHasParent); 1563 } 1564 1565 SmallVector<FunctionSummary::EdgeTy, 0> Edges; 1566 // create edges to all roots in the Index 1567 for (auto &P : FunctionHasParent) { 1568 if (P.second) 1569 continue; // skip over non-root nodes 1570 Edges.push_back(std::make_pair(P.first, CalleeInfo{})); 1571 } 1572 return FunctionSummary::makeDummyFunctionSummary(std::move(Edges)); 1573 } 1574 1575 bool withGlobalValueDeadStripping() const { 1576 return WithGlobalValueDeadStripping; 1577 } 1578 void setWithGlobalValueDeadStripping() { 1579 WithGlobalValueDeadStripping = true; 1580 } 1581 1582 bool withAttributePropagation() const { return WithAttributePropagation; } 1583 void setWithAttributePropagation() { 1584 WithAttributePropagation = true; 1585 } 1586 1587 bool withDSOLocalPropagation() const { return WithDSOLocalPropagation; } 1588 void setWithDSOLocalPropagation() { WithDSOLocalPropagation = true; } 1589 1590 bool withWholeProgramVisibility() const { return WithWholeProgramVisibility; } 1591 void setWithWholeProgramVisibility() { WithWholeProgramVisibility = true; } 1592 1593 bool isReadOnly(const GlobalVarSummary *GVS) const { 1594 return WithAttributePropagation && GVS->maybeReadOnly(); 1595 } 1596 bool isWriteOnly(const GlobalVarSummary *GVS) const { 1597 return WithAttributePropagation && GVS->maybeWriteOnly(); 1598 } 1599 1600 bool withSupportsHotColdNew() const { return WithSupportsHotColdNew; } 1601 void setWithSupportsHotColdNew() { WithSupportsHotColdNew = true; } 1602 1603 bool skipModuleByDistributedBackend() const { 1604 return SkipModuleByDistributedBackend; 1605 } 1606 void setSkipModuleByDistributedBackend() { 1607 SkipModuleByDistributedBackend = true; 1608 } 1609 1610 bool enableSplitLTOUnit() const { return EnableSplitLTOUnit; } 1611 void setEnableSplitLTOUnit() { EnableSplitLTOUnit = true; } 1612 1613 bool hasUnifiedLTO() const { return UnifiedLTO; } 1614 void setUnifiedLTO() { UnifiedLTO = true; } 1615 1616 bool partiallySplitLTOUnits() const { return PartiallySplitLTOUnits; } 1617 void setPartiallySplitLTOUnits() { PartiallySplitLTOUnits = true; } 1618 1619 bool hasParamAccess() const { return HasParamAccess; } 1620 1621 bool isGlobalValueLive(const GlobalValueSummary *GVS) const { 1622 return !WithGlobalValueDeadStripping || GVS->isLive(); 1623 } 1624 bool isGUIDLive(GlobalValue::GUID GUID) const; 1625 1626 /// Return a ValueInfo for the index value_type (convenient when iterating 1627 /// index). 1628 ValueInfo getValueInfo(const GlobalValueSummaryMapTy::value_type &R) const { 1629 return ValueInfo(HaveGVs, &R); 1630 } 1631 1632 /// Return a ValueInfo for GUID if it exists, otherwise return ValueInfo(). 1633 ValueInfo getValueInfo(GlobalValue::GUID GUID) const { 1634 auto I = GlobalValueMap.find(GUID); 1635 return ValueInfo(HaveGVs, I == GlobalValueMap.end() ? nullptr : &*I); 1636 } 1637 1638 /// Return a ValueInfo for \p GUID. 1639 ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID) { 1640 return ValueInfo(HaveGVs, getOrInsertValuePtr(GUID)); 1641 } 1642 1643 // Save a string in the Index. Use before passing Name to 1644 // getOrInsertValueInfo when the string isn't owned elsewhere (e.g. on the 1645 // module's Strtab). 1646 StringRef saveString(StringRef String) { return Saver.save(String); } 1647 1648 /// Return a ValueInfo for \p GUID setting value \p Name. 1649 ValueInfo getOrInsertValueInfo(GlobalValue::GUID GUID, StringRef Name) { 1650 assert(!HaveGVs); 1651 auto VP = getOrInsertValuePtr(GUID); 1652 VP->second.U.Name = Name; 1653 return ValueInfo(HaveGVs, VP); 1654 } 1655 1656 /// Return a ValueInfo for \p GV and mark it as belonging to GV. 1657 ValueInfo getOrInsertValueInfo(const GlobalValue *GV) { 1658 assert(HaveGVs); 1659 auto VP = getOrInsertValuePtr(GV->getGUID()); 1660 VP->second.U.GV = GV; 1661 return ValueInfo(HaveGVs, VP); 1662 } 1663 1664 /// Return the GUID for \p OriginalId in the OidGuidMap. 1665 GlobalValue::GUID getGUIDFromOriginalID(GlobalValue::GUID OriginalID) const { 1666 const auto I = OidGuidMap.find(OriginalID); 1667 return I == OidGuidMap.end() ? 0 : I->second; 1668 } 1669 1670 std::set<std::string, std::less<>> &cfiFunctionDefs() { 1671 return CfiFunctionDefs; 1672 } 1673 const std::set<std::string, std::less<>> &cfiFunctionDefs() const { 1674 return CfiFunctionDefs; 1675 } 1676 1677 std::set<std::string, std::less<>> &cfiFunctionDecls() { 1678 return CfiFunctionDecls; 1679 } 1680 const std::set<std::string, std::less<>> &cfiFunctionDecls() const { 1681 return CfiFunctionDecls; 1682 } 1683 1684 /// Add a global value summary for a value. 1685 void addGlobalValueSummary(const GlobalValue &GV, 1686 std::unique_ptr<GlobalValueSummary> Summary) { 1687 addGlobalValueSummary(getOrInsertValueInfo(&GV), std::move(Summary)); 1688 } 1689 1690 /// Add a global value summary for a value of the given name. 1691 void addGlobalValueSummary(StringRef ValueName, 1692 std::unique_ptr<GlobalValueSummary> Summary) { 1693 addGlobalValueSummary(getOrInsertValueInfo(GlobalValue::getGUID(ValueName)), 1694 std::move(Summary)); 1695 } 1696 1697 /// Add a global value summary for the given ValueInfo. 1698 void addGlobalValueSummary(ValueInfo VI, 1699 std::unique_ptr<GlobalValueSummary> Summary) { 1700 if (const FunctionSummary *FS = dyn_cast<FunctionSummary>(Summary.get())) 1701 HasParamAccess |= !FS->paramAccesses().empty(); 1702 addOriginalName(VI.getGUID(), Summary->getOriginalName()); 1703 // Here we have a notionally const VI, but the value it points to is owned 1704 // by the non-const *this. 1705 const_cast<GlobalValueSummaryMapTy::value_type *>(VI.getRef()) 1706 ->second.SummaryList.push_back(std::move(Summary)); 1707 } 1708 1709 /// Add an original name for the value of the given GUID. 1710 void addOriginalName(GlobalValue::GUID ValueGUID, 1711 GlobalValue::GUID OrigGUID) { 1712 if (OrigGUID == 0 || ValueGUID == OrigGUID) 1713 return; 1714 auto [It, Inserted] = OidGuidMap.try_emplace(OrigGUID, ValueGUID); 1715 if (!Inserted && It->second != ValueGUID) 1716 It->second = 0; 1717 } 1718 1719 /// Find the summary for ValueInfo \p VI in module \p ModuleId, or nullptr if 1720 /// not found. 1721 GlobalValueSummary *findSummaryInModule(ValueInfo VI, StringRef ModuleId) const { 1722 auto SummaryList = VI.getSummaryList(); 1723 auto Summary = 1724 llvm::find_if(SummaryList, 1725 [&](const std::unique_ptr<GlobalValueSummary> &Summary) { 1726 return Summary->modulePath() == ModuleId; 1727 }); 1728 if (Summary == SummaryList.end()) 1729 return nullptr; 1730 return Summary->get(); 1731 } 1732 1733 /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if 1734 /// not found. 1735 GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID, 1736 StringRef ModuleId) const { 1737 auto CalleeInfo = getValueInfo(ValueGUID); 1738 if (!CalleeInfo) 1739 return nullptr; // This function does not have a summary 1740 return findSummaryInModule(CalleeInfo, ModuleId); 1741 } 1742 1743 /// Returns the first GlobalValueSummary for \p GV, asserting that there 1744 /// is only one if \p PerModuleIndex. 1745 GlobalValueSummary *getGlobalValueSummary(const GlobalValue &GV, 1746 bool PerModuleIndex = true) const { 1747 assert(GV.hasName() && "Can't get GlobalValueSummary for GV with no name"); 1748 return getGlobalValueSummary(GV.getGUID(), PerModuleIndex); 1749 } 1750 1751 /// Returns the first GlobalValueSummary for \p ValueGUID, asserting that 1752 /// there 1753 /// is only one if \p PerModuleIndex. 1754 GlobalValueSummary *getGlobalValueSummary(GlobalValue::GUID ValueGUID, 1755 bool PerModuleIndex = true) const; 1756 1757 /// Table of modules, containing module hash and id. 1758 const StringMap<ModuleHash> &modulePaths() const { 1759 return ModulePathStringTable; 1760 } 1761 1762 /// Table of modules, containing hash and id. 1763 StringMap<ModuleHash> &modulePaths() { return ModulePathStringTable; } 1764 1765 /// Get the module SHA1 hash recorded for the given module path. 1766 const ModuleHash &getModuleHash(const StringRef ModPath) const { 1767 auto It = ModulePathStringTable.find(ModPath); 1768 assert(It != ModulePathStringTable.end() && "Module not registered"); 1769 return It->second; 1770 } 1771 1772 /// Convenience method for creating a promoted global name 1773 /// for the given value name of a local, and its original module's ID. 1774 static std::string getGlobalNameForLocal(StringRef Name, ModuleHash ModHash) { 1775 std::string Suffix = utostr((uint64_t(ModHash[0]) << 32) | 1776 ModHash[1]); // Take the first 64 bits 1777 return getGlobalNameForLocal(Name, Suffix); 1778 } 1779 1780 static std::string getGlobalNameForLocal(StringRef Name, StringRef Suffix) { 1781 SmallString<256> NewName(Name); 1782 NewName += ".llvm."; 1783 NewName += Suffix; 1784 return std::string(NewName); 1785 } 1786 1787 /// Helper to obtain the unpromoted name for a global value (or the original 1788 /// name if not promoted). Split off the rightmost ".llvm.${hash}" suffix, 1789 /// because it is possible in certain clients (not clang at the moment) for 1790 /// two rounds of ThinLTO optimization and therefore promotion to occur. 1791 static StringRef getOriginalNameBeforePromote(StringRef Name) { 1792 std::pair<StringRef, StringRef> Pair = Name.rsplit(".llvm."); 1793 return Pair.first; 1794 } 1795 1796 typedef ModulePathStringTableTy::value_type ModuleInfo; 1797 1798 /// Add a new module with the given \p Hash, mapped to the given \p 1799 /// ModID, and return a reference to the module. 1800 ModuleInfo *addModule(StringRef ModPath, ModuleHash Hash = ModuleHash{{0}}) { 1801 return &*ModulePathStringTable.insert({ModPath, Hash}).first; 1802 } 1803 1804 /// Return module entry for module with the given \p ModPath. 1805 ModuleInfo *getModule(StringRef ModPath) { 1806 auto It = ModulePathStringTable.find(ModPath); 1807 assert(It != ModulePathStringTable.end() && "Module not registered"); 1808 return &*It; 1809 } 1810 1811 /// Return module entry for module with the given \p ModPath. 1812 const ModuleInfo *getModule(StringRef ModPath) const { 1813 auto It = ModulePathStringTable.find(ModPath); 1814 assert(It != ModulePathStringTable.end() && "Module not registered"); 1815 return &*It; 1816 } 1817 1818 /// Check if the given Module has any functions available for exporting 1819 /// in the index. We consider any module present in the ModulePathStringTable 1820 /// to have exported functions. 1821 bool hasExportedFunctions(const Module &M) const { 1822 return ModulePathStringTable.count(M.getModuleIdentifier()); 1823 } 1824 1825 const TypeIdSummaryMapTy &typeIds() const { return TypeIdMap; } 1826 1827 /// Return an existing or new TypeIdSummary entry for \p TypeId. 1828 /// This accessor can mutate the map and therefore should not be used in 1829 /// the ThinLTO backends. 1830 TypeIdSummary &getOrInsertTypeIdSummary(StringRef TypeId) { 1831 auto TidIter = TypeIdMap.equal_range(GlobalValue::getGUID(TypeId)); 1832 for (auto &[GUID, TypeIdPair] : make_range(TidIter)) 1833 if (TypeIdPair.first == TypeId) 1834 return TypeIdPair.second; 1835 auto It = TypeIdMap.insert({GlobalValue::getGUID(TypeId), 1836 {TypeIdSaver.save(TypeId), TypeIdSummary()}}); 1837 return It->second.second; 1838 } 1839 1840 /// This returns either a pointer to the type id summary (if present in the 1841 /// summary map) or null (if not present). This may be used when importing. 1842 const TypeIdSummary *getTypeIdSummary(StringRef TypeId) const { 1843 auto TidIter = TypeIdMap.equal_range(GlobalValue::getGUID(TypeId)); 1844 for (const auto &[GUID, TypeIdPair] : make_range(TidIter)) 1845 if (TypeIdPair.first == TypeId) 1846 return &TypeIdPair.second; 1847 return nullptr; 1848 } 1849 1850 TypeIdSummary *getTypeIdSummary(StringRef TypeId) { 1851 return const_cast<TypeIdSummary *>( 1852 static_cast<const ModuleSummaryIndex *>(this)->getTypeIdSummary( 1853 TypeId)); 1854 } 1855 1856 const auto &typeIdCompatibleVtableMap() const { 1857 return TypeIdCompatibleVtableMap; 1858 } 1859 1860 /// Return an existing or new TypeIdCompatibleVtableMap entry for \p TypeId. 1861 /// This accessor can mutate the map and therefore should not be used in 1862 /// the ThinLTO backends. 1863 TypeIdCompatibleVtableInfo & 1864 getOrInsertTypeIdCompatibleVtableSummary(StringRef TypeId) { 1865 return TypeIdCompatibleVtableMap[TypeIdSaver.save(TypeId)]; 1866 } 1867 1868 /// For the given \p TypeId, this returns the TypeIdCompatibleVtableMap 1869 /// entry if present in the summary map. This may be used when importing. 1870 std::optional<TypeIdCompatibleVtableInfo> 1871 getTypeIdCompatibleVtableSummary(StringRef TypeId) const { 1872 auto I = TypeIdCompatibleVtableMap.find(TypeId); 1873 if (I == TypeIdCompatibleVtableMap.end()) 1874 return std::nullopt; 1875 return I->second; 1876 } 1877 1878 /// Collect for the given module the list of functions it defines 1879 /// (GUID -> Summary). 1880 void collectDefinedFunctionsForModule(StringRef ModulePath, 1881 GVSummaryMapTy &GVSummaryMap) const; 1882 1883 /// Collect for each module the list of Summaries it defines (GUID -> 1884 /// Summary). 1885 template <class Map> 1886 void 1887 collectDefinedGVSummariesPerModule(Map &ModuleToDefinedGVSummaries) const { 1888 for (const auto &GlobalList : *this) { 1889 auto GUID = GlobalList.first; 1890 for (const auto &Summary : GlobalList.second.SummaryList) { 1891 ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get(); 1892 } 1893 } 1894 } 1895 1896 /// Print to an output stream. 1897 void print(raw_ostream &OS, bool IsForDebug = false) const; 1898 1899 /// Dump to stderr (for debugging). 1900 void dump() const; 1901 1902 /// Export summary to dot file for GraphViz. 1903 void 1904 exportToDot(raw_ostream &OS, 1905 const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) const; 1906 1907 /// Print out strongly connected components for debugging. 1908 void dumpSCCs(raw_ostream &OS); 1909 1910 /// Do the access attribute and DSOLocal propagation in combined index. 1911 void propagateAttributes(const DenseSet<GlobalValue::GUID> &PreservedSymbols); 1912 1913 /// Checks if we can import global variable from another module. 1914 bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs) const; 1915 1916 /// Same as above but checks whether the global var is importable as a 1917 /// declaration. 1918 bool canImportGlobalVar(const GlobalValueSummary *S, bool AnalyzeRefs, 1919 bool &CanImportDecl) const; 1920 }; 1921 1922 /// GraphTraits definition to build SCC for the index 1923 template <> struct GraphTraits<ValueInfo> { 1924 typedef ValueInfo NodeRef; 1925 using EdgeRef = FunctionSummary::EdgeTy &; 1926 1927 static NodeRef valueInfoFromEdge(FunctionSummary::EdgeTy &P) { 1928 return P.first; 1929 } 1930 using ChildIteratorType = 1931 mapped_iterator<SmallVector<FunctionSummary::EdgeTy, 0>::iterator, 1932 decltype(&valueInfoFromEdge)>; 1933 1934 using ChildEdgeIteratorType = 1935 SmallVector<FunctionSummary::EdgeTy, 0>::iterator; 1936 1937 static NodeRef getEntryNode(ValueInfo V) { return V; } 1938 1939 static ChildIteratorType child_begin(NodeRef N) { 1940 if (!N.getSummaryList().size()) // handle external function 1941 return ChildIteratorType( 1942 FunctionSummary::ExternalNode.CallGraphEdgeList.begin(), 1943 &valueInfoFromEdge); 1944 FunctionSummary *F = 1945 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject()); 1946 return ChildIteratorType(F->CallGraphEdgeList.begin(), &valueInfoFromEdge); 1947 } 1948 1949 static ChildIteratorType child_end(NodeRef N) { 1950 if (!N.getSummaryList().size()) // handle external function 1951 return ChildIteratorType( 1952 FunctionSummary::ExternalNode.CallGraphEdgeList.end(), 1953 &valueInfoFromEdge); 1954 FunctionSummary *F = 1955 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject()); 1956 return ChildIteratorType(F->CallGraphEdgeList.end(), &valueInfoFromEdge); 1957 } 1958 1959 static ChildEdgeIteratorType child_edge_begin(NodeRef N) { 1960 if (!N.getSummaryList().size()) // handle external function 1961 return FunctionSummary::ExternalNode.CallGraphEdgeList.begin(); 1962 1963 FunctionSummary *F = 1964 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject()); 1965 return F->CallGraphEdgeList.begin(); 1966 } 1967 1968 static ChildEdgeIteratorType child_edge_end(NodeRef N) { 1969 if (!N.getSummaryList().size()) // handle external function 1970 return FunctionSummary::ExternalNode.CallGraphEdgeList.end(); 1971 1972 FunctionSummary *F = 1973 cast<FunctionSummary>(N.getSummaryList().front()->getBaseObject()); 1974 return F->CallGraphEdgeList.end(); 1975 } 1976 1977 static NodeRef edge_dest(EdgeRef E) { return E.first; } 1978 }; 1979 1980 template <> 1981 struct GraphTraits<ModuleSummaryIndex *> : public GraphTraits<ValueInfo> { 1982 static NodeRef getEntryNode(ModuleSummaryIndex *I) { 1983 std::unique_ptr<GlobalValueSummary> Root = 1984 std::make_unique<FunctionSummary>(I->calculateCallGraphRoot()); 1985 GlobalValueSummaryInfo G(I->haveGVs()); 1986 G.SummaryList.push_back(std::move(Root)); 1987 static auto P = 1988 GlobalValueSummaryMapTy::value_type(GlobalValue::GUID(0), std::move(G)); 1989 return ValueInfo(I->haveGVs(), &P); 1990 } 1991 }; 1992 } // end namespace llvm 1993 1994 #endif // LLVM_IR_MODULESUMMARYINDEX_H 1995