1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===// 2 // 3 // The LLVM Compiler Infrastructure 4 // 5 // This file is distributed under the University of Illinois Open Source 6 // License. See LICENSE.TXT for details. 7 // 8 //===----------------------------------------------------------------------===// 9 // 10 // This file implements the pruning of a directory based on least recently used. 11 // 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Support/CachePruning.h" 15 16 #include "llvm/Support/Debug.h" 17 #include "llvm/Support/Errc.h" 18 #include "llvm/Support/Error.h" 19 #include "llvm/Support/FileSystem.h" 20 #include "llvm/Support/Path.h" 21 #include "llvm/Support/raw_ostream.h" 22 23 #define DEBUG_TYPE "cache-pruning" 24 25 #include <set> 26 #include <system_error> 27 28 using namespace llvm; 29 30 /// Write a new timestamp file with the given path. This is used for the pruning 31 /// interval option. 32 static void writeTimestampFile(StringRef TimestampFile) { 33 std::error_code EC; 34 raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::F_None); 35 } 36 37 static Expected<std::chrono::seconds> parseDuration(StringRef Duration) { 38 if (Duration.empty()) 39 return make_error<StringError>("Duration must not be empty", 40 inconvertibleErrorCode()); 41 42 StringRef NumStr = Duration.slice(0, Duration.size()-1); 43 uint64_t Num; 44 if (NumStr.getAsInteger(0, Num)) 45 return make_error<StringError>("'" + NumStr + "' not an integer", 46 inconvertibleErrorCode()); 47 48 switch (Duration.back()) { 49 case 's': 50 return std::chrono::seconds(Num); 51 case 'm': 52 return std::chrono::minutes(Num); 53 case 'h': 54 return std::chrono::hours(Num); 55 default: 56 return make_error<StringError>("'" + Duration + 57 "' must end with one of 's', 'm' or 'h'", 58 inconvertibleErrorCode()); 59 } 60 } 61 62 Expected<CachePruningPolicy> 63 llvm::parseCachePruningPolicy(StringRef PolicyStr) { 64 CachePruningPolicy Policy; 65 std::pair<StringRef, StringRef> P = {"", PolicyStr}; 66 while (!P.second.empty()) { 67 P = P.second.split(':'); 68 69 StringRef Key, Value; 70 std::tie(Key, Value) = P.first.split('='); 71 if (Key == "prune_interval") { 72 auto DurationOrErr = parseDuration(Value); 73 if (!DurationOrErr) 74 return DurationOrErr.takeError(); 75 Policy.Interval = *DurationOrErr; 76 } else if (Key == "prune_after") { 77 auto DurationOrErr = parseDuration(Value); 78 if (!DurationOrErr) 79 return DurationOrErr.takeError(); 80 Policy.Expiration = *DurationOrErr; 81 } else if (Key == "cache_size") { 82 if (Value.back() != '%') 83 return make_error<StringError>("'" + Value + "' must be a percentage", 84 inconvertibleErrorCode()); 85 StringRef SizeStr = Value.slice(0, Value.size() - 1); 86 uint64_t Size; 87 if (SizeStr.getAsInteger(0, Size)) 88 return make_error<StringError>("'" + SizeStr + "' not an integer", 89 inconvertibleErrorCode()); 90 if (Size > 100) 91 return make_error<StringError>("'" + SizeStr + 92 "' must be between 0 and 100", 93 inconvertibleErrorCode()); 94 Policy.PercentageOfAvailableSpace = Size; 95 } else { 96 return make_error<StringError>("Unknown key: '" + Key + "'", 97 inconvertibleErrorCode()); 98 } 99 } 100 101 return Policy; 102 } 103 104 /// Prune the cache of files that haven't been accessed in a long time. 105 bool llvm::pruneCache(StringRef Path, CachePruningPolicy Policy) { 106 using namespace std::chrono; 107 108 if (Path.empty()) 109 return false; 110 111 bool isPathDir; 112 if (sys::fs::is_directory(Path, isPathDir)) 113 return false; 114 115 if (!isPathDir) 116 return false; 117 118 Policy.PercentageOfAvailableSpace = 119 std::min(Policy.PercentageOfAvailableSpace, 100u); 120 121 if (Policy.Expiration == seconds(0) && 122 Policy.PercentageOfAvailableSpace == 0) { 123 DEBUG(dbgs() << "No pruning settings set, exit early\n"); 124 // Nothing will be pruned, early exit 125 return false; 126 } 127 128 // Try to stat() the timestamp file. 129 SmallString<128> TimestampFile(Path); 130 sys::path::append(TimestampFile, "llvmcache.timestamp"); 131 sys::fs::file_status FileStatus; 132 const auto CurrentTime = system_clock::now(); 133 if (auto EC = sys::fs::status(TimestampFile, FileStatus)) { 134 if (EC == errc::no_such_file_or_directory) { 135 // If the timestamp file wasn't there, create one now. 136 writeTimestampFile(TimestampFile); 137 } else { 138 // Unknown error? 139 return false; 140 } 141 } else { 142 if (Policy.Interval == seconds(0)) { 143 // Check whether the time stamp is older than our pruning interval. 144 // If not, do nothing. 145 const auto TimeStampModTime = FileStatus.getLastModificationTime(); 146 auto TimeStampAge = CurrentTime - TimeStampModTime; 147 if (TimeStampAge <= Policy.Interval) { 148 DEBUG(dbgs() << "Timestamp file too recent (" 149 << duration_cast<seconds>(TimeStampAge).count() 150 << "s old), do not prune.\n"); 151 return false; 152 } 153 } 154 // Write a new timestamp file so that nobody else attempts to prune. 155 // There is a benign race condition here, if two processes happen to 156 // notice at the same time that the timestamp is out-of-date. 157 writeTimestampFile(TimestampFile); 158 } 159 160 bool ShouldComputeSize = (Policy.PercentageOfAvailableSpace > 0); 161 162 // Keep track of space 163 std::set<std::pair<uint64_t, std::string>> FileSizes; 164 uint64_t TotalSize = 0; 165 // Helper to add a path to the set of files to consider for size-based 166 // pruning, sorted by size. 167 auto AddToFileListForSizePruning = 168 [&](StringRef Path) { 169 if (!ShouldComputeSize) 170 return; 171 TotalSize += FileStatus.getSize(); 172 FileSizes.insert( 173 std::make_pair(FileStatus.getSize(), std::string(Path))); 174 }; 175 176 // Walk the entire directory cache, looking for unused files. 177 std::error_code EC; 178 SmallString<128> CachePathNative; 179 sys::path::native(Path, CachePathNative); 180 // Walk all of the files within this directory. 181 for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd; 182 File != FileEnd && !EC; File.increment(EC)) { 183 // Do not touch the timestamp. 184 if (File->path() == TimestampFile) 185 continue; 186 187 // Look at this file. If we can't stat it, there's nothing interesting 188 // there. 189 if (sys::fs::status(File->path(), FileStatus)) { 190 DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n"); 191 continue; 192 } 193 194 // If the file hasn't been used recently enough, delete it 195 const auto FileAccessTime = FileStatus.getLastAccessedTime(); 196 auto FileAge = CurrentTime - FileAccessTime; 197 if (FileAge > Policy.Expiration) { 198 DEBUG(dbgs() << "Remove " << File->path() << " (" 199 << duration_cast<seconds>(FileAge).count() << "s old)\n"); 200 sys::fs::remove(File->path()); 201 continue; 202 } 203 204 // Leave it here for now, but add it to the list of size-based pruning. 205 AddToFileListForSizePruning(File->path()); 206 } 207 208 // Prune for size now if needed 209 if (ShouldComputeSize) { 210 auto ErrOrSpaceInfo = sys::fs::disk_space(Path); 211 if (!ErrOrSpaceInfo) { 212 report_fatal_error("Can't get available size"); 213 } 214 sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get(); 215 auto AvailableSpace = TotalSize + SpaceInfo.free; 216 auto FileAndSize = FileSizes.rbegin(); 217 DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace) 218 << "% target is: " << Policy.PercentageOfAvailableSpace 219 << "\n"); 220 // Remove the oldest accessed files first, till we get below the threshold 221 while (((100 * TotalSize) / AvailableSpace) > 222 Policy.PercentageOfAvailableSpace && 223 FileAndSize != FileSizes.rend()) { 224 // Remove the file. 225 sys::fs::remove(FileAndSize->second); 226 // Update size 227 TotalSize -= FileAndSize->first; 228 DEBUG(dbgs() << " - Remove " << FileAndSize->second << " (size " 229 << FileAndSize->first << "), new occupancy is " << TotalSize 230 << "%\n"); 231 ++FileAndSize; 232 } 233 } 234 return true; 235 } 236