LLVM  14.0.0git
CachePruning.cpp
Go to the documentation of this file.
1 //===-CachePruning.cpp - LLVM Cache Directory Pruning ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the pruning of a directory based on least recently used.
10 //
11 //===----------------------------------------------------------------------===//
12 
14 #include "llvm/ADT/StringRef.h"
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/Errc.h"
17 #include "llvm/Support/Error.h"
19 #include "llvm/Support/Path.h"
21 
22 #define DEBUG_TYPE "cache-pruning"
23 
24 #include <set>
25 #include <system_error>
26 
27 using namespace llvm;
28 
29 namespace {
30 struct FileInfo {
31  sys::TimePoint<> Time;
32  uint64_t Size;
33  std::string Path;
34 
35  /// Used to determine which files to prune first. Also used to determine
36  /// set membership, so must take into account all fields.
37  bool operator<(const FileInfo &Other) const {
38  return std::tie(Time, Other.Size, Path) <
39  std::tie(Other.Time, Size, Other.Path);
40  }
41 };
42 } // anonymous namespace
43 
44 /// Write a new timestamp file with the given path. This is used for the pruning
45 /// interval option.
46 static void writeTimestampFile(StringRef TimestampFile) {
47  std::error_code EC;
48  raw_fd_ostream Out(TimestampFile.str(), EC, sys::fs::OF_None);
49 }
50 
52  if (Duration.empty())
53  return make_error<StringError>("Duration must not be empty",
55 
56  StringRef NumStr = Duration.slice(0, Duration.size()-1);
57  uint64_t Num;
58  if (NumStr.getAsInteger(0, Num))
59  return make_error<StringError>("'" + NumStr + "' not an integer",
61 
62  switch (Duration.back()) {
63  case 's':
64  return std::chrono::seconds(Num);
65  case 'm':
66  return std::chrono::minutes(Num);
67  case 'h':
68  return std::chrono::hours(Num);
69  default:
70  return make_error<StringError>("'" + Duration +
71  "' must end with one of 's', 'm' or 'h'",
73  }
74 }
75 
78  CachePruningPolicy Policy;
79  std::pair<StringRef, StringRef> P = {"", PolicyStr};
80  while (!P.second.empty()) {
81  P = P.second.split(':');
82 
84  std::tie(Key, Value) = P.first.split('=');
85  if (Key == "prune_interval") {
86  auto DurationOrErr = parseDuration(Value);
87  if (!DurationOrErr)
88  return DurationOrErr.takeError();
89  Policy.Interval = *DurationOrErr;
90  } else if (Key == "prune_after") {
91  auto DurationOrErr = parseDuration(Value);
92  if (!DurationOrErr)
93  return DurationOrErr.takeError();
94  Policy.Expiration = *DurationOrErr;
95  } else if (Key == "cache_size") {
96  if (Value.back() != '%')
97  return make_error<StringError>("'" + Value + "' must be a percentage",
99  StringRef SizeStr = Value.drop_back();
100  uint64_t Size;
101  if (SizeStr.getAsInteger(0, Size))
102  return make_error<StringError>("'" + SizeStr + "' not an integer",
104  if (Size > 100)
105  return make_error<StringError>("'" + SizeStr +
106  "' must be between 0 and 100",
109  } else if (Key == "cache_size_bytes") {
110  uint64_t Mult = 1;
111  switch (tolower(Value.back())) {
112  case 'k':
113  Mult = 1024;
114  Value = Value.drop_back();
115  break;
116  case 'm':
117  Mult = 1024 * 1024;
118  Value = Value.drop_back();
119  break;
120  case 'g':
121  Mult = 1024 * 1024 * 1024;
122  Value = Value.drop_back();
123  break;
124  }
125  uint64_t Size;
126  if (Value.getAsInteger(0, Size))
127  return make_error<StringError>("'" + Value + "' not an integer",
129  Policy.MaxSizeBytes = Size * Mult;
130  } else if (Key == "cache_size_files") {
131  if (Value.getAsInteger(0, Policy.MaxSizeFiles))
132  return make_error<StringError>("'" + Value + "' not an integer",
134  } else {
135  return make_error<StringError>("Unknown key: '" + Key + "'",
137  }
138  }
139 
140  return Policy;
141 }
142 
143 /// Prune the cache of files that haven't been accessed in a long time.
145  using namespace std::chrono;
146 
147  if (Path.empty())
148  return false;
149 
150  bool isPathDir;
151  if (sys::fs::is_directory(Path, isPathDir))
152  return false;
153 
154  if (!isPathDir)
155  return false;
156 
159 
160  if (Policy.Expiration == seconds(0) &&
161  Policy.MaxSizePercentageOfAvailableSpace == 0 &&
162  Policy.MaxSizeBytes == 0 && Policy.MaxSizeFiles == 0) {
163  LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");
164  // Nothing will be pruned, early exit
165  return false;
166  }
167 
168  // Try to stat() the timestamp file.
169  SmallString<128> TimestampFile(Path);
170  sys::path::append(TimestampFile, "llvmcache.timestamp");
171  sys::fs::file_status FileStatus;
172  const auto CurrentTime = system_clock::now();
173  if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {
175  // If the timestamp file wasn't there, create one now.
176  writeTimestampFile(TimestampFile);
177  } else {
178  // Unknown error?
179  return false;
180  }
181  } else {
182  if (!Policy.Interval)
183  return false;
184  if (Policy.Interval != seconds(0)) {
185  // Check whether the time stamp is older than our pruning interval.
186  // If not, do nothing.
187  const auto TimeStampModTime = FileStatus.getLastModificationTime();
188  auto TimeStampAge = CurrentTime - TimeStampModTime;
189  if (TimeStampAge <= *Policy.Interval) {
190  LLVM_DEBUG(dbgs() << "Timestamp file too recent ("
191  << duration_cast<seconds>(TimeStampAge).count()
192  << "s old), do not prune.\n");
193  return false;
194  }
195  }
196  // Write a new timestamp file so that nobody else attempts to prune.
197  // There is a benign race condition here, if two processes happen to
198  // notice at the same time that the timestamp is out-of-date.
199  writeTimestampFile(TimestampFile);
200  }
201 
202  // Keep track of files to delete to get below the size limit.
203  // Order by time of last use so that recently used files are preserved.
204  std::set<FileInfo> FileInfos;
205  uint64_t TotalSize = 0;
206 
207  // Walk the entire directory cache, looking for unused files.
208  std::error_code EC;
209  SmallString<128> CachePathNative;
210  sys::path::native(Path, CachePathNative);
211  // Walk all of the files within this directory.
212  for (sys::fs::directory_iterator File(CachePathNative, EC), FileEnd;
213  File != FileEnd && !EC; File.increment(EC)) {
214  // Ignore filenames not beginning with "llvmcache-" or "Thin-". This
215  // includes the timestamp file as well as any files created by the user.
216  // This acts as a safeguard against data loss if the user specifies the
217  // wrong directory as their cache directory.
219  if (!filename.startswith("llvmcache-") && !filename.startswith("Thin-"))
220  continue;
221 
222  // Look at this file. If we can't stat it, there's nothing interesting
223  // there.
224  ErrorOr<sys::fs::basic_file_status> StatusOrErr = File->status();
225  if (!StatusOrErr) {
226  LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");
227  continue;
228  }
229 
230  // If the file hasn't been used recently enough, delete it
231  const auto FileAccessTime = StatusOrErr->getLastAccessedTime();
232  auto FileAge = CurrentTime - FileAccessTime;
233  if (Policy.Expiration != seconds(0) && FileAge > Policy.Expiration) {
234  LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("
235  << duration_cast<seconds>(FileAge).count()
236  << "s old)\n");
237  sys::fs::remove(File->path());
238  continue;
239  }
240 
241  // Leave it here for now, but add it to the list of size-based pruning.
242  TotalSize += StatusOrErr->getSize();
243  FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});
244  }
245 
246  auto FileInfo = FileInfos.begin();
247  size_t NumFiles = FileInfos.size();
248 
249  auto RemoveCacheFile = [&]() {
250  // Remove the file.
251  sys::fs::remove(FileInfo->Path);
252  // Update size
253  TotalSize -= FileInfo->Size;
254  NumFiles--;
255  LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "
256  << FileInfo->Size << "), new occupancy is " << TotalSize
257  << "%\n");
258  ++FileInfo;
259  };
260 
261  // Prune for number of files.
262  if (Policy.MaxSizeFiles)
263  while (NumFiles > Policy.MaxSizeFiles)
264  RemoveCacheFile();
265 
266  // Prune for size now if needed
267  if (Policy.MaxSizePercentageOfAvailableSpace > 0 || Policy.MaxSizeBytes > 0) {
268  auto ErrOrSpaceInfo = sys::fs::disk_space(Path);
269  if (!ErrOrSpaceInfo) {
270  report_fatal_error("Can't get available size");
271  }
272  sys::fs::space_info SpaceInfo = ErrOrSpaceInfo.get();
273  auto AvailableSpace = TotalSize + SpaceInfo.free;
274 
275  if (Policy.MaxSizePercentageOfAvailableSpace == 0)
277  if (Policy.MaxSizeBytes == 0)
278  Policy.MaxSizeBytes = AvailableSpace;
279  auto TotalSizeTarget = std::min<uint64_t>(
280  AvailableSpace * Policy.MaxSizePercentageOfAvailableSpace / 100ull,
281  Policy.MaxSizeBytes);
282 
283  LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)
284  << "% target is: "
285  << Policy.MaxSizePercentageOfAvailableSpace << "%, "
286  << Policy.MaxSizeBytes << " bytes\n");
287 
288  // Remove the oldest accessed files first, till we get below the threshold.
289  while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())
290  RemoveCacheFile();
291  }
292  return true;
293 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::StringRef::back
LLVM_NODISCARD char back() const
back - Get the last character in the string.
Definition: StringRef.h:167
llvm::StringRef::startswith
LLVM_NODISCARD bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:285
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:23
llvm::StringRef::empty
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:152
FileSystem.h
llvm::sys::fs::OF_None
@ OF_None
Definition: FileSystem.h:757
llvm::sys::fs::space_info
space_info - Self explanatory.
Definition: FileSystem.h:80
StringRef.h
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
Path.h
Error.h
Errc.h
llvm::sys::path::native
void native(const Twine &path, SmallVectorImpl< char > &result, Style style=Style::native)
Convert path to the native form.
Definition: Path.cpp:540
llvm::errc::no_such_file_or_directory
@ no_such_file_or_directory
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
llvm::StringRef::slice
LLVM_NODISCARD StringRef slice(size_t Start, size_t End) const
Return a reference to the substring from [Start, End).
Definition: StringRef.h:731
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::sys::TimePoint
std::chrono::time_point< std::chrono::system_clock, D > TimePoint
A time point on the system clock.
Definition: Chrono.h:33
llvm::sys::path::append
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:456
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
CachePruning.h
llvm::CachePruningPolicy::MaxSizeBytes
uint64_t MaxSizeBytes
The maximum size for the cache directory in bytes.
Definition: CachePruning.h:49
llvm::AMDGPU::PALMD::Key
Key
PAL metadata keys.
Definition: AMDGPUMetadata.h:481
llvm::CachePruningPolicy::Expiration
std::chrono::seconds Expiration
The expiration for a file.
Definition: CachePruning.h:37
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
llvm::StringRef::str
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:244
llvm::StringRef::getAsInteger
std::enable_if_t< std::numeric_limits< T >::is_signed, bool > getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:509
now
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
Definition: ArchiveWriter.cpp:259
llvm::SmallString< 128 >
writeTimestampFile
static void writeTimestampFile(StringRef TimestampFile)
Write a new timestamp file with the given path.
Definition: CachePruning.cpp:46
llvm::sys::fs::file_status
Represents the result of a call to sys::fs::status().
Definition: FileSystem.h:226
llvm::count
auto count(R &&Range, const E &Element)
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:1683
uint64_t
llvm::operator<
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:338
llvm::sys::fs::remove
std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)
Remove path.
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::CachePruningPolicy::MaxSizePercentageOfAvailableSpace
unsigned MaxSizePercentageOfAvailableSpace
The maximum size for the cache directory, in terms of percentage of the available space on the disk.
Definition: CachePruning.h:44
llvm::CachePruningPolicy
Policy for the pruneCache() function.
Definition: CachePruning.h:27
llvm::raw_fd_ostream
A raw_ostream that writes to a file descriptor.
Definition: raw_ostream.h:443
llvm::sys::fs::space_info::free
uint64_t free
Definition: FileSystem.h:82
llvm::MipsISD::Mult
@ Mult
Definition: MipsISelLowering.h:132
llvm::CachePruningPolicy::MaxSizeFiles
uint64_t MaxSizeFiles
The maximum number of files in the cache directory.
Definition: CachePruning.h:60
llvm::inconvertibleErrorCode
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Definition: Error.cpp:77
llvm::CachePruningPolicy::Interval
llvm::Optional< std::chrono::seconds > Interval
The pruning interval.
Definition: CachePruning.h:32
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::sys::fs::is_directory
bool is_directory(const basic_file_status &status)
Does status represent a directory?
Definition: Path.cpp:1088
llvm::sys::fs::directory_iterator
directory_iterator - Iterates through the entries in path.
Definition: FileSystem.h:1402
llvm::sys::path::filename
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
Definition: Path.cpp:577
llvm::sys::fs::status
std::error_code status(const Twine &path, file_status &result, bool follow=true)
Get file status as if by POSIX stat().
llvm::ErrorOr
Represents either an error or a value T.
Definition: ErrorOr.h:56
llvm::pruneCache
bool pruneCache(StringRef Path, CachePruningPolicy Policy)
Peform pruning using the supplied policy, returns true if pruning occurred, i.e.
Definition: CachePruning.cpp:144
parseDuration
static Expected< std::chrono::seconds > parseDuration(StringRef Duration)
Definition: CachePruning.cpp:51
llvm::sys::fs::basic_file_status::getLastModificationTime
TimePoint getLastModificationTime() const
The file modification time as reported from the underlying file system.
raw_ostream.h
llvm::StringRef::size
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:156
llvm::parseCachePruningPolicy
Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)
Parse the given string as a cache pruning policy.
Definition: CachePruning.cpp:77
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
File
Instrumentation for Order File
Definition: InstrOrderFile.cpp:205
llvm::sys::fs::disk_space
ErrorOr< space_info > disk_space(const Twine &Path)
Get disk space usage information.
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1195