LLVM  14.0.0git
Caching.cpp
Go to the documentation of this file.
1 //===-Caching.cpp - LLVM Link Time Optimizer Cache Handling ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Caching for ThinLTO.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/LTO/Caching.h"
14 #include "llvm/ADT/StringExtras.h"
15 #include "llvm/Support/Errc.h"
18 #include "llvm/Support/Path.h"
19 #include "llvm/Support/Process.h"
21 
22 #if !defined(_MSC_VER) && !defined(__MINGW32__)
23 #include <unistd.h>
24 #else
25 #include <io.h>
26 #endif
27 
28 using namespace llvm;
29 using namespace llvm::lto;
30 
32  AddBufferFn AddBuffer) {
33  if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPath))
34  return errorCodeToError(EC);
35 
36  return [=](unsigned Task, StringRef Key) -> AddStreamFn {
37  // This choice of file name allows the cache to be pruned (see pruneCache()
38  // in include/llvm/Support/CachePruning.h).
39  SmallString<64> EntryPath;
40  sys::path::append(EntryPath, CacheDirectoryPath, "llvmcache-" + Key);
41  // First, see if we have a cache hit.
42  SmallString<64> ResultPath;
44  Twine(EntryPath), sys::fs::OF_UpdateAtime, &ResultPath);
45  std::error_code EC;
46  if (FDOrErr) {
48  MemoryBuffer::getOpenFile(*FDOrErr, EntryPath,
49  /*FileSize=*/-1,
50  /*RequiresNullTerminator=*/false);
51  sys::fs::closeFile(*FDOrErr);
52  if (MBOrErr) {
53  AddBuffer(Task, std::move(*MBOrErr));
54  return AddStreamFn();
55  }
56  EC = MBOrErr.getError();
57  } else {
58  EC = errorToErrorCode(FDOrErr.takeError());
59  }
60 
61  // On Windows we can fail to open a cache file with a permission denied
62  // error. This generally means that another process has requested to delete
63  // the file while it is still open, but it could also mean that another
64  // process has opened the file without the sharing permissions we need.
65  // Since the file is probably being deleted we handle it in the same way as
66  // if the file did not exist at all.
68  report_fatal_error(Twine("Failed to open cache file ") + EntryPath +
69  ": " + EC.message() + "\n");
70 
71  // This native object stream is responsible for commiting the resulting
72  // file to the cache and calling AddBuffer to add it to the link.
73  struct CacheStream : NativeObjectStream {
74  AddBufferFn AddBuffer;
75  sys::fs::TempFile TempFile;
76  std::string EntryPath;
77  unsigned Task;
78 
79  CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
80  sys::fs::TempFile TempFile, std::string EntryPath,
81  unsigned Task)
82  : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
83  TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
84  Task(Task) {}
85 
86  ~CacheStream() {
87  // Make sure the stream is closed before committing it.
88  OS.reset();
89 
90  // Open the file first to avoid racing with a cache pruner.
93  sys::fs::convertFDToNativeFile(TempFile.FD), TempFile.TmpName,
94  /*FileSize=*/-1, /*RequiresNullTerminator=*/false);
95  if (!MBOrErr)
96  report_fatal_error(Twine("Failed to open new cache file ") +
97  TempFile.TmpName + ": " +
98  MBOrErr.getError().message() + "\n");
99 
100  // On POSIX systems, this will atomically replace the destination if
101  // it already exists. We try to emulate this on Windows, but this may
102  // fail with a permission denied error (for example, if the destination
103  // is currently opened by another process that does not give us the
104  // sharing permissions we need). Since the existing file should be
105  // semantically equivalent to the one we are trying to write, we give
106  // AddBuffer a copy of the bytes we wrote in that case. We do this
107  // instead of just using the existing file, because the pruner might
108  // delete the file before we get a chance to use it.
109  Error E = TempFile.keep(EntryPath);
110  E = handleErrors(std::move(E), [&](const ECError &E) -> Error {
111  std::error_code EC = E.convertToErrorCode();
112  if (EC != errc::permission_denied)
113  return errorCodeToError(EC);
114 
115  auto MBCopy = MemoryBuffer::getMemBufferCopy((*MBOrErr)->getBuffer(),
116  EntryPath);
117  MBOrErr = std::move(MBCopy);
118 
119  // FIXME: should we consume the discard error?
120  consumeError(TempFile.discard());
121 
122  return Error::success();
123  });
124 
125  if (E)
126  report_fatal_error(Twine("Failed to rename temporary file ") +
127  TempFile.TmpName + " to " + EntryPath + ": " +
128  toString(std::move(E)) + "\n");
129 
130  AddBuffer(Task, std::move(*MBOrErr));
131  }
132  };
133 
134  return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
135  // Write to a temporary to avoid race condition
136  SmallString<64> TempFilenameModel;
137  sys::path::append(TempFilenameModel, CacheDirectoryPath, "Thin-%%%%%%.tmp.o");
139  TempFilenameModel, sys::fs::owner_read | sys::fs::owner_write);
140  if (!Temp) {
141  errs() << "Error: " << toString(Temp.takeError()) << "\n";
142  report_fatal_error("ThinLTO: Can't get a temporary file");
143  }
144 
145  // This CacheStream will move the temporary file into the cache when done.
146  return std::make_unique<CacheStream>(
147  std::make_unique<raw_fd_ostream>(Temp->FD, /* ShouldClose */ false),
148  AddBuffer, std::move(*Temp), std::string(EntryPath.str()), Task);
149  };
150  };
151 }
MemoryBuffer.h
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::handleErrors
Error handleErrors(Error E, HandlerTs &&... Hs)
Pass the ErrorInfo(s) contained in E to their respective handlers.
Definition: Error.h:935
llvm::MemoryBuffer::getOpenFile
static ErrorOr< std::unique_ptr< MemoryBuffer > > getOpenFile(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize, bool RequiresNullTerminator=true, bool IsVolatile=false)
Given an already-open file descriptor, read the file and return a MemoryBuffer.
Definition: MemoryBuffer.cpp:503
FileSystem.h
llvm::sys::fs::TempFile::TmpName
std::string TmpName
Definition: FileSystem.h:863
llvm::lto::localCache
Expected< NativeObjectCache > localCache(StringRef CacheDirectoryPath, AddBufferFn AddBuffer)
Create a local file system cache which uses the given cache directory and file callback.
Definition: Caching.cpp:31
llvm::toString
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:1020
Path.h
llvm::Error::success
static ErrorSuccess success()
Create a success value.
Definition: Error.h:331
Errc.h
llvm::errc::no_such_file_or_directory
@ no_such_file_or_directory
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:892
llvm::Expected
Tagged union holding either a T or a Error.
Definition: APFloat.h:42
llvm::ECError
This class wraps a std::error_code in a Error.
Definition: Error.h:1133
llvm::consumeError
void consumeError(Error Err)
Consume a Error without doing anything.
Definition: Error.h:1035
llvm::sys::fs::TempFile::discard
Error discard()
Definition: Path.cpp:1198
llvm::errc::permission_denied
@ permission_denied
llvm::sys::path::append
void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")
Append to path.
Definition: Path.cpp:454
Caching.h
Process.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::errorCodeToError
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
Definition: Error.cpp:87
llvm::AMDGPU::PALMD::Key
Key
PAL metadata keys.
Definition: AMDGPUMetadata.h:481
llvm::ErrorOr::getError
std::error_code getError() const
Definition: ErrorOr.h:153
llvm::sys::fs::create_directories
std::error_code create_directories(const Twine &path, bool IgnoreExisting=true, perms Perms=owner_all|group_all)
Create all the non-existent directories in path.
Definition: Path.cpp:963
llvm::lto
Definition: Caching.h:20
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::sys::fs::TempFile::keep
Error keep(const Twine &Name)
Definition: Path.cpp:1223
llvm::sys::fs::openNativeFileForRead
Expected< file_t > openNativeFileForRead(const Twine &Name, OpenFlags Flags=OF_None, SmallVectorImpl< char > *RealPath=nullptr)
Opens the file with the given name in a read-only mode, returning its open file descriptor.
llvm::lto::NativeObjectStream
This class wraps an output stream for a native object.
Definition: LTO.h:194
llvm::SmallString< 64 >
llvm::sys::fs::owner_write
@ owner_write
Definition: FileSystem.h:89
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
StringExtras.h
llvm::sys::fs::closeFile
std::error_code closeFile(file_t &F)
Close the file object.
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::sys::fs::OF_UpdateAtime
@ OF_UpdateAtime
Force files Atime to be updated on access.
Definition: FileSystem.h:784
llvm::sys::fs::convertFDToNativeFile
file_t convertFDToNativeFile(int FD)
Converts from a Posix file descriptor number to a native file handle.
Definition: FileSystem.h:987
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::lto::AddStreamFn
std::function< std::unique_ptr< NativeObjectStream >(unsigned Task)> AddStreamFn
This type defines the callback to add a native object that is generated on the fly.
Definition: LTO.h:206
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
llvm::sys::fs::owner_read
@ owner_read
Definition: FileSystem.h:88
llvm::SmallString::str
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:259
llvm::errorToErrorCode
std::error_code errorToErrorCode(Error Err)
Helper for converting an ECError to a std::error_code.
Definition: Error.cpp:93
llvm::Expected::takeError
Error takeError()
Take ownership of the stored error.
Definition: Error.h:589
llvm::lto::AddBufferFn
std::function< void(unsigned Task, std::unique_ptr< MemoryBuffer > MB)> AddBufferFn
This type defines the callback to add a pre-existing native object file (e.g.
Definition: Caching.h:27
llvm::ErrorOr
Represents either an error or a value T.
Definition: ErrorOr.h:56
llvm::sys::fs::TempFile
Represents a temporary file.
Definition: FileSystem.h:849
llvm::sys::fs::TempFile::create
static Expected< TempFile > create(const Twine &Model, unsigned Mode=all_read|all_write, OpenFlags ExtraFlags=OF_None)
This creates a temporary file with createUniqueFile and schedules it for deletion with sys::RemoveFil...
Definition: Path.cpp:1291
raw_ostream.h
llvm::MemoryBuffer::getMemBufferCopy
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
Definition: MemoryBuffer.cpp:136
llvm::sys::fs::TempFile::FD
int FD
Definition: FileSystem.h:866