30 "Timer group for offload bundler");
41 return HeaderOrErr->FileSize;
51 size_t NextbundleStart = 0;
53 std::unique_ptr<MemoryBuffer> Buffer;
67 if (std::optional<size_t>
Size =
70 NextbundleStart = (*Buffer).getBuffer().find(Magic, *
Size);
73 NextbundleStart = (*Buffer).getBuffer().find(Magic, Magic.size());
74 CurBundleEnd = NextbundleStart;
77 NextbundleStart = (*Buffer).getBuffer().size();
79 CurBundleEnd = (*Buffer).getBuffer().size();
84 (*Buffer).getBuffer().take_front(CurBundleEnd), FileName,
false);
85 if (std::error_code EC = CodeOrErr.
getError())
90 if (!DecompressedBufferOrErr)
95 **DecompressedBufferOrErr,
Offset, FileName,
true);
97 return FatBundleOrErr.takeError();
107 *Buffer, SectionOffset +
Offset, FileName);
109 return FatBundleOrErr.takeError();
114 Magic =
"__CLANG_OFFLOAD_BUNDLE__";
115 NextbundleStart = (*Buffer).getBuffer().find(Magic, Magic.size());
119 Offset += NextbundleStart;
140 NumberOfEntries = NumOfEntries;
161 auto Entry = std::make_unique<OffloadBundleEntry>(
162 EntryOffset + SectionOffset, EntrySize, EntryIDSize, EntryID);
164 Entries.push_back(*Entry);
182 std::unique_ptr<OffloadBundleFatBin> TheBundle(
187 TheBundle->readEntries(Buf.
getBuffer(), Decompress ? 0 : SectionOffset);
191 return std::move(TheBundle);
204 "-size" +
itostr(Entry.Size) +
".co";
216 if (!Obj.isELF() && !Obj.isCOFF())
233 }
else if (Obj.isCOFF()) {
236 COFF.getSection(
COFF.getSectionID(Sec));
239 SectionOffset = (*SecOrErr)->PointerToRawData;
244 Obj.getFileName(), Bundles))
263 if (
Size > InputBuffOrErr->getBufferSize())
265 Size, InputBuffOrErr->getBufferSize());
267 if (
Offset > InputBuffOrErr->getBufferSize())
269 "offset in URI (%zu) is beyond the end of the source (%zu)",
Offset,
270 InputBuffOrErr->getBufferSize());
272 if (
Offset +
Size > InputBuffOrErr->getBufferSize())
274 "offset + size (%zu) in URI is beyond the end of the source (%zu)",
275 Offset +
Size, InputBuffOrErr->getBufferSize());
277 std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
278 std::copy(InputBuffOrErr->getBufferStart() +
Offset,
279 InputBuffOrErr->getBufferStart() +
Offset +
Size,
280 Buf->getBufferStart());
281 if (
Error E = Buf->commit())
294 std::unique_ptr<FileOutputBuffer> Buf = std::move(*BufferOrErr);
298 return Buf->commit();
319 return ObjOrErr.takeError();
321 auto Obj = ObjOrErr->getBinary();
331 std::string Num = std::to_string(
Value);
340Expected<std::unique_ptr<MemoryBuffer>>
346 Timer HashTimer(
"Hash Calculation Timer",
"Hash calculation time",
354 uint64_t TruncatedHash = Result.low();
360 reinterpret_cast<const uint8_t *
>(
Input.getBuffer().data()),
361 Input.getBuffer().size());
362 Timer CompressTimer(
"Compression Timer",
"Compression time",
379 if (UncompressedSize64 > std::numeric_limits<uint32_t>::max())
381 "unsigned 32-bit integer limit",
383 TotalFileSize64 = MagicNumber.size() +
sizeof(
uint32_t) +
sizeof(
Version) +
384 sizeof(CompressionMethod) +
sizeof(
uint32_t) +
385 sizeof(TruncatedHash) + CompressedBuffer.
size();
386 if (TotalFileSize64 > std::numeric_limits<uint32_t>::max())
388 "unsigned 32-bit integer limit",
392 TotalFileSize64 = MagicNumber.size() +
sizeof(
uint64_t) +
sizeof(
Version) +
393 sizeof(CompressionMethod) +
sizeof(
uint64_t) +
394 sizeof(TruncatedHash) + CompressedBuffer.
size();
403 Writer.
write(CompressionMethod);
410 Writer.
write(TotalFileSize64);
411 Writer.
write(UncompressedSize64);
414 Writer.
write(TruncatedHash);
415 OS.
write(
reinterpret_cast<const char *
>(CompressedBuffer.
data()),
416 CompressedBuffer.
size());
420 double CompressionRate =
421 static_cast<double>(UncompressedSize64) / CompressedBuffer.
size();
423 double CompressionSpeedMBs =
424 (UncompressedSize64 / (1024.0 * 1024.0)) / CompressionTimeSeconds;
425 *VerboseStream <<
"Compressed bundle format version: " <<
Version <<
"\n"
426 <<
"Total file size (including headers): "
428 <<
"Compression method used: " << MethodUsed <<
"\n"
429 <<
"Compression level: " <<
P.level <<
"\n"
430 <<
"Binary size before compression: "
432 <<
"Binary size after compression: "
434 <<
"Compression rate: " <<
format(
"%.2lf", CompressionRate)
436 <<
"Compression ratio: "
437 <<
format(
"%.2lf%%", 100.0 / CompressionRate) <<
"\n"
438 <<
"Compression speed: "
439 <<
format(
"%.2lf MB/s", CompressionSpeedMBs) <<
"\n"
440 <<
"Truncated MD5 hash: " <<
format_hex(TruncatedHash, 16)
499Expected<CompressedOffloadBundle::CompressedBundleHeader>
505 std::memcpy(&Header, Blob.
data(), std::min(Blob.
size(),
sizeof(Header)));
508 Normalized.
Version = Header.Common.Version;
512 if (Blob.
size() < RequiredSize)
518 Normalized.
Hash = Header.V1.Hash;
521 Normalized.
FileSize = Header.V2.FileSize;
523 Normalized.
Hash = Header.V2.Hash;
526 Normalized.
FileSize = Header.V3.FileSize;
528 Normalized.
Hash = Header.V3.Hash;
535 switch (Header.Common.Method) {
559 *VerboseStream <<
"Uncompressed bundle\n";
569 unsigned ThisVersion = Normalized.
Version;
574 size_t TotalFileSize = Normalized.
FileSize.value_or(0);
576 auto StoredHash = Normalized.
Hash;
578 Timer DecompressTimer(
"Decompression Timer",
"Decompression time",
585 Blob.
substr(HeaderSize, TotalFileSize - HeaderSize);
589 DecompressedData, UncompressedSize))
591 toString(std::move(DecompressionError)));
596 double DecompressionTimeSeconds =
600 Timer HashRecalcTimer(
"Hash Recalculation Timer",
"Hash recalculation time",
607 uint64_t RecalculatedHash = Result.low();
609 bool HashMatch = (StoredHash == RecalculatedHash);
611 double CompressionRate =
612 static_cast<double>(UncompressedSize) / CompressedData.
size();
613 double DecompressionSpeedMBs =
614 (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;
616 *VerboseStream <<
"Compressed bundle format version: " << ThisVersion
618 if (ThisVersion >= 2)
619 *VerboseStream <<
"Total file size (from header): "
622 <<
"Decompression method: "
625 <<
"Size before decompression: "
629 <<
"Compression rate: " <<
format(
"%.2lf", CompressionRate) <<
"\n"
630 <<
"Compression ratio: " <<
format(
"%.2lf%%", 100.0 / CompressionRate)
632 <<
"Decompression speed: "
633 <<
format(
"%.2lf MB/s", DecompressionSpeedMBs) <<
"\n"
634 <<
"Stored hash: " <<
format_hex(StoredHash, 16) <<
"\n"
635 <<
"Recalculated hash: " <<
format_hex(RecalculatedHash, 16) <<
"\n"
636 <<
"Hashes match: " << (HashMatch ?
"Yes" :
"No") <<
"\n";
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
#define LLVM_PACKED_START
Module.h This file contains the declarations for the Module class.
static LLVM_PACKED_END size_t getHeaderSize(uint16_t Version)
Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset, StringRef FileName, SmallVectorImpl< OffloadBundleFatBin > &Bundles)
static std::string formatWithCommas(unsigned long long Value)
static TimerGroup OffloadBundlerTimerGroup("Offload Bundler Timer Group", "Timer group for offload bundler")
static std::optional< size_t > getCompressedBundleSize(StringRef Blob)
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Provides read only access to a subclass of BinaryStream.
Error readInteger(T &Dest)
Read an integer of the specified endianness into Dest and update the stream's offset.
LLVM_ABI Error readFixedString(StringRef &Dest, uint32_t Length)
Read a Length byte string into Dest.
Represents either an error or a value T.
std::error_code getError() const
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
Error takeError()
Take ownership of the stored error.
static LLVM_ABI Expected< std::unique_ptr< FileOutputBuffer > > create(StringRef FilePath, size_t Size, unsigned Flags=0)
Factory method to create an OutputBuffer object which manages a read/write buffer of the specified si...
LLVM_ABI void update(ArrayRef< uint8_t > Data)
Updates the hash for the byte stream provided.
LLVM_ABI void final(MD5Result &Result)
Finishes off the hash and puts the result in result.
size_t getBufferSize() const
const char * getBufferStart() const
StringRef getBuffer() const
This interface provides simple read-only access to a block of memory, and provides simple methods for...
static std::unique_ptr< MemoryBuffer > getMemBuffer(StringRef InputData, StringRef BufferName="", bool RequiresNullTerminator=true)
Open the specified memory range as a MemoryBuffer.
static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")
Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
pointer data()
Return a pointer to the vector's buffer, even if empty().
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Represent a constant reference to a string, i.e.
static constexpr size_t npos
std::string str() const
Get the contents as an std::string.
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
StringRef drop_front(size_t N=1) const
Return a StringRef equal to 'this' but with the first N elements dropped.
constexpr size_t size() const
Get the string size.
constexpr const char * data() const
Get a pointer to the start of the string (which may not be null terminated).
double getWallTime() const
The TimerGroup class is used to group together related timers into a single report that is printed wh...
This class is used to track the amount of time spent between invocations of its startTimer()/stopTime...
LLVM_ABI void stopTimer()
Stop the timer.
LLVM_ABI void startTimer()
Start the timer running.
TimeRecord getTotalTime() const
Return the duration for which this timer has been running.
LLVM Value Representation.
static LLVM_ABI llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > decompress(const llvm::MemoryBuffer &Input, raw_ostream *VerboseStream=nullptr)
static LLVM_ABI llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, uint16_t Version, raw_ostream *VerboseStream=nullptr)
uint64_t getOffset() const
This class is the base class for all object file types.
static Expected< OwningBinary< ObjectFile > > createObjectFile(StringRef ObjectPath)
LLVM_ABI Error readEntries(StringRef Section, uint64_t SectionOffset)
OffloadBundleFatBin(MemoryBufferRef Source, StringRef File, bool Decompress=false)
LLVM_ABI Error extractBundle(const ObjectFile &Source)
StringRef getFileName() const
static LLVM_ABI Expected< std::unique_ptr< OffloadBundleFatBin > > create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName, bool Decompress=false)
This is a value type class that represents a single section in the list of sections in the object fil...
This class implements an extremely fast bulk output stream that can only output to a stream.
raw_ostream & write(unsigned char C)
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
LLVM_ABI bool isAvailable()
LLVM_ABI bool isAvailable()
LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef< uint8_t > Input, uint8_t *Output, size_t UncompressedSize)
LLVM_ABI void compress(Params P, ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &Output)
LLVM_ABI Error extractOffloadBundleByURI(StringRef URIstr)
Extracts an Offload Bundle Entry given by URI.
LLVM_ABI Error extractOffloadBundleFatBinary(const ObjectFile &Obj, SmallVectorImpl< OffloadBundleFatBin > &Bundles)
Extracts fat binary in binary clang-offload-bundler format from object Obj and return it in Bundles.
LLVM_ABI Error extractCodeObject(const ObjectFile &Source, size_t Offset, size_t Size, StringRef OutputFileName)
Extract code object memory from the given Source object file at Offset and of Size,...
detail::packed_endian_specific_integral< uint64_t, llvm::endianness::little, unaligned > ulittle64_t
detail::packed_endian_specific_integral< uint32_t, llvm::endianness::little, unaligned > ulittle32_t
detail::packed_endian_specific_integral< uint16_t, llvm::endianness::little, unaligned > ulittle16_t
This is an optimization pass for GlobalISel generic memory operations.
LLVM_ABI file_magic identify_magic(StringRef magic)
Identify the type of a binary file based on how magical it is.
Error createFileError(const Twine &F, Error E)
Concatenate a source file path and/or name with an Error.
ArrayRef< CharT > arrayRefFromStringRef(StringRef Input)
Construct an array ref of bytes from a string ref.
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)
format_hex - Output N as a fixed width hexadecimal.
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
LLVM_ABI Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
void consumeError(Error Err)
Consume a Error without doing anything.
StringRef toStringRef(bool B)
Construct a string ref from a boolean.
std::string itostr(int64_t X)
@ offload_bundle
Clang offload bundle file.
@ offload_bundle_compressed
Compressed clang offload bundle file.
Bundle entry in binary clang-offload-bundler format.
static Expected< std::unique_ptr< OffloadBundleURI > > createOffloadBundleURI(StringRef Str, UriTypeT Type)
Adapter to write values to a stream in a particular byte order.
void write(ArrayRef< value_type > Val)