44#define DEBUG_TYPE "memprof"
49template <
class T = u
int64_t>
inline T alignedRead(
const char *
Ptr) {
50 static_assert(std::is_pod<T>::value,
"Not a pod type.");
51 assert(
reinterpret_cast<size_t>(
Ptr) %
sizeof(
T) == 0 &&
"Unaligned Read");
52 return *
reinterpret_cast<const T *
>(
Ptr);
55Error checkBuffer(
const MemoryBuffer &Buffer) {
59 if (Buffer.getBufferSize() == 0)
62 if (Buffer.getBufferSize() <
sizeof(Header)) {
69 const char *Next = Buffer.getBufferStart();
70 while (Next < Buffer.getBufferEnd()) {
71 auto *
H =
reinterpret_cast<const Header *
>(Next);
72 if (
H->Version != MEMPROF_RAW_VERSION) {
76 TotalSize +=
H->TotalSize;
80 if (Buffer.getBufferSize() != TotalSize) {
87 using namespace support;
90 endian::readNext<uint64_t, little, unaligned>(
Ptr);
93 Items.
push_back(*
reinterpret_cast<const SegmentEntry *
>(
94 Ptr +
I *
sizeof(SegmentEntry)));
100readMemInfoBlocks(
const char *
Ptr) {
101 using namespace support;
104 endian::readNext<uint64_t, little, unaligned>(
Ptr);
107 const uint64_t Id = endian::readNext<uint64_t, little, unaligned>(
Ptr);
108 const MemInfoBlock MIB = *
reinterpret_cast<const MemInfoBlock *
>(
Ptr);
111 Ptr +=
sizeof(MemInfoBlock);
117 using namespace support;
120 endian::readNext<uint64_t, little, unaligned>(
Ptr);
124 const uint64_t StackId = endian::readNext<uint64_t, little, unaligned>(
Ptr);
125 const uint64_t NumPCs = endian::readNext<uint64_t, little, unaligned>(
Ptr);
127 SmallVector<uint64_t> CallStack;
128 for (
uint64_t J = 0; J < NumPCs; J++) {
129 CallStack.push_back(endian::readNext<uint64_t, little, unaligned>(
Ptr));
132 Items[StackId] = CallStack;
141 for (
const auto &IdStack :
From) {
142 auto I = To.find(IdStack.first);
144 To[IdStack.first] = IdStack.second;
147 if (IdStack.second !=
I->second)
154Error report(Error
E,
const StringRef
Context) {
159bool isRuntimePath(
const StringRef Path) {
163 return Filename.equals(
"memprof_malloc_linux.cpp") ||
164 Filename.equals(
"memprof_interceptors.cpp") ||
165 Filename.equals(
"memprof_new_delete.cpp");
168std::string getBuildIdString(
const SegmentEntry &Entry) {
170 if (Entry.BuildIdSize == 0)
174 raw_string_ostream
OS(Str);
175 for (
size_t I = 0;
I < Entry.BuildIdSize;
I++) {
182Expected<std::unique_ptr<RawMemProfReader>>
186 if (std::error_code EC = BufferOr.getError())
189 std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
190 return create(std::move(Buffer), ProfiledBinary, KeepName);
195 const StringRef ProfiledBinary,
bool KeepName) {
196 if (
Error E = checkBuffer(*Buffer))
197 return report(std::move(
E), Buffer->getBufferIdentifier());
199 if (ProfiledBinary.
empty()) {
201 const std::vector<std::string> BuildIds =
peekBuildIds(Buffer.get());
202 std::string ErrorMessage(
203 R
"(Path to profiled binary is empty, expected binary with one of the following build ids:
205 for (
const auto &Id : BuildIds) {
206 ErrorMessage +=
"\n BuildId: ";
216 return report(BinaryOr.takeError(), ProfiledBinary);
220 std::unique_ptr<RawMemProfReader> Reader(
222 if (
Error E = Reader->initialize(std::move(Buffer))) {
225 return std::move(Reader);
233 std::unique_ptr<MemoryBuffer> Buffer(BufferOr.get().release());
243 return Magic == MEMPROF_RAW_MAGIC_64;
247 uint64_t NumAllocFunctions = 0, NumMibInfo = 0;
249 const size_t NumAllocSites = KV.second.AllocSites.size();
250 if (NumAllocSites > 0) {
252 NumMibInfo += NumAllocSites;
256 OS <<
"MemprofProfile:\n";
258 OS <<
" Version: " << MEMPROF_RAW_VERSION <<
"\n";
259 OS <<
" NumSegments: " << SegmentInfo.
size() <<
"\n";
260 OS <<
" NumMibInfo: " << NumMibInfo <<
"\n";
261 OS <<
" NumAllocFunctions: " << NumAllocFunctions <<
"\n";
262 OS <<
" NumStackOffsets: " << StackMap.
size() <<
"\n";
264 OS <<
" Segments:\n";
265 for (
const auto &Entry : SegmentInfo) {
267 OS <<
" BuildId: " << getBuildIdString(Entry) <<
"\n";
268 OS <<
" Start: 0x" << llvm::utohexstr(Entry.Start) <<
"\n";
269 OS <<
" End: 0x" << llvm::utohexstr(Entry.End) <<
"\n";
270 OS <<
" Offset: 0x" << llvm::utohexstr(Entry.Offset) <<
"\n";
274 for (
const auto &Entry : *
this) {
276 OS <<
" FunctionGUID: " << Entry.first <<
"\n";
277 Entry.second.print(
OS);
281Error RawMemProfReader::initialize(std::unique_ptr<MemoryBuffer> DataBuffer) {
282 const StringRef FileName = Binary.getBinary()->getFileName();
284 auto *ElfObject = dyn_cast<object::ELFObjectFileBase>(Binary.getBinary());
286 return report(make_error<StringError>(
Twine(
"Not an ELF file: "),
294 auto* Elf64LEObject = llvm::cast<llvm::object::ELF64LEObjectFile>(ElfObject);
296 auto PHdrsOr = ElfFile.program_headers();
299 make_error<StringError>(
Twine(
"Could not read program headers: "),
303 int NumExecutableSegments = 0;
304 for (
const auto &Phdr : *PHdrsOr) {
309 if (++NumExecutableSegments > 1) {
311 make_error<StringError>(
312 "Expect only one executable load segment in the binary",
321 PreferredTextSegmentAddress = Phdr.p_vaddr;
322 assert(Phdr.p_vaddr == (Phdr.p_vaddr & ~(0x1000 - 1U)) &&
323 "Expect p_vaddr to always be page aligned");
324 assert(Phdr.p_offset == 0 &&
"Expect p_offset = 0 for symbolization.");
329 auto Triple = ElfObject->makeTriple();
331 return report(make_error<StringError>(Twine(
"Unsupported target: ") +
332 Triple.getArchName(),
336 auto *
Object = cast<object::ObjectFile>(Binary.getBinary());
341 Object, std::move(Context),
false);
343 return report(SOFOr.takeError(), FileName);
344 Symbolizer = std::move(SOFOr.get());
347 if (Error
E = readRawProfile(std::move(DataBuffer)))
350 if (Error
E = setupForSymbolization())
353 if (Error
E = symbolizeAndFilterStackFrames())
356 return mapRawProfileToRecords();
359Error RawMemProfReader::setupForSymbolization() {
360 auto *
Object = cast<object::ObjectFile>(Binary.getBinary());
362 if (BinaryId.empty())
363 return make_error<StringError>(Twine(
"No build id found in binary ") +
364 Binary.getBinary()->getFileName(),
368 for (
const auto &Entry : SegmentInfo) {
370 if (BinaryId == SegmentId) {
373 if (++NumMatched > 1) {
374 return make_error<StringError>(
375 "We expect only one executable segment in the profiled binary",
378 ProfiledTextSegmentStart = Entry.Start;
379 ProfiledTextSegmentEnd = Entry.End;
382 assert(NumMatched != 0 &&
"No matching executable segments in segment info.");
383 assert((PreferredTextSegmentAddress == 0 ||
384 (PreferredTextSegmentAddress == ProfiledTextSegmentStart)) &&
385 "Expect text segment address to be 0 or equal to profiled text "
390Error RawMemProfReader::mapRawProfileToRecords() {
396 PerFunctionCallSites;
400 for (
const auto &Entry : CallstackProfileData) {
401 const uint64_t StackId = Entry.first;
403 auto It = StackMap.
find(StackId);
404 if (It == StackMap.
end())
405 return make_error<InstrProfError>(
407 "memprof callstack record does not contain id: " + Twine(StackId));
411 Callstack.
reserve(It->getSecond().size());
414 for (
size_t I = 0;
I < Addresses.
size();
I++) {
417 "Address not found in SymbolizedFrame map");
418 const SmallVector<FrameId> &Frames = SymbolizedFrame[
Address];
421 "The last frame should not be inlined");
426 for (
size_t J = 0; J < Frames.size(); J++) {
427 if (
I == 0 && J == 0)
434 PerFunctionCallSites[Guid].
insert(&Frames);
438 Callstack.
append(Frames.begin(), Frames.end());
443 for (
size_t I = 0; ;
I++) {
447 IndexedMemProfRecord &Record =
Result.first->second;
448 Record.AllocSites.emplace_back(Callstack, Entry.second);
450 if (!
F.IsInlineFrame)
456 for (
const auto &[Id, Locs] : PerFunctionCallSites) {
460 IndexedMemProfRecord &Record =
Result.first->second;
461 for (LocationPtr Loc : Locs) {
462 Record.CallSites.push_back(*Loc);
469Error RawMemProfReader::symbolizeAndFilterStackFrames() {
471 const DILineInfoSpecifier Specifier(
472 DILineInfoSpecifier::FileLineInfoKind::RawValue,
473 DILineInfoSpecifier::FunctionNameKind::LinkageName);
481 for (
auto &Entry : StackMap) {
482 for (
const uint64_t VAddr : Entry.getSecond()) {
486 if (SymbolizedFrame.count(VAddr) > 0 ||
490 Expected<DIInliningInfo> DIOr = Symbolizer->symbolizeInlinedCode(
491 getModuleOffset(VAddr), Specifier,
false);
493 return DIOr.takeError();
494 DIInliningInfo DI = DIOr.get();
498 isRuntimePath(DI.getFrame(0).FileName)) {
499 AllVAddrsToDiscard.
insert(VAddr);
503 for (
size_t I = 0, NumFrames = DI.getNumberOfFrames();
I < NumFrames;
505 const auto &DIFrame = DI.getFrame(
I);
508 const Frame
F(Guid, DIFrame.Line - DIFrame.StartLine, DIFrame.Column,
515 if (KeepSymbolName) {
516 StringRef CanonicalName =
518 DIFrame.FunctionName);
519 GuidToSymbolName.
insert({Guid, CanonicalName.str()});
524 SymbolizedFrame[VAddr].push_back(Hash);
528 auto &CallStack = Entry.getSecond();
532 if (CallStack.empty())
533 EntriesToErase.
push_back(Entry.getFirst());
537 for (
const uint64_t Id : EntriesToErase) {
539 CallstackProfileData.erase(Id);
542 if (StackMap.empty())
543 return make_error<InstrProfError>(
545 "no entries in callstack map after symbolization");
550std::vector<std::string>
559 std::vector<std::string> BuildIds;
561 while (Next < DataBuffer->getBufferEnd()) {
562 auto *Header =
reinterpret_cast<const memprof::Header *
>(Next);
565 readSegmentEntries(Next + Header->SegmentOffset);
567 for (
const auto &Entry : Entries) {
568 const std::string Id = getBuildIdString(Entry);
571 BuildIds.push_back(Id);
575 Next += Header->TotalSize;
580Error RawMemProfReader::readRawProfile(
581 std::unique_ptr<MemoryBuffer> DataBuffer) {
582 const char *Next = DataBuffer->getBufferStart();
584 while (Next < DataBuffer->getBufferEnd()) {
585 auto *Header =
reinterpret_cast<const memprof::Header *
>(Next);
590 readSegmentEntries(Next + Header->SegmentOffset);
591 if (!SegmentInfo.empty() && SegmentInfo != Entries) {
595 return make_error<InstrProfError>(
597 "memprof raw profile has different segment information");
599 SegmentInfo.assign(Entries.begin(), Entries.end());
604 for (
const auto &
Value : readMemInfoBlocks(Next + Header->MIBOffset)) {
605 if (CallstackProfileData.count(
Value.first)) {
606 CallstackProfileData[
Value.first].Merge(
Value.second);
608 CallstackProfileData[Value.first] = Value.second;
614 const CallStackMap CSM = readStackInfo(Next + Header->StackOffset);
615 if (StackMap.empty()) {
618 if (mergeStackMap(CSM, StackMap))
619 return make_error<InstrProfError>(
621 "memprof raw profile got different call stack for same id");
624 Next += Header->TotalSize;
630object::SectionedAddress
631RawMemProfReader::getModuleOffset(
const uint64_t VirtualAddress) {
632 if (VirtualAddress > ProfiledTextSegmentStart &&
633 VirtualAddress <= ProfiledTextSegmentEnd) {
639 VirtualAddress + PreferredTextSegmentAddress - ProfiledTextSegmentStart;
640 return object::SectionedAddress{AdjustedAddress};
645 return object::SectionedAddress{VirtualAddress};
654 auto IdToFrameCallback = [
this](
const FrameId Id) {
656 if (!this->KeepSymbolName)
658 auto Iter = this->GuidToSymbolName.
find(
F.Function);
660 F.SymbolName =
Iter->getSecond();
BlockVerifier::State From
This file declares a library for handling Build IDs and using them to find debug info.
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file defines DenseMapInfo traits for DenseMap.
This file defines the DenseMap class.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallSet class.
This file defines the SmallVector class.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
static std::unique_ptr< DWARFContext > create(const object::ObjectFile &Obj, ProcessDebugRelocations RelocAction=ProcessDebugRelocations::Process, const LoadedObjectInfo *L=nullptr, std::string DWPName="", std::function< void(Error)> RecoverableErrorHandler=WithColor::defaultErrorHandler, std::function< void(Error)> WarningHandler=WithColor::defaultWarningHandler, bool ThreadSafe=false)
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Implements a dense probed hash-table based set.
Lightweight error class with error context and mandatory checking.
static ErrorSuccess success()
Create a success value.
Tagged union holding either a T or a Error.
uint64_t GUID
Declare a type to represent a global unique identifier for a global value.
This class implements a map that also provides access to all stored values in a deterministic order.
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
This interface provides simple read-only access to a block of memory, and provides simple methods for...
size_t getBufferSize() const
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".
const char * getBufferStart() const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
bool contains(const T &V) const
Check if the SmallSet contains the given element.
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr bool empty() const
empty - Check if the string is empty.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
LLVM Value Representation.
std::pair< iterator, bool > insert(const ValueT &V)
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
const Frame & idToFrame(const FrameId Id) const
virtual Error readNextRecord(GuidMemProfRecordPair &GuidRecord, std::function< const Frame(const FrameId)> Callback=nullptr)
llvm::DenseMap< FrameId, Frame > IdToFrame
llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord >::iterator Iter
llvm::MapVector< GlobalValue::GUID, IndexedMemProfRecord > FunctionProfileData
std::pair< GlobalValue::GUID, MemProfRecord > GuidMemProfRecordPair
void printYAML(raw_ostream &OS)
static Expected< std::unique_ptr< RawMemProfReader > > create(const Twine &Path, StringRef ProfiledBinary, bool KeepName=false)
static std::vector< std::string > peekBuildIds(MemoryBuffer *DataBuffer)
virtual Error readNextRecord(GuidMemProfRecordPair &GuidRecord, std::function< const Frame(const FrameId)> Callback) override
static bool hasFormat(const MemoryBuffer &DataBuffer)
This class implements an extremely fast bulk output stream that can only output to a stream.
static StringRef getCanonicalFnName(const Function &F)
Return the canonical name for a function, taking into account suffix elision policy attributes.
static Expected< std::unique_ptr< SymbolizableObjectFile > > create(const object::ObjectFile *Obj, std::unique_ptr< DIContext > DICtx, bool UntagAddresses)
llvm::DenseMap< uint64_t, llvm::SmallVector< uint64_t > > CallStackMap
BuildIDRef getBuildID(const ObjectFile *Obj)
Returns the build ID, if any, contained in the given object file.
ArrayRef< uint8_t > BuildIDRef
A reference to a BuildID in binary form.
Expected< std::unique_ptr< Binary > > createBinary(MemoryBufferRef Source, LLVMContext *Context=nullptr, bool InitContent=true)
Create a Binary from Source, autodetecting the file type.
StringRef filename(StringRef path, Style style=Style::native)
Get filename.
This is an optimization pass for GlobalISel generic memory operations.
std::error_code inconvertibleErrorCode()
The value returned by this function can be returned from convertToErrorCode for Error values where no...
Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)
Create formatted StringError object.
Error joinErrors(Error E1, Error E2)
Concatenate errors.
FormattedNumber format_hex_no_prefix(uint64_t N, unsigned Width, bool Upper=false)
format_hex_no_prefix - Output N as a fixed width hexadecimal.
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Error errorCodeToError(std::error_code EC)
Helper for converting an std::error_code to a Error.
static constexpr const char *const BadString
GlobalValue::GUID Function
static GlobalValue::GUID getGUID(const StringRef FunctionName)