51 #define DEBUG_TYPE "insert-gcov-profiling"
67 cl::desc(
"Make counter updates atomic"));
72 return (
s.size() / 4) + 2;
95 GCOVProfiler() : GCOVProfiler(
GCOVOptions::getDefault()) {}
109 os->write(
s.data(),
s.size());
110 os->write_zeros(4 -
s.size() % 4);
112 void writeBytes(
const char *Bytes,
int Size) { os->write(Bytes, Size); }
117 emitProfileNotes(
NamedMDNode *CUNode,
bool HasExecOrFork,
123 void emitGlobalConstructor(
126 bool isFunctionInstrumented(
const Function &
F);
127 std::vector<Regex> createRegexesFromString(
StringRef RegexesStr);
128 static bool doesFilenameMatchARegex(
StringRef Filename,
129 std::vector<Regex> &Regexes);
141 insertCounterWriteout(
ArrayRef<std::pair<GlobalVariable *, MDNode *>>);
144 bool AddFlushBeforeForkAndExec();
146 enum class GCovFileType { GCNO, GCDA };
160 std::vector<Regex> FilterRe;
161 std::vector<Regex> ExcludeRe;
172 std::string infoString()
const {
186 bool Removed =
false;
187 bool IsCritical =
false;
190 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
193 std::string infoString()
const {
194 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
195 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight))
202 if (!SP->getLinkageName().empty())
203 return SP->getLinkageName();
227 GCOVRecord(GCOVProfiler *
P) :
P(
P) {}
231 void writeBytes(
const char *Bytes,
int Size) {
P->writeBytes(Bytes, Size); }
240 class GCOVLines :
public GCOVRecord {
243 assert(Line != 0 &&
"Line zero is not a valid real line number.");
244 Lines.push_back(Line);
253 writeString(Filename);
254 for (
int i = 0,
e =
Lines.size();
i !=
e; ++
i)
262 std::string Filename;
273 return LinesByFile.try_emplace(Filename,
P, Filename).first->second;
277 OutEdges.emplace_back(&
Successor, Flags);
283 for (
auto &
I : LinesByFile) {
284 Len +=
I.second.length();
285 SortedLinesByFile.push_back(&
I);
294 return LHS->getKey() <
RHS->getKey();
296 for (
auto &
I : SortedLinesByFile)
297 I->getValue().writeOut();
306 assert(LinesByFile.empty());
329 : GCOVRecord(
P), SP(SP), EndLine(EndLine), Ident(Ident),
332 bool ExitBlockBeforeBody =
Version >= 48;
333 uint32_t i = ExitBlockBeforeBody ? 2 : 1;
336 if (!ExitBlockBeforeBody)
337 ReturnBlock.Number =
i;
339 std::string FunctionNameAndLine;
343 FuncChecksum =
hash_value(FunctionNameAndLine);
347 return Blocks.find(
const_cast<BasicBlock *
>(
BB))->second;
350 GCOVBlock &getEntryBlock() {
return EntryBlock; }
359 void writeOut(
uint32_t CfgChecksum) {
376 writeString(Filename);
377 write(SP->getLine());
379 write(SP->isArtificial());
380 writeString(Filename);
381 write(SP->getLine());
393 write(Blocks.size() + 2);
394 for (
int i = Blocks.size() + 2;
i; --
i)
398 write(Blocks.size() + 2);
403 const uint32_t Outgoing = EntryBlock.OutEdges.size();
406 write(Outgoing * 2 + 1);
407 write(EntryBlock.Number);
408 for (
const auto &
E : EntryBlock.OutEdges) {
413 for (
auto &It : Blocks) {
415 if (
Block.OutEdges.empty())
continue;
420 for (
const auto &
E :
Block.OutEdges) {
427 for (
auto &It : Blocks)
428 It.second.writeOut();
445 std::vector<Regex> GCOVProfiler::createRegexesFromString(
StringRef RegexesStr) {
446 std::vector<Regex> Regexes;
447 while (!RegexesStr.
empty()) {
448 std::pair<StringRef, StringRef> HeadTail = RegexesStr.
split(
';');
449 if (!HeadTail.first.empty()) {
450 Regex Re(HeadTail.first);
452 if (!Re.isValid(Err)) {
453 Ctx->emitError(
Twine(
"Regex ") + HeadTail.first +
454 " is not valid: " + Err);
458 RegexesStr = HeadTail.second;
463 bool GCOVProfiler::doesFilenameMatchARegex(
StringRef Filename,
464 std::vector<Regex> &Regexes) {
465 for (
Regex &Re : Regexes)
466 if (Re.match(Filename))
471 bool GCOVProfiler::isFunctionInstrumented(
const Function &
F) {
472 if (FilterRe.empty() && ExcludeRe.empty()) {
476 auto It = InstrumentedFiles.find(Filename);
477 if (It != InstrumentedFiles.end()) {
489 RealFilename = Filename;
491 RealFilename = RealPath;
494 bool ShouldInstrument;
495 if (FilterRe.empty()) {
496 ShouldInstrument = !doesFilenameMatchARegex(RealFilename, ExcludeRe);
497 }
else if (ExcludeRe.empty()) {
498 ShouldInstrument = doesFilenameMatchARegex(RealFilename, FilterRe);
500 ShouldInstrument = doesFilenameMatchARegex(RealFilename, FilterRe) &&
501 !doesFilenameMatchARegex(RealFilename, ExcludeRe);
503 InstrumentedFiles[Filename] = ShouldInstrument;
504 return ShouldInstrument;
508 GCovFileType OutputType) {
509 bool Notes = OutputType == GCovFileType::GCNO;
511 if (
NamedMDNode *GCov =
M->getNamedMetadata(
"llvm.gcov")) {
512 for (
int i = 0,
e = GCov->getNumOperands();
i !=
e; ++
i) {
514 bool ThreeElement =
N->getNumOperands() == 3;
515 if (!ThreeElement &&
N->getNumOperands() != 2)
517 if (dyn_cast<MDNode>(
N->getOperand(ThreeElement ? 2 : 1)) !=
CU)
523 MDString *NotesFile = dyn_cast<MDString>(
N->getOperand(0));
524 MDString *DataFile = dyn_cast<MDString>(
N->getOperand(1));
525 if (!NotesFile || !DataFile)
527 return std::string(Notes ? NotesFile->
getString()
531 MDString *GCovFile = dyn_cast<MDString>(
N->getOperand(0));
537 return std::string(Filename.
str());
546 return std::string(FName);
548 return std::string(CurPath.
str());
551 bool GCOVProfiler::runOnModule(
557 Ctx = &
M.getContext();
559 NamedMDNode *CUNode =
M.getNamedMetadata(
"llvm.dbg.cu");
563 bool HasExecOrFork = AddFlushBeforeForkAndExec();
565 FilterRe = createRegexesFromString(
Options.Filter);
566 ExcludeRe = createRegexesFromString(
Options.Exclude);
567 emitProfileNotes(CUNode, HasExecOrFork, GetBFI, GetBPI, this->GetTLI);
574 GCOVProfiler Profiler(GCOVOpts);
588 if (!Profiler.runOnModule(
M, GetBFI, GetBPI, GetTLI))
602 if (isa<DbgInfoIntrinsic>(&
I))
continue;
609 if (Loc.
getLine() == 0)
continue;
619 if (!
F.hasPersonalityFn())
return false;
625 bool GCOVProfiler::AddFlushBeforeForkAndExec() {
628 for (
auto &
F :
M->functions()) {
629 auto *TLI = &GetTLI(
F);
631 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
632 if (
Function *Callee = CI->getCalledFunction()) {
634 if (TLI->getLibFunc(*Callee, LF)) {
635 if (LF == LibFunc_fork) {
639 }
else if (LF == LibFunc_execl || LF == LibFunc_execle ||
640 LF == LibFunc_execlp || LF == LibFunc_execv ||
641 LF == LibFunc_execvp || LF == LibFunc_execve ||
642 LF == LibFunc_execvpe || LF == LibFunc_execvP) {
651 for (
auto F : Forks) {
654 auto NextInst = ++
F->getIterator();
659 F->setCalledFunction(GCOVFork);
676 for (
auto E : Execs) {
679 auto NextInst = ++
E->getIterator();
685 M->getOrInsertFunction(
"llvm_writeout_files", FTy);
689 Builder.SetInsertPoint(&*NextInst);
692 FunctionCallee ResetF =
M->getOrInsertFunction(
"llvm_reset_counters", FTy);
693 Builder.CreateCall(ResetF)->setDebugLoc(Loc);
694 ExecBlocks.insert(Parent);
699 return !Forks.empty() || !Execs.empty();
704 if (
E.InMST ||
E.Removed)
710 if (SrcBB ==
nullptr)
712 if (DestBB ==
nullptr)
718 if (
BB->getFirstInsertionPt() ==
BB->end())
727 return CanInstrument(SrcBB);
729 return CanInstrument(DestBB);
739 MST.
addEdge(SrcBB, InstrBB, 0);
740 MST.
addEdge(InstrBB, DestBB, 0).InMST =
true;
743 return CanInstrument(InstrBB);
750 GCOVBlock &Src =
E.SrcBB ? GF.getBlock(
E.SrcBB) : GF.getEntryBlock();
751 GCOVBlock &Dst =
E.DestBB ? GF.getBlock(
E.DestBB) : GF.getReturnBlock();
752 dbgs() <<
" Edge " <<
ID++ <<
": " << Src.Number <<
"->" << Dst.Number
753 <<
E.infoString() <<
"\n";
758 bool GCOVProfiler::emitProfileNotes(
765 uint8_t c3 =
Options.Version[0];
767 uint8_t c1 =
Options.Version[2];
768 Version = c3 >=
'A' ? (c3 -
'A') * 100 + (
c2 -
'0') * 10 + c1 -
'0'
769 : (c3 -
'0') * 10 + c1 -
'0';
772 bool EmitGCDA =
Options.EmitData;
784 std::vector<uint8_t> EdgeDestinations;
789 unsigned FunctionIdent = 0;
790 for (
auto &
F :
M->functions()) {
798 if (
F.hasFnAttribute(llvm::Attribute::NoProfile))
817 for (
size_t I : llvm::seq<size_t>(0, MST.AllEdges.size())) {
818 auto &
E = *MST.AllEdges[
I];
827 Funcs.push_back(std::make_unique<GCOVFunction>(
this, &
F, SP, EndLine,
834 return E->Removed || (!E->InMST && !E->Place);
836 const size_t Measured =
837 std::stable_partition(
838 MST.AllEdges.begin(), MST.AllEdges.end(),
839 [](std::unique_ptr<Edge> &
E) { return E->Place; }) -
840 MST.AllEdges.begin();
841 for (
size_t I : llvm::seq<size_t>(0, Measured)) {
842 Edge &
E = *MST.AllEdges[
I];
844 E.SrcBB ?
Func.getBlock(
E.SrcBB) :
Func.getEntryBlock();
846 E.DestBB ?
Func.getBlock(
E.DestBB) :
Func.getReturnBlock();
847 E.SrcNumber = Src.Number;
848 E.DstNumber = Dst.Number;
851 MST.AllEdges.begin(), MST.AllEdges.begin() + Measured,
852 [](
const std::unique_ptr<Edge> &L,
const std::unique_ptr<Edge> &R) {
853 return L->SrcNumber != R->SrcNumber ? L->SrcNumber < R->SrcNumber
854 : L->DstNumber < R->DstNumber;
859 E.SrcBB ?
Func.getBlock(
E.SrcBB) :
Func.getEntryBlock();
861 E.DestBB ?
Func.getBlock(
E.DestBB) :
Func.getReturnBlock();
866 if (!SP->isArtificial())
867 Func.getBlock(&EntryBlock).getFile(Filename).addLine(Line);
871 for (
auto &GB :
Func.Blocks) {
873 auto &
Block = GB.second;
874 for (
auto Succ :
Block.OutEdges) {
876 do EdgeDestinations.push_back(Idx & 255);
877 while ((Idx >>= 8) > 0);
883 if (isa<DbgInfoIntrinsic>(&
I))
continue;
893 if (Line == Loc.
getLine())
continue;
911 for (
size_t I : llvm::seq<size_t>(0, Measured)) {
912 const Edge &
E = *MST.AllEdges[
I];
931 JC.
update(EdgeDestinations);
941 Twine(
"failed to open coverage notes file for writing: ") +
947 out.write(
"gcno", 4);
950 out.write(
"oncg", 4);
960 for (
auto &Func : Funcs)
961 Func->writeOut(Stamp);
969 emitGlobalConstructor(CountersBySP);
981 F->addFnAttr(Attribute::NoUnwind);
983 F->addFnAttr(Attribute::NoRedZone);
987 void GCOVProfiler::emitGlobalConstructor(
988 SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP) {
989 Function *WriteoutF = insertCounterWriteout(CountersBySP);
990 Function *ResetF = insertReset(CountersBySP);
996 Function *
F = createInternalFunction(FTy,
"__llvm_gcov_init");
997 F->addFnAttr(Attribute::NoInline);
1008 FunctionCallee GCOVInit =
M->getOrInsertFunction(
"llvm_gcov_init", FTy);
1009 Builder.CreateCall(GCOVInit, {WriteoutF, ResetF});
1024 AL =
AL.addParamAttribute(*Ctx, 2, AK);
1038 AL =
AL.addParamAttribute(*Ctx, 0, AK);
1039 AL =
AL.addParamAttribute(*Ctx, 1, AK);
1040 AL =
AL.addParamAttribute(*Ctx, 2, AK);
1042 return M->getOrInsertFunction(
"llvm_gcda_emit_function", FTy);
1053 AL =
AL.addParamAttribute(*Ctx, 0, AK);
1054 return M->getOrInsertFunction(
"llvm_gcda_emit_arcs", FTy,
AL);
1059 return M->getOrInsertFunction(
"llvm_gcda_summary_info", FTy);
1064 return M->getOrInsertFunction(
"llvm_gcda_end_file", FTy);
1067 Function *GCOVProfiler::insertCounterWriteout(
1068 ArrayRef<std::pair<GlobalVariable *, MDNode *> > CountersBySP) {
1070 Function *WriteoutF =
M->getFunction(
"__llvm_gcov_writeout");
1072 WriteoutF = createInternalFunction(WriteoutFTy,
"__llvm_gcov_writeout");
1073 WriteoutF->
addFnAttr(Attribute::NoInline);
1078 auto *TLI = &GetTLI(*WriteoutF);
1086 NamedMDNode *CUNodes =
M->getNamedMetadata(
"llvm.dbg.cu");
1096 "start_file_args_ty");
1099 "emit_function_args_ty");
1102 "emit_arcs_args_ty");
1105 EmitFunctionCallArgsTy->getPointerTo(),
1106 EmitArcsCallArgsTy->getPointerTo()},
1111 Constant *TwoZero32s[] = {Zero32, Zero32};
1121 std::string FilenameGcda = mangleName(
CU, GCovFileType::GCDA);
1124 StartFileCallArgsTy,
1125 {
Builder.CreateGlobalStringPtr(FilenameGcda),
1127 Builder.getInt32(CfgChecksum)});
1131 for (
int j : llvm::seq<int>(0, CountersBySP.size())) {
1132 uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[
j]->getFuncChecksum();
1134 EmitFunctionCallArgsTy,
1136 Builder.getInt32(FuncChecksum),
1137 Builder.getInt32(CfgChecksum)}));
1140 unsigned Arcs = cast<ArrayType>(GV->
getValueType())->getNumElements();
1147 int CountersSize = CountersBySP.size();
1148 assert(CountersSize == (
int)EmitFunctionCallArgsArray.size() &&
1149 "Mismatched array size!");
1150 assert(CountersSize == (
int)EmitArcsCallArgsArray.size() &&
1151 "Mismatched array size!");
1152 auto *EmitFunctionCallArgsArrayTy =
1155 *M, EmitFunctionCallArgsArrayTy,
true,
1158 EmitFunctionCallArgsArray),
1159 Twine(
"__llvm_internal_gcov_emit_function_args.") +
Twine(
i));
1160 auto *EmitArcsCallArgsArrayTy =
1162 EmitFunctionCallArgsArrayGV->setUnnamedAddr(
1165 *M, EmitArcsCallArgsArrayTy,
true,
1168 Twine(
"__llvm_internal_gcov_emit_arcs_args.") +
Twine(
i));
1173 {StartFileCallArgs,
Builder.getInt32(CountersSize),
1175 EmitFunctionCallArgsArrayGV,
1178 EmitArcsCallArgsArrayTy, EmitArcsCallArgsArrayGV, TwoZero32s)}));
1182 if (FileInfos.empty()) {
1192 if ((int64_t)FileInfos.size() > (int64_t)INT_MAX)
1193 FileInfos.
resize(INT_MAX);
1197 auto *FileInfoArrayTy =
ArrayType::get(FileInfoTy, FileInfos.size());
1201 "__llvm_internal_gcov_emit_file_info");
1205 auto *FileLoopHeader =
1207 auto *CounterLoopHeader =
1213 Builder.CreateBr(FileLoopHeader);
1216 Builder.SetInsertPoint(FileLoopHeader);
1220 auto *FileInfoPtr =
Builder.CreateInBoundsGEP(
1221 FileInfoArrayTy, FileInfoArrayGV, {
Builder.getInt32(0),
IV});
1222 auto *StartFileCallArgsPtr =
1223 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 0,
"start_file_args");
1224 auto *StartFileCall =
Builder.CreateCall(
1227 Builder.CreateStructGEP(StartFileCallArgsTy,
1228 StartFileCallArgsPtr, 0),
1231 Builder.CreateStructGEP(StartFileCallArgsTy,
1232 StartFileCallArgsPtr, 1),
1235 Builder.CreateStructGEP(StartFileCallArgsTy,
1236 StartFileCallArgsPtr, 2),
1239 StartFileCall->addParamAttr(2, AK);
1240 auto *NumCounters =
Builder.CreateLoad(
1241 FileInfoTy->getElementType(1),
1242 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 1),
"num_ctrs");
1243 auto *EmitFunctionCallArgsArray =
1244 Builder.CreateLoad(FileInfoTy->getElementType(2),
1245 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 2),
1246 "emit_function_args");
1247 auto *EmitArcsCallArgsArray =
Builder.CreateLoad(
1248 FileInfoTy->getElementType(3),
1249 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 3),
"emit_arcs_args");
1250 auto *EnterCounterLoopCond =
1252 Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch);
1254 Builder.SetInsertPoint(CounterLoopHeader);
1257 JV->addIncoming(
Builder.getInt32(0), FileLoopHeader);
1258 auto *EmitFunctionCallArgsPtr =
Builder.CreateInBoundsGEP(
1259 EmitFunctionCallArgsTy, EmitFunctionCallArgsArray, JV);
1260 auto *EmitFunctionCall =
Builder.CreateCall(
1262 {
Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(0),
1263 Builder.CreateStructGEP(EmitFunctionCallArgsTy,
1264 EmitFunctionCallArgsPtr, 0),
1266 Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(1),
1267 Builder.CreateStructGEP(EmitFunctionCallArgsTy,
1268 EmitFunctionCallArgsPtr, 1),
1270 Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(2),
1271 Builder.CreateStructGEP(EmitFunctionCallArgsTy,
1272 EmitFunctionCallArgsPtr, 2),
1275 EmitFunctionCall->addParamAttr(0, AK);
1276 EmitFunctionCall->addParamAttr(1, AK);
1277 EmitFunctionCall->addParamAttr(2, AK);
1279 auto *EmitArcsCallArgsPtr =
1280 Builder.CreateInBoundsGEP(EmitArcsCallArgsTy, EmitArcsCallArgsArray, JV);
1281 auto *EmitArcsCall =
Builder.CreateCall(
1284 EmitArcsCallArgsTy->getElementType(0),
1285 Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 0),
1288 EmitArcsCallArgsTy->getElementType(1),
1289 Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 1),
1292 EmitArcsCall->addParamAttr(0, AK);
1294 auto *CounterLoopCond =
Builder.CreateICmpSLT(NextJV, NumCounters);
1295 Builder.CreateCondBr(CounterLoopCond, CounterLoopHeader, FileLoopLatch);
1296 JV->addIncoming(NextJV, CounterLoopHeader);
1298 Builder.SetInsertPoint(FileLoopLatch);
1299 Builder.CreateCall(SummaryInfo, {});
1300 Builder.CreateCall(EndFile, {});
1301 auto *NextIV =
Builder.CreateAdd(
IV,
Builder.getInt32(1),
"next_file_idx");
1302 auto *FileLoopCond =
1303 Builder.CreateICmpSLT(NextIV,
Builder.getInt32(FileInfos.size()));
1304 Builder.CreateCondBr(FileLoopCond, FileLoopHeader, ExitBB);
1305 IV->addIncoming(NextIV, FileLoopLatch);
1307 Builder.SetInsertPoint(ExitBB);
1313 Function *GCOVProfiler::insertReset(
1314 ArrayRef<std::pair<GlobalVariable *, MDNode *>> CountersBySP) {
1316 Function *ResetF =
M->getFunction(
"__llvm_gcov_reset");
1318 ResetF = createInternalFunction(FTy,
"__llvm_gcov_reset");
1326 for (
const auto &
I : CountersBySP) {
1330 GVTy->getNumElements() *
1331 GVTy->getElementType()->getScalarSizeInBits() / 8,