30 #define DEBUG_TYPE "openmp-ir-builder"
37 cl::desc(
"Use optimistic attributes describing "
38 "'as-if' properties of runtime calls."),
46 auto FnAttrs =
Attrs.getFnAttributes();
47 auto RetAttrs =
Attrs.getRetAttributes();
49 for (
size_t ArgNo = 0; ArgNo < Fn.
arg_size(); ++ArgNo)
52 #define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
53 #include "llvm/Frontend/OpenMP/OMPKinds.def"
57 #define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
59 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
60 RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet); \
61 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
63 ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]); \
64 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
66 #include "llvm/Frontend/OpenMP/OMPKinds.def"
80 #define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
82 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
84 Fn = M.getFunction(Str); \
86 #include "llvm/Frontend/OpenMP/OMPKinds.def"
92 #define OMP_RTL(Enum, Str, ...) \
94 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
96 #include "llvm/Frontend/OpenMP/OMPKinds.def"
100 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
110 LLVMContext::MD_callback,
112 2, {-1, -1},
true)}));
118 addAttributes(FnID, *Fn);
125 assert(Fn &&
"Failed to create OpenMP runtime function");
134 auto *Fn = dyn_cast<llvm::Function>(RTLFn.
getCallee());
135 assert(Fn &&
"Failed to create OpenMP runtime function pointer");
149 DeferredOutlines.push_back(OI);
153 ParallelRegionBlockSet.
clear();
155 OI.collectBlocks(ParallelRegionBlockSet, Blocks);
170 <<
" Exit: " << OI.ExitBB->getName() <<
"\n");
172 "Expected OpenMP outlining to be possible!");
177 LLVM_DEBUG(
dbgs() <<
" Outlined function: " << *OutlinedFn <<
"\n");
179 "OpenMP outlined functions should not return a value!");
184 M.getFunctionList().insertAfter(OuterFn->
getIterator(), OutlinedFn);
191 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);
192 if (AllowExtractorSinking) {
198 "Expected instructions to sink in the outlined region");
200 End = ArtificialEntry.
end();
205 if (
I.isTerminator())
208 I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
211 OI.EntryBB->moveBefore(&ArtificialEntry);
218 if (OI.PostOutlineCB)
219 OI.PostOutlineCB(*OutlinedFn);
223 OutlineInfos =
std::move(DeferredOutlines);
227 assert(OutlineInfos.empty() &&
"There must be no outstanding outlinings");
232 unsigned Reserve2Flags) {
234 LocFlags |= OMP_IDENT_FLAG_KMPC;
237 IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
244 cast<StructType>(IdentPtr->getPointerElementType()), IdentData);
249 if (GV.getType() == IdentPtr && GV.hasInitializer())
250 if (GV.getInitializer() == Initializer)
257 GV->setAlignment(
Align(8));
260 return Builder.CreatePointerCast(Ident, IdentPtr);
265 Triple triple(
M.getTargetTriple());
272 Constant *&SrcLocStr = SrcLocStrMap[LocStr];
280 if (GV.isConstant() && GV.hasInitializer() &&
281 GV.getInitializer() == Initializer)
284 SrcLocStr =
Builder.CreateGlobalStringPtr(LocStr,
"",
295 Buffer.push_back(
';');
297 Buffer.push_back(
';');
298 Buffer.
append(FunctionName);
299 Buffer.push_back(
';');
301 Buffer.push_back(
';');
303 Buffer.push_back(
';');
304 Buffer.push_back(
';');
305 return getOrCreateSrcLocStr(Buffer.
str());
309 return getOrCreateSrcLocStr(
";unknown;unknown;0;0;;");
316 return getOrCreateDefaultSrcLocStr();
318 if (
DIFile *DIF = DIL->getFile())
324 return getOrCreateSrcLocStr(
Function, FileName, DIL->getLine(),
330 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
331 "omp_global_thread_num");
336 bool ForceSimpleCall,
bool CheckCancelFlag) {
337 if (!updateToLocation(Loc))
339 return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
344 bool ForceSimpleCall,
bool CheckCancelFlag) {
351 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
354 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
357 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
360 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
363 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
367 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
368 Value *
Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags),
369 getOrCreateThreadID(getOrCreateIdent(SrcLocStr))};
374 bool UseCancelBarrier =
375 !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
378 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
379 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
380 : OMPRTL___kmpc_barrier),
383 if (UseCancelBarrier && CheckCancelFlag)
384 emitCancelationCheckImpl(Result, OMPD_parallel);
392 omp::Directive CanceledDirective) {
393 if (!updateToLocation(Loc))
397 auto *UI =
Builder.CreateUnreachable();
402 Builder.SetInsertPoint(ThenTI);
404 Value *CancelKind =
nullptr;
405 switch (CanceledDirective) {
406 #define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
407 case DirectiveEnum: \
408 CancelKind = Builder.getInt32(Value); \
410 #include "llvm/Frontend/OpenMP/OMPKinds.def"
415 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
416 Value *Ident = getOrCreateIdent(SrcLocStr);
417 Value *
Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
419 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel),
Args);
422 emitCancelationCheckImpl(Result, CanceledDirective);
425 Builder.SetInsertPoint(UI->getParent());
426 UI->eraseFromParent();
432 Value *CancelFlag, omp::Directive CanceledDirective) {
433 assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
434 "Unexpected cancellation!");
439 if (
Builder.GetInsertPoint() ==
BB->end()) {
443 BB->getContext(),
BB->getName() +
".cont",
BB->getParent());
446 BB->getTerminator()->eraseFromParent();
450 BB->getContext(),
BB->getName() +
".cncl",
BB->getParent());
454 Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
459 Builder.SetInsertPoint(CancellationBlock);
460 auto &FI = FinalizationStack.back();
464 Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->
begin());
471 omp::ProcBindKind ProcBind,
bool IsCancellable) {
472 if (!updateToLocation(Loc))
475 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
476 Value *Ident = getOrCreateIdent(SrcLocStr);
477 Value *ThreadID = getOrCreateThreadID(Ident);
483 Builder.CreateIntCast(NumThreads, Int32,
false)};
485 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads),
Args);
488 if (ProcBind != OMP_PROC_BIND_default) {
494 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind),
Args);
502 BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
510 Builder.restoreIP(OuterAllocaIP);
522 ToBeDeleted.push_back(TIDAddr);
523 ToBeDeleted.push_back(ZeroAddr);
546 if (
IP.getBlock()->end() ==
IP.getPoint()) {
552 assert(
IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
553 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
554 "Unexpected insertion point for finalization call!");
558 FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
566 Builder.CreateAlloca(Int32,
nullptr,
"tid.addr.local");
570 ToBeDeleted.push_back(
Builder.CreateLoad(Int32, TIDAddr,
"tid.addr.use"));
573 ToBeDeleted.push_back(ZeroAddrUse);
590 LLVM_DEBUG(
dbgs() <<
"Before body codegen: " << *OuterFn <<
"\n");
593 assert(BodyGenCB &&
"Expected body generation callback!");
595 BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB);
599 FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
600 if (
auto *
F = dyn_cast<llvm::Function>(RTLFn.
getCallee())) {
601 if (!
F->hasMetadata(llvm::LLVMContext::MD_callback)) {
610 llvm::LLVMContext::MD_callback,
620 OutlinedFn.addParamAttr(0, Attribute::NoAlias);
621 OutlinedFn.addParamAttr(1, Attribute::NoAlias);
622 OutlinedFn.addFnAttr(Attribute::NoUnwind);
623 OutlinedFn.addFnAttr(Attribute::NoRecurse);
625 assert(OutlinedFn.arg_size() >= 2 &&
626 "Expected at least tid and bounded tid as arguments");
627 unsigned NumCapturedVars =
628 OutlinedFn.arg_size() - 2;
630 CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
635 Value *ForkCallArgs[] = {
636 Ident,
Builder.getInt32(NumCapturedVars),
637 Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};
643 Builder.CreateCall(RTLFn, RealArgs);
646 << *
Builder.GetInsertBlock()->getParent() <<
"\n");
651 Builder.SetInsertPoint(PrivTID);
653 Builder.CreateStore(
Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);
663 Builder.SetInsertPoint(ElseTI);
666 Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
668 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
669 SerializedParallelCallArgs);
676 Value *EndArgs[] = {Ident, ThreadID};
678 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
682 << *
Builder.GetInsertBlock()->getParent() <<
"\n");
686 I->eraseFromParent();
692 auto FiniInfo = FinalizationStack.pop_back_val();
694 assert(FiniInfo.DK == OMPD_parallel &&
695 "Unexpected finalization stack state!");
714 PRegOutlinedExitBB->
setName(
"omp.par.outlined.exit");
715 Blocks.push_back(PRegOutlinedExitBB);
730 Extractor.
findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
733 LLVM_DEBUG(
dbgs() <<
"Before privatization: " << *OuterFn <<
"\n");
736 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
738 auto PrivHelper = [&](
Value &V) {
739 if (&V == TIDAddr || &V == ZeroAddr)
743 for (
Use &U : V.uses())
744 if (
auto *UserI = dyn_cast<Instruction>(U.getUser()))
745 if (ParallelRegionBlockSet.
count(UserI->getParent()))
755 if (!V.getType()->isPointerTy()) {
759 Builder.restoreIP(OuterAllocaIP);
761 Builder.CreateAlloca(V.getType(),
nullptr, V.getName() +
".reloaded");
765 Builder.SetInsertPoint(InsertBB,
770 Builder.restoreIP(InnerAllocaIP);
771 Inner =
Builder.CreateLoad(V.getType(), Ptr);
774 Value *ReplacementValue =
nullptr;
775 CallInst *CI = dyn_cast<CallInst>(&V);
777 ReplacementValue = PrivTID;
780 PrivCB(InnerAllocaIP,
Builder.saveIP(), V, *Inner, ReplacementValue));
781 assert(ReplacementValue &&
782 "Expected copy/create callback to set replacement value!");
783 if (ReplacementValue == &V)
788 UPtr->set(ReplacementValue);
805 for (
Value *Input : Inputs) {
810 for (
Value *Output : Outputs)
814 "OpenMP outlining should not produce live-out values!");
816 LLVM_DEBUG(
dbgs() <<
"After privatization: " << *OuterFn <<
"\n");
818 for (
auto *
BB : Blocks)
819 dbgs() <<
" PBR: " <<
BB->getName() <<
"\n";
825 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
826 UI->eraseFromParent();
833 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
834 Value *
Args[] = {getOrCreateIdent(SrcLocStr)};
836 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush),
Args);
840 if (!updateToLocation(Loc))
848 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
849 Value *Ident = getOrCreateIdent(SrcLocStr);
850 Value *
Args[] = {Ident, getOrCreateThreadID(Ident)};
853 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
858 if (!updateToLocation(Loc))
860 emitTaskwaitImpl(Loc);
865 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
866 Value *Ident = getOrCreateIdent(SrcLocStr);
868 Value *
Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
870 Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
875 if (!updateToLocation(Loc))
877 emitTaskyieldImpl(Loc);
885 if (!updateToLocation(Loc))
889 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
890 Value *Ident = getOrCreateIdent(SrcLocStr);
891 Value *ThreadId = getOrCreateThreadID(Ident);
894 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
897 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
900 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
930 Builder.SetInsertPoint(Preheader);
933 Builder.SetInsertPoint(Header);
940 Builder.CreateICmpULT(IndVarPHI, TripCount,
"omp_" +
Name +
".cmp");
941 Builder.CreateCondBr(Cmp, Body, Exit);
948 "omp_" +
Name +
".next",
true);
956 LoopInfos.emplace_front();
959 CL->Preheader = Preheader;
983 NextBB, NextBB,
Name);
987 if (updateToLocation(Loc)) {
991 Builder.CreateBr(CL->Preheader);
1009 Value *Start,
Value *Stop,
Value *Step,
bool IsSigned,
bool InclusiveStop,
1019 auto *IndVarTy = cast<IntegerType>(Start->getType());
1020 assert(IndVarTy == Stop->getType() &&
"Stop type mismatch");
1021 assert(IndVarTy == Step->
getType() &&
"Step type mismatch");
1025 updateToLocation(ComputeLoc);
1043 Incr =
Builder.CreateSelect(IsNeg,
Builder.CreateNeg(Step), Step);
1046 Span =
Builder.CreateSub(UB, LB,
"",
false,
true);
1050 Span =
Builder.CreateSub(Stop, Start,
"",
true);
1055 Value *CountIfLooping;
1056 if (InclusiveStop) {
1057 CountIfLooping =
Builder.CreateAdd(
Builder.CreateUDiv(Span, Incr), One);
1064 CountIfLooping =
Builder.CreateSelect(OneCmp, One, CountIfTwo);
1066 Value *TripCount =
Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
1067 "omp_" +
Name +
".tripcount");
1073 BodyGenCB(
Builder.saveIP(), IndVar);
1076 return createCanonicalLoop(LoopLoc, BodyGen, TripCount,
Name);
1088 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
1091 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
1100 assert(isa<CmpInst>(CmpI) &&
"First inst must compare IV with TripCount");
1109 if (!updateToLocation(Loc))
1112 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1113 Value *SrcLoc = getOrCreateIdent(SrcLocStr);
1120 getOrCreateRuntimeFunction(
M, omp::OMPRTL___kmpc_for_static_fini);
1125 Value *PLastIter =
Builder.CreateAlloca(I32Type,
nullptr,
"p.lastiter");
1126 Value *PLowerBound =
Builder.CreateAlloca(IVTy,
nullptr,
"p.lowerbound");
1127 Value *PUpperBound =
Builder.CreateAlloca(IVTy,
nullptr,
"p.upperbound");
1128 Value *PStride =
Builder.CreateAlloca(IVTy,
nullptr,
"p.stride");
1137 Builder.CreateStore(Zero, PLowerBound);
1139 Builder.CreateStore(UpperBound, PUpperBound);
1140 Builder.CreateStore(One, PStride);
1145 Value *ThreadNum = getOrCreateThreadID(SrcLoc);
1149 constexpr
int StaticSchedType = 34;
1154 Builder.CreateCall(StaticInit,
1155 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
1156 PUpperBound, PStride, One, Chunk});
1157 Value *LowerBound =
Builder.CreateLoad(IVTy, PLowerBound);
1158 Value *InclusiveUpperBound =
Builder.CreateLoad(IVTy, PUpperBound);
1159 Value *TripCountMinusOne =
Builder.CreateSub(InclusiveUpperBound, LowerBound);
1160 Value *TripCount =
Builder.CreateAdd(TripCountMinusOne, One);
1171 auto *Instr = dyn_cast<Instruction>(U.
getUser());
1173 (Instr->getParent() != CLI->
getCond() &&
1180 Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
1185 omp::Directive::OMPD_for,
false,
1196 return createStaticWorkshareLoop(Loc, CLI, AllocaIP, NeedsBarrier);
1207 auto *Br = cast<BranchInst>(
Term);
1208 assert(!Br->isConditional() &&
1209 "BB's terminator must be an unconditional branch (or degenerate)");
1212 Br->setSuccessor(0,
Target);
1217 NewBr->setDebugLoc(
DL);
1232 auto HasRemainingUses = [&BBsToErase](
BasicBlock *
BB) {
1233 for (
Use &U :
BB->uses()) {
1234 auto *UseInst = dyn_cast<Instruction>(U.getUser());
1237 if (BBsToErase.count(UseInst->getParent()))
1245 bool Changed =
false;
1247 if (HasRemainingUses(
BB)) {
1248 BBsToErase.erase(
BB);
1263 assert(
Loops.size() >= 1 &&
"At least one loop required");
1264 size_t NumLoops =
Loops.size();
1268 return Loops.front();
1278 if (ComputeIP.isSet())
1285 Value *CollapsedTripCount =
nullptr;
1287 Value *OrigTripCount = L->getTripCount();
1288 if (!CollapsedTripCount) {
1289 CollapsedTripCount = OrigTripCount;
1294 CollapsedTripCount =
Builder.CreateMul(CollapsedTripCount, OrigTripCount,
1300 createLoopSkeleton(
DL, CollapsedTripCount,
F,
1301 OrigPreheader->
getNextNode(), OrigAfter,
"collapsed");
1307 Builder.restoreIP(Result->getBodyIP());
1309 Value *Leftover = Result->getIndVar();
1311 NewIndVars.set_size(NumLoops);
1312 for (
int i = NumLoops - 1;
i >= 1; --
i) {
1315 Value *NewIndVar =
Builder.CreateURem(Leftover, OrigTripCount);
1316 NewIndVars[
i] = NewIndVar;
1318 Leftover =
Builder.CreateUDiv(Leftover, OrigTripCount);
1321 NewIndVars[0] = Leftover;
1330 BasicBlock *ContinueBlock = Result->getBody();
1332 auto ContinueWith = [&ContinueBlock, &ContinuePred,
DL](
BasicBlock *Dest,
1339 ContinueBlock =
nullptr;
1340 ContinuePred = NextSrc;
1347 for (
size_t i = 0;
i < NumLoops - 1; ++
i)
1348 ContinueWith(
Loops[
i]->getBody(),
Loops[
i + 1]->getHeader());
1354 for (
size_t i = NumLoops - 1;
i > 0; --
i)
1355 ContinueWith(
Loops[
i]->getAfter(),
Loops[
i - 1]->getLatch());
1358 ContinueWith(Result->getLatch(),
nullptr);
1365 for (
size_t i = 0;
i < NumLoops; ++
i)
1366 Loops[
i]->getIndVar()->replaceAllUsesWith(NewIndVars[
i]);
1372 Loop->collectControlBlocks(OldControlBBs);
1381 std::vector<CanonicalLoopInfo *>
1385 "Must pass as many tile sizes as there are loops");
1386 int NumLoops =
Loops.size();
1387 assert(NumLoops >= 1 &&
"At least one loop to tile required");
1401 OrigTripCounts.push_back(L->getTripCount());
1402 OrigIndVars.push_back(L->getIndVar());
1412 for (
int i = 0;
i < NumLoops - 1; ++
i) {
1425 for (
int i = 0;
i < NumLoops; ++
i) {
1427 Value *OrigTripCount = OrigTripCounts[
i];
1440 Value *FloorTripOverflow =
1443 FloorTripOverflow =
Builder.CreateZExt(FloorTripOverflow, IVType);
1445 Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
1446 "omp_floor" +
Twine(
i) +
".tripcount",
true);
1449 FloorCount.push_back(FloorTripCount);
1450 FloorRems.push_back(FloorTripRem);
1454 std::vector<CanonicalLoopInfo *> Result;
1455 Result.reserve(NumLoops * 2);
1468 auto EmbeddNewLoop =
1469 [
this,
DL,
F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
1472 DL, TripCount,
F, InnerEnter, OutroInsertBefore,
Name);
1477 Enter = EmbeddedLoop->
getBody();
1478 Continue = EmbeddedLoop->
getLatch();
1479 OutroInsertBefore = EmbeddedLoop->
getLatch();
1480 return EmbeddedLoop;
1484 const Twine &NameBase) {
1487 EmbeddNewLoop(
P.value(), NameBase +
Twine(
P.index()));
1488 Result.push_back(EmbeddedLoop);
1492 EmbeddNewLoops(FloorCount,
"floor");
1496 Builder.SetInsertPoint(Enter->getTerminator());
1498 for (
int i = 0;
i < NumLoops; ++
i) {
1502 Value *FloorIsEpilogue =
1504 Value *TileTripCount =
1507 TileCounts.push_back(TileTripCount);
1511 EmbeddNewLoops(TileCounts,
"tile");
1516 for (std::pair<BasicBlock *, BasicBlock *>
P : InbetweenCode) {
1525 BodyEnter =
nullptr;
1526 BodyEntered = ExitBB;
1538 Builder.restoreIP(Result.back()->getBodyIP());
1539 for (
int i = 0;
i < NumLoops; ++
i) {
1542 Value *OrigIndVar = OrigIndVars[
i];
1556 Loop->collectControlBlocks(OldControlBBs);
1570 if (!updateToLocation(Loc))
1573 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1574 Value *Ident = getOrCreateIdent(SrcLocStr);
1575 Value *ThreadId = getOrCreateThreadID(Ident);
1579 Value *
Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
1581 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
1592 if (!updateToLocation(Loc))
1600 Directive OMPD = Directive::OMPD_single;
1601 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1602 Value *Ident = getOrCreateIdent(SrcLocStr);
1603 Value *ThreadId = getOrCreateThreadID(Ident);
1606 Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
1609 Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
1618 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1626 if (!updateToLocation(Loc))
1629 Directive OMPD = Directive::OMPD_critical;
1630 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1631 Value *Ident = getOrCreateIdent(SrcLocStr);
1632 Value *ThreadId = getOrCreateThreadID(Ident);
1633 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
1634 Value *
Args[] = {Ident, ThreadId, LockVar};
1640 EnterArgs.push_back(HintInst);
1641 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
1643 RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
1648 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
1651 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1657 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB,
bool Conditional,
1661 FinalizationStack.push_back({FiniCB, OMPD,
false});
1667 if (!isa_and_nonnull<BranchInst>(SplitPos))
1674 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
1677 BodyGenCB( InsertPointTy(),
1684 if (SkipEmittingRegion) {
1689 assert(!FinalizationStack.empty() &&
1690 "Unexpected finalization stack state!");
1691 FinalizationStack.pop_back();
1698 "Unexpected control flow graph state!!");
1699 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
1701 "Unexpected Control Flow State!");
1708 "Unexpected Insertion point location!");
1709 if (!Conditional && SkipEmittingRegion) {
1711 Builder.ClearInsertionPoint();
1715 auto InsertBB = merged ? ExitPredBB : ExitBB;
1716 if (!isa_and_nonnull<BranchInst>(SplitPos))
1718 Builder.SetInsertPoint(InsertBB);
1744 Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
1748 UI->eraseFromParent();
1749 Builder.SetInsertPoint(ThenBB->getTerminator());
1756 omp::Directive OMPD, InsertPointTy FinIP,
Instruction *ExitCall,
1763 assert(!FinalizationStack.empty() &&
1764 "Unexpected finalization stack state!");
1766 FinalizationInfo Fi = FinalizationStack.pop_back_val();
1767 assert(Fi.DK == OMPD &&
"Unexpected Directive for Finalization call!");
1775 Builder.SetInsertPoint(FiniBBTI);
1814 if (isa_and_nonnull<BranchInst>(OMP_Entry->
getTerminator())) {
1816 "copyin.not.master.end");
1823 Builder.SetInsertPoint(OMP_Entry);
1824 Value *MasterPtr =
Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
1825 Value *PrivatePtr =
Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
1827 Builder.CreateCondBr(
cmp, CopyBegin, CopyEnd);
1829 Builder.SetInsertPoint(CopyBegin);
1842 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1843 Value *Ident = getOrCreateIdent(SrcLocStr);
1844 Value *ThreadId = getOrCreateThreadID(Ident);
1847 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
1858 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1859 Value *Ident = getOrCreateIdent(SrcLocStr);
1860 Value *ThreadId = getOrCreateThreadID(Ident);
1862 Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
1872 Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1873 Value *Ident = getOrCreateIdent(SrcLocStr);
1874 Value *ThreadId = getOrCreateThreadID(Ident);
1876 getOrCreateOMPInternalVariable(Int8PtrPtr,
Name);
1880 getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
1895 return OS.
str().str();
1898 Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
1909 auto &Elem = *InternalVars.try_emplace(RuntimeName,
nullptr).first;
1911 assert(Elem.second->getType()->getPointerElementType() == Ty &&
1912 "OMP internal variable has different type than requested");
1928 Value *OpenMPIRBuilder::getOMPCriticalRegionLock(
StringRef CriticalName) {
1929 std::string
Prefix =
Twine(
"gomp_critical_user_", CriticalName).
str();
1930 std::string
Name = getNameWithSeparators({
Prefix,
"var"},
".",
".");
1931 return getOrCreateOMPInternalVariable(KmpCriticalNameTy,
Name);
1936 void OpenMPIRBuilder::initializeTypes(
Module &M) {
1939 #define OMP_TYPE(VarName, InitValue) VarName = InitValue;
1940 #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
1941 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
1942 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
1943 #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
1944 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
1945 VarName##Ptr = PointerType::getUnqual(VarName);
1946 #define OMP_STRUCT_TYPE(VarName, StructName, ...) \
1947 T = StructType::getTypeByName(Ctx, StructName); \
1949 T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \
1951 VarName##Ptr = PointerType::getUnqual(T);
1952 #include "llvm/Frontend/OpenMP/OMPKinds.def"
1959 BlockSet.
insert(EntryBB);
1962 Worklist.push_back(EntryBB);
1963 while (!Worklist.empty()) {
1965 BlockVector.push_back(
BB);
1967 if (BlockSet.
insert(SuccBB).second)
1968 Worklist.push_back(SuccBB);
1972 void CanonicalLoopInfo::collectControlBlocks(
1979 BBs.
append({Preheader, Header,
Cond, Latch, Exit, After});
1990 "Preheader must terminate with unconditional branch");
1992 "Preheader must jump to header");
1996 "Header must terminate with unconditional branch");
1998 "Header must jump to exiting block");
2001 assert(
Cond->getSinglePredecessor() == Header &&
2002 "Exiting block only reachable from header");
2004 assert(isa<BranchInst>(
Cond->getTerminator()) &&
2005 "Exiting block must terminate with conditional branch");
2007 "Exiting block must have two successors");
2008 assert(cast<BranchInst>(
Cond->getTerminator())->getSuccessor(0) == Body &&
2009 "Exiting block's first successor jump to the body");
2010 assert(cast<BranchInst>(
Cond->getTerminator())->getSuccessor(1) == Exit &&
2011 "Exiting block's second successor must exit the loop");
2015 "Body only reachable from exiting block");
2020 "Latch must terminate with unconditional branch");
2029 "Exit block must terminate with unconditional branch");
2031 "Exit block must jump to after block");
2035 "After block only reachable from exit block");
2039 assert(IndVar &&
"Canonical induction variable not found?");
2041 "Induction variable must be an integer");
2043 "Induction variable must be a PHI in the loop header");
2044 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
2046 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->
isZero());
2047 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
2049 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
2052 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
2053 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
2056 Value *TripCount = getTripCount();
2057 assert(TripCount &&
"Loop trip count not found?");
2059 "Trip count and induction variable must have the same type");
2061 auto *CmpI = cast<CmpInst>(&
Cond->front());
2063 "Exit condition must be a signed less-than comparison");
2064 assert(CmpI->getOperand(0) == IndVar &&
2065 "Exit condition must compare the induction variable");
2066 assert(CmpI->getOperand(1) == TripCount &&
2067 "Exit condition must compare with the trip count");