62#define DEBUG_TYPE "openmp-ir-builder"
69 cl::desc(
"Use optimistic attributes describing "
70 "'as-if' properties of runtime calls."),
74 "openmp-ir-builder-unroll-threshold-factor",
cl::Hidden,
75 cl::desc(
"Factor for the unroll threshold to account for code "
76 "simplifications still taking place"),
87 if (!IP1.isSet() || !IP2.isSet())
89 return IP1.getBlock() == IP2.getBlock() && IP1.getPoint() == IP2.getPoint();
94 switch (SchedType & ~OMPScheduleType::MonotonicityMask) {
95 case OMPScheduleType::UnorderedStaticChunked:
96 case OMPScheduleType::UnorderedStatic:
97 case OMPScheduleType::UnorderedDynamicChunked:
98 case OMPScheduleType::UnorderedGuidedChunked:
99 case OMPScheduleType::UnorderedRuntime:
100 case OMPScheduleType::UnorderedAuto:
101 case OMPScheduleType::UnorderedTrapezoidal:
102 case OMPScheduleType::UnorderedGreedy:
103 case OMPScheduleType::UnorderedBalanced:
104 case OMPScheduleType::UnorderedGuidedIterativeChunked:
105 case OMPScheduleType::UnorderedGuidedAnalyticalChunked:
106 case OMPScheduleType::UnorderedSteal:
107 case OMPScheduleType::UnorderedStaticBalancedChunked:
108 case OMPScheduleType::UnorderedGuidedSimd:
109 case OMPScheduleType::UnorderedRuntimeSimd:
110 case OMPScheduleType::OrderedStaticChunked:
111 case OMPScheduleType::OrderedStatic:
112 case OMPScheduleType::OrderedDynamicChunked:
113 case OMPScheduleType::OrderedGuidedChunked:
114 case OMPScheduleType::OrderedRuntime:
115 case OMPScheduleType::OrderedAuto:
116 case OMPScheduleType::OrderdTrapezoidal:
117 case OMPScheduleType::NomergeUnorderedStaticChunked:
118 case OMPScheduleType::NomergeUnorderedStatic:
119 case OMPScheduleType::NomergeUnorderedDynamicChunked:
120 case OMPScheduleType::NomergeUnorderedGuidedChunked:
121 case OMPScheduleType::NomergeUnorderedRuntime:
122 case OMPScheduleType::NomergeUnorderedAuto:
123 case OMPScheduleType::NomergeUnorderedTrapezoidal:
124 case OMPScheduleType::NomergeUnorderedGreedy:
125 case OMPScheduleType::NomergeUnorderedBalanced:
126 case OMPScheduleType::NomergeUnorderedGuidedIterativeChunked:
127 case OMPScheduleType::NomergeUnorderedGuidedAnalyticalChunked:
128 case OMPScheduleType::NomergeUnorderedSteal:
129 case OMPScheduleType::NomergeOrderedStaticChunked:
130 case OMPScheduleType::NomergeOrderedStatic:
131 case OMPScheduleType::NomergeOrderedDynamicChunked:
132 case OMPScheduleType::NomergeOrderedGuidedChunked:
133 case OMPScheduleType::NomergeOrderedRuntime:
134 case OMPScheduleType::NomergeOrderedAuto:
135 case OMPScheduleType::NomergeOrderedTrapezoidal:
143 SchedType & OMPScheduleType::MonotonicityMask;
144 if (MonotonicityFlags == OMPScheduleType::MonotonicityMask)
155 if (Features.
count(
"+wavefrontsize64"))
156 return omp::getAMDGPUGridValues<64>();
157 return omp::getAMDGPUGridValues<32>();
168 bool HasSimdModifier) {
170 switch (ClauseKind) {
171 case OMP_SCHEDULE_Default:
172 case OMP_SCHEDULE_Static:
173 return HasChunks ? OMPScheduleType::BaseStaticChunked
174 : OMPScheduleType::BaseStatic;
175 case OMP_SCHEDULE_Dynamic:
176 return OMPScheduleType::BaseDynamicChunked;
177 case OMP_SCHEDULE_Guided:
178 return HasSimdModifier ? OMPScheduleType::BaseGuidedSimd
179 : OMPScheduleType::BaseGuidedChunked;
180 case OMP_SCHEDULE_Auto:
182 case OMP_SCHEDULE_Runtime:
183 return HasSimdModifier ? OMPScheduleType::BaseRuntimeSimd
184 : OMPScheduleType::BaseRuntime;
192 bool HasOrderedClause) {
193 assert((BaseScheduleType & OMPScheduleType::ModifierMask) ==
194 OMPScheduleType::None &&
195 "Must not have ordering nor monotonicity flags already set");
198 ? OMPScheduleType::ModifierOrdered
199 : OMPScheduleType::ModifierUnordered;
200 OMPScheduleType OrderingScheduleType = BaseScheduleType | OrderingModifier;
203 if (OrderingScheduleType ==
204 (OMPScheduleType::BaseGuidedSimd | OMPScheduleType::ModifierOrdered))
205 return OMPScheduleType::OrderedGuidedChunked;
206 else if (OrderingScheduleType == (OMPScheduleType::BaseRuntimeSimd |
207 OMPScheduleType::ModifierOrdered))
208 return OMPScheduleType::OrderedRuntime;
210 return OrderingScheduleType;
216 bool HasSimdModifier,
bool HasMonotonic,
217 bool HasNonmonotonic,
bool HasOrderedClause) {
218 assert((ScheduleType & OMPScheduleType::MonotonicityMask) ==
219 OMPScheduleType::None &&
220 "Must not have monotonicity flags already set");
221 assert((!HasMonotonic || !HasNonmonotonic) &&
222 "Monotonic and Nonmonotonic are contradicting each other");
225 return ScheduleType | OMPScheduleType::ModifierMonotonic;
226 }
else if (HasNonmonotonic) {
227 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
237 if ((BaseScheduleType == OMPScheduleType::BaseStatic) ||
238 (BaseScheduleType == OMPScheduleType::BaseStaticChunked) ||
244 return ScheduleType | OMPScheduleType::ModifierNonmonotonic;
252 bool HasSimdModifier,
bool HasMonotonicModifier,
253 bool HasNonmonotonicModifier,
bool HasOrderedClause) {
259 OrderedSchedule, HasSimdModifier, HasMonotonicModifier,
260 HasNonmonotonicModifier, HasOrderedClause);
274 auto *Br = cast<BranchInst>(Term);
275 assert(!Br->isConditional() &&
276 "BB's terminator must be an unconditional branch (or degenerate)");
279 Br->setSuccessor(0,
Target);
284 NewBr->setDebugLoc(
DL);
289 assert(New->getFirstInsertionPt() == New->begin() &&
290 "Target BB must not have PHI nodes");
294 New->splice(New->begin(), Old, IP.
getPoint(), Old->
end());
322 New->replaceSuccessorsPhiUsesWith(Old, New);
367 const Twine &
Name =
"",
bool AsPtr =
true) {
375 FakeVal = FakeValAddr;
403enum OpenMPOffloadingRequiresDirFlags {
405 OMP_REQ_UNDEFINED = 0x000,
407 OMP_REQ_NONE = 0x001,
409 OMP_REQ_REVERSE_OFFLOAD = 0x002,
411 OMP_REQ_UNIFIED_ADDRESS = 0x004,
413 OMP_REQ_UNIFIED_SHARED_MEMORY = 0x008,
415 OMP_REQ_DYNAMIC_ALLOCATORS = 0x010,
422 : RequiresFlags(OMP_REQ_UNDEFINED) {}
425 bool IsTargetDevice,
bool IsGPU,
bool OpenMPOffloadMandatory,
426 bool HasRequiresReverseOffload,
bool HasRequiresUnifiedAddress,
427 bool HasRequiresUnifiedSharedMemory,
bool HasRequiresDynamicAllocators)
428 : IsTargetDevice(IsTargetDevice), IsGPU(IsGPU),
429 OpenMPOffloadMandatory(OpenMPOffloadMandatory),
430 RequiresFlags(OMP_REQ_UNDEFINED) {
431 if (HasRequiresReverseOffload)
432 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
433 if (HasRequiresUnifiedAddress)
434 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
435 if (HasRequiresUnifiedSharedMemory)
436 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
437 if (HasRequiresDynamicAllocators)
438 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
442 return RequiresFlags & OMP_REQ_REVERSE_OFFLOAD;
446 return RequiresFlags & OMP_REQ_UNIFIED_ADDRESS;
450 return RequiresFlags & OMP_REQ_UNIFIED_SHARED_MEMORY;
454 return RequiresFlags & OMP_REQ_DYNAMIC_ALLOCATORS;
459 :
static_cast<int64_t
>(OMP_REQ_NONE);
464 RequiresFlags |= OMP_REQ_REVERSE_OFFLOAD;
466 RequiresFlags &= ~OMP_REQ_REVERSE_OFFLOAD;
471 RequiresFlags |= OMP_REQ_UNIFIED_ADDRESS;
473 RequiresFlags &= ~OMP_REQ_UNIFIED_ADDRESS;
478 RequiresFlags |= OMP_REQ_UNIFIED_SHARED_MEMORY;
480 RequiresFlags &= ~OMP_REQ_UNIFIED_SHARED_MEMORY;
485 RequiresFlags |= OMP_REQ_DYNAMIC_ALLOCATORS;
487 RequiresFlags &= ~OMP_REQ_DYNAMIC_ALLOCATORS;
500 constexpr const size_t MaxDim = 3;
508 for (
unsigned I = 1;
I < std::min(KernelArgs.
NumTeams.size(), MaxDim); ++
I)
511 Value *NumThreads3D =
512 Builder.CreateInsertValue(ZeroArray, KernelArgs.NumThreads, {0});
516 KernelArgs.RTArgs.BasePointersArray,
517 KernelArgs.RTArgs.PointersArray,
518 KernelArgs.RTArgs.SizesArray,
519 KernelArgs.RTArgs.MapTypesArray,
520 KernelArgs.RTArgs.MapNamesArray,
521 KernelArgs.RTArgs.MappersArray,
522 KernelArgs.NumIterations,
526 KernelArgs.DynCGGroupMem};
534 auto FnAttrs = Attrs.getFnAttrs();
535 auto RetAttrs = Attrs.getRetAttrs();
537 for (
size_t ArgNo = 0; ArgNo < Fn.
arg_size(); ++ArgNo)
542 bool Param =
true) ->
void {
543 bool HasSignExt = AS.hasAttribute(Attribute::SExt);
544 bool HasZeroExt = AS.hasAttribute(Attribute::ZExt);
545 if (HasSignExt || HasZeroExt) {
546 assert(AS.getNumAttributes() == 1 &&
547 "Currently not handling extension attr combined with others.");
549 if (
auto AK = TargetLibraryInfo::getExtAttrForI32Param(
T, HasSignExt))
552 TargetLibraryInfo::getExtAttrForI32Return(
T, HasSignExt))
559#define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
560#include "llvm/Frontend/OpenMP/OMPKinds.def"
564#define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
566 FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
567 addAttrSet(RetAttrs, RetAttrSet, false); \
568 for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
569 addAttrSet(ArgAttrs[ArgNo], ArgAttrSets[ArgNo]); \
570 Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
572#include "llvm/Frontend/OpenMP/OMPKinds.def"
586#define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
588 FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
590 Fn = M.getFunction(Str); \
592#include "llvm/Frontend/OpenMP/OMPKinds.def"
598#define OMP_RTL(Enum, Str, ...) \
600 Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
602#include "llvm/Frontend/OpenMP/OMPKinds.def"
606 if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
616 LLVMContext::MD_callback,
618 2, {-1, -1},
true)}));
631 assert(Fn &&
"Failed to create OpenMP runtime function");
638 auto *Fn = dyn_cast<llvm::Function>(RTLFn.
getCallee());
639 assert(Fn &&
"Failed to create OpenMP runtime function pointer");
654 for (
auto Inst =
Block->getReverseIterator()->begin();
655 Inst !=
Block->getReverseIterator()->end();) {
656 if (
auto *
AllocaInst = dyn_cast_if_present<llvm::AllocaInst>(Inst)) {
680 ParallelRegionBlockSet.
clear();
682 OI.collectBlocks(ParallelRegionBlockSet,
Blocks);
701 ".omp_par", ArgsInZeroAddressSpace);
705 <<
" Exit: " << OI.ExitBB->getName() <<
"\n");
707 "Expected OpenMP outlining to be possible!");
709 for (
auto *V : OI.ExcludeArgsFromAggregate)
716 if (TargetCpuAttr.isStringAttribute())
719 auto TargetFeaturesAttr = OuterFn->
getFnAttribute(
"target-features");
720 if (TargetFeaturesAttr.isStringAttribute())
721 OutlinedFn->
addFnAttr(TargetFeaturesAttr);
724 LLVM_DEBUG(
dbgs() <<
" Outlined function: " << *OutlinedFn <<
"\n");
726 "OpenMP outlined functions should not return a value!");
738 assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);
745 "Expected instructions to add in the outlined region entry");
752 if (
I.isTerminator())
755 I.moveBeforePreserving(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
758 OI.EntryBB->moveBefore(&ArtificialEntry);
765 if (OI.PostOutlineCB)
766 OI.PostOutlineCB(*OutlinedFn);
797 errs() <<
"Error of kind: " << Kind
798 <<
" when emitting offload entries and metadata during "
799 "OMPIRBuilder finalization \n";
806 std::vector<WeakTrackingVH> LLVMCompilerUsed = {
808 emitUsed(
"llvm.compiler.used", LLVMCompilerUsed);
830 unsigned Reserve2Flags) {
832 LocFlags |= OMP_IDENT_FLAG_KMPC;
840 ConstantInt::get(
Int32, Reserve2Flags),
841 ConstantInt::get(
Int32, SrcLocStrSize), SrcLocStr};
848 if (
GV.getValueType() == OpenMPIRBuilder::Ident &&
GV.hasInitializer())
849 if (
GV.getInitializer() == Initializer)
854 M, OpenMPIRBuilder::Ident,
869 SrcLocStrSize = LocStr.
size();
878 if (
GV.isConstant() &&
GV.hasInitializer() &&
879 GV.getInitializer() == Initializer)
890 unsigned Line,
unsigned Column,
896 Buffer.
append(FunctionName);
898 Buffer.
append(std::to_string(Line));
900 Buffer.
append(std::to_string(Column));
908 StringRef UnknownLoc =
";unknown;unknown;0;0;;";
919 if (
DIFile *DIF = DIL->getFile())
920 if (std::optional<StringRef> Source = DIF->getSource())
926 DIL->getColumn(), SrcLocStrSize);
938 "omp_global_thread_num");
943 bool ForceSimpleCall,
bool CheckCancelFlag) {
953 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
956 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
959 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
962 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
965 BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
978 bool UseCancelBarrier =
983 UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
984 : OMPRTL___kmpc_barrier),
987 if (UseCancelBarrier && CheckCancelFlag)
996 omp::Directive CanceledDirective) {
1008 Value *CancelKind =
nullptr;
1009 switch (CanceledDirective) {
1010#define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
1011 case DirectiveEnum: \
1012 CancelKind = Builder.getInt32(Value); \
1014#include "llvm/Frontend/OpenMP/OMPKinds.def"
1025 auto ExitCB = [
this, CanceledDirective, Loc](
InsertPointTy IP) {
1026 if (CanceledDirective == OMPD_parallel) {
1030 omp::Directive::OMPD_unknown,
false,
1040 UI->eraseFromParent();
1053 auto *KernelArgsPtr =
1066 NumThreads, HostPtr, KernelArgsPtr};
1094 assert(OutlinedFnID &&
"Invalid outlined function ID!");
1098 Value *Return =
nullptr;
1118 Args.NumTeams.front(), Args.NumThreads,
1119 OutlinedFnID, ArgsVector));
1132 emitBlock(OffloadContBlock, CurFn,
true);
1137 omp::Directive CanceledDirective,
1140 "Unexpected cancellation!");
1190 OutlinedFn.
addFnAttr(Attribute::NoUnwind);
1193 "Expected at least tid and bounded tid as arguments");
1194 unsigned NumCapturedVars = OutlinedFn.
arg_size() - 2;
1197 assert(CI &&
"Expected call instruction to outlined function");
1198 CI->
getParent()->setName(
"omp_parallel");
1201 Type *PtrTy = OMPIRBuilder->VoidPtr;
1205 OpenMPIRBuilder ::InsertPointTy CurrentIP = Builder.
saveIP();
1209 Value *Args = ArgsAlloca;
1217 for (
unsigned Idx = 0;
Idx < NumCapturedVars;
Idx++) {
1229 Value *Parallel51CallArgs[] = {
1233 NumThreads ? NumThreads : Builder.
getInt32(-1),
1236 Builder.
CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr),
1239 Builder.
getInt64(NumCapturedVars)};
1244 Builder.
CreateCall(RTLFn, Parallel51CallArgs);
1259 I->eraseFromParent();
1281 if (
auto *
F = dyn_cast<Function>(RTLFn.
getCallee())) {
1282 if (!
F->hasMetadata(LLVMContext::MD_callback)) {
1290 F->addMetadata(LLVMContext::MD_callback,
1299 OutlinedFn.
addFnAttr(Attribute::NoUnwind);
1302 "Expected at least tid and bounded tid as arguments");
1303 unsigned NumCapturedVars = OutlinedFn.
arg_size() - 2;
1306 CI->
getParent()->setName(
"omp_parallel");
1310 Value *ForkCallArgs[] = {
1311 Ident, Builder.
getInt32(NumCapturedVars),
1312 Builder.
CreateBitCast(&OutlinedFn, OMPIRBuilder->ParallelTaskPtr)};
1315 RealArgs.
append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
1324 auto PtrTy = OMPIRBuilder->VoidPtr;
1325 if (IfCondition && NumCapturedVars == 0) {
1329 if (IfCondition && RealArgs.
back()->getType() != PtrTy)
1347 I->eraseFromParent();
1355 omp::ProcBindKind ProcBind,
bool IsCancellable) {
1382 if (ProcBind != OMP_PROC_BIND_default) {
1386 ConstantInt::get(
Int32,
unsigned(ProcBind),
true)};
1414 TIDAddrAlloca, PointerType ::get(
M.
getContext(), 0),
"tid.addr.ascast");
1419 "zero.addr.ascast");
1443 if (IP.getBlock()->end() == IP.getPoint()) {
1449 assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
1450 IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
1451 "Unexpected insertion point for finalization call!");
1487 LLVM_DEBUG(
dbgs() <<
"Before body codegen: " << *OuterFn <<
"\n");
1490 assert(BodyGenCB &&
"Expected body generation callback!");
1492 BodyGenCB(InnerAllocaIP, CodeGenIP);
1494 LLVM_DEBUG(
dbgs() <<
"After body codegen: " << *OuterFn <<
"\n");
1500 std::move(ToBeDeleted)](
Function &OutlinedFn) {
1502 IfCondition, NumThreads, PrivTID, PrivTIDAddr,
1503 ThreadID, ToBeDeletedVec);
1508 std::move(ToBeDeleted)](
Function &OutlinedFn) {
1510 PrivTID, PrivTIDAddr, ToBeDeletedVec);
1527 PRegOutlinedExitBB->
setName(
"omp.par.outlined.exit");
1528 Blocks.push_back(PRegOutlinedExitBB);
1539 ".omp_par", ArgsInZeroAddressSpace);
1544 Extractor.
findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
1547 LLVM_DEBUG(
dbgs() <<
"Before privatization: " << *OuterFn <<
"\n");
1552 auto PrivHelper = [&](
Value &V) {
1553 if (&V == TIDAddr || &V == ZeroAddr) {
1559 for (
Use &U : V.uses())
1560 if (
auto *UserI = dyn_cast<Instruction>(U.getUser()))
1561 if (ParallelRegionBlockSet.
count(UserI->getParent()))
1571 if (!V.getType()->isPointerTy()) {
1590 Value *ReplacementValue =
nullptr;
1591 CallInst *CI = dyn_cast<CallInst>(&V);
1593 ReplacementValue = PrivTID;
1596 PrivCB(InnerAllocaIP,
Builder.
saveIP(), V, *Inner, ReplacementValue));
1601 assert(ReplacementValue &&
1602 "Expected copy/create callback to set replacement value!");
1603 if (ReplacementValue == &V)
1608 UPtr->set(ReplacementValue);
1625 for (
Value *Input : Inputs) {
1630 for (
Value *Output : Outputs)
1634 "OpenMP outlining should not produce live-out values!");
1636 LLVM_DEBUG(
dbgs() <<
"After privatization: " << *OuterFn <<
"\n");
1639 dbgs() <<
" PBR: " << BB->getName() <<
"\n";
1647 assert(FiniInfo.DK == OMPD_parallel &&
1648 "Unexpected finalization stack state!");
1658 InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
1659 UI->eraseFromParent();
1725 if (Dependencies.
empty())
1745 Type *DependInfo = OMPBuilder.DependInfo;
1748 Value *DepArray =
nullptr;
1754 DepArray = Builder.
CreateAlloca(DepArrayTy,
nullptr,
".dep.arr.addr");
1756 for (
const auto &[DepIdx, Dep] :
enumerate(Dependencies)) {
1762 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
1767 DependInfo,
Base,
static_cast<unsigned int>(RTLDependInfoFields::Len));
1769 Builder.
getInt64(M.getDataLayout().getTypeStoreSize(Dep.DepValueType)),
1774 static_cast<unsigned int>(RTLDependInfoFields::Flags));
1777 static_cast<unsigned int>(Dep.DepKind)),
1820 BodyGenCB(TaskAllocaIP, TaskBodyIP);
1830 Builder, AllocaIP, ToBeDeleted, TaskAllocaIP,
"global.tid",
false));
1832 OI.
PostOutlineCB = [
this, Ident, Tied, Final, IfCondition, Dependencies,
1833 TaskAllocaBB, ToBeDeleted](
Function &OutlinedFn)
mutable {
1835 assert(OutlinedFn.getNumUses() == 1 &&
1836 "there must be a single user for the outlined function");
1837 CallInst *StaleCI = cast<CallInst>(OutlinedFn.user_back());
1841 bool HasShareds = StaleCI->
arg_size() > 1;
1880 assert(ArgStructAlloca &&
1881 "Unable to find the alloca instruction corresponding to arguments "
1882 "for extracted function");
1885 assert(ArgStructType &&
"Unable to find struct type corresponding to "
1886 "arguments for extracted function");
1894 TaskAllocFn, {Ident, ThreadID,
Flags,
1895 TaskSize, SharedsSize,
1907 Value *DepArray =
nullptr;
1908 if (Dependencies.
size()) {
1923 static_cast<unsigned int>(RTLDependInfoFields::BaseAddr));
1930 static_cast<unsigned int>(RTLDependInfoFields::Len));
1937 static_cast<unsigned int>(RTLDependInfoFields::Flags));
1940 static_cast<unsigned int>(Dep.DepKind)),
1971 Instruction *ThenTI = IfTerminator, *ElseTI =
nullptr;
1977 if (Dependencies.
size()) {
2001 if (Dependencies.
size()) {
2022 Shareds, [Shareds](
Use &U) {
return U.getUser() != Shareds; });
2026 I->eraseFromParent();
2074 if (IP.getBlock()->end() != IP.getPoint())
2085 auto *CaseBB = IP.getBlock()->getSinglePredecessor();
2086 auto *CondBB = CaseBB->getSinglePredecessor()->getSinglePredecessor();
2087 auto *ExitBB = CondBB->getTerminator()->getSuccessor(1);
2118 unsigned CaseNumber = 0;
2119 for (
auto SectionCB : SectionCBs) {
2135 Value *LB = ConstantInt::get(I32Ty, 0);
2136 Value *UB = ConstantInt::get(I32Ty, SectionCBs.
size());
2137 Value *ST = ConstantInt::get(I32Ty, 1);
2139 Loc, LoopBodyGenCB, LB, UB, ST,
true,
false, AllocaIP,
"section_loop");
2141 applyStaticWorkshareLoop(Loc.
DL,
LoopInfo, AllocaIP, !IsNowait);
2145 assert(FiniInfo.DK == OMPD_sections &&
2146 "Unexpected finalization stack state!");
2152 AfterIP = {FiniBB, FiniBB->
begin()};
2166 if (IP.getBlock()->end() != IP.getPoint())
2185 Directive OMPD = Directive::OMPD_sections;
2188 return EmitOMPInlinedRegion(OMPD,
nullptr,
nullptr, BodyGenCB, FiniCBWrapper,
2200 std::vector<WeakTrackingVH> &
List) {
2207 for (
unsigned I = 0, E =
List.size();
I != E; ++
I)
2211 if (UsedArray.
empty())
2218 GV->setSection(
"llvm.metadata");
2221Value *OpenMPIRBuilder::getGPUThreadID() {
2224 OMPRTL___kmpc_get_hardware_thread_id_in_block),
2228Value *OpenMPIRBuilder::getGPUWarpSize() {
2233Value *OpenMPIRBuilder::getNVPTXWarpID() {
2238Value *OpenMPIRBuilder::getNVPTXLaneID() {
2240 assert(LaneIDBits < 32 &&
"Invalid LaneIDBits size in NVPTX device.");
2241 unsigned LaneIDMask = ~0
u >> (32u - LaneIDBits);
2246Value *OpenMPIRBuilder::castValueToType(InsertPointTy AllocaIP,
Value *
From,
2251 assert(FromSize > 0 &&
"From size must be greater than zero");
2252 assert(ToSize > 0 &&
"To size must be greater than zero");
2253 if (FromType == ToType)
2255 if (FromSize == ToSize)
2265 CastItem,
FromType->getPointerTo());
2270Value *OpenMPIRBuilder::createRuntimeShuffleFunction(InsertPointTy AllocaIP,
2275 assert(
Size <= 8 &&
"Unsupported bitwidth in shuffle instruction");
2279 Value *ElemCast = castValueToType(AllocaIP, Element, CastTy);
2283 Size <= 4 ? RuntimeFunction::OMPRTL___kmpc_shuffle_int32
2284 : RuntimeFunction::OMPRTL___kmpc_shuffle_int64);
2285 Value *WarpSizeCast =
2287 Value *ShuffleCall =
2289 return castValueToType(AllocaIP, ShuffleCall, CastTy);
2292void OpenMPIRBuilder::shuffleAndStore(InsertPointTy AllocaIP,
Value *SrcAddr,
2308 Value *ElemPtr = DstAddr;
2310 for (
unsigned IntSize = 8; IntSize >= 1; IntSize /= 2) {
2322 if ((
Size / IntSize) > 1) {
2346 Value *Res = createRuntimeShuffleFunction(
2355 Value *LocalElemPtr =
2362 Value *Res = createRuntimeShuffleFunction(
2376void OpenMPIRBuilder::emitReductionListCopy(
2377 InsertPointTy AllocaIP, CopyAction Action,
Type *ReductionArrayTy,
2379 CopyOptionsTy CopyOptions) {
2382 Value *RemoteLaneOffset = CopyOptions.RemoteLaneOffset;
2386 for (
auto En :
enumerate(ReductionInfos)) {
2387 const ReductionInfo &RI = En.value();
2388 Value *SrcElementAddr =
nullptr;
2389 Value *DestElementAddr =
nullptr;
2390 Value *DestElementPtrAddr =
nullptr;
2392 bool ShuffleInElement =
false;
2395 bool UpdateDestListPtr =
false;
2399 ReductionArrayTy, SrcBase,
2400 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
2406 ReductionArrayTy, DestBase,
2407 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
2413 ".omp.reduction.element");
2416 DestElementAddr = DestAlloca;
2419 DestElementAddr->
getName() +
".ascast");
2421 ShuffleInElement =
true;
2422 UpdateDestListPtr =
true;
2434 if (ShuffleInElement) {
2435 shuffleAndStore(AllocaIP, SrcElementAddr, DestElementAddr, RI.ElementType,
2436 RemoteLaneOffset, ReductionArrayTy);
2438 switch (RI.EvaluationKind) {
2447 RI.ElementType, SrcElementAddr, 0, 0,
".realp");
2449 RI.ElementType->getStructElementType(0), SrcRealPtr,
".real");
2451 RI.ElementType, SrcElementAddr, 0, 1,
".imagp");
2453 RI.ElementType->getStructElementType(1), SrcImgPtr,
".imag");
2456 RI.ElementType, DestElementAddr, 0, 0,
".realp");
2458 RI.ElementType, DestElementAddr, 0, 1,
".imagp");
2480 if (UpdateDestListPtr) {
2483 DestElementAddr->
getName() +
".ascast");
2489Function *OpenMPIRBuilder::emitInterWarpCopyFunction(
2499 "_omp_reduction_inter_warp_copy_func", &
M);
2522 "__openmp_nvptx_data_transfer_temporary_storage";
2526 if (!TransferMedium) {
2535 Value *GPUThreadID = getGPUThreadID();
2537 Value *LaneID = getNVPTXLaneID();
2539 Value *WarpID = getNVPTXWarpID();
2548 Arg0Type,
nullptr, ReduceListArg->
getName() +
".addr");
2552 ReduceListAlloca, Arg0Type, ReduceListAlloca->
getName() +
".ascast");
2555 NumWarpsAlloca->
getName() +
".ascast");
2566 for (
auto En :
enumerate(ReductionInfos)) {
2571 const ReductionInfo &RI = En.value();
2573 for (
unsigned TySize = 4; TySize > 0 && RealTySize > 0; TySize /= 2) {
2576 unsigned NumIters = RealTySize / TySize;
2579 Value *Cnt =
nullptr;
2580 Value *CntAddr =
nullptr;
2590 CntAddr->
getName() +
".ascast");
2609 omp::Directive::OMPD_unknown,
2622 auto *RedListArrayTy =
2628 {ConstantInt::get(IndexTy, 0),
2629 ConstantInt::get(IndexTy, En.index())});
2654 omp::Directive::OMPD_unknown,
2663 Value *NumWarpsVal =
2666 Value *IsActiveThread =
2677 Value *TargetElemPtrPtr =
2679 {ConstantInt::get(IndexTy, 0),
2680 ConstantInt::get(IndexTy, En.index())});
2681 Value *TargetElemPtrVal =
2683 Value *TargetElemPtr = TargetElemPtrVal;
2689 Value *SrcMediumValue =
2708 RealTySize %= TySize;
2718Function *OpenMPIRBuilder::emitShuffleAndReduceFunction(
2724 {Builder.getPtrTy(), Builder.getInt16Ty(),
2725 Builder.getInt16Ty(), Builder.getInt16Ty()},
2729 "_omp_reduction_shuffle_and_reduce_func", &
M);
2750 Type *ReduceListArgType = ReduceListArg->
getType();
2754 ReduceListArgType,
nullptr, ReduceListArg->
getName() +
".addr");
2756 LaneIDArg->
getName() +
".addr");
2758 LaneIDArgType,
nullptr, RemoteLaneOffsetArg->
getName() +
".addr");
2760 AlgoVerArg->
getName() +
".addr");
2767 RedListArrayTy,
nullptr,
".omp.reduction.remote_reduce_list");
2770 ReduceListAlloca, ReduceListArgType,
2771 ReduceListAlloca->
getName() +
".ascast");
2773 LaneIdAlloca, LaneIDArgPtrType, LaneIdAlloca->
getName() +
".ascast");
2775 RemoteLaneOffsetAlloca, LaneIDArgPtrType,
2776 RemoteLaneOffsetAlloca->
getName() +
".ascast");
2778 AlgoVerAlloca, LaneIDArgPtrType, AlgoVerAlloca->
getName() +
".ascast");
2781 RemoteReductionListAlloca->
getName() +
".ascast");
2790 Value *RemoteLaneOffset =
2799 emitReductionListCopy(
2801 ReduceList, RemoteListAddrCast, {RemoteLaneOffset,
nullptr,
nullptr});
2832 Value *RemoteOffsetComp =
2849 ->addFnAttr(Attribute::NoUnwind);
2870 ReductionInfos, RemoteListAddrCast, ReduceList);
2883Function *OpenMPIRBuilder::emitListToGlobalCopyFunction(
2890 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
2894 "_omp_reduction_list_to_global_copy_func", &
M);
2911 BufferArg->
getName() +
".addr");
2918 BufferArgAlloca->
getName() +
".ascast");
2923 ReduceListArgAlloca->
getName() +
".ascast");
2929 Value *LocalReduceList =
2931 Value *BufferArgVal =
2936 for (
auto En :
enumerate(ReductionInfos)) {
2937 const ReductionInfo &RI = En.value();
2938 auto *RedListArrayTy =
2942 RedListArrayTy, LocalReduceList,
2943 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
2951 ReductionsBufferTy, BufferVD, 0, En.index());
2953 switch (RI.EvaluationKind) {
2961 RI.ElementType, ElemPtr, 0, 0,
".realp");
2963 RI.ElementType->getStructElementType(0), SrcRealPtr,
".real");
2965 RI.ElementType, ElemPtr, 0, 1,
".imagp");
2967 RI.ElementType->getStructElementType(1), SrcImgPtr,
".imag");
2970 RI.ElementType, GlobVal, 0, 0,
".realp");
2972 RI.ElementType, GlobVal, 0, 1,
".imagp");
2993Function *OpenMPIRBuilder::emitListToGlobalReduceFunction(
3000 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3004 "_omp_reduction_list_to_global_reduce_func", &
M);
3021 BufferArg->
getName() +
".addr");
3026 auto *RedListArrayTy =
3031 Value *LocalReduceList =
3036 BufferArgAlloca->
getName() +
".ascast");
3041 ReduceListArgAlloca->
getName() +
".ascast");
3044 LocalReduceList->
getName() +
".ascast");
3054 for (
auto En :
enumerate(ReductionInfos)) {
3056 RedListArrayTy, LocalReduceListAddrCast,
3057 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3062 ReductionsBufferTy, BufferVD, 0, En.index());
3070 ->addFnAttr(Attribute::NoUnwind);
3076Function *OpenMPIRBuilder::emitGlobalToListCopyFunction(
3083 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3087 "_omp_reduction_global_to_list_copy_func", &
M);
3104 BufferArg->
getName() +
".addr");
3111 BufferArgAlloca->
getName() +
".ascast");
3116 ReduceListArgAlloca->
getName() +
".ascast");
3121 Value *LocalReduceList =
3127 for (
auto En :
enumerate(ReductionInfos)) {
3129 auto *RedListArrayTy =
3133 RedListArrayTy, LocalReduceList,
3134 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3141 ReductionsBufferTy, BufferVD, 0, En.index());
3184Function *OpenMPIRBuilder::emitGlobalToListReduceFunction(
3191 {Builder.getPtrTy(), Builder.getInt32Ty(), Builder.getPtrTy()},
3195 "_omp_reduction_global_to_list_reduce_func", &
M);
3212 BufferArg->
getName() +
".addr");
3222 Value *LocalReduceList =
3227 BufferArgAlloca->
getName() +
".ascast");
3232 ReduceListArgAlloca->
getName() +
".ascast");
3235 LocalReduceList->
getName() +
".ascast");
3245 for (
auto En :
enumerate(ReductionInfos)) {
3247 RedListArrayTy, ReductionList,
3248 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3253 ReductionsBufferTy, BufferVD, 0, En.index());
3261 ->addFnAttr(Attribute::NoUnwind);
3267std::string OpenMPIRBuilder::getReductionFuncName(
StringRef Name)
const {
3268 std::string Suffix =
3270 return (
Name + Suffix).str();
3273Function *OpenMPIRBuilder::createReductionFunction(
3275 ReductionGenCBKind ReductionGenCBKind,
AttributeList FuncAttrs) {
3277 {Builder.getPtrTy(), Builder.getPtrTy()},
3279 std::string
Name = getReductionFuncName(ReducerName);
3291 Value *LHSArrayPtr =
nullptr;
3292 Value *RHSArrayPtr =
nullptr;
3303 LHSAlloca, Arg0Type, LHSAlloca->
getName() +
".ascast");
3305 RHSAlloca, Arg1Type, RHSAlloca->
getName() +
".ascast");
3315 for (
auto En :
enumerate(ReductionInfos)) {
3316 const ReductionInfo &RI = En.value();
3318 RedArrayTy, RHSArrayPtr,
3319 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3322 RHSI8Ptr, RI.PrivateVariable->getType(),
3323 RHSI8Ptr->
getName() +
".ascast");
3326 RedArrayTy, LHSArrayPtr,
3327 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3330 LHSI8Ptr, RI.Variable->getType(), LHSI8Ptr->
getName() +
".ascast");
3341 return ReductionFunc;
3347 for (
auto En :
enumerate(ReductionInfos)) {
3348 unsigned Index = En.index();
3349 const ReductionInfo &RI = En.value();
3350 Value *LHSFixupPtr, *RHSFixupPtr;
3357 LHSPtrs[
Index], [ReductionFunc](
const Use &U) {
3358 return cast<Instruction>(
U.getUser())->getParent()->getParent() ==
3362 RHSPtrs[
Index], [ReductionFunc](
const Use &U) {
3363 return cast<Instruction>(
U.getUser())->getParent()->getParent() ==
3369 return ReductionFunc;
3377 assert(RI.Variable &&
"expected non-null variable");
3378 assert(RI.PrivateVariable &&
"expected non-null private variable");
3379 assert((RI.ReductionGen || RI.ReductionGenClang) &&
3380 "expected non-null reduction generator callback");
3383 RI.Variable->getType() == RI.PrivateVariable->getType() &&
3384 "expected variables and their private equivalents to have the same "
3387 assert(RI.Variable->getType()->isPointerTy() &&
3388 "expected variables to be pointers");
3395 bool IsNoWait,
bool IsTeamsReduction,
bool HasDistribute,
3397 unsigned ReductionBufNum,
Value *SrcLocInfo) {
3411 if (ReductionInfos.
size() == 0)
3430 if (GridValue.has_value())
3448 Value *ReductionListAlloca =
3451 ReductionListAlloca, PtrTy, ReductionListAlloca->
getName() +
".ascast");
3455 for (
auto En :
enumerate(ReductionInfos)) {
3458 RedArrayTy, ReductionList,
3459 {ConstantInt::get(IndexTy, 0), ConstantInt::get(IndexTy, En.index())});
3466 emitShuffleAndReduceFunction(ReductionInfos, ReductionFunc, FuncAttrs);
3467 Function *WcFunc = emitInterWarpCopyFunction(Loc, ReductionInfos, FuncAttrs);
3472 unsigned MaxDataSize = 0;
3474 for (
auto En :
enumerate(ReductionInfos)) {
3476 if (
Size > MaxDataSize)
3478 ReductionTypeArgs.
emplace_back(En.value().ElementType);
3480 Value *ReductionDataSize =
3482 if (!IsTeamsReduction) {
3483 Value *SarFuncCast =
3487 Value *Args[] = {SrcLocInfo, ReductionDataSize, RL, SarFuncCast,
3490 RuntimeFunction::OMPRTL___kmpc_nvptx_parallel_reduce_nowait_v2);
3495 Ctx, ReductionTypeArgs,
"struct._globalized_locals_ty");
3497 RuntimeFunction::OMPRTL___kmpc_reduction_get_fixed_buffer);
3498 Function *LtGCFunc = emitListToGlobalCopyFunction(
3499 ReductionInfos, ReductionsBufferTy, FuncAttrs);
3500 Function *LtGRFunc = emitListToGlobalReduceFunction(
3501 ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
3502 Function *GtLCFunc = emitGlobalToListCopyFunction(
3503 ReductionInfos, ReductionsBufferTy, FuncAttrs);
3504 Function *GtLRFunc = emitGlobalToListReduceFunction(
3505 ReductionInfos, ReductionFunc, ReductionsBufferTy, FuncAttrs);
3509 RedFixedBuferFn, {},
"_openmp_teams_reductions_buffer_$_$ptr");
3511 Value *Args3[] = {SrcLocInfo,
3512 KernelTeamsReductionPtr,
3524 RuntimeFunction::OMPRTL___kmpc_nvptx_teams_reduce_nowait_v2);
3541 for (
auto En :
enumerate(ReductionInfos)) {
3548 Value *LHSPtr, *RHSPtr;
3550 &LHSPtr, &RHSPtr, CurFunc));
3555 return cast<Instruction>(U.getUser())->getParent()->getParent() ==
3559 return cast<Instruction>(U.getUser())->getParent()->getParent() ==
3563 assert(
false &&
"Unhandled ReductionGenCBKind");
3579 ".omp.reduction.func", &M);
3590 assert(RI.Variable &&
"expected non-null variable");
3591 assert(RI.PrivateVariable &&
"expected non-null private variable");
3592 assert(RI.ReductionGen &&
"expected non-null reduction generator callback");
3593 assert(RI.Variable->getType() == RI.PrivateVariable->getType() &&
3594 "expected variables and their private equivalents to have the same "
3596 assert(RI.Variable->getType()->isPointerTy() &&
3597 "expected variables to be pointers");
3610 unsigned NumReductions = ReductionInfos.
size();
3617 for (
auto En :
enumerate(ReductionInfos)) {
3618 unsigned Index = En.index();
3636 ? IdentFlag::OMP_IDENT_FLAG_ATOMIC_REDUCE
3641 unsigned RedArrayByteSize =
DL.getTypeStoreSize(RedArrayTy);
3644 Value *Lock = getOMPCriticalRegionLock(
".reduction");
3646 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_reduce_nowait
3647 : RuntimeFunction::OMPRTL___kmpc_reduce);
3650 {Ident, ThreadId, NumVariables, RedArraySize, RedArray,
3651 ReductionFunc, Lock},
3670 for (
auto En :
enumerate(ReductionInfos)) {
3675 Value *RedValue =
nullptr;
3676 if (!IsByRef[En.index()]) {
3678 "red.value." +
Twine(En.index()));
3680 Value *PrivateRedValue =
3682 "red.private.value." +
Twine(En.index()));
3684 if (IsByRef[En.index()]) {
3686 PrivateRedValue, Reduced));
3689 PrivateRedValue, Reduced));
3694 if (!IsByRef[En.index()])
3698 IsNoWait ? RuntimeFunction::OMPRTL___kmpc_end_reduce_nowait
3699 : RuntimeFunction::OMPRTL___kmpc_end_reduce);
3707 if (CanGenerateAtomic &&
llvm::none_of(IsByRef, [](
bool P) {
return P; })) {
3728 for (
auto En :
enumerate(ReductionInfos)) {
3731 RedArrayTy, LHSArrayPtr, 0, En.index());
3736 RedArrayTy, RHSArrayPtr, 0, En.index());
3746 if (!IsByRef[En.index()])
3763 Directive OMPD = Directive::OMPD_master;
3768 Value *Args[] = {Ident, ThreadId};
3776 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3787 Directive OMPD = Directive::OMPD_masked;
3793 Value *ArgsEnd[] = {Ident, ThreadId};
3801 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
3836 IndVarPHI->
addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
3849 "omp_" +
Name +
".next",
true);
3860 CL->Header = Header;
3879 NextBB, NextBB,
Name);
3903 Value *Start,
Value *Stop,
Value *Step,
bool IsSigned,
bool InclusiveStop,
3913 auto *IndVarTy = cast<IntegerType>(Start->getType());
3914 assert(IndVarTy == Stop->
getType() &&
"Stop type mismatch");
3915 assert(IndVarTy == Step->
getType() &&
"Step type mismatch");
3921 ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
3949 Value *CountIfLooping;
3950 if (InclusiveStop) {
3960 "omp_" +
Name +
".tripcount");
3981 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
3984 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
3990 InsertPointTy AllocaIP,
3991 bool NeedsBarrier) {
3992 assert(CLI->
isValid() &&
"Requires a valid canonical loop");
3994 "Require dedicated allocate IP");
4006 Type *IVTy =
IV->getType();
4026 Constant *One = ConstantInt::get(IVTy, 1);
4034 Constant *SchedulingType = ConstantInt::get(
4035 I32Type,
static_cast<int>(OMPScheduleType::UnorderedStatic));
4040 {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
4041 PUpperBound, PStride, One,
Zero});
4046 CLI->setTripCount(TripCount);
4067 omp::Directive::OMPD_for,
false,
4078 bool NeedsBarrier,
Value *ChunkSize) {
4079 assert(CLI->
isValid() &&
"Requires a valid canonical loop");
4080 assert(ChunkSize &&
"Chunk size is required");
4085 Type *IVTy =
IV->getType();
4087 "Max supported tripcount bitwidth is 64 bits");
4089 :
Type::getInt64Ty(Ctx);
4092 Constant *One = ConstantInt::get(InternalIVTy, 1);
4104 Value *PLowerBound =
4106 Value *PUpperBound =
4115 Value *CastedChunkSize =
4117 Value *CastedTripCount =
4120 Constant *SchedulingType = ConstantInt::get(
4121 I32Type,
static_cast<int>(OMPScheduleType::UnorderedStaticChunked));
4135 SchedulingType, PLastIter,
4136 PLowerBound, PUpperBound,
4141 Value *FirstChunkStart =
4143 Value *FirstChunkStop =
4148 Value *NextChunkStride =
4153 Value *DispatchCounter;
4157 FirstChunkStart, CastedTripCount, NextChunkStride,
4181 Value *IsLastChunk =
4183 Value *CountUntilOrigTripCount =
4186 IsLastChunk, CountUntilOrigTripCount, ChunkRange,
"omp_chunk.tripcount");
4187 Value *BackcastedChunkTC =
4189 CLI->setTripCount(BackcastedChunkTC);
4194 Value *BackcastedDispatchCounter =
4228 case WorksharingLoopType::ForStaticLoop:
4231 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_4u);
4234 M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_loop_8u);
4236 case WorksharingLoopType::DistributeStaticLoop:
4239 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_4u);
4242 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_static_loop_8u);
4244 case WorksharingLoopType::DistributeForStaticLoop:
4247 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_4u);
4250 M, omp::RuntimeFunction::OMPRTL___kmpc_distribute_for_static_loop_8u);
4253 if (Bitwidth != 32 && Bitwidth != 64) {
4275 if (LoopType == WorksharingLoopType::DistributeStaticLoop) {
4276 RealArgs.
push_back(ConstantInt::get(TripCountTy, 0));
4281 M, omp::RuntimeFunction::OMPRTL_omp_get_num_threads);
4282 Builder.
restoreIP({InsertBlock, std::prev(InsertBlock->
end())});
4287 RealArgs.
push_back(ConstantInt::get(TripCountTy, 0));
4288 if (LoopType == WorksharingLoopType::DistributeForStaticLoop) {
4289 RealArgs.
push_back(ConstantInt::get(TripCountTy, 0));
4325 CleanUpInfo.
collectBlocks(RegionBlockSet, BlocksToBeRemoved);
4333 "Expected unique undroppable user of outlined function");
4334 CallInst *OutlinedFnCallInstruction = dyn_cast<CallInst>(OutlinedFnUser);
4335 assert(OutlinedFnCallInstruction &&
"Expected outlined function call");
4337 "Expected outlined function call to be located in loop preheader");
4339 if (OutlinedFnCallInstruction->
arg_size() > 1)
4346 LoopBodyArg, ParallelTaskPtr, TripCount,
4349 for (
auto &ToBeDeletedItem : ToBeDeleted)
4350 ToBeDeletedItem->eraseFromParent();
4356 InsertPointTy AllocaIP,
4369 OI.OuterAllocaBB = AllocaIP.getBlock();
4374 "omp.prelatch",
true);
4394 OI.collectBlocks(ParallelRegionBlockSet,
Blocks);
4396 ParallelRegionBlockSet.
end());
4416 Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
4425 if (ParallelRegionBlockSet.
count(Inst->getParent())) {
4426 Inst->replaceUsesOfWith(CLI->
getIndVar(), NewLoopCntLoad);
4432 OI.ExcludeArgsFromAggregate.push_back(NewLoopCntLoad);
4439 OI.PostOutlineCB = [=, ToBeDeletedVec =
4440 std::move(ToBeDeleted)](
Function &OutlinedFn) {
4442 ToBeDeletedVec, LoopType);
4450 bool NeedsBarrier, omp::ScheduleKind SchedKind,
Value *ChunkSize,
4451 bool HasSimdModifier,
bool HasMonotonicModifier,
4452 bool HasNonmonotonicModifier,
bool HasOrderedClause,
4455 return applyWorkshareLoopTarget(
DL, CLI, AllocaIP, LoopType);
4457 SchedKind, ChunkSize, HasSimdModifier, HasMonotonicModifier,
4458 HasNonmonotonicModifier, HasOrderedClause);
4460 bool IsOrdered = (EffectiveScheduleType & OMPScheduleType::ModifierOrdered) ==
4461 OMPScheduleType::ModifierOrdered;
4462 switch (EffectiveScheduleType & ~OMPScheduleType::ModifierMask) {
4463 case OMPScheduleType::BaseStatic:
4464 assert(!ChunkSize &&
"No chunk size with static-chunked schedule");
4466 return applyDynamicWorkshareLoop(
DL, CLI, AllocaIP, EffectiveScheduleType,
4467 NeedsBarrier, ChunkSize);
4469 return applyStaticWorkshareLoop(
DL, CLI, AllocaIP, NeedsBarrier);
4471 case OMPScheduleType::BaseStaticChunked:
4473 return applyDynamicWorkshareLoop(
DL, CLI, AllocaIP, EffectiveScheduleType,
4474 NeedsBarrier, ChunkSize);
4476 return applyStaticChunkedWorkshareLoop(
DL, CLI, AllocaIP, NeedsBarrier,
4479 case OMPScheduleType::BaseRuntime:
4480 case OMPScheduleType::BaseAuto:
4481 case OMPScheduleType::BaseGreedy:
4482 case OMPScheduleType::BaseBalanced:
4483 case OMPScheduleType::BaseSteal:
4484 case OMPScheduleType::BaseGuidedSimd:
4485 case OMPScheduleType::BaseRuntimeSimd:
4487 "schedule type does not support user-defined chunk sizes");
4489 case OMPScheduleType::BaseDynamicChunked:
4490 case OMPScheduleType::BaseGuidedChunked:
4491 case OMPScheduleType::BaseGuidedIterativeChunked:
4492 case OMPScheduleType::BaseGuidedAnalyticalChunked:
4493 case OMPScheduleType::BaseStaticBalancedChunked:
4494 return applyDynamicWorkshareLoop(
DL, CLI, AllocaIP, EffectiveScheduleType,
4495 NeedsBarrier, ChunkSize);
4511 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_4u);
4514 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_init_8u);
4527 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_4u);
4530 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_next_8u);
4542 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_4u);
4545 M, omp::RuntimeFunction::OMPRTL___kmpc_dispatch_fini_8u);
4552 assert(CLI->
isValid() &&
"Requires a valid canonical loop");
4554 "Require dedicated allocate IP");
4556 "Require valid schedule type");
4558 bool Ordered = (SchedType & OMPScheduleType::ModifierOrdered) ==
4559 OMPScheduleType::ModifierOrdered;
4570 Type *IVTy =
IV->getType();
4588 Constant *One = ConstantInt::get(IVTy, 1);
4609 ConstantInt::get(I32Type,
static_cast<int>(SchedType));
4613 {SrcLoc, ThreadNum, SchedulingType, One,
4614 UpperBound, One, Chunk});
4624 PLowerBound, PUpperBound, PStride});
4625 Constant *Zero32 = ConstantInt::get(I32Type, 0);
4634 auto *PI = cast<PHINode>(Phi);
4635 PI->setIncomingBlock(0, OuterCond);
4636 PI->setIncomingValue(0, LowerBound);
4640 auto *Br = cast<BranchInst>(Term);
4641 Br->setSuccessor(0, OuterCond);
4649 auto *CI = cast<CmpInst>(Comp);
4650 CI->setOperand(1, UpperBound);
4653 auto *BI = cast<BranchInst>(Branch);
4654 assert(BI->getSuccessor(1) == Exit);
4655 BI->setSuccessor(1, OuterCond);
4668 omp::Directive::OMPD_for,
false,
4688 auto HasRemainingUses = [&BBsToErase](
BasicBlock *BB) {
4689 for (
Use &U : BB->uses()) {
4690 auto *UseInst = dyn_cast<Instruction>(U.getUser());
4693 if (BBsToErase.count(UseInst->getParent()))
4700 while (BBsToErase.remove_if(HasRemainingUses)) {
4711 assert(
Loops.size() >= 1 &&
"At least one loop required");
4712 size_t NumLoops =
Loops.size();
4716 return Loops.front();
4728 Loop->collectControlBlocks(OldControlBBs);
4732 if (ComputeIP.
isSet())
4739 Value *CollapsedTripCount =
nullptr;
4742 "All loops to collapse must be valid canonical loops");
4743 Value *OrigTripCount = L->getTripCount();
4744 if (!CollapsedTripCount) {
4745 CollapsedTripCount = OrigTripCount;
4757 OrigPreheader->
getNextNode(), OrigAfter,
"collapsed");
4765 Value *Leftover = Result->getIndVar();
4767 NewIndVars.
resize(NumLoops);
4768 for (
int i = NumLoops - 1; i >= 1; --i) {
4769 Value *OrigTripCount =
Loops[i]->getTripCount();
4772 NewIndVars[i] = NewIndVar;
4777 NewIndVars[0] = Leftover;
4786 BasicBlock *ContinueBlock = Result->getBody();
4788 auto ContinueWith = [&ContinueBlock, &ContinuePred,
DL](
BasicBlock *Dest,
4795 ContinueBlock =
nullptr;
4796 ContinuePred = NextSrc;
4803 for (
size_t i = 0; i < NumLoops - 1; ++i)
4804 ContinueWith(
Loops[i]->getBody(),
Loops[i + 1]->getHeader());
4810 for (
size_t i = NumLoops - 1; i > 0; --i)
4811 ContinueWith(
Loops[i]->getAfter(),
Loops[i - 1]->getLatch());
4814 ContinueWith(Result->getLatch(),
nullptr);
4821 for (
size_t i = 0; i < NumLoops; ++i)
4822 Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
4836std::vector<CanonicalLoopInfo *>
4840 "Must pass as many tile sizes as there are loops");
4841 int NumLoops =
Loops.size();
4842 assert(NumLoops >= 1 &&
"At least one loop to tile required");
4854 Loop->collectControlBlocks(OldControlBBs);
4862 assert(L->isValid() &&
"All input loops must be valid canonical loops");
4863 OrigTripCounts.
push_back(L->getTripCount());
4874 for (
int i = 0; i < NumLoops - 1; ++i) {
4887 for (
int i = 0; i < NumLoops; ++i) {
4889 Value *OrigTripCount = OrigTripCounts[i];
4902 Value *FloorTripOverflow =
4908 "omp_floor" +
Twine(i) +
".tripcount",
true);
4916 std::vector<CanonicalLoopInfo *> Result;
4917 Result.reserve(NumLoops * 2);
4930 auto EmbeddNewLoop =
4931 [
this,
DL,
F, InnerEnter, &Enter, &
Continue, &OutroInsertBefore](
4934 DL, TripCount,
F, InnerEnter, OutroInsertBefore,
Name);
4939 Enter = EmbeddedLoop->
getBody();
4941 OutroInsertBefore = EmbeddedLoop->
getLatch();
4942 return EmbeddedLoop;
4946 const Twine &NameBase) {
4949 EmbeddNewLoop(
P.value(), NameBase +
Twine(
P.index()));
4950 Result.push_back(EmbeddedLoop);
4954 EmbeddNewLoops(FloorCount,
"floor");
4960 for (
int i = 0; i < NumLoops; ++i) {
4964 Value *FloorIsEpilogue =
4966 Value *TileTripCount =
4973 EmbeddNewLoops(TileCounts,
"tile");
4978 for (std::pair<BasicBlock *, BasicBlock *>
P : InbetweenCode) {
4987 BodyEnter =
nullptr;
4988 BodyEntered = ExitBB;
5001 for (
int i = 0; i < NumLoops; ++i) {
5004 Value *OrigIndVar = OrigIndVars[i];
5032 if (Properties.
empty())
5055 assert(
Loop->isValid() &&
"Expecting a valid CanonicalLoopInfo");
5059 assert(Latch &&
"A valid CanonicalLoopInfo must have a unique latch");
5067 if (
I.mayReadOrWriteMemory()) {
5071 I.setMetadata(LLVMContext::MD_access_group, AccessGroup);
5093 const Twine &NamePrefix) {
5099 SplitBefore = dyn_cast<Instruction>(IfCond);
5145 VMap[
Block] = NewBB;
5155 if (TargetTriple.
isX86()) {
5156 if (Features.
lookup(
"avx512f"))
5158 else if (Features.
lookup(
"avx"))
5162 if (TargetTriple.
isPPC())
5164 if (TargetTriple.
isWasm())
5171 Value *IfCond, OrderKind Order,
5190 if (AlignedVars.
size()) {
5193 for (
auto &AlignedItem : AlignedVars) {
5194 Value *AlignedPtr = AlignedItem.first;
5195 Value *Alignment = AlignedItem.second;
5197 AlignedPtr, Alignment);
5204 createIfVersion(CanonicalLoop, IfCond, VMap,
"simd");
5208 "Cannot find value which corresponds to original loop latch");
5209 assert(isa<BasicBlock>(MappedLatch) &&
5210 "Cannot cast mapped latch block value to BasicBlock");
5211 BasicBlock *NewLatchBlock = dyn_cast<BasicBlock>(MappedLatch);
5240 if ((Safelen ==
nullptr) || (Order == OrderKind::OMP_ORDER_concurrent)) {
5248 Ctx, {
MDString::get(Ctx,
"llvm.loop.parallel_accesses"), AccessGroup}));
5256 Ctx, {
MDString::get(Ctx,
"llvm.loop.vectorize.enable"), BoolConst}));
5258 if (Simdlen || Safelen) {
5262 ConstantInt *VectorizeWidth = Simdlen ==
nullptr ? Safelen : Simdlen;
5288static std::unique_ptr<TargetMachine>
5292 StringRef CPU =
F->getFnAttribute(
"target-cpu").getValueAsString();
5293 StringRef Features =
F->getFnAttribute(
"target-features").getValueAsString();
5294 const std::string &
Triple = M->getTargetTriple();
5304 std::nullopt, OptLevel));
5328 [&](
const Function &
F) {
return TM->getTargetTransformInfo(
F); });
5343 assert(L &&
"Expecting CanonicalLoopInfo to be recognized as a loop");
5348 nullptr, ORE,
static_cast<int>(OptLevel),
5369 <<
" Threshold=" << UP.
Threshold <<
"\n"
5372 <<
" PartialOptSizeThreshold="
5391 if (
auto *Load = dyn_cast<LoadInst>(&
I)) {
5392 Ptr = Load->getPointerOperand();
5393 }
else if (
auto *Store = dyn_cast<StoreInst>(&
I)) {
5394 Ptr = Store->getPointerOperand();
5398 Ptr =
Ptr->stripPointerCasts();
5400 if (
auto *Alloca = dyn_cast<AllocaInst>(
Ptr)) {
5401 if (Alloca->getParent() == &
F->getEntryBlock())
5421 int MaxTripCount = 0;
5422 bool MaxOrZero =
false;
5423 unsigned TripMultiple = 0;
5425 bool UseUpperBound =
false;
5427 MaxTripCount, MaxOrZero, TripMultiple, UCE, UP, PP,
5429 unsigned Factor = UP.
Count;
5430 LLVM_DEBUG(
dbgs() <<
"Suggesting unroll factor of " << Factor <<
"\n");
5441 assert(Factor >= 0 &&
"Unroll factor must not be negative");
5457 Ctx, {
MDString::get(Ctx,
"llvm.loop.unroll.count"), FactorConst}));
5470 *UnrolledCLI =
Loop;
5475 "unrolling only makes sense with a factor of 2 or larger");
5477 Type *IndVarTy =
Loop->getIndVarType();
5484 std::vector<CanonicalLoopInfo *>
LoopNest =
5499 Ctx, {
MDString::get(Ctx,
"llvm.loop.unroll.count"), FactorConst})});
5502 (*UnrolledCLI)->assertOK();
5520 Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
5539 if (!CPVars.
empty()) {
5544 Directive OMPD = Directive::OMPD_single;
5549 Value *Args[] = {Ident, ThreadId};
5575 EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCBWrapper,
5580 for (
size_t I = 0, E = CPVars.
size();
I < E; ++
I)
5583 ConstantInt::get(
Int64, 0), CPVars[
I],
5586 }
else if (!IsNowait)
5588 omp::Directive::OMPD_unknown,
false,
5600 Directive OMPD = Directive::OMPD_critical;
5605 Value *LockVar = getOMPCriticalRegionLock(CriticalName);
5606 Value *Args[] = {Ident, ThreadId, LockVar};
5623 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5631 const Twine &
Name,
bool IsDependSource) {
5634 [](
Value *SV) {
return SV->
getType()->isIntegerTy(64); }) &&
5635 "OpenMP runtime requires depend vec with i64 type");
5648 for (
unsigned I = 0;
I < NumLoops; ++
I) {
5662 Value *Args[] = {Ident, ThreadId, DependBaseAddrGEP};
5680 Directive OMPD = Directive::OMPD_ordered;
5689 Value *Args[] = {Ident, ThreadId};
5699 return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
5705 BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB,
bool Conditional,
5706 bool HasFinalize,
bool IsCancellable) {
5715 if (!isa_and_nonnull<BranchInst>(SplitPos))
5722 emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
5732 "Unexpected control flow graph state!!");
5733 emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
5735 "Unexpected Control Flow State!");
5741 "Unexpected Insertion point location!");
5744 auto InsertBB = merged ? ExitPredBB : ExitBB;
5745 if (!isa_and_nonnull<BranchInst>(SplitPos))
5755 if (!Conditional || !EntryCall)
5775 UI->eraseFromParent();
5783 omp::Directive OMPD, InsertPointTy FinIP,
Instruction *ExitCall,
5791 "Unexpected finalization stack state!");
5794 assert(Fi.DK == OMPD &&
"Unexpected Directive for Finalization call!");
5844 if (isa_and_nonnull<BranchInst>(OMP_Entry->
getTerminator())) {
5846 "copyin.not.master.end");
5901 Value *DependenceAddress,
bool HaveNowaitClause) {
5909 if (Device ==
nullptr)
5911 Constant *InteropTypeVal = ConstantInt::get(
Int32, (
int)InteropType);
5912 if (NumDependences ==
nullptr) {
5913 NumDependences = ConstantInt::get(
Int32, 0);
5917 Value *HaveNowaitClauseVal = ConstantInt::get(
Int32, HaveNowaitClause);
5919 Ident, ThreadId, InteropVar, InteropTypeVal,
5920 Device, NumDependences, DependenceAddress, HaveNowaitClauseVal};
5929 Value *NumDependences,
Value *DependenceAddress,
bool HaveNowaitClause) {
5937 if (Device ==
nullptr)
5939 if (NumDependences ==
nullptr) {
5940 NumDependences = ConstantInt::get(
Int32, 0);
5944 Value *HaveNowaitClauseVal = ConstantInt::get(
Int32, HaveNowaitClause);
5946 Ident, ThreadId, InteropVar, Device,
5947 NumDependences, DependenceAddress, HaveNowaitClauseVal};
5956 Value *NumDependences,
5957 Value *DependenceAddress,
5958 bool HaveNowaitClause) {
5965 if (Device ==
nullptr)
5967 if (NumDependences ==
nullptr) {
5968 NumDependences = ConstantInt::get(
Int32, 0);
5972 Value *HaveNowaitClauseVal = ConstantInt::get(
Int32, HaveNowaitClause);
5974 Ident, ThreadId, InteropVar, Device,
5975 NumDependences, DependenceAddress, HaveNowaitClauseVal};
6004 int32_t MinThreadsVal, int32_t MaxThreadsVal,
6005 int32_t MinTeamsVal, int32_t MaxTeamsVal) {
6023 const std::string DebugPrefix =
"_debug__";
6024 if (KernelName.
ends_with(DebugPrefix)) {
6025 KernelName = KernelName.
drop_back(DebugPrefix.length());
6032 if (MinTeamsVal > 1 || MaxTeamsVal > 0)
6036 if (MaxThreadsVal < 0)
6037 MaxThreadsVal = std::max(
6040 if (MaxThreadsVal > 0)
6051 omp::RuntimeFunction::OMPRTL___kmpc_target_init);
6054 Twine DynamicEnvironmentName = KernelName +
"_dynamic_environment";
6055 Constant *DynamicEnvironmentInitializer =
6059 DynamicEnvironmentInitializer, DynamicEnvironmentName,
6061 DL.getDefaultGlobalsAddressSpace());
6065 DynamicEnvironmentGV->
getType() == DynamicEnvironmentPtr
6066 ? DynamicEnvironmentGV
6068 DynamicEnvironmentPtr);
6071 ConfigurationEnvironment, {
6072 UseGenericStateMachineVal,
6073 MayUseNestedParallelismVal,
6080 ReductionBufferLength,
6083 KernelEnvironment, {
6084 ConfigurationEnvironmentInitializer,
6088 std::string KernelEnvironmentName =
6089 (KernelName +
"_kernel_environment").str();
6092 KernelEnvironmentInitializer, KernelEnvironmentName,
6094 DL.getDefaultGlobalsAddressSpace());
6098 KernelEnvironmentGV->
getType() == KernelEnvironmentPtr
6099 ? KernelEnvironmentGV
6101 KernelEnvironmentPtr);
6102 Value *KernelLaunchEnvironment = DebugKernelWrapper->
getArg(0);
6130 UI->eraseFromParent();
6138 int32_t TeamsReductionDataSize,
6139 int32_t TeamsReductionBufferLength) {
6144 omp::RuntimeFunction::OMPRTL___kmpc_target_deinit);
6148 if (!TeamsReductionBufferLength || !TeamsReductionDataSize)
6154 const std::string DebugPrefix =
"_debug__";
6156 KernelName = KernelName.
drop_back(DebugPrefix.length());
6157 auto *KernelEnvironmentGV =
6159 assert(KernelEnvironmentGV &&
"Expected kernel environment global\n");
6160 auto *KernelEnvironmentInitializer = KernelEnvironmentGV->getInitializer();
6162 KernelEnvironmentInitializer,
6163 ConstantInt::get(
Int32, TeamsReductionDataSize), {0, 7});
6165 NewInitializer, ConstantInt::get(
Int32, TeamsReductionBufferLength),
6167 KernelEnvironmentGV->setInitializer(NewInitializer);
6172 NamedMDNode *MD = M.getOrInsertNamedMetadata(
"nvvm.annotations");
6176 auto *KernelOp = dyn_cast<ConstantAsMetadata>(
Op->getOperand(0));
6177 if (!KernelOp || KernelOp->getValue() != &
Kernel)
6179 auto *Prop = dyn_cast<MDString>(
Op->getOperand(1));
6180 if (!Prop || Prop->getString() !=
Name)
6192 auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->
getOperand(2));
6193 int32_t OldLimit = cast<ConstantInt>(OldVal->getValue())->getZExtValue();
6196 OldVal->getValue()->getType(),
6197 Min ? std::min(OldLimit,
Value) : std::max(OldLimit,
Value))));
6206 NamedMDNode *MD = M.getOrInsertNamedMetadata(
"nvvm.annotations");
6211std::pair<int32_t, int32_t>
6213 int32_t ThreadLimit =
6218 if (!Attr.isValid() || !Attr.isStringAttribute())
6219 return {0, ThreadLimit};
6222 if (!llvm::to_integer(UBStr, UB, 10))
6223 return {0, ThreadLimit};
6224 UB = ThreadLimit ? std::min(ThreadLimit, UB) : UB;
6225 if (!llvm::to_integer(LBStr, LB, 10))
6231 auto *OldVal = cast<ConstantAsMetadata>(ExistingOp->getOperand(2));
6232 int32_t UB = cast<ConstantInt>(OldVal->getValue())->getZExtValue();
6233 return {0, ThreadLimit ? std::min(ThreadLimit, UB) : UB};
6235 return {0, ThreadLimit};
6245 llvm::utostr(LB) +
"," + llvm::utostr(UB));
6252std::pair<int32_t, int32_t>
6259 int32_t LB, int32_t UB) {
6264 Kernel.
addFnAttr(
"amdgpu-max-num-workgroups", llvm::utostr(LB) +
",1,1");
6269void OpenMPIRBuilder::setOutlinedTargetRegionFunctionAttributes(
6284 assert(OutlinedFn &&
"The outlined function must exist if embedded");
6293Constant *OpenMPIRBuilder::createTargetRegionEntryAddr(
Function *OutlinedFn,
6299 "Named kernel already exists?");
6314 ? GenerateFunctionCallback(EntryFnName)
6320 if (!IsOffloadEntry)
6323 std::string EntryFnIDName =
6325 ? std::string(EntryFnName)
6329 EntryFnName, EntryFnIDName);
6336 setOutlinedTargetRegionFunctionAttributes(OutlinedFn);
6337 auto OutlinedFnID = createOutlinedFunctionID(OutlinedFn, EntryFnIDName);
6338 auto EntryAddr = createTargetRegionEntryAddr(OutlinedFn, EntryFnName);
6340 EntryInfo, EntryAddr, OutlinedFnID,
6342 return OutlinedFnID;
6365 bool IsStandAlone = !BodyGenCB;
6389 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
6396 assert(MapperFunc &&
"MapperFunc missing for standalone target data");
6401 omp::OMPRTL___tgt_target_data_begin_mapper);
6405 for (
auto DeviceMap :
Info.DevicePtrInfoMap) {
6406 if (isa<AllocaInst>(DeviceMap.second.second)) {
6443 Value *OffloadingArgs[] = {SrcLocInfo, DeviceID,
6460 emitIfClause(IfCond, BeginThenGen, BeginElseGen, AllocaIP);
6470 emitIfClause(IfCond, EndThenGen, EndElseGen, AllocaIP);
6476 emitIfClause(IfCond, BeginThenGen, EndElseGen, AllocaIP);
6487 bool IsGPUDistribute) {
6488 assert((IVSize == 32 || IVSize == 64) &&
6489 "IV size is not compatible with the omp runtime");
6491 if (IsGPUDistribute)
6493 ? (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_4
6494 : omp::OMPRTL___kmpc_distribute_static_init_4u)
6495 : (IVSigned ? omp::OMPRTL___kmpc_distribute_static_init_8
6496 : omp::OMPRTL___kmpc_distribute_static_init_8u);
6498 Name = IVSize == 32 ? (IVSigned ? omp::OMPRTL___kmpc_for_static_init_4
6499 : omp::OMPRTL___kmpc_for_static_init_4u)
6500 : (IVSigned ? omp::OMPRTL___kmpc_for_static_init_8
6501 : omp::OMPRTL___kmpc_for_static_init_8u);
6508 assert((IVSize == 32 || IVSize == 64) &&
6509 "IV size is not compatible with the omp runtime");
6511 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_4
6512 : omp::OMPRTL___kmpc_dispatch_init_4u)
6513 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_init_8
6514 : omp::OMPRTL___kmpc_dispatch_init_8u);
6521 assert((IVSize == 32 || IVSize == 64) &&
6522 "IV size is not compatible with the omp runtime");
6524 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_4
6525 : omp::OMPRTL___kmpc_dispatch_next_4u)
6526 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_next_8
6527 : omp::OMPRTL___kmpc_dispatch_next_8u);
6534 assert((IVSize == 32 || IVSize == 64) &&
6535 "IV size is not compatible with the omp runtime");
6537 ? (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_4
6538 : omp::OMPRTL___kmpc_dispatch_fini_4u)
6539 : (IVSigned ? omp::OMPRTL___kmpc_dispatch_fini_8
6540 : omp::OMPRTL___kmpc_dispatch_fini_8u);
6563 for (
auto &Arg : Inputs)
6564 ParameterTypes.
push_back(Arg->getType()->isPointerTy()
6568 for (
auto &Arg : Inputs)
6569 ParameterTypes.
push_back(Arg->getType());
6578 auto OldInsertPoint = Builder.
saveIP();
6606 auto AllocaIP = Builder.
saveIP();
6611 const auto &ArgRange =
6613 ?
make_range(Func->arg_begin() + 1, Func->arg_end())
6634 if (
auto *Const = dyn_cast<Constant>(Input))
6639 if (
auto *Instr = dyn_cast<Instruction>(
User))
6640 if (Instr->getFunction() == Func)
6641 Instr->replaceUsesOfWith(Input, InputCopy);
6647 for (
auto InArg :
zip(Inputs, ArgRange)) {
6648 Value *Input = std::get<0>(InArg);
6649 Argument &Arg = std::get<1>(InArg);
6650 Value *InputCopy =
nullptr;
6653 ArgAccessorFuncCB(Arg, Input, InputCopy, AllocaIP, Builder.
saveIP()));
6671 if (llvm::isa<llvm::GlobalValue>(std::get<0>(InArg)) ||
6672 llvm::isa<llvm::GlobalObject>(std::get<0>(InArg)) ||
6673 llvm::isa<llvm::GlobalVariable>(std::get<0>(InArg))) {
6674 DeferredReplacement.
push_back(std::make_pair(Input, InputCopy));
6678 ReplaceValue(Input, InputCopy, Func);
6682 for (
auto Deferred : DeferredReplacement)
6683 ReplaceValue(std::get<0>(Deferred), std::get<1>(Deferred), Func);
6726 Type *TaskPtrTy = OMPBuilder.TaskPtr;
6727 Type *TaskTy = OMPBuilder.Task;
6732 ".omp_target_task_proxy_func",
6734 ProxyFn->getArg(0)->setName(
"thread.id");
6735 ProxyFn->getArg(1)->setName(
"task");
6741 bool HasShareds = StaleCI->
arg_size() > 1;
6747 "StaleCI with shareds should have exactly two arguments.");
6749 auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->
getArgOperand(1));
6750 assert(ArgStructAlloca &&
6751 "Unable to find the alloca instruction corresponding to arguments "
6752 "for extracted function");
6753 auto *ArgStructType =
6754 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
6757 Builder.
CreateAlloca(ArgStructType,
nullptr,
"structArg");
6758 Value *TaskT = ProxyFn->getArg(1);
6759 Value *ThreadId = ProxyFn->getArg(0);
6760 Value *SharedsSize =
6761 Builder.
getInt64(M.getDataLayout().getTypeStoreSize(ArgStructType));
6768 NewArgStructAlloca, NewArgStructAlloca->
getAlign(), LoadShared,
6771 Builder.
CreateCall(KernelLaunchFunction, {ThreadId, NewArgStructAlloca});
6784 [&OMPBuilder, &Builder, &Inputs, &CBFunc,
6785 &ArgAccessorFuncCB](
StringRef EntryFnName) {
6787 CBFunc, ArgAccessorFuncCB);
6791 IsOffloadEntry, OutlinedFn, OutlinedFnID);
6893 TargetTaskAllocaBB->
begin());
6897 OI.
EntryBB = TargetTaskAllocaBB;
6903 Builder, AllocaIP, ToBeDeleted, TargetTaskAllocaIP,
"global.tid",
false));
6916 EmitTargetCallFallbackCB, Args, DeviceID,
6917 RTLoc, TargetTaskAllocaIP));
6926 HasNoWait](
Function &OutlinedFn)
mutable {
6928 "there must be a single user for the outlined function");
6931 bool HasShareds = StaleCI->
arg_size() > 1;
6935 LLVM_DEBUG(
dbgs() <<
"Proxy task entry function created: " << *ProxyFn
6966 auto *ArgStructAlloca = dyn_cast<AllocaInst>(StaleCI->
getArgOperand(1));
6967 assert(ArgStructAlloca &&
6968 "Unable to find the alloca instruction corresponding to arguments "
6969 "for extracted function");
6970 auto *ArgStructType =
6971 dyn_cast<StructType>(ArgStructAlloca->getAllocatedType());
6972 assert(ArgStructType &&
"Unable to find struct type corresponding to "
6973 "arguments for extracted function");
6990 TaskAllocFn, {Ident, ThreadID,
Flags,
6991 TaskSize, SharedsSize,
7034 }
else if (DepArray) {
7053 I->eraseFromParent();
7071 DeviceAddrCB, CustomMapperCB);
7085 auto &&EmitTargetCallFallbackCB =
7092 bool HasNoWait =
false;
7093 bool HasDependencies = Dependencies.size() > 0;
7094 bool RequiresOuterTargetTask = HasNoWait || HasDependencies;
7098 if (!OutlinedFnID) {
7099 if (RequiresOuterTargetTask) {
7105 OutlinedFn,
nullptr, EmitTargetCallFallbackCB, KArgs,
7106 nullptr,
nullptr, AllocaIP, Dependencies,
7126 for (
auto V : NumTeams)
7129 unsigned NumTargetItems =
Info.NumberOfPtrs;
7143 NumTeamsC, NumThreadsVal,
7144 DynCGGroupMem, HasNoWait);
7148 if (RequiresOuterTargetTask) {
7150 OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs, DeviceID,
7151 RTLoc, AllocaIP, Dependencies, HasNoWait));
7154 Builder, OutlinedFn, OutlinedFnID, EmitTargetCallFallbackCB, KArgs,
7155 DeviceID, RTLoc, AllocaIP));
7179 OutlinedFn, OutlinedFnID, Args, CBFunc,
7187 NumThreads, Args, GenMapInfoCB, Dependenciess);
7201 return OS.str().str();
7215 assert(Elem.second->getValueType() == Ty &&
7216 "OMP internal variable has different type than requested");
7232 GV->setAlignment(std::max(TypeAlign, PtrAlign));
7239Value *OpenMPIRBuilder::getOMPCriticalRegionLock(
StringRef CriticalName) {
7240 std::string Prefix =
Twine(
"gomp_critical_user_", CriticalName).
str();
7241 std::string
Name = getNameWithSeparators({Prefix,
"var"},
".",
".");
7252 return SizePtrToInt;
7257 std::string VarName) {
7265 return MaptypesArrayGlobal;
7270 unsigned NumOperands,
7279 ArrI8PtrTy,
nullptr,
".offload_baseptrs");
7283 ArrI64Ty,
nullptr,
".offload_sizes");
7294 int64_t DeviceID,
unsigned NumOperands) {
7300 Value *ArgsBaseGEP =
7302 {Builder.getInt32(0), Builder.getInt32(0)});
7305 {Builder.getInt32(0), Builder.getInt32(0)});
7306 Value *ArgSizesGEP =
7308 {Builder.getInt32(0), Builder.getInt32(0)});
7314 ArgSizesGEP, MaptypesArg, MapnamesArg, NullPtr});
7321 assert((!ForEndCall ||
Info.separateBeginEndCalls()) &&
7322 "expected region end call to runtime only when end call is separate");
7324 auto VoidPtrTy = UnqualPtrTy;
7325 auto VoidPtrPtrTy = UnqualPtrTy;
7327 auto Int64PtrTy = UnqualPtrTy;
7329 if (!
Info.NumberOfPtrs) {
7341 Info.RTArgs.BasePointersArray,
7352 ForEndCall &&
Info.RTArgs.MapTypesArrayEnd ?
Info.RTArgs.MapTypesArrayEnd
7353 :
Info.RTArgs.MapTypesArray,
7359 if (!
Info.EmitDebug)
7368 if (!
Info.HasMapper)
7393 "struct.descriptor_dim");
7395 enum { OffsetFD = 0, CountFD, StrideFD };
7399 for (
unsigned I = 0, L = 0, E = NonContigInfo.
Dims.
size();
I < E; ++
I) {
7402 if (NonContigInfo.
Dims[
I] == 1)
7409 for (
unsigned II = 0, EE = NonContigInfo.
Dims[
I];
II < EE; ++
II) {
7410 unsigned RevIdx = EE -
II - 1;
7413 {Builder.getInt64(0), Builder.getInt64(II)});
7417 NonContigInfo.
Offsets[L][RevIdx], OffsetLVal,
7422 NonContigInfo.
Counts[L][RevIdx], CountLVal,
7427 NonContigInfo.
Strides[L][RevIdx], StrideLVal,
7436 Info.RTArgs.PointersArray, 0,
I);
7450 Info.clearArrayInfo();
7453 if (
Info.NumberOfPtrs == 0)
7463 PointerArrayType,
nullptr,
".offload_baseptrs");
7466 PointerArrayType,
nullptr,
".offload_ptrs");
7468 PointerArrayType,
nullptr,
".offload_mappers");
7469 Info.RTArgs.MappersArray = MappersArray;
7476 ConstantInt::get(Int64Ty, 0));
7478 for (
unsigned I = 0, E = CombinedInfo.
Sizes.
size();
I < E; ++
I) {
7479 if (
auto *CI = dyn_cast<Constant>(CombinedInfo.
Sizes[
I])) {
7480 if (!isa<ConstantExpr>(CI) && !isa<GlobalValue>(CI)) {
7481 if (IsNonContiguous &&
7482 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7484 OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG))
7492 RuntimeSizes.
set(
I);
7495 if (RuntimeSizes.
all()) {
7498 SizeArrayType,
nullptr,
".offload_sizes");
7504 auto *SizesArrayGbl =
7509 if (!RuntimeSizes.
any()) {
7510 Info.RTArgs.SizesArray = SizesArrayGbl;
7516 SizeArrayType,
nullptr,
".offload_sizes");
7521 SizesArrayGbl, OffloadSizeAlign,
7526 Info.RTArgs.SizesArray = Buffer;
7534 for (
auto mapFlag : CombinedInfo.
Types)
7536 static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7540 Info.RTArgs.MapTypesArray = MapTypesArrayGbl;
7545 auto *MapNamesArrayGbl =
7547 Info.RTArgs.MapNamesArray = MapNamesArrayGbl;
7548 Info.EmitDebug =
true;
7550 Info.RTArgs.MapNamesArray =
7552 Info.EmitDebug =
false;
7557 if (
Info.separateBeginEndCalls()) {
7558 bool EndMapTypesDiffer =
false;
7560 if (
Type &
static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>
>(
7561 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT)) {
7562 Type &= ~static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(
7563 OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);
7564 EndMapTypesDiffer =
true;
7567 if (EndMapTypesDiffer) {
7569 Info.RTArgs.MapTypesArrayEnd = MapTypesArrayGbl;
7574 for (
unsigned I = 0;
I <
Info.NumberOfPtrs; ++
I) {
7582 if (
Info.requiresDevicePointerInfo()) {
7589 DeviceAddrCB(
I,
Info.DevicePtrInfoMap[BPVal].second);
7591 Info.DevicePtrInfoMap[BPVal] = {BP, BP};
7593 DeviceAddrCB(
I, BP);
7605 if (RuntimeSizes.
test(
I)) {
7619 if (
Value *CustomMFunc = CustomMapperCB(
I))
7623 {Builder.getIntN(IndexSize, 0), Builder.getIntN(IndexSize, I)});
7629 Info.NumberOfPtrs == 0)
7674 if (
auto *CI = dyn_cast<ConstantInt>(
Cond)) {
7675 auto CondConstant = CI->getSExtValue();
7705bool OpenMPIRBuilder::checkAndEmitFlushAfterAtomic(
7709 "Unexpected Atomic Ordering.");
7773 assert(
X.Var->getType()->isPointerTy() &&
7774 "OMP Atomic expects a pointer to target memory");
7775 Type *XElemTy =
X.ElemTy;
7778 "OMP atomic read expected a scalar type");
7780 Value *XRead =
nullptr;
7786 XRead = cast<Value>(XLD);
7800 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Read);
7812 assert(
X.Var->getType()->isPointerTy() &&
7813 "OMP Atomic expects a pointer to target memory");
7814 Type *XElemTy =
X.ElemTy;
7817 "OMP atomic write expected a scalar type");
7832 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Write);
7845 Type *XTy =
X.Var->getType();
7847 "OMP Atomic expects a pointer to target memory");
7848 Type *XElemTy =
X.ElemTy;
7851 "OMP atomic update expected a scalar type");
7854 "OpenMP atomic does not support LT or GT operations");
7857 emitAtomicUpdate(AllocaIP,
X.Var,
X.ElemTy, Expr, AO, RMWOp, UpdateOp,
7858 X.IsVolatile, IsXBinopExpr);
7859 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Update);
7864Value *OpenMPIRBuilder::emitRMWOpAsInstruction(
Value *Src1,
Value *Src2,
7896std::pair<Value *, Value *> OpenMPIRBuilder::emitAtomicUpdate(
7899 AtomicUpdateCallbackTy &UpdateOp,
bool VolatileX,
bool IsXBinopExpr) {
7902 bool emitRMWOp =
false;
7910 emitRMWOp = XElemTy;
7913 emitRMWOp = (IsXBinopExpr && XElemTy);
7920 std::pair<Value *, Value *> Res;
7927 Res.second = Res.first;
7929 Res.second = emitRMWOpAsInstruction(Res.first, Expr, RMWOp);
7947 X->getName() +
".atomic.cont");
7951 NewAtomicAddr->
setName(
X->getName() +
"x.new.val");
7954 PHI->addIncoming(OldVal, CurBB);
7960 X->getName() +
".atomic.fltCast");
7963 X->getName() +
".atomic.ptrCast");
7974 Result->setVolatile(VolatileX);
7980 Res.first = OldExprVal;
8000 bool UpdateExpr,
bool IsPostfixUpdate,
bool IsXBinopExpr) {
8005 Type *XTy =
X.Var->getType();
8007 "OMP Atomic expects a pointer to target memory");
8008 Type *XElemTy =
X.ElemTy;
8011 "OMP atomic capture expected a scalar type");
8013 "OpenMP atomic does not support LT or GT operations");
8019 std::pair<Value *, Value *> Result =
8020 emitAtomicUpdate(AllocaIP,
X.Var,
X.ElemTy, Expr, AO, AtomicOp, UpdateOp,
8021 X.IsVolatile, IsXBinopExpr);
8023 Value *CapturedVal = (IsPostfixUpdate ? Result.first : Result.second);
8026 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Capture);
8038 IsPostfixUpdate, IsFailOnly, Failure);
8050 assert(
X.Var->getType()->isPointerTy() &&
8051 "OMP atomic expects a pointer to target memory");
8054 assert(V.Var->getType()->isPointerTy() &&
"v.var must be of pointer type");
8055 assert(V.ElemTy ==
X.ElemTy &&
"x and v must be of same type");
8060 if (
Op == OMPAtomicCompareOp::EQ) {
8079 "OldValue and V must be of same type");
8080 if (IsPostfixUpdate) {
8098 CurBBTI,
X.Var->getName() +
".atomic.exit");
8118 Value *CapturedValue =
8126 assert(R.Var->getType()->isPointerTy() &&
8127 "r.var must be of pointer type");
8128 assert(R.ElemTy->isIntegerTy() &&
"r must be of integral type");
8131 Value *ResultCast = R.IsSigned
8137 assert((
Op == OMPAtomicCompareOp::MAX ||
Op == OMPAtomicCompareOp::MIN) &&
8138 "Op should be either max or min at this point");
8139 assert(!IsFailOnly &&
"IsFailOnly is only valid when the comparison is ==");
8179 Value *CapturedValue =
nullptr;
8180 if (IsPostfixUpdate) {
8181 CapturedValue = OldValue;
8213 checkAndEmitFlushAfterAtomic(Loc, AO, AtomicKind::Compare);
8260 bool SubClausesPresent =
8261 (NumTeamsLower || NumTeamsUpper || ThreadLimit || IfExpr);
8264 assert((NumTeamsLower ==
nullptr || NumTeamsUpper !=
nullptr) &&
8265 "if lowerbound is non-null, then upperbound must also be non-null "
8266 "for bounds on num_teams");
8268 if (NumTeamsUpper ==
nullptr)
8271 if (NumTeamsLower ==
nullptr)
8272 NumTeamsLower = NumTeamsUpper;
8276 "argument to if clause must be an integer value");
8281 ConstantInt::get(IfExpr->
getType(), 0));
8290 if (ThreadLimit ==
nullptr)
8296 {Ident, ThreadNum, NumTeamsLower, NumTeamsUpper, ThreadLimit});
8301 BodyGenCB(AllocaIP, CodeGenIP);
8312 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP,
"gid",
true));
8314 Builder, OuterAllocaIP, ToBeDeleted, AllocaIP,
"tid",
true));
8316 auto HostPostOutlineCB = [
this, Ident,
8317 ToBeDeleted](
Function &OutlinedFn)
mutable {
8322 "there must be a single user for the outlined function");
8327 "Outlined function must have two or three arguments only");
8329 bool HasShared = OutlinedFn.
arg_size() == 3;
8337 assert(StaleCI &&
"Error while outlining - no CallInst user found for the "
8338 "outlined function.");
8345 omp::RuntimeFunction::OMPRTL___kmpc_fork_teams),
8349 I->eraseFromParent();
8364 std::string VarName) {
8373 return MapNamesArrayGlobal;
8378void OpenMPIRBuilder::initializeTypes(
Module &M) {
8381#define OMP_TYPE(VarName, InitValue) VarName = InitValue;
8382#define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
8383 VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
8384 VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
8385#define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
8386 VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
8387 VarName##Ptr = PointerType::getUnqual(VarName);
8388#define OMP_STRUCT_TYPE(VarName, StructName, Packed, ...) \
8389 T = StructType::getTypeByName(Ctx, StructName); \
8391 T = StructType::create(Ctx, {__VA_ARGS__}, StructName, Packed); \
8393 VarName##Ptr = PointerType::getUnqual(T);
8394#include "llvm/Frontend/OpenMP/OMPKinds.def"
8405 while (!Worklist.
empty()) {
8409 if (BlockSet.
insert(SuccBB).second)
8421 "omp_offloading_entries");
8445 Fn->
addFnAttr(
"uniform-work-group-size",
"true");
8464 auto &&GetMDInt = [
this](
unsigned V) {
8472 auto &&TargetRegionMetadataEmitter =
8473 [&
C, MD, &OrderedEntries, &GetMDInt, &GetMDString](
8488 GetMDInt(E.getKind()), GetMDInt(EntryInfo.DeviceID),
8489 GetMDInt(EntryInfo.FileID), GetMDString(EntryInfo.ParentName),
8490 GetMDInt(EntryInfo.Line), GetMDInt(EntryInfo.Count),
8491 GetMDInt(E.getOrder())};
8494 OrderedEntries[E.getOrder()] = std::make_pair(&E, EntryInfo);
8503 auto &&DeviceGlobalVarMetadataEmitter =
8504 [&
C, &OrderedEntries, &GetMDInt, &GetMDString, MD](
8514 Metadata *Ops[] = {GetMDInt(E.getKind()), GetMDString(MangledName),
8515 GetMDInt(E.getFlags()), GetMDInt(E.getOrder())};
8519 OrderedEntries[E.getOrder()] = std::make_pair(&E, varInfo);
8526 DeviceGlobalVarMetadataEmitter);
8528 for (
const auto &E : OrderedEntries) {
8529 assert(E.first &&
"All ordered entries must exist!");
8530 if (
const auto *CE =
8531 dyn_cast<OffloadEntriesInfoManager::OffloadEntryInfoTargetRegion>(
8533 if (!CE->getID() || !CE->getAddress()) {
8545 }
else if (
const auto *CE =
dyn_cast<
8556 if (!CE->getAddress()) {
8561 if (CE->getVarSize() == 0)
8567 "Declaret target link address is set.");
8570 if (!CE->getAddress()) {
8582 if (
auto *
GV = dyn_cast<GlobalValue>(CE->getAddress()))
8583 if ((
GV->hasLocalLinkage() ||
GV->hasHiddenVisibility()) &&
8591 Flags, CE->getLinkage(), CE->getVarName());
8594 Flags, CE->getLinkage());
8615 unsigned FileID,
unsigned Line,
unsigned Count) {
8618 <<
llvm::format(
"_%x_", FileID) << ParentName <<
"_l" << Line;
8625 unsigned NewCount = getTargetRegionEntryInfoCount(EntryInfo);
8628 EntryInfo.
Line, NewCount);
8635 auto FileIDInfo = CallBack();
8638 "getTargetEntryUniqueInfo, error message: " +
8644 std::get<1>(FileIDInfo));
8650 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>
>(
8652 !(Remain & 1); Remain = Remain >> 1)
8670 if (
static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>
>(
8672 static_cast<std::underlying_type_t<omp::OpenMPOffloadMappingFlags>
>(
8679 Flags &= ~omp::OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;
8680 Flags |= MemberOfFlag;
8686 bool IsDeclaration,
bool IsExternallyVisible,
8688 std::vector<GlobalVariable *> &GeneratedRefs,
bool OpenMPSIMD,
8689 std::vector<Triple> TargetTriple,
Type *LlvmPtrTy,
8690 std::function<
Constant *()> GlobalInitializer,
8706 if (!IsExternallyVisible)
8708 OS <<
"_decl_tgt_ref_ptr";
8717 auto *
GV = cast<GlobalVariable>(
Ptr);
8721 if (GlobalInitializer)
8722 GV->setInitializer(GlobalInitializer());
8728 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
8729 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
8730 GlobalInitializer, VariableLinkage, LlvmPtrTy, cast<Constant>(
Ptr));
8733 return cast<Constant>(
Ptr);
8742 bool IsDeclaration,
bool IsExternallyVisible,
8744 std::vector<GlobalVariable *> &GeneratedRefs,
bool OpenMPSIMD,
8745 std::vector<Triple> TargetTriple,
8746 std::function<
Constant *()> GlobalInitializer,
8763 VarName = MangledName;
8771 Linkage = (VariableLinkage) ? VariableLinkage() : LlvmVal->
getLinkage();
8787 auto *GvAddrRef = cast<GlobalVariable>(AddrRef);
8788 GvAddrRef->setConstant(
true);
8790 GvAddrRef->setInitializer(
Addr);
8791 GeneratedRefs.push_back(GvAddrRef);
8801 VarName = (
Addr) ?
Addr->getName() :
"";
8805 CaptureClause, DeviceClause, IsDeclaration, IsExternallyVisible,
8806 EntryInfo, MangledName, GeneratedRefs, OpenMPSIMD, TargetTriple,
8807 LlvmPtrTy, GlobalInitializer, VariableLinkage);
8808 VarName = (
Addr) ?
Addr->getName() :
"";
8829 auto &&GetMDInt = [MN](
unsigned Idx) {
8830 auto *V = cast<ConstantAsMetadata>(MN->getOperand(
Idx));
8831 return cast<ConstantInt>(V->getValue())->getZExtValue();
8834 auto &&GetMDString = [MN](
unsigned Idx) {
8835 auto *V = cast<MDString>(MN->getOperand(
Idx));
8836 return V->getString();
8839 switch (GetMDInt(0)) {
8867 if (HostFilePath.
empty())
8871 if (std::error_code Err = Buf.getError()) {
8873 "OpenMPIRBuilder: " +
8881 if (std::error_code Err =
M.getError()) {
8883 (
"error parsing host file inside of OpenMPIRBuilder: " + Err.message())
8895 return OffloadEntriesTargetRegion.empty() &&
8896 OffloadEntriesDeviceGlobalVar.empty();
8899unsigned OffloadEntriesInfoManager::getTargetRegionEntryInfoCount(
8901 auto It = OffloadEntriesTargetRegionCount.find(
8902 getTargetRegionEntryCountKey(EntryInfo));
8903 if (It == OffloadEntriesTargetRegionCount.end())
8908void OffloadEntriesInfoManager::incrementTargetRegionEntryInfoCount(
8910 OffloadEntriesTargetRegionCount[getTargetRegionEntryCountKey(EntryInfo)] =
8911 EntryInfo.
Count + 1;
8917 OffloadEntriesTargetRegion[EntryInfo] =
8919 OMPTargetRegionEntryTargetRegion);
8920 ++OffloadingEntriesNum;
8926 assert(EntryInfo.
Count == 0 &&
"expected default EntryInfo");
8929 EntryInfo.
Count = getTargetRegionEntryInfoCount(EntryInfo);
8935 if (!hasTargetRegionEntryInfo(EntryInfo)) {
8938 auto &Entry = OffloadEntriesTargetRegion[EntryInfo];
8939 Entry.setAddress(
Addr);
8941 Entry.setFlags(
Flags);
8944 hasTargetRegionEntryInfo(EntryInfo,
true))
8946 assert(!hasTargetRegionEntryInfo(EntryInfo) &&
8947 "Target region entry already registered!");
8949 OffloadEntriesTargetRegion[EntryInfo] = Entry;
8950 ++OffloadingEntriesNum;
8952 incrementTargetRegionEntryInfoCount(EntryInfo);
8959 EntryInfo.
Count = getTargetRegionEntryInfoCount(EntryInfo);
8961 auto It = OffloadEntriesTargetRegion.find(EntryInfo);
8962 if (It == OffloadEntriesTargetRegion.end()) {
8966 if (!IgnoreAddressId && (It->second.getAddress() || It->second.getID()))
8974 for (
const auto &It : OffloadEntriesTargetRegion) {
8975 Action(It.first, It.second);
8981 OffloadEntriesDeviceGlobalVar.try_emplace(
Name, Order,
Flags);
8982 ++OffloadingEntriesNum;
8990 if (!hasDeviceGlobalVarEntryInfo(VarName))
8992 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
8993 if (Entry.getAddress() && hasDeviceGlobalVarEntryInfo(VarName)) {
8994 if (Entry.getVarSize() == 0) {
8995 Entry.setVarSize(VarSize);
8996 Entry.setLinkage(Linkage);
9000 Entry.setVarSize(VarSize);
9001 Entry.setLinkage(Linkage);
9002 Entry.setAddress(
Addr);
9004 if (hasDeviceGlobalVarEntryInfo(VarName)) {
9005 auto &Entry = OffloadEntriesDeviceGlobalVar[VarName];
9006 assert(Entry.isValid() && Entry.getFlags() ==
Flags &&
9007 "Entry not initialized!");
9008 if (Entry.getVarSize() == 0) {
9009 Entry.setVarSize(VarSize);
9010 Entry.setLinkage(Linkage);
9015 OffloadEntriesDeviceGlobalVar.try_emplace(VarName, OffloadingEntriesNum,
9019 OffloadEntriesDeviceGlobalVar.try_emplace(
9020 VarName, OffloadingEntriesNum,
Addr, VarSize,
Flags, Linkage,
"");
9021 ++OffloadingEntriesNum;
9028 for (
const auto &E : OffloadEntriesDeviceGlobalVar)
9029 Action(E.getKey(), E.getValue());
9036void CanonicalLoopInfo::collectControlBlocks(
9043 BBs.
append({getPreheader(), Header,
Cond, Latch, Exit, getAfter()});
9055void CanonicalLoopInfo::setTripCount(
Value *TripCount) {
9059 assert(isa<CmpInst>(CmpI) &&
"First inst must compare IV with TripCount");
9067void CanonicalLoopInfo::mapIndVar(
9077 for (
Use &U : OldIV->
uses()) {
9078 auto *
User = dyn_cast<Instruction>(U.getUser());
9081 if (
User->getParent() == getCond())
9083 if (
User->getParent() == getLatch())
9089 Value *NewIV = Updater(OldIV);
9092 for (
Use *U : ReplacableUses)
9113 "Preheader must terminate with unconditional branch");
9115 "Preheader must jump to header");
9118 assert(isa<BranchInst>(Header->getTerminator()) &&
9119 "Header must terminate with unconditional branch");
9120 assert(Header->getSingleSuccessor() ==
Cond &&
9121 "Header must jump to exiting block");
9124 assert(
Cond->getSinglePredecessor() == Header &&
9125 "Exiting block only reachable from header");
9127 assert(isa<BranchInst>(
Cond->getTerminator()) &&
9128 "Exiting block must terminate with conditional branch");
9130 "Exiting block must have two successors");
9131 assert(cast<BranchInst>(
Cond->getTerminator())->getSuccessor(0) == Body &&
9132 "Exiting block's first successor jump to the body");
9133 assert(cast<BranchInst>(
Cond->getTerminator())->getSuccessor(1) == Exit &&
9134 "Exiting block's second successor must exit the loop");
9138 "Body only reachable from exiting block");
9143 "Latch must terminate with unconditional branch");
9151 assert(isa<BranchInst>(Exit->getTerminator()) &&
9152 "Exit block must terminate with unconditional branch");
9154 "Exit block must jump to after block");
9158 "After block only reachable from exit block");
9162 assert(IndVar &&
"Canonical induction variable not found?");
9164 "Induction variable must be an integer");
9166 "Induction variable must be a PHI in the loop header");
9167 assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
9169 cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->
isZero());
9170 assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
9172 auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
9174 assert(cast<BinaryOperator>(NextIndVar)->
getOpcode() == BinaryOperator::Add);
9175 assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
9176 assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
9179 Value *TripCount = getTripCount();
9180 assert(TripCount &&
"Loop trip count not found?");
9182 "Trip count and induction variable must have the same type");
9184 auto *CmpI = cast<CmpInst>(&
Cond->front());
9186 "Exit condition must be a signed less-than comparison");
9188 "Exit condition must compare the induction variable");
9190 "Exit condition must compare with the trip count");
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))
This file contains the simple types necessary to represent the attributes associated with functions a...
static const Function * getParent(const Value *V)
#define LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE()
LLVM_ENABLE_BITMASK_ENUMS_IN_NAMESPACE() pulls the operator overloads used by LLVM_MARK_AS_BITMASK_EN...
#define LLVM_MARK_AS_BITMASK_ENUM(LargestValue)
LLVM_MARK_AS_BITMASK_ENUM lets you opt in an individual enum type so you can perform bitwise operatio...
BlockVerifier::State From
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
DenseMap< Block *, BlockRelaxAux > Blocks
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
iv Induction Variable Users
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static cl::opt< unsigned > TileSize("fuse-matrix-tile-size", cl::init(4), cl::Hidden, cl::desc("Tile size for matrix instruction fusion using square-shaped tiles."))
uint64_t IntrinsicInst * II
#define OMP_KERNEL_ARG_VERSION
Provides definitions for Target specific Grid Values.
static OMPScheduleType getOpenMPBaseScheduleType(llvm::omp::ScheduleKind ClauseKind, bool HasChunks, bool HasSimdModifier)
Determine which scheduling algorithm to use, determined from schedule clause arguments.
static Function * createOutlinedFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, StringRef FuncName, SmallVectorImpl< Value * > &Inputs, OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc, OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB)
static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL)
Make Source branch to Target.
Value * createFakeIntVal(IRBuilderBase &Builder, OpenMPIRBuilder::InsertPointTy OuterAllocaIP, llvm::SmallVectorImpl< Instruction * > &ToBeDeleted, OpenMPIRBuilder::InsertPointTy InnerAllocaIP, const Twine &Name="", bool AsPtr=true)
static FunctionCallee getKmpcForDynamicFiniForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Returns an LLVM function to call for finalizing the dynamic loop using depending on type.
static Function * emitTargetTaskProxyFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, CallInst *StaleCI)
Create an entry point for a target task with the following.
static void updateNVPTXMetadata(Function &Kernel, StringRef Name, int32_t Value, bool Min)
static OMPScheduleType getOpenMPOrderingScheduleType(OMPScheduleType BaseScheduleType, bool HasOrderedClause)
Adds ordering modifier flags to schedule type.
static OMPScheduleType getOpenMPMonotonicityScheduleType(OMPScheduleType ScheduleType, bool HasSimdModifier, bool HasMonotonic, bool HasNonmonotonic, bool HasOrderedClause)
Adds monotonicity modifier flags to schedule type.
static void addSimdMetadata(BasicBlock *Block, MDNode *AccessGroup, LoopInfo &LI)
Attach llvm.access.group metadata to the memref instructions of Block.
static OMPScheduleType computeOpenMPScheduleType(ScheduleKind ClauseKind, bool HasChunks, bool HasSimdModifier, bool HasMonotonicModifier, bool HasNonmonotonicModifier, bool HasOrderedClause)
Determine the schedule type using schedule and ordering clause arguments.
static bool isValidWorkshareLoopScheduleType(OMPScheduleType SchedType)
static Function * getFreshReductionFunc(Module &M)
static void raiseUserConstantDataAllocasToEntryBlock(IRBuilderBase &Builder, Function *Function)
static MDNode * getNVPTXMDNode(Function &Kernel, StringRef Name)
static FunctionCallee getKmpcForDynamicNextForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Returns an LLVM function to call for updating the next loop using OpenMP dynamic scheduling depending...
static bool isConflictIP(IRBuilder<>::InsertPoint IP1, IRBuilder<>::InsertPoint IP2)
Return whether IP1 and IP2 are ambiguous, i.e.
static void checkReductionInfos(ArrayRef< OpenMPIRBuilder::ReductionInfo > ReductionInfos, bool IsGPU)
static FunctionCallee getKmpcForDynamicInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Returns an LLVM function to call for initializing loop bounds using OpenMP dynamic scheduling dependi...
static void emitTargetOutlinedFunction(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, bool IsOffloadEntry, TargetRegionEntryInfo &EntryInfo, Function *&OutlinedFn, Constant *&OutlinedFnID, SmallVectorImpl< Value * > &Inputs, OpenMPIRBuilder::TargetBodyGenCallbackTy &CBFunc, OpenMPIRBuilder::TargetGenArgAccessorsCallbackTy &ArgAccessorFuncCB)
static cl::opt< double > UnrollThresholdFactor("openmp-ir-builder-unroll-threshold-factor", cl::Hidden, cl::desc("Factor for the unroll threshold to account for code " "simplifications still taking place"), cl::init(1.5))
static int32_t computeHeuristicUnrollFactor(CanonicalLoopInfo *CLI)
Heuristically determine the best-performant unroll factor for CLI.
static void workshareLoopTargetCallback(OpenMPIRBuilder *OMPIRBuilder, CanonicalLoopInfo *CLI, Value *Ident, Function &OutlinedFn, Type *ParallelTaskPtr, const SmallVector< Instruction *, 4 > &ToBeDeleted, WorksharingLoopType LoopType)
static void emitTargetCall(OpenMPIRBuilder &OMPBuilder, IRBuilderBase &Builder, OpenMPIRBuilder::InsertPointTy AllocaIP, Function *OutlinedFn, Constant *OutlinedFnID, ArrayRef< int32_t > NumTeams, int32_t NumThreads, SmallVectorImpl< Value * > &Args, OpenMPIRBuilder::GenMapInfoCallbackTy GenMapInfoCB, SmallVector< llvm::OpenMPIRBuilder::DependData > Dependencies={})
static OpenMPIRBuilder::InsertPointTy getInsertPointAfterInstr(Instruction *I)
static void redirectAllPredecessorsTo(BasicBlock *OldTarget, BasicBlock *NewTarget, DebugLoc DL)
Redirect all edges that branch to OldTarget to NewTarget.
static std::unique_ptr< TargetMachine > createTargetMachine(Function *F, CodeGenOptLevel OptLevel)
Create the TargetMachine object to query the backend for optimization preferences.
static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
static void addBasicBlockMetadata(BasicBlock *BB, ArrayRef< Metadata * > Properties)
Attach metadata Properties to the basic block described by BB.
static cl::opt< bool > OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden, cl::desc("Use optimistic attributes describing " "'as-if' properties of runtime calls."), cl::init(false))
static FunctionCallee getKmpcForStaticLoopForType(Type *Ty, OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType)
static void createTargetLoopWorkshareCall(OpenMPIRBuilder *OMPBuilder, WorksharingLoopType LoopType, BasicBlock *InsertBlock, Value *Ident, Value *LoopBodyArg, Type *ParallelTaskPtr, Value *TripCount, Function &LoopBodyFn)
static Value * emitTaskDependencies(OpenMPIRBuilder &OMPBuilder, SmallVectorImpl< OpenMPIRBuilder::DependData > &Dependencies)
static const omp::GV & getGridValue(const Triple &T, Function *Kernel)
static void addLoopMetadata(CanonicalLoopInfo *Loop, ArrayRef< Metadata * > Properties)
Attach loop metadata Properties to the loop described by Loop.
static void removeUnusedBlocksFromParent(ArrayRef< BasicBlock * > BBs)
Determine which blocks in BBs are reachable from outside and remove the ones that are not reachable f...
static void targetParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn, BasicBlock *OuterAllocaBB, Value *Ident, Value *IfCondition, Value *NumThreads, Instruction *PrivTID, AllocaInst *PrivTIDAddr, Value *ThreadID, const SmallVector< Instruction *, 4 > &ToBeDeleted)
static void hostParallelCallback(OpenMPIRBuilder *OMPIRBuilder, Function &OutlinedFn, Function *OuterFn, Value *Ident, Value *IfCondition, Instruction *PrivTID, AllocaInst *PrivTIDAddr, const SmallVector< Instruction *, 4 > &ToBeDeleted)
FunctionAnalysisManager FAM
const char LLVMTargetMachineRef TM
This file defines the Pass Instrumentation classes that provide instrumentation points into the pass ...
This header defines various interfaces for pass management in LLVM.
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallSet class.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
static cl::opt< unsigned > MaxThreads("xcore-max-threads", cl::Optional, cl::desc("Maximum number of threads (for emulation thread-local storage)"), cl::Hidden, cl::value_desc("number"), cl::init(8))
static const uint32_t IV[8]
Class for arbitrary precision integers.
This class represents a conversion between pointers from one address space to another.
an instruction to allocate memory on the stack
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
PointerType * getType() const
Overload to return most specific pointer type.
Type * getAllocatedType() const
Return the type that is being allocated by the instruction.
unsigned getAddressSpace() const
Return the address space for the allocation.
std::optional< TypeSize > getAllocationSize(const DataLayout &DL) const
Get allocation size in bytes.
void setAlignment(Align Align)
const Value * getArraySize() const
Get the number of elements allocated.
A container for analyses that lazily runs them and caches their results.
bool registerPass(PassBuilderT &&PassBuilder)
Register an analysis pass with the manager.
This class represents an incoming formal argument to a Function.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
Class to represent array types.
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
A function analysis which provides an AssumptionCache.
AssumptionCache run(Function &F, FunctionAnalysisManager &)
A cache of @llvm.assume calls within a function.
An instruction that atomically checks whether a specified value is in a memory location,...
static AtomicOrdering getStrongestFailureOrdering(AtomicOrdering SuccessOrdering)
Returns the strongest permitted ordering on failure, given the desired ordering on success.
an instruction that atomically reads a memory location, combines it with another value,...
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ FMin
*p = minnum(old, v) minnum matches the behavior of llvm.minnum.
@ UMax
*p = old >unsigned v ? old : v
@ FMax
*p = maxnum(old, v) maxnum matches the behavior of llvm.maxnum.
@ UDecWrap
Decrement one until a minimum value or zero.
AttrBuilder & addAttribute(Attribute::AttrKind Val)
Add an attribute to the builder.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
AttributeSet getFnAttrs() const
The function attributes are returned.
AttributeList addFnAttributes(LLVMContext &C, const AttrBuilder &B) const
Add function attribute to the list.
AttributeSet addAttributes(LLVMContext &C, AttributeSet AS) const
Add attributes to the attribute set.
AttributeSet addAttribute(LLVMContext &C, Attribute::AttrKind Kind) const
Add an argument attribute.
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New)
Update all phi nodes in this basic block's successors to refer to basic block New instead of basic bl...
iterator begin()
Instruction iterator methods.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
reverse_iterator rbegin()
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
const Instruction & front() const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
const BasicBlock * getUniqueSuccessor() const
Return the successor of this block if it has a unique successor.
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
InstListType::reverse_iterator reverse_iterator
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
const Function * getParent() const
Return the enclosing method, or null if none.
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
InstListType::iterator iterator
Instruction iterators...
LLVMContext & getContext() const
Get the context in which this basic block lives.
void moveBefore(BasicBlock *MovePos)
Unlink this basic block from its current function and insert it into the function that MovePos lives ...
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB)
Transfer all instructions from FromBB to this basic block at ToIt.
const Instruction & back() const
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, InsertPosition InsertBefore=nullptr)
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Value * getArgOperand(unsigned i) const
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
unsigned arg_size() const
This class represents a function call, abstracting a target machine's calling convention.
Class to represented the control flow structure of an OpenMP canonical loop.
Value * getTripCount() const
Returns the llvm::Value containing the number of loop iterations.
BasicBlock * getHeader() const
The header is the entry for each iteration.
void assertOK() const
Consistency self-check.
Type * getIndVarType() const
Return the type of the induction variable (and the trip count).
BasicBlock * getBody() const
The body block is the single entry for a loop iteration and not controlled by CanonicalLoopInfo.
bool isValid() const
Returns whether this object currently represents the IR of a loop.
OpenMPIRBuilder::InsertPointTy getAfterIP() const
Return the insertion point for user code after the loop.
OpenMPIRBuilder::InsertPointTy getBodyIP() const
Return the insertion point for user code in the body.
BasicBlock * getAfter() const
The after block is intended for clean-up code such as lifetime end markers.
Function * getFunction() const
void invalidate()
Invalidate this loop.
BasicBlock * getLatch() const
Reaching the latch indicates the end of the loop body code.
OpenMPIRBuilder::InsertPointTy getPreheaderIP() const
Return the insertion point for user code before the loop.
BasicBlock * getCond() const
The condition block computes whether there is another loop iteration.
BasicBlock * getExit() const
Reaching the exit indicates no more iterations are being executed.
BasicBlock * getPreheader() const
The preheader ensures that there is only a single edge entering the loop.
Instruction * getIndVar() const
Returns the instruction representing the current logical induction variable.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ ICMP_ULT
unsigned less than
@ ICMP_ULE
unsigned less or equal
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
static Constant * get(LLVMContext &Context, ArrayRef< ElementTy > Elts)
get() constructor - Return a constant with array type with an element count and element type matching...
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
static Constant * getPointerBitCastOrAddrSpaceCast(Constant *C, Type *Ty)
Create a BitCast or AddrSpaceCast for a pointer type depending on the address space.
static Constant * getAddrSpaceCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
This is the shared class of boolean and integer constants.
static ConstantInt * getTrue(LLVMContext &Context)
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static ConstantInt * getFalse(LLVMContext &Context)
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
static Constant * get(StructType *T, ArrayRef< Constant * > V)
This is an important base class in LLVM.
static Constant * getAllOnesValue(Type *Ty)
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
This class represents an Operation in the Expression.
uint64_t getNumOperands() const
A parsed version of the target data layout string in and methods for querying it.
unsigned getDefaultGlobalsAddressSpace() const
Align getABIIntegerTypeAlignment(unsigned BitWidth) const
Returns the minimum ABI-required alignment for an integer type of the specified bitwidth.
unsigned getAllocaAddrSpace() const
TypeSize getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
unsigned getPointerSize(unsigned AS=0) const
Layout pointer size in bytes, rounded up to a whole number of bytes.
unsigned getIndexSizeInBits(unsigned AS) const
Size in bits of index used for address calculation in getelementptr.
TypeSize getTypeSizeInBits(Type *Ty) const
Size examples:
TypeSize getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type.
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Analysis pass which computes a DominatorTree.
DominatorTree run(Function &F, FunctionAnalysisManager &)
Run the analysis pass over a function and produce a dominator tree.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Lightweight error class with error context and mandatory checking.
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Class to represent function types.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
const BasicBlock & getEntryBlock() const
FunctionType * getFunctionType() const
Returns the FunctionType for me.
void removeFromParent()
removeFromParent - This method unlinks 'this' from the containing module, but does not delete it.
const DataLayout & getDataLayout() const
Get the data layout of the module this function belongs to.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
AttributeList getAttributes() const
Return the attribute list for this Function.
const Function & getFunction() const
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
adds the attribute to the list of attributes for the given arg.
Function::iterator insert(Function::iterator Position, BasicBlock *BB)
Insert BB in the basic block list at Position.
Type * getReturnType() const
Returns the type of the ret val.
void setCallingConv(CallingConv::ID CC)
Argument * getArg(unsigned i) const
bool hasMetadata() const
Return true if this value has any metadata attached to it.
void addMetadata(unsigned KindID, MDNode &MD)
Add a metadata attachment.
LinkageTypes getLinkage() const
void setLinkage(LinkageTypes LT)
Module * getParent()
Get the module that this global value is contained inside of...
void setDSOLocal(bool Local)
PointerType * getType() const
Global values are always pointers.
@ HiddenVisibility
The GV is hidden.
@ ProtectedVisibility
The GV is protected.
void setVisibility(VisibilityTypes V)
LinkageTypes
An enumeration for the kinds of linkage for global values.
@ PrivateLinkage
Like Internal, but omit from symbol table.
@ CommonLinkage
Tentative definitions.
@ InternalLinkage
Rename collisions when linking (static functions).
@ WeakODRLinkage
Same, but only replaced by something equivalent.
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
@ AppendingLinkage
Special purpose, only applies to global arrays.
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Type * getValueType() const
InsertPoint - A saved insertion point.
BasicBlock * getBlock() const
bool isSet() const
Returns true if this insert point is set.
BasicBlock::iterator getPoint() const
Common base class shared among various IRBuilders.
Value * CreatePtrDiff(Type *ElemTy, Value *LHS, Value *RHS, const Twine &Name="")
Return the i64 difference between two pointer values, dividing out the size of the pointed-to objects...
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
AtomicCmpXchgInst * CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, MaybeAlign Align, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SyncScope::ID SSID=SyncScope::System)
AllocaInst * CreateAlloca(Type *Ty, unsigned AddrSpace, Value *ArraySize=nullptr, const Twine &Name="")
Value * CreateInsertValue(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &Name="")
IntegerType * getIntNTy(unsigned N)
Fetch the type representing an N-bit integer.
Value * CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name="")
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Value * CreateZExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a ZExt or Trunc from the integer value V to DestTy.
UnreachableInst * CreateUnreachable()
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")
CallInst * CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, unsigned Alignment, Value *OffsetValue=nullptr)
Create an assume intrinsic call that represents an alignment assumption on the provided pointer.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, const Twine &Name="")
IntegerType * getIndexTy(const DataLayout &DL, unsigned AddrSpace)
Fetch the type of an integer that should be used to index GEP operations within AddressSpace.
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name="")
BasicBlock * GetInsertBlock() const
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Value * CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="")
Value * CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateUDiv(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt16Ty()
Fetch the type representing a 16-bit integer.
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Value * CreateNeg(Value *V, const Twine &Name="", bool HasNSW=false)
InsertPoint saveIP() const
Returns the current insert point.
Constant * CreateGlobalStringPtr(StringRef Str, const Twine &Name="", unsigned AddressSpace=0, Module *M=nullptr, bool AddNull=true)
Same as CreateGlobalString, but return a pointer with "i8*" type instead of a pointer to array of i8.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
SwitchInst * CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases=10, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a switch instruction with the specified value, default dest, and with a hint for the number of...
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
DebugLoc getCurrentDebugLocation() const
Get location information used by debugging information.
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Value * CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, const Twine &Name="")
Value * CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg != 0.
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
PointerType * getPtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer.
BranchInst * CreateBr(BasicBlock *Dest)
Create an unconditional 'br label X' instruction.
void ClearInsertionPoint()
Clear the insertion point: created instructions will not be inserted into a block.
Value * CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name="")
ConstantInt * getInt16(uint16_t C)
Get a constant 16-bit value.
Value * CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
Value * CreateIsNull(Value *Arg, const Twine &Name="")
Return a boolean value testing if Arg == 0.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Type * getVoidTy()
Fetch the type representing void.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateAShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Value * CreateXor(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
CallInst * CreateMemCpy(Value *Dst, MaybeAlign DstAlign, Value *Src, MaybeAlign SrcAlign, uint64_t Size, bool isVolatile=false, MDNode *TBAATag=nullptr, MDNode *TBAAStructTag=nullptr, MDNode *ScopeTag=nullptr, MDNode *NoAliasTag=nullptr)
Create and insert a memcpy between the specified pointers.
Value * CreateSExtOrTrunc(Value *V, Type *DestTy, const Twine &Name="")
Create a SExt or Trunc from the integer value V to DestTy.
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveBeforePreserving(Instruction *MovePos)
Perform a moveBefore operation, while signalling that the caller intends to preserve the original ord...
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static bool classof(const Value *V)
Methods for support type inquiry through isa, cast, and dyn_cast:
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
BasicBlock * getSuccessor(unsigned Idx) const LLVM_READONLY
Return the specified successor. This instruction must be a terminator.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
void insertAfter(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately after the specified instruction.
Class to represent integer types.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this load instruction.
Analysis pass that exposes the LoopInfo for a function.
LoopInfo run(Function &F, FunctionAnalysisManager &AM)
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
This class represents a loop nest and can be used to query its properties.
Represents a single loop in the control flow graph.
MDNode * createCallbackEncoding(unsigned CalleeArgNo, ArrayRef< int > Arguments, bool VarArgsArePassed)
Return metadata describing a callback (see llvm::AbstractCallSite).
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
static MDTuple * getDistinct(LLVMContext &Context, ArrayRef< Metadata * > MDs)
const MDOperand & getOperand(unsigned I) const
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDString * get(LLVMContext &Context, StringRef Str)
This class implements a map that also provides access to all stored values in a deterministic order.
static ErrorOr< std::unique_ptr< MemoryBuffer > > getFile(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, bool IsVolatile=false, std::optional< Align > Alignment=std::nullopt)
Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...
A Module instance is used to store all the information related to an LLVM module.
NamedMDNode * getNamedMetadata(StringRef Name) const
Return the first NamedMDNode in the module with the specified name.
LLVMContext & getContext() const
Get the global data context.
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
StringRef getName() const
Get a short "name" for the module.
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
iterator_range< global_iterator > globals()
const FunctionListType & getFunctionList() const
Get the Module's list of functions (constant).
GlobalVariable * getGlobalVariable(StringRef Name) const
Look up the specified global variable in the module symbol table.
GlobalValue * getNamedValue(StringRef Name) const
Return the global value in the module with the specified name, of arbitrary type.
NamedMDNode * getOrInsertNamedMetadata(StringRef Name)
Return the named MDNode in the module with the specified name.
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
iterator_range< op_iterator > operands()
void addOperand(MDNode *M)
Device global variable entries info.
Target region entries info.
Base class of the entries info.
@ OffloadingEntryInfoTargetRegion
Entry is a target region.
@ OffloadingEntryInfoDeviceGlobalVar
Entry is a declare target variable.
OMPTargetDeviceClauseKind
Kind of device clause for declare target variables and functions NOTE: Currently not used as a part o...
@ OMPTargetDeviceClauseAny
The target is marked for all devices.
void registerDeviceGlobalVarEntryInfo(StringRef VarName, Constant *Addr, int64_t VarSize, OMPTargetGlobalVarEntryKind Flags, GlobalValue::LinkageTypes Linkage)
Register device global variable entry.
void initializeDeviceGlobalVarEntryInfo(StringRef Name, OMPTargetGlobalVarEntryKind Flags, unsigned Order)
Initialize device global variable entry.
void actOnDeviceGlobalVarEntriesInfo(const OffloadDeviceGlobalVarEntryInfoActTy &Action)
OMPTargetRegionEntryKind
Kind of the target registry entry.
@ OMPTargetRegionEntryTargetRegion
Mark the entry as target region.
void getTargetRegionEntryFnName(SmallVectorImpl< char > &Name, const TargetRegionEntryInfo &EntryInfo)
bool hasTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo, bool IgnoreAddressId=false) const
Return true if a target region entry with the provided information exists.
void registerTargetRegionEntryInfo(TargetRegionEntryInfo EntryInfo, Constant *Addr, Constant *ID, OMPTargetRegionEntryKind Flags)
Register target region entry.
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
unsigned size() const
Return number of entries defined so far.
void initializeTargetRegionEntryInfo(const TargetRegionEntryInfo &EntryInfo, unsigned Order)
Initialize target region entry.
OMPTargetGlobalVarEntryKind
Kind of the global variable entry..
@ OMPTargetGlobalVarEntryEnter
Mark the entry as a declare target enter.
@ OMPTargetGlobalRegisterRequires
Mark the entry as a register requires global.
@ OMPTargetGlobalVarEntryIndirect
Mark the entry as a declare target indirect global.
@ OMPTargetGlobalVarEntryLink
Mark the entry as a to declare target link.
@ OMPTargetGlobalVarEntryTo
Mark the entry as a to declare target.
bool hasDeviceGlobalVarEntryInfo(StringRef VarName) const
Checks if the variable with the given name has been registered already.
bool empty() const
Return true if a there are no entries defined.
std::optional< bool > IsTargetDevice
Flag to define whether to generate code for the role of the OpenMP host (if set to false) or device (...
void setGridValue(omp::GV G)
StringRef separator() const
int64_t getRequiresFlags() const
Returns requires directive clauses as flags compatible with those expected by libomptarget.
StringRef firstSeparator() const
std::optional< bool > EmitLLVMUsedMetaInfo
Flag for specifying if LLVMUsed information should be emitted.
omp::GV getGridValue() const
void setHasRequiresReverseOffload(bool Value)
bool hasRequiresUnifiedSharedMemory() const
void setHasRequiresUnifiedSharedMemory(bool Value)
bool hasRequiresDynamicAllocators() const
bool openMPOffloadMandatory() const
void setHasRequiresUnifiedAddress(bool Value)
bool isTargetDevice() const
void setHasRequiresDynamicAllocators(bool Value)
void setEmitLLVMUsed(bool Value=true)
bool hasRequiresReverseOffload() const
bool hasRequiresFlags() const
bool hasRequiresUnifiedAddress() const
Struct that keeps the information that should be kept throughout a 'target data' region.
An interface to create LLVM-IR for OpenMP directives.
Constant * getOrCreateIdent(Constant *SrcLocStr, uint32_t SrcLocStrSize, omp::IdentFlag Flags=omp::IdentFlag(0), unsigned Reserve2Flags=0)
Return an ident_t* encoding the source location SrcLocStr and Flags.
FunctionCallee getOrCreateRuntimeFunction(Module &M, omp::RuntimeFunction FnID)
Return the function declaration for the runtime function with FnID.
std::function< void(InsertPointTy CodeGenIP)> FinalizeCallbackTy
Callback type for variable finalization (think destructors).
InsertPointTy createTargetInit(const LocationDescription &Loc, bool IsSPMD, int32_t MinThreadsVal=0, int32_t MaxThreadsVal=0, int32_t MinTeamsVal=0, int32_t MaxTeamsVal=0)
The omp target interface.
void emitIfClause(Value *Cond, BodyGenCallbackTy ThenGen, BodyGenCallbackTy ElseGen, InsertPointTy AllocaIP={})
Emits code for OpenMP 'if' clause using specified BodyGenCallbackTy Here is the logic: if (Cond) { Th...
ReductionGenCBKind
Enum class for the RedctionGen CallBack type to be used.
CanonicalLoopInfo * collapseLoops(DebugLoc DL, ArrayRef< CanonicalLoopInfo * > Loops, InsertPointTy ComputeIP)
Collapse a loop nest into a single loop.
void createTaskyield(const LocationDescription &Loc)
Generator for '#omp taskyield'.
void emitBranch(BasicBlock *Target)
InsertPointTy createAtomicWrite(const LocationDescription &Loc, AtomicOpValue &X, Value *Expr, AtomicOrdering AO)
Emit atomic write for : X = Expr — Only Scalar data types.
static void writeThreadBoundsForKernel(const Triple &T, Function &Kernel, int32_t LB, int32_t UB)
InsertPointTy createCritical(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst)
Generator for '#omp critical'.
static TargetRegionEntryInfo getTargetEntryUniqueInfo(FileIdentifierInfoCallbackTy CallBack, StringRef ParentName="")
Creates a unique info for a target entry when provided a filename and line number from.
void emitTaskwaitImpl(const LocationDescription &Loc)
Generate a taskwait runtime call.
Constant * registerTargetRegionFunction(TargetRegionEntryInfo &EntryInfo, Function *OutlinedFunction, StringRef EntryFnName, StringRef EntryFnIDName)
Registers the given function and sets up the attribtues of the function Returns the FunctionID.
InsertPointTy createAtomicCapture(const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, AtomicOpValue &V, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool UpdateExpr, bool IsPostfixUpdate, bool IsXBinopExpr)
Emit atomic update for constructs: — Only Scalar data types V = X; X = X BinOp Expr ,...
void initialize()
Initialize the internal state, this will put structures types and potentially other helpers into the ...
void createTargetDeinit(const LocationDescription &Loc, int32_t TeamsReductionDataSize=0, int32_t TeamsReductionBufferLength=1024)
Create a runtime call for kmpc_target_deinit.
CanonicalLoopInfo * createCanonicalLoop(const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB, Value *TripCount, const Twine &Name="loop")
Generator for the control flow structure of an OpenMP canonical loop.
void loadOffloadInfoMetadata(Module &M)
Loads all the offload entries information from the host IR metadata.
InsertPointTy createAtomicUpdate(const LocationDescription &Loc, InsertPointTy AllocaIP, AtomicOpValue &X, Value *Expr, AtomicOrdering AO, AtomicRMWInst::BinOp RMWOp, AtomicUpdateCallbackTy &UpdateOp, bool IsXBinopExpr)
Emit atomic update for constructs: X = X BinOp Expr ,or X = Expr BinOp X For complex Operations: X = ...
void unrollLoopFull(DebugLoc DL, CanonicalLoopInfo *Loop)
Fully unroll a loop.
void emitFlush(const LocationDescription &Loc)
Generate a flush runtime call.
InsertPointTy createBarrier(const LocationDescription &Loc, omp::Directive Kind, bool ForceSimpleCall=false, bool CheckCancelFlag=true)
Emitter methods for OpenMP directives.
InsertPointTy emitKernelLaunch(const LocationDescription &Loc, Function *OutlinedFn, Value *OutlinedFnID, EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP)
Generate a target region entry call and host fallback call.
InsertPointTy createCancel(const LocationDescription &Loc, Value *IfCondition, omp::Directive CanceledDirective)
Generator for '#omp cancel'.
static std::pair< int32_t, int32_t > readThreadBoundsForKernel(const Triple &T, Function &Kernel)
}
OpenMPIRBuilderConfig Config
The OpenMPIRBuilder Configuration.
CallInst * createOMPInteropDestroy(const LocationDescription &Loc, Value *InteropVar, Value *Device, Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause)
Create a runtime call for __tgt_interop_destroy.
InsertPointTy createAtomicRead(const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V, AtomicOrdering AO)
Emit atomic Read for : V = X — Only Scalar data types.
std::function< void(EmitMetadataErrorKind, TargetRegionEntryInfo)> EmitMetadataErrorReportFunctionTy
Callback function type.
InsertPointTy createOrderedThreadsSimd(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool IsThreads)
Generator for '#omp ordered [threads | simd]'.
OpenMPIRBuilder::InsertPointTy createTargetData(const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, Value *DeviceID, Value *IfCond, TargetDataInfo &Info, GenMapInfoCallbackTy GenMapInfoCB, omp::RuntimeFunction *MapperFunc=nullptr, function_ref< InsertPointTy(InsertPointTy CodeGenIP, BodyGenTy BodyGenType)> BodyGenCB=nullptr, function_ref< void(unsigned int, Value *)> DeviceAddrCB=nullptr, function_ref< Value *(unsigned int)> CustomMapperCB=nullptr, Value *SrcLocInfo=nullptr)
Generator for '#omp target data'.
std::forward_list< CanonicalLoopInfo > LoopInfos
Collection of owned canonical loop objects that eventually need to be free'd.
void createTaskwait(const LocationDescription &Loc)
Generator for '#omp taskwait'.
CanonicalLoopInfo * createLoopSkeleton(DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore, BasicBlock *PostInsertBefore, const Twine &Name={})
Create the control flow structure of a canonical OpenMP loop.
std::string createPlatformSpecificName(ArrayRef< StringRef > Parts) const
Get the create a name using the platform specific separators.
FunctionCallee createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned.
static void getKernelArgsVector(TargetKernelArgs &KernelArgs, IRBuilderBase &Builder, SmallVector< Value * > &ArgsVector)
Create the kernel args vector used by emitTargetKernel.
void unrollLoopHeuristic(DebugLoc DL, CanonicalLoopInfo *Loop)
Fully or partially unroll a loop.
omp::OpenMPOffloadMappingFlags getMemberOfFlag(unsigned Position)
Get OMP_MAP_MEMBER_OF flag with extra bits reserved based on the position given.
InsertPointTy createReductionsGPU(const LocationDescription &Loc, InsertPointTy AllocaIP, InsertPointTy CodeGenIP, ArrayRef< ReductionInfo > ReductionInfos, bool IsNoWait=false, bool IsTeamsReduction=false, bool HasDistribute=false, ReductionGenCBKind ReductionGenCBKind=ReductionGenCBKind::MLIR, std::optional< omp::GV > GridValue={}, unsigned ReductionBufNum=1024, Value *SrcLocInfo=nullptr)
Design of OpenMP reductions on the GPU.
void addAttributes(omp::RuntimeFunction FnID, Function &Fn)
Add attributes known for FnID to Fn.
Module & M
The underlying LLVM-IR module.
StringMap< Constant * > SrcLocStrMap
Map to remember source location strings.
void createMapperAllocas(const LocationDescription &Loc, InsertPointTy AllocaIP, unsigned NumOperands, struct MapperAllocas &MapperAllocas)
Create the allocas instruction used in call to mapper functions.
Constant * getOrCreateSrcLocStr(StringRef LocStr, uint32_t &SrcLocStrSize)
Return the (LLVM-IR) string describing the source location LocStr.
void addOutlineInfo(OutlineInfo &&OI)
Add a new region that will be outlined later.
FunctionCallee createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned.
void unrollLoopPartial(DebugLoc DL, CanonicalLoopInfo *Loop, int32_t Factor, CanonicalLoopInfo **UnrolledCLI)
Partially unroll a loop.
InsertPointTy createSections(const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef< StorableBodyGenCallbackTy > SectionCBs, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, bool IsCancellable, bool IsNowait)
Generator for '#omp sections'.
InsertPointTy createTask(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, bool Tied=true, Value *Final=nullptr, Value *IfCondition=nullptr, SmallVector< DependData > Dependencies={})
Generator for #omp task
void emitTaskyieldImpl(const LocationDescription &Loc)
Generate a taskyield runtime call.
void emitMapperCall(const LocationDescription &Loc, Function *MapperFunc, Value *SrcLocInfo, Value *MaptypesArg, Value *MapnamesArg, struct MapperAllocas &MapperAllocas, int64_t DeviceID, unsigned NumOperands)
Create the call for the target mapper function.
InsertPointTy createAtomicCompare(const LocationDescription &Loc, AtomicOpValue &X, AtomicOpValue &V, AtomicOpValue &R, Value *E, Value *D, AtomicOrdering AO, omp::OMPAtomicCompareOp Op, bool IsXBinopExpr, bool IsPostfixUpdate, bool IsFailOnly)
Emit atomic compare for constructs: — Only scalar data types cond-expr-stmt: x = x ordop expr ?...
InsertPointTy createOrderedDepend(const LocationDescription &Loc, InsertPointTy AllocaIP, unsigned NumLoops, ArrayRef< llvm::Value * > StoreValues, const Twine &Name, bool IsDependSource)
Generator for '#omp ordered depend (source | sink)'.
InsertPointTy createCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, llvm::IntegerType *IntPtrTy, bool BranchtoEnd=true)
Generate conditional branch and relevant BasicBlocks through which private threads copy the 'copyin' ...
void emitOffloadingArrays(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo, TargetDataInfo &Info, bool IsNonContiguous=false, function_ref< void(unsigned int, Value *)> DeviceAddrCB=nullptr, function_ref< Value *(unsigned int)> CustomMapperCB=nullptr)
Emit the arrays used to pass the captures and map information to the offloading runtime library.
SmallVector< FinalizationInfo, 8 > FinalizationStack
The finalization stack made up of finalize callbacks currently in-flight, wrapped into FinalizationIn...
std::vector< CanonicalLoopInfo * > tileLoops(DebugLoc DL, ArrayRef< CanonicalLoopInfo * > Loops, ArrayRef< Value * > TileSizes)
Tile a loop nest.
CallInst * createOMPInteropInit(const LocationDescription &Loc, Value *InteropVar, omp::OMPInteropType InteropType, Value *Device, Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause)
Create a runtime call for __tgt_interop_init.
void finalize(Function *Fn=nullptr)
Finalize the underlying module, e.g., by outlining regions.
SmallVector< OutlineInfo, 16 > OutlineInfos
Collection of regions that need to be outlined during finalization.
Function * getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID)
const Triple T
The target triple of the underlying module.
DenseMap< std::pair< Constant *, uint64_t >, Constant * > IdentMap
Map to remember existing ident_t*.
CallInst * createOMPFree(const LocationDescription &Loc, Value *Addr, Value *Allocator, std::string Name="")
Create a runtime call for kmpc_free.
FunctionCallee createForStaticInitFunction(unsigned IVSize, bool IVSigned, bool IsGPUDistribute)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned.
CallInst * createOMPAlloc(const LocationDescription &Loc, Value *Size, Value *Allocator, std::string Name="")
Create a runtime call for kmpc_Alloc.
void emitNonContiguousDescriptor(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, MapInfosTy &CombinedInfo, TargetDataInfo &Info)
Emit an array of struct descriptors to be assigned to the offload args.
InsertPointTy createSection(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB)
Generator for '#omp section'.
InsertPointTy applyWorkshareLoop(DebugLoc DL, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier, llvm::omp::ScheduleKind SchedKind=llvm::omp::OMP_SCHEDULE_Default, Value *ChunkSize=nullptr, bool HasSimdModifier=false, bool HasMonotonicModifier=false, bool HasNonmonotonicModifier=false, bool HasOrderedClause=false, omp::WorksharingLoopType LoopType=omp::WorksharingLoopType::ForStaticLoop)
Modifies the canonical loop to be a workshare loop.
void emitBlock(BasicBlock *BB, Function *CurFn, bool IsFinished=false)
Value * getOrCreateThreadID(Value *Ident)
Return the current thread ID.
void emitOffloadingArraysAndArgs(InsertPointTy AllocaIP, InsertPointTy CodeGenIP, TargetDataInfo &Info, TargetDataRTArgs &RTArgs, MapInfosTy &CombinedInfo, bool IsNonContiguous=false, bool ForEndCall=false, function_ref< void(unsigned int, Value *)> DeviceAddrCB=nullptr, function_ref< Value *(unsigned int)> CustomMapperCB=nullptr)
Allocates memory for and populates the arrays required for offloading (offload_{baseptrs|ptrs|mappers...
InsertPointTy createMaster(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB)
Generator for '#omp master'.
IRBuilder ::InsertPoint createParallel(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable)
Generator for '#omp parallel'.
StringMap< GlobalVariable *, BumpPtrAllocator > InternalVars
An ordered map of auto-generated variables to their unique names.
GlobalVariable * getOrCreateInternalVariable(Type *Ty, const StringRef &Name, unsigned AddressSpace=0)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
FunctionCallee createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned.
InsertPointTy createSingle(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool IsNowait, ArrayRef< llvm::Value * > CPVars={}, ArrayRef< llvm::Function * > CPFuncs={})
Generator for '#omp single'.
CallInst * createOMPInteropUse(const LocationDescription &Loc, Value *InteropVar, Value *Device, Value *NumDependences, Value *DependenceAddress, bool HaveNowaitClause)
Create a runtime call for __tgt_interop_use.
IRBuilder<>::InsertPoint InsertPointTy
Type used throughout for insertion points.
GlobalVariable * createOffloadMapnames(SmallVectorImpl< llvm::Constant * > &Names, std::string VarName)
Create the global variable holding the offload names information.
static void writeTeamsForKernel(const Triple &T, Function &Kernel, int32_t LB, int32_t UB)
std::function< Function *(StringRef FunctionName)> FunctionGenCallback
Functions used to generate a function with the given name.
void setCorrectMemberOfFlag(omp::OpenMPOffloadMappingFlags &Flags, omp::OpenMPOffloadMappingFlags MemberOfFlag)
Given an initial flag set, this function modifies it to contain the passed in MemberOfFlag generated ...
void emitCancelationCheckImpl(Value *CancelFlag, omp::Directive CanceledDirective, FinalizeCallbackTy ExitCB={})
Generate control flow and cleanup for cancellation.
Constant * getOrCreateDefaultSrcLocStr(uint32_t &SrcLocStrSize)
Return the (LLVM-IR) string describing the default source location.
InsertPointTy createMasked(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, Value *Filter)
Generator for '#omp masked'.
void createOffloadEntry(Constant *ID, Constant *Addr, uint64_t Size, int32_t Flags, GlobalValue::LinkageTypes, StringRef Name="")
Creates offloading entry for the provided entry ID ID, address Addr, size Size, and flags Flags.
static unsigned getOpenMPDefaultSimdAlign(const Triple &TargetTriple, const StringMap< bool > &Features)
Get the default alignment value for given target.
unsigned getFlagMemberOffset()
Get the offset of the OMP_MAP_MEMBER_OF field.
InsertPointTy createTaskgroup(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB)
Generator for the taskgroup construct.
void createOffloadEntriesAndInfoMetadata(EmitMetadataErrorReportFunctionTy &ErrorReportFunction)
void applySimd(CanonicalLoopInfo *Loop, MapVector< Value *, Value * > AlignedVars, Value *IfCond, omp::OrderKind Order, ConstantInt *Simdlen, ConstantInt *Safelen)
Add metadata to simd-ize a loop.
void emitTargetRegionFunction(TargetRegionEntryInfo &EntryInfo, FunctionGenCallback &GenerateFunctionCallback, bool IsOffloadEntry, Function *&OutlinedFn, Constant *&OutlinedFnID)
Create a unique name for the entry function using the source location information of the current targ...
bool isLastFinalizationInfoCancellable(omp::Directive DK)
Return true if the last entry in the finalization stack is of kind DK and cancellable.
InsertPointTy emitTargetKernel(const LocationDescription &Loc, InsertPointTy AllocaIP, Value *&Return, Value *Ident, Value *DeviceID, Value *NumTeams, Value *NumThreads, Value *HostPtr, ArrayRef< Value * > KernelArgs)
Generate a target region entry call.
GlobalVariable * createOffloadMaptypes(SmallVectorImpl< uint64_t > &Mappings, std::string VarName)
Create the global variable holding the offload mappings information.
CallInst * createCachedThreadPrivate(const LocationDescription &Loc, llvm::Value *Pointer, llvm::ConstantInt *Size, const llvm::Twine &Name=Twine(""))
Create a runtime call for kmpc_threadprivate_cached.
IRBuilder Builder
The LLVM-IR Builder used to create IR.
GlobalValue * createGlobalFlag(unsigned Value, StringRef Name)
Create a hidden global flag Name in the module with initial value Value.
void emitOffloadingArraysArgument(IRBuilderBase &Builder, OpenMPIRBuilder::TargetDataRTArgs &RTArgs, OpenMPIRBuilder::TargetDataInfo &Info, bool ForEndCall=false)
Emit the arguments to be passed to the runtime library based on the arrays of base pointers,...
Value * getSizeInBytes(Value *BasePtr)
Computes the size of type in bytes.
FunctionCallee createDispatchDeinitFunction()
Returns __kmpc_dispatch_deinit runtime function.
void registerTargetGlobalVariable(OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause, OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause, bool IsDeclaration, bool IsExternallyVisible, TargetRegionEntryInfo EntryInfo, StringRef MangledName, std::vector< GlobalVariable * > &GeneratedRefs, bool OpenMPSIMD, std::vector< Triple > TargetTriple, std::function< Constant *()> GlobalInitializer, std::function< GlobalValue::LinkageTypes()> VariableLinkage, Type *LlvmPtrTy, Constant *Addr)
Registers a target variable for device or host.
InsertPointTy createTeams(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, Value *NumTeamsLower=nullptr, Value *NumTeamsUpper=nullptr, Value *ThreadLimit=nullptr, Value *IfExpr=nullptr)
Generator for #omp teams
BodyGenTy
Type of BodyGen to use for region codegen.
InsertPointTy createTarget(const LocationDescription &Loc, bool IsOffloadEntry, OpenMPIRBuilder::InsertPointTy AllocaIP, OpenMPIRBuilder::InsertPointTy CodeGenIP, TargetRegionEntryInfo &EntryInfo, ArrayRef< int32_t > NumTeams, int32_t NumThreads, SmallVectorImpl< Value * > &Inputs, GenMapInfoCallbackTy GenMapInfoCB, TargetBodyGenCallbackTy BodyGenCB, TargetGenArgAccessorsCallbackTy ArgAccessorFuncCB, SmallVector< DependData > Dependencies={})
Generator for '#omp target'.
SmallVector< llvm::Function *, 16 > ConstantAllocaRaiseCandidates
A collection of candidate target functions that's constant allocas will attempt to be raised on a cal...
OffloadEntriesInfoManager OffloadInfoManager
Info manager to keep track of target regions.
static std::pair< int32_t, int32_t > readTeamBoundsForKernel(const Triple &T, Function &Kernel)
Read/write a bounds on teams for Kernel.
std::function< std::tuple< std::string, uint64_t >()> FileIdentifierInfoCallbackTy
const std::string ompOffloadInfoName
OMP Offload Info Metadata name string.
InsertPointTy createCopyPrivate(const LocationDescription &Loc, llvm::Value *BufSize, llvm::Value *CpyBuf, llvm::Value *CpyFn, llvm::Value *DidIt)
Generator for __kmpc_copyprivate.
InsertPointTy createReductions(const LocationDescription &Loc, InsertPointTy AllocaIP, ArrayRef< ReductionInfo > ReductionInfos, ArrayRef< bool > IsByRef, bool IsNoWait=false)
Generator for '#omp reduction'.
bool updateToLocation(const LocationDescription &Loc)
Update the internal location to Loc.
void createFlush(const LocationDescription &Loc)
Generator for '#omp flush'.
Constant * getAddrOfDeclareTargetVar(OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind CaptureClause, OffloadEntriesInfoManager::OMPTargetDeviceClauseKind DeviceClause, bool IsDeclaration, bool IsExternallyVisible, TargetRegionEntryInfo EntryInfo, StringRef MangledName, std::vector< GlobalVariable * > &GeneratedRefs, bool OpenMPSIMD, std::vector< Triple > TargetTriple, Type *LlvmPtrTy, std::function< Constant *()> GlobalInitializer, std::function< GlobalValue::LinkageTypes()> VariableLinkage)
Retrieve (or create if non-existent) the address of a declare target variable, used in conjunction wi...
InsertPointTy emitTargetTask(Function *OutlinedFn, Value *OutlinedFnID, EmitFallbackCallbackTy EmitTargetCallFallbackCB, TargetKernelArgs &Args, Value *DeviceID, Value *RTLoc, InsertPointTy AllocaIP, SmallVector< OpenMPIRBuilder::DependData > &Dependencies, bool HasNoWait)
Generate a target-task for the target construct.
EmitMetadataErrorKind
The kind of errors that can occur when emitting the offload entries and metadata.
@ EMIT_MD_DECLARE_TARGET_ERROR
@ EMIT_MD_GLOBAL_VAR_LINK_ERROR
@ EMIT_MD_TARGET_REGION_ERROR
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Pseudo-analysis pass that exposes the PassInstrumentation to pass managers.
Class to represent pointers.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Analysis pass that exposes the ScalarEvolution for a function.
ScalarEvolution run(Function &F, FunctionAnalysisManager &AM)
The main scalar evolution driver.
A vector that has set insertion semantics.
bool empty() const
Determine if the SetVector is empty or not.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool test(unsigned Idx) const
bool all() const
Returns true if all bits are set.
bool any() const
Returns true if any bit is set.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
void append(StringRef RHS)
Append from a StringRef.
StringRef str() const
Explicit conversion to StringRef.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
void setAlignment(Align Align)
void setAtomic(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Sets the ordering constraint and the synchronization scope ID of this store instruction.
StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...
ValueTy lookup(StringRef Key) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
StringRef - Represent a constant reference to a string, i.e.
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
constexpr bool empty() const
empty - Check if the string is empty.
constexpr size_t size() const
size - Get the string size.
size_t count(char C) const
Return the number of occurrences of C in the string.
bool ends_with(StringRef Suffix) const
Check if this string ends with the given Suffix.
StringRef drop_back(size_t N=1) const
Return a StringRef equal to 'this' but with the last N elements dropped.
Class to represent struct types.
static StructType * create(LLVMContext &Context, StringRef Name)
This creates an identified struct.
void addCase(ConstantInt *OnVal, BasicBlock *Dest)
Add an entry to the switch instruction.
Analysis pass providing the TargetTransformInfo.
Result run(const Function &F, FunctionAnalysisManager &)
Analysis pass providing the TargetLibraryInfo.
Target - Wrapper for Target specific information.
TargetMachine * createTargetMachine(StringRef TT, StringRef CPU, StringRef Features, const TargetOptions &Options, std::optional< Reloc::Model > RM, std::optional< CodeModel::Model > CM=std::nullopt, CodeGenOptLevel OL=CodeGenOptLevel::Default, bool JIT=false) const
createTargetMachine - Create a target specific machine implementation for the specified Triple.
Triple - Helper class for working with autoconf configuration names.
bool isPPC() const
Tests whether the target is PowerPC (32- or 64-bit LE or BE).
bool isX86() const
Tests whether the target is x86 (32- or 64-bit).
bool isWasm() const
Tests whether the target is wasm (32- and 64-bit).
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getIntegerBitWidth() const
Type * getStructElementType(unsigned N) const
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
bool isPointerTy() const
True if this is an instance of PointerType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
This function has undefined behavior.
Produce an estimate of the unrolled cost of the specified loop.
bool canUnroll() const
Whether it is legal to unroll this loop.
uint64_t getRolledLoopSize() const
A Use represents the edge between a Value definition and its users.
void setOperand(unsigned i, Value *Val)
Value * getOperand(unsigned i) const
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
user_iterator user_begin()
void setName(const Twine &Name)
Change the name of the value.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
iterator_range< user_iterator > users()
Align getPointerAlignment(const DataLayout &DL) const
Returns an alignment of the pointer value.
void replaceUsesWithIf(Value *New, llvm::function_ref< bool(Use &U)> ShouldReplace)
Go through the uses list for this definition and make each use point to "V" if the callback ShouldRep...
User * getUniqueUndroppableUser()
Return true if there is exactly one unique user of this value that cannot be dropped (that user can h...
unsigned getNumUses() const
This method computes the number of uses of this Value.
iterator_range< use_iterator > uses()
StringRef getName() const
Return a constant reference to the value's name.
An efficient, type-erasing, non-owning reference to a callable.
const ParentTy * getParent() const
self_iterator getIterator()
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
iterator insertAfter(iterator where, pointer New)
A raw_ostream that writes to an SmallVector or SmallString.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AMDGPU_KERNEL
Used for AMDGPU code object kernels.
@ C
The default llvm calling convention, compatible with C.
initializer< Ty > init(const Ty &Val)
void emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, uint64_t Size, int32_t Flags, int32_t Data, StringRef SectionName)
Create an offloading section struct used to register this global at runtime.
OpenMPOffloadMappingFlags
Values for bit flags used to specify the mapping type for offloading.
@ OMP_MAP_PTR_AND_OBJ
The element being mapped is a pointer-pointee pair; both the pointer and the pointee should be mapped...
@ OMP_MAP_MEMBER_OF
The 16 MSBs of the flags indicate whether the entry is member of some struct/class.
@ OMP_DEVICEID_UNDEF
Device ID if the device was not defined, runtime should get it from environment variables in the spec...
IdentFlag
IDs for all omp runtime library ident_t flag encodings (see their defintion in openmp/runtime/src/kmp...
RuntimeFunction
IDs for all omp runtime library (RTL) functions.
static constexpr GV NVPTXGridValues
For Nvidia GPUs.
@ OMP_TGT_EXEC_MODE_GENERIC
WorksharingLoopType
A type of worksharing loop construct.
OMPAtomicCompareOp
Atomic compare operations. Currently OpenMP only supports ==, >, and <.
NodeAddr< PhiNode * > Phi
std::error_code getUniqueID(const Twine Path, UniqueID &Result)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
BasicBlock * splitBBWithSuffix(IRBuilderBase &Builder, bool CreateBranch, llvm::Twine Suffix=".split")
Like splitBB, but reuses the current block's name for the new name.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Expected< std::unique_ptr< Module > > parseBitcodeFile(MemoryBufferRef Buffer, LLVMContext &Context, ParserCallbacks Callbacks={})
Read the specified bitcode file, returning the module.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
auto successors(const MachineBasicBlock *BB)
testing::Matcher< const detail::ErrorHolder & > Failed()
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
ErrorOr< T > expectedToErrorOrAndEmitErrors(LLVMContext &Ctx, Expected< T > Val)
bool convertUsersOfConstantsToInstructions(ArrayRef< Constant * > Consts, Function *RestrictToFunc=nullptr, bool RemoveDeadConstants=true, bool IncludeSelf=false)
Replace constant expressions users of the given constants with instructions.
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
auto reverse(ContainerTy &&C)
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr, DebugInfoFinder *DIFinder=nullptr)
Return a copy of the specified basic block, but without embedding the block into a particular functio...
TargetTransformInfo::PeelingPreferences gatherPeelingPreferences(Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, std::optional< bool > UserAllowPeeling, std::optional< bool > UserAllowProfileBasedPeeling, bool UnrollingSpecficValues=false)
void spliceBB(IRBuilderBase::InsertPoint IP, BasicBlock *New, bool CreateBranch)
Move the instruction after an InsertPoint to the beginning of another BasicBlock.
void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
BasicBlock * splitBB(IRBuilderBase::InsertPoint IP, bool CreateBranch, llvm::Twine Name={})
Split a BasicBlock at an InsertPoint, even if the block is degenerate (missing the terminator).
CodeGenOptLevel
Code generation optimization level.
bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI, AssumptionCache *AC, ScalarEvolution &SE, const SmallPtrSetImpl< const Value * > &EphValues, OptimizationRemarkEmitter *ORE, unsigned TripCount, unsigned MaxTripCount, bool MaxOrZero, unsigned TripMultiple, const UnrollCostEstimator &UCE, TargetTransformInfo::UnrollingPreferences &UP, TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound)
format_object< Ts... > format(const char *Fmt, const Ts &... Vals)
These are helper functions used to produce formatted output.
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
AtomicOrdering
Atomic ordering for LLVM's memory model.
constexpr T divideCeil(U Numerator, V Denominator)
Returns the integer ceil(Numerator / Denominator).
bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
DWARFExpression::Operation Op
void remapInstructionsInBlocks(ArrayRef< BasicBlock * > Blocks, ValueToValueMapTy &VMap)
Remaps instructions in Blocks using the mapping in VMap.
TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, llvm::OptimizationRemarkEmitter &ORE, int OptLevel, std::optional< unsigned > UserThreshold, std::optional< unsigned > UserCount, std::optional< bool > UserAllowPartial, std::optional< bool > UserRuntime, std::optional< bool > UserUpperBound, std::optional< unsigned > UserFullUnrollMaxCount)
Gather the various unrolling parameters based on the defaults, compiler flags, TTI overrides and user...
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
Constant * ConstantFoldInsertValueInstruction(Constant *Agg, Constant *Val, ArrayRef< unsigned > Idxs)
ConstantFoldInsertValueInstruction - Attempt to constant fold an insertvalue instruction with the spe...
void DeleteDeadBlocks(ArrayRef< BasicBlock * > BBs, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified blocks from BB.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static void collectEphemeralValues(const Loop *L, AssumptionCache *AC, SmallPtrSetImpl< const Value * > &EphValues)
Collect a loop's ephemeral values (those used only by an assume or similar intrinsics in the loop).
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
a struct to pack relevant information while generating atomic Ops
A struct to pack the relevant information for an OpenMP depend clause.
Description of a LLVM-IR insertion point (IP) and a debug/source location (filename,...
MapNonContiguousArrayTy Offsets
MapNonContiguousArrayTy Counts
MapNonContiguousArrayTy Strides
This structure contains combined information generated for mappable clauses, including base pointers,...
MapDeviceInfoArrayTy DevicePointers
MapValuesArrayTy BasePointers
MapValuesArrayTy Pointers
StructNonContiguousInfo NonContigInfo
Helper that contains information about regions we need to outline during finalization.
PostOutlineCBTy PostOutlineCB
void collectBlocks(SmallPtrSetImpl< BasicBlock * > &BlockSet, SmallVectorImpl< BasicBlock * > &BlockVector)
Collect all blocks in between EntryBB and ExitBB in both the given vector and set.
SmallVector< Value *, 2 > ExcludeArgsFromAggregate
BasicBlock * OuterAllocaBB
Information about an OpenMP reduction.
EvalKind EvaluationKind
Reduction evaluation kind - scalar, complex or aggregate.
ReductionGenAtomicCBTy AtomicReductionGen
Callback for generating the atomic reduction body, may be null.
ReductionGenCBTy ReductionGen
Callback for generating the reduction body.
Value * Variable
Reduction variable of pointer type.
Value * PrivateVariable
Thread-private partial reduction variable.
ReductionGenClangCBTy ReductionGenClang
Clang callback for generating the reduction body.
Type * ElementType
Reduction element type, must match pointee type of variable.
Container for the arguments used to pass data to the runtime library.
Value * SizesArray
The array of sizes passed to the runtime library.
Value * PointersArray
The array of section pointers passed to the runtime library.
Value * MappersArray
The array of user-defined mappers passed to the runtime library.
Value * BasePointersArray
The array of base pointer passed to the runtime library.
Value * MapTypesArray
The array of map types passed to the runtime library for the beginning of the region or for the entir...
Value * MapNamesArray
The array of original declaration names of mapped pointers sent to the runtime library for debugging.
Data structure that contains the needed information to construct the kernel args vector.
unsigned NumTargetItems
Number of arguments passed to the runtime library.
bool HasNoWait
True if the kernel has 'no wait' clause.
ArrayRef< Value * > NumTeams
The number of teams.
Data structure to contain the information needed to uniquely identify a target entry.
static void getTargetRegionEntryFnName(SmallVectorImpl< char > &Name, StringRef ParentName, unsigned DeviceID, unsigned FileID, unsigned Line, unsigned Count)
static const Target * lookupTarget(StringRef Triple, std::string &Error)
lookupTarget - Lookup a target based on a target triple.
Defines various target-specific GPU grid values that must be consistent between host RTL (plugin),...
unsigned GV_Warp_Size
The default value of maximum number of threads in a worker warp.