20 #include "llvm/ADT/ArrayRef.h"
21 #include "llvm/Bitcode/ReaderWriter.h"
22 #include "llvm/IR/CallSite.h"
23 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/IR/GlobalValue.h"
25 #include "llvm/IR/Value.h"
26 #include "llvm/Support/Format.h"
27 #include "llvm/Support/raw_ostream.h"
30 using namespace clang;
31 using namespace CodeGen;
38 enum CGOpenMPRegionKind {
41 ParallelOutlinedRegion,
52 const CGOpenMPRegionKind RegionKind,
55 : CGCapturedStmtInfo(CS,
CR_OpenMP), RegionKind(RegionKind),
56 CodeGen(CodeGen), Kind(Kind), HasCancel(HasCancel) {}
58 CGOpenMPRegionInfo(
const CGOpenMPRegionKind RegionKind,
61 : CGCapturedStmtInfo(
CR_OpenMP), RegionKind(RegionKind), CodeGen(CodeGen),
62 Kind(Kind), HasCancel(HasCancel) {}
66 virtual const VarDecl *getThreadIDVariable()
const = 0;
77 CGOpenMPRegionKind getRegionKind()
const {
return RegionKind; }
81 bool hasCancel()
const {
return HasCancel; }
83 static bool classof(
const CGCapturedStmtInfo *Info) {
87 ~CGOpenMPRegionInfo()
override =
default;
90 CGOpenMPRegionKind RegionKind;
97 class CGOpenMPOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
102 : CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen, Kind,
104 ThreadIDVar(ThreadIDVar) {
105 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
110 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
113 StringRef getHelperName()
const override {
return ".omp_outlined."; }
115 static bool classof(
const CGCapturedStmtInfo *Info) {
117 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
118 ParallelOutlinedRegion;
128 class CGOpenMPTaskOutlinedRegionInfo final :
public CGOpenMPRegionInfo {
134 llvm::SwitchInst *UntiedSwitch =
nullptr;
137 UntiedTaskActionTy(
bool Tied,
const VarDecl *PartIDVar,
139 : Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}
148 UntiedSwitch = CGF.
Builder.CreateSwitch(Res, DoneBB);
152 UntiedSwitch->addCase(CGF.
Builder.getInt32(0),
154 emitUntiedSwitch(CGF);
169 UntiedSwitch->addCase(CGF.
Builder.getInt32(UntiedSwitch->getNumCases()),
175 unsigned getNumberOfParts()
const {
return UntiedSwitch->getNumCases(); }
181 const UntiedTaskActionTy &
Action)
182 : CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen, Kind, HasCancel),
183 ThreadIDVar(ThreadIDVar), Action(Action) {
184 assert(ThreadIDVar !=
nullptr &&
"No ThreadID in OpenMP region.");
189 const VarDecl *getThreadIDVariable()
const override {
return ThreadIDVar; }
195 StringRef getHelperName()
const override {
return ".omp_outlined."; }
198 Action.emitUntiedSwitch(CGF);
201 static bool classof(
const CGCapturedStmtInfo *Info) {
203 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==
212 const UntiedTaskActionTy &
Action;
217 class CGOpenMPInlinedRegionInfo :
public CGOpenMPRegionInfo {
222 : CGOpenMPRegionInfo(InlinedRegion, CodeGen, Kind, HasCancel),
224 OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}
229 return OuterRegionInfo->getContextValue();
230 llvm_unreachable(
"No context value for inlined OpenMP region");
234 if (OuterRegionInfo) {
235 OuterRegionInfo->setContextValue(V);
238 llvm_unreachable(
"No context value for inlined OpenMP region");
244 return OuterRegionInfo->lookup(VD);
250 FieldDecl *getThisFieldDecl()
const override {
252 return OuterRegionInfo->getThisFieldDecl();
258 const VarDecl *getThreadIDVariable()
const override {
260 return OuterRegionInfo->getThreadIDVariable();
265 StringRef getHelperName()
const override {
266 if (
auto *OuterRegionInfo = getOldCSI())
267 return OuterRegionInfo->getHelperName();
268 llvm_unreachable(
"No helper name for inlined OpenMP construct");
273 OuterRegionInfo->emitUntiedSwitch(CGF);
278 static bool classof(
const CGCapturedStmtInfo *Info) {
280 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;
283 ~CGOpenMPInlinedRegionInfo()
override =
default;
288 CGOpenMPRegionInfo *OuterRegionInfo;
296 class CGOpenMPTargetRegionInfo final :
public CGOpenMPRegionInfo {
300 : CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,
302 HelperName(HelperName) {}
306 const VarDecl *getThreadIDVariable()
const override {
return nullptr; }
309 StringRef getHelperName()
const override {
return HelperName; }
311 static bool classof(
const CGCapturedStmtInfo *Info) {
313 cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;
317 StringRef HelperName;
321 llvm_unreachable(
"No codegen for expressions");
325 class CGOpenMPInnerExprInfo final :
public CGOpenMPInlinedRegionInfo {
328 : CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,
336 if (!
C.capturesVariable() && !
C.capturesVariableByCopy())
339 const VarDecl *VD =
C.getCapturedVar();
347 PrivScope.addPrivate(VD, [&CGF, &DRE]() ->
Address {
351 (void)PrivScope.Privatize();
356 if (
auto *FD = CGOpenMPInlinedRegionInfo::lookup(VD))
363 llvm_unreachable(
"No body for expressions");
368 const VarDecl *getThreadIDVariable()
const override {
369 llvm_unreachable(
"No thread id for expressions");
373 StringRef getHelperName()
const override {
374 llvm_unreachable(
"No helper name for expressions");
377 static bool classof(
const CGCapturedStmtInfo *Info) {
return false; }
385 class InlinedOpenMPRegionRAII {
387 llvm::DenseMap<const VarDecl *, FieldDecl *> LambdaCaptureFields;
388 FieldDecl *LambdaThisCaptureField =
nullptr;
406 ~InlinedOpenMPRegionRAII() {
422 OMP_IDENT_IMD = 0x01,
424 OMP_IDENT_KMPC = 0x02,
426 OMP_ATOMIC_REDUCE = 0x10,
428 OMP_IDENT_BARRIER_EXPL = 0x20,
430 OMP_IDENT_BARRIER_IMPL = 0x40,
432 OMP_IDENT_BARRIER_IMPL_FOR = 0x40,
434 OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,
436 OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140
690 Callback(CodeGen, CGF, *PrePostAction);
693 Callback(CodeGen, CGF, Action);
716 LValue CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(
719 getThreadIDVariable()->
getType(),
729 KmpCriticalNameTy = llvm::ArrayType::get(CGM.
Int32Ty, 8);
735 InternalVars.clear();
738 static llvm::Function *
740 const Expr *CombinerInitializer,
const VarDecl *In,
741 const VarDecl *Out,
bool IsCombiner) {
744 QualType PtrTy = C.getPointerType(Ty).withRestrict();
750 Args.push_back(&OmpOutParm);
751 Args.push_back(&OmpInParm);
757 IsCombiner ?
".omp_combiner." :
".omp_initializer.", &CGM.
getModule());
759 Fn->addFnAttr(llvm::Attribute::AlwaysInline);
784 if (UDRMap.count(D) > 0)
786 auto &C = CGM.getContext();
788 In = &C.Idents.get(
"omp_in");
789 Out = &C.Idents.get(
"omp_out");
795 llvm::Function *Initializer =
nullptr;
797 if (!Priv || !Orig) {
798 Priv = &C.Idents.get(
"omp_priv");
799 Orig = &C.Idents.get(
"omp_orig");
806 UDRMap.insert(std::make_pair(D, std::make_pair(Combiner, Initializer)));
808 auto &Decls = FunctionUDRMap.FindAndConstruct(CGF->
CurFn);
809 Decls.second.push_back(D);
813 std::pair<llvm::Function *, llvm::Function *>
815 auto I = UDRMap.find(D);
816 if (
I != UDRMap.end())
819 return UDRMap.lookup(D);
836 const llvm::Twine &
Name =
"") {
845 "thread id variable must be of type kmp_int32 *");
848 bool HasCancel =
false;
849 if (
auto *OPD = dyn_cast<OMPParallelDirective>(&D))
850 HasCancel = OPD->hasCancel();
851 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&D))
852 HasCancel = OPSD->hasCancel();
853 else if (
auto *OPFD = dyn_cast<OMPParallelForDirective>(&D))
854 HasCancel = OPFD->hasCancel();
855 CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,
865 bool Tied,
unsigned &NumberOfParts) {
872 CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),
877 CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy
Action(Tied, PartIDVar,
881 "thread id variable must be of type kmp_int32 for tasks");
885 CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,
887 TD ? TD->hasCancel() :
false,
Action);
889 auto *Res = CGF.GenerateCapturedStmtFunction(*CS);
891 NumberOfParts = Action.getNumberOfParts();
895 Address CGOpenMPRuntime::getOrCreateDefaultLocation(
unsigned Flags) {
897 llvm::Value *Entry = OpenMPDefaultLocMap.lookup(Flags);
899 if (!DefaultOpenMPPSource) {
904 DefaultOpenMPPSource =
905 CGM.GetAddrOfConstantCString(
";unknown;unknown;0;0;;").getPointer();
906 DefaultOpenMPPSource =
907 llvm::ConstantExpr::getBitCast(DefaultOpenMPPSource, CGM.Int8PtrTy);
909 auto DefaultOpenMPLocation =
new llvm::GlobalVariable(
910 CGM.getModule(), IdentTy,
true,
911 llvm::GlobalValue::PrivateLinkage,
nullptr);
912 DefaultOpenMPLocation->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
913 DefaultOpenMPLocation->setAlignment(Align.
getQuantity());
915 llvm::Constant *Zero = llvm::ConstantInt::get(CGM.Int32Ty, 0,
true);
916 llvm::Constant *Values[] = {Zero,
917 llvm::ConstantInt::get(CGM.Int32Ty, Flags),
918 Zero, Zero, DefaultOpenMPPSource};
919 llvm::Constant *Init = llvm::ConstantStruct::get(IdentTy, Values);
920 DefaultOpenMPLocation->setInitializer(Init);
921 OpenMPDefaultLocMap[Flags] = Entry = DefaultOpenMPLocation;
929 Flags |= OMP_IDENT_KMPC;
933 return getOrCreateDefaultLocation(Flags).getPointer();
935 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
937 Address LocValue = Address::invalid();
938 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
939 if (
I != OpenMPLocThreadIDMap.end())
948 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
952 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
961 auto OMPDebugLoc = OpenMPDebugLocMap.lookup(Loc.
getRawEncoding());
962 if (OMPDebugLoc ==
nullptr) {
964 llvm::raw_svector_ostream OS2(Buffer2);
970 OS2 << FD->getQualifiedNameAsString();
973 OMPDebugLoc = CGF.
Builder.CreateGlobalStringPtr(OS2.str());
986 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
991 auto I = OpenMPLocThreadIDMap.find(CGF.
CurFn);
992 if (
I != OpenMPLocThreadIDMap.end()) {
993 ThreadID =
I->second.ThreadID;
994 if (ThreadID !=
nullptr)
997 if (
auto *OMPRegionInfo =
999 if (OMPRegionInfo->getThreadIDVariable()) {
1001 auto LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);
1006 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1007 Elem.second.ThreadID = ThreadID;
1017 CGBuilderTy::InsertPointGuard IPG(CGF.
Builder);
1022 auto &Elem = OpenMPLocThreadIDMap.FindAndConstruct(CGF.
CurFn);
1023 Elem.second.ThreadID = ThreadID;
1028 assert(CGF.
CurFn &&
"No function in current CodeGenFunction.");
1029 if (OpenMPLocThreadIDMap.count(CGF.
CurFn))
1030 OpenMPLocThreadIDMap.erase(CGF.
CurFn);
1031 if (FunctionUDRMap.count(CGF.
CurFn) > 0) {
1032 for(
auto *D : FunctionUDRMap[CGF.
CurFn]) {
1035 FunctionUDRMap.erase(CGF.
CurFn);
1042 return llvm::PointerType::getUnqual(IdentTy);
1046 if (!Kmpc_MicroTy) {
1048 llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(CGM.Int32Ty),
1049 llvm::PointerType::getUnqual(CGM.Int32Ty)};
1050 Kmpc_MicroTy = llvm::FunctionType::get(CGM.VoidTy, MicroParams,
true);
1052 return llvm::PointerType::getUnqual(Kmpc_MicroTy);
1057 llvm::Constant *RTLFn =
nullptr;
1058 switch (static_cast<OpenMPRTLFunction>(Function)) {
1064 llvm::FunctionType *FnTy =
1065 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
true);
1066 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_fork_call");
1072 llvm::FunctionType *FnTy =
1073 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1074 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_global_thread_num");
1081 CGM.VoidPtrTy, CGM.SizeTy,
1082 CGM.VoidPtrTy->getPointerTo()->getPointerTo()};
1083 llvm::FunctionType *FnTy =
1084 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams,
false);
1085 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_threadprivate_cached");
1091 llvm::Type *TypeParams[] = {
1093 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1094 llvm::FunctionType *FnTy =
1095 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1096 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_critical");
1103 llvm::PointerType::getUnqual(KmpCriticalNameTy),
1105 llvm::FunctionType *FnTy =
1106 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1107 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_critical_with_hint");
1115 llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1116 false)->getPointerTo();
1118 llvm::Type *KmpcCopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1119 auto KmpcCopyCtorTy =
1120 llvm::FunctionType::get(CGM.VoidPtrTy, KmpcCopyCtorTyArgs,
1121 false)->getPointerTo();
1124 llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
false)
1127 KmpcCopyCtorTy, KmpcDtorTy};
1128 auto FnTy = llvm::FunctionType::get(CGM.VoidTy, FnTyArgs,
1130 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_threadprivate_register");
1136 llvm::Type *TypeParams[] = {
1138 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1139 llvm::FunctionType *FnTy =
1140 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1141 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_critical");
1148 llvm::FunctionType *FnTy =
1149 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1150 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_cancel_barrier");
1156 llvm::FunctionType *FnTy =
1157 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1158 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_barrier");
1164 llvm::FunctionType *FnTy =
1165 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1166 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_for_static_fini");
1174 llvm::FunctionType *FnTy =
1175 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1176 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_push_num_threads");
1183 llvm::FunctionType *FnTy =
1184 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1185 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_serialized_parallel");
1192 llvm::FunctionType *FnTy =
1193 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1194 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_serialized_parallel");
1200 llvm::FunctionType *FnTy =
1201 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1202 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_flush");
1208 llvm::FunctionType *FnTy =
1209 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1210 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_master");
1216 llvm::FunctionType *FnTy =
1217 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1218 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_master");
1225 llvm::FunctionType *FnTy =
1226 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1227 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_taskyield");
1233 llvm::FunctionType *FnTy =
1234 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1235 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_single");
1241 llvm::FunctionType *FnTy =
1242 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1243 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_single");
1250 assert(KmpRoutineEntryPtrTy !=
nullptr &&
1251 "Type kmp_routine_entry_t must be created.");
1253 CGM.SizeTy, CGM.SizeTy, KmpRoutineEntryPtrTy};
1255 llvm::FunctionType *FnTy =
1256 llvm::FunctionType::get(CGM.VoidPtrTy, TypeParams,
false);
1257 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task_alloc");
1265 llvm::FunctionType *FnTy =
1266 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1267 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task");
1274 llvm::Type *CpyTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1276 llvm::FunctionType::get(CGM.VoidTy, CpyTypeParams,
false);
1278 CGM.VoidPtrTy, CpyFnTy->getPointerTo(),
1280 llvm::FunctionType *FnTy =
1281 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1282 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_copyprivate");
1289 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1290 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1292 llvm::Type *TypeParams[] = {
1294 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1295 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1296 llvm::FunctionType *FnTy =
1297 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1298 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_reduce");
1306 llvm::Type *ReduceTypeParams[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1307 auto *ReduceFnTy = llvm::FunctionType::get(CGM.VoidTy, ReduceTypeParams,
1309 llvm::Type *TypeParams[] = {
1311 CGM.VoidPtrTy, ReduceFnTy->getPointerTo(),
1312 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1313 llvm::FunctionType *FnTy =
1314 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1315 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_reduce_nowait");
1321 llvm::Type *TypeParams[] = {
1323 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1324 llvm::FunctionType *FnTy =
1325 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1326 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_reduce");
1332 llvm::Type *TypeParams[] = {
1334 llvm::PointerType::getUnqual(KmpCriticalNameTy)};
1335 llvm::FunctionType *FnTy =
1336 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1338 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_reduce_nowait");
1346 llvm::FunctionType *FnTy =
1347 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1349 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task_begin_if0");
1357 llvm::FunctionType *FnTy =
1358 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1359 RTLFn = CGM.CreateRuntimeFunction(FnTy,
1360 "__kmpc_omp_task_complete_if0");
1366 llvm::FunctionType *FnTy =
1367 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1368 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_ordered");
1374 llvm::FunctionType *FnTy =
1375 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1376 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_ordered");
1382 llvm::FunctionType *FnTy =
1383 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1384 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_taskwait");
1390 llvm::FunctionType *FnTy =
1391 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1392 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_taskgroup");
1398 llvm::FunctionType *FnTy =
1399 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1400 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_end_taskgroup");
1407 llvm::FunctionType *FnTy =
1408 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1409 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_push_proc_bind");
1416 llvm::Type *TypeParams[] = {
1418 CGM.VoidPtrTy, CGM.Int32Ty, CGM.VoidPtrTy};
1419 llvm::FunctionType *FnTy =
1420 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1422 CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_task_with_deps");
1430 CGM.Int32Ty, CGM.VoidPtrTy,
1431 CGM.Int32Ty, CGM.VoidPtrTy};
1432 llvm::FunctionType *FnTy =
1433 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1434 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_omp_wait_deps");
1441 llvm::FunctionType *FnTy =
1442 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1443 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_cancellationpoint");
1450 llvm::FunctionType *FnTy =
1451 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1452 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_cancel");
1460 llvm::FunctionType *FnTy =
1461 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1462 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_push_num_teams");
1470 llvm::FunctionType *FnTy =
1471 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
true);
1472 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_fork_teams");
1483 CGM.Int64Ty->getPointerTo(),
1484 CGM.Int64Ty->getPointerTo(),
1490 llvm::FunctionType *FnTy =
1491 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1492 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_taskloop");
1502 llvm::FunctionType *FnTy =
1503 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1504 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_init");
1510 llvm::FunctionType *FnTy =
1511 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1512 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_fini");
1519 CGM.Int64Ty->getPointerTo()};
1520 llvm::FunctionType *FnTy =
1521 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1522 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_post");
1529 CGM.Int64Ty->getPointerTo()};
1530 llvm::FunctionType *FnTy =
1531 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1532 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__kmpc_doacross_wait");
1539 llvm::Type *TypeParams[] = {CGM.Int32Ty,
1544 CGM.SizeTy->getPointerTo(),
1545 CGM.Int32Ty->getPointerTo()};
1546 llvm::FunctionType *FnTy =
1547 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1548 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target");
1555 llvm::Type *TypeParams[] = {CGM.Int32Ty,
1560 CGM.SizeTy->getPointerTo(),
1561 CGM.Int32Ty->getPointerTo(),
1564 llvm::FunctionType *FnTy =
1565 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1566 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_teams");
1573 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
1574 llvm::FunctionType *FnTy =
1575 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1576 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_register_lib");
1583 llvm::Type *TypeParams[] = {CGM.getTypes().ConvertTypeForMem(ParamTy)};
1584 llvm::FunctionType *FnTy =
1585 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1586 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_unregister_lib");
1592 llvm::Type *TypeParams[] = {CGM.Int32Ty,
1596 CGM.SizeTy->getPointerTo(),
1597 CGM.Int32Ty->getPointerTo()};
1598 llvm::FunctionType *FnTy =
1599 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1600 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_data_begin");
1606 llvm::Type *TypeParams[] = {CGM.Int32Ty,
1610 CGM.SizeTy->getPointerTo(),
1611 CGM.Int32Ty->getPointerTo()};
1612 llvm::FunctionType *FnTy =
1613 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1614 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_data_end");
1620 llvm::Type *TypeParams[] = {CGM.Int32Ty,
1624 CGM.SizeTy->getPointerTo(),
1625 CGM.Int32Ty->getPointerTo()};
1626 llvm::FunctionType *FnTy =
1627 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1628 RTLFn = CGM.CreateRuntimeFunction(FnTy,
"__tgt_target_data_update");
1632 assert(RTLFn &&
"Unable to find OpenMP runtime function");
1638 assert((IVSize == 32 || IVSize == 64) &&
1639 "IV size is not compatible with the omp runtime");
1640 auto Name = IVSize == 32 ? (IVSigned ?
"__kmpc_for_static_init_4"
1641 :
"__kmpc_for_static_init_4u")
1642 : (IVSigned ?
"__kmpc_for_static_init_8"
1643 :
"__kmpc_for_static_init_8u");
1644 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1645 auto PtrTy = llvm::PointerType::getUnqual(ITy);
1646 llvm::Type *TypeParams[] = {
1650 llvm::PointerType::getUnqual(CGM.Int32Ty),
1657 llvm::FunctionType *FnTy =
1658 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1659 return CGM.CreateRuntimeFunction(FnTy,
Name);
1664 assert((IVSize == 32 || IVSize == 64) &&
1665 "IV size is not compatible with the omp runtime");
1668 ? (IVSigned ?
"__kmpc_dispatch_init_4" :
"__kmpc_dispatch_init_4u")
1669 : (IVSigned ?
"__kmpc_dispatch_init_8" :
"__kmpc_dispatch_init_8u");
1670 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1679 llvm::FunctionType *FnTy =
1680 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1681 return CGM.CreateRuntimeFunction(FnTy,
Name);
1686 assert((IVSize == 32 || IVSize == 64) &&
1687 "IV size is not compatible with the omp runtime");
1690 ? (IVSigned ?
"__kmpc_dispatch_fini_4" :
"__kmpc_dispatch_fini_4u")
1691 : (IVSigned ?
"__kmpc_dispatch_fini_8" :
"__kmpc_dispatch_fini_8u");
1692 llvm::Type *TypeParams[] = {
1696 llvm::FunctionType *FnTy =
1697 llvm::FunctionType::get(CGM.VoidTy, TypeParams,
false);
1698 return CGM.CreateRuntimeFunction(FnTy,
Name);
1703 assert((IVSize == 32 || IVSize == 64) &&
1704 "IV size is not compatible with the omp runtime");
1707 ? (IVSigned ?
"__kmpc_dispatch_next_4" :
"__kmpc_dispatch_next_4u")
1708 : (IVSigned ?
"__kmpc_dispatch_next_8" :
"__kmpc_dispatch_next_8u");
1709 auto ITy = IVSize == 32 ? CGM.Int32Ty : CGM.Int64Ty;
1710 auto PtrTy = llvm::PointerType::getUnqual(ITy);
1711 llvm::Type *TypeParams[] = {
1714 llvm::PointerType::getUnqual(CGM.Int32Ty),
1719 llvm::FunctionType *FnTy =
1720 llvm::FunctionType::get(CGM.Int32Ty, TypeParams,
false);
1721 return CGM.CreateRuntimeFunction(FnTy,
Name);
1726 assert(!CGM.getLangOpts().OpenMPUseTLS ||
1727 !CGM.getContext().getTargetInfo().isTLSSupported());
1730 Twine(CGM.getMangledName(VD)) +
".cache.");
1737 if (CGM.getLangOpts().OpenMPUseTLS &&
1738 CGM.getContext().getTargetInfo().isTLSSupported())
1745 CGM.
getSize(CGM.GetTargetTypeStoreSize(VarTy)),
1765 Ctor, CopyCtor, Dtor};
1773 if (CGM.getLangOpts().OpenMPUseTLS &&
1774 CGM.getContext().getTargetInfo().isTLSSupported())
1782 llvm::Value *Ctor =
nullptr, *CopyCtor =
nullptr, *Dtor =
nullptr;
1784 if (CGM.getLangOpts().CPlusPlus && PerformInit) {
1790 nullptr, CGM.getContext().VoidPtrTy);
1791 Args.push_back(&Dst);
1793 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1794 CGM.getContext().VoidPtrTy, Args);
1795 auto FTy = CGM.getTypes().GetFunctionType(FI);
1796 auto Fn = CGM.CreateGlobalInitOrDestructFunction(
1797 FTy,
".__kmpc_global_ctor_.", FI, Loc);
1802 CGM.getContext().VoidPtrTy, Dst.getLocation());
1810 CGM.getContext().VoidPtrTy, Dst.getLocation());
1821 nullptr, CGM.getContext().VoidPtrTy);
1822 Args.push_back(&Dst);
1824 auto &FI = CGM.getTypes().arrangeBuiltinFunctionDeclaration(
1825 CGM.getContext().VoidTy, Args);
1826 auto FTy = CGM.getTypes().GetFunctionType(FI);
1827 auto Fn = CGM.CreateGlobalInitOrDestructFunction(
1828 FTy,
".__kmpc_global_dtor_.", FI, Loc);
1833 auto AL = ApplyDebugLocation::CreateArtificial(DtorCGF);
1836 false, CGM.getContext().VoidPtrTy, Dst.getLocation());
1847 llvm::Type *CopyCtorTyArgs[] = {CGM.VoidPtrTy, CGM.VoidPtrTy};
1849 llvm::FunctionType::get(CGM.VoidPtrTy, CopyCtorTyArgs,
1850 false)->getPointerTo();
1854 CopyCtor = llvm::Constant::getNullValue(CopyCtorTy);
1855 if (Ctor ==
nullptr) {
1856 auto CtorTy = llvm::FunctionType::get(CGM.VoidPtrTy, CGM.VoidPtrTy,
1857 false)->getPointerTo();
1858 Ctor = llvm::Constant::getNullValue(CtorTy);
1860 if (Dtor ==
nullptr) {
1861 auto DtorTy = llvm::FunctionType::get(CGM.VoidTy, CGM.VoidPtrTy,
1862 false)->getPointerTo();
1863 Dtor = llvm::Constant::getNullValue(DtorTy);
1866 auto InitFunctionTy =
1867 llvm::FunctionType::get(CGM.VoidTy,
false);
1868 auto InitFunction = CGM.CreateGlobalInitOrDestructFunction(
1869 InitFunctionTy,
".__omp_threadprivate_init_.",
1870 CGM.getTypes().arrangeNullaryFunction());
1874 CGM.getTypes().arrangeNullaryFunction(), ArgList,
1878 return InitFunction;
1933 ArrayRef<llvm::Value *> CapturedVars,
1934 const Expr *IfCond) {
1938 auto &&ThenGen = [OutlinedFn, CapturedVars, RTLoc](
CodeGenFunction &CGF,
1941 auto &RT = CGF.CGM.getOpenMPRuntime();
1944 CGF.Builder.getInt32(CapturedVars.size()),
1945 CGF.Builder.CreateBitCast(OutlinedFn, RT.getKmpc_MicroPointerTy())};
1948 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
1951 CGF.EmitRuntimeCall(RTLFn, RealArgs);
1953 auto &&ElseGen = [OutlinedFn, CapturedVars, RTLoc, Loc](
CodeGenFunction &CGF,
1955 auto &RT = CGF.CGM.getOpenMPRuntime();
1956 auto ThreadID = RT.getThreadID(CGF, Loc);
1960 CGF.EmitRuntimeCall(
1964 auto ThreadIDAddr = RT.emitThreadIDAddress(CGF, Loc);
1968 CGF.InitTempAlloca(ZeroAddr, CGF.Builder.getInt32( 0));
1970 OutlinedFnArgs.push_back(ThreadIDAddr.getPointer());
1971 OutlinedFnArgs.push_back(ZeroAddr.
getPointer());
1972 OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());
1973 CGF.EmitCallOrInvoke(OutlinedFn, OutlinedFnArgs);
1976 llvm::Value *EndArgs[] = {RT.emitUpdateLocation(CGF, Loc), ThreadID};
1977 CGF.EmitRuntimeCall(
1997 if (
auto *OMPRegionInfo =
1999 if (OMPRegionInfo->getThreadIDVariable())
2000 return OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();
2005 auto ThreadIDTemp = CGF.
CreateMemTemp(Int32Ty,
".threadid_temp.");
2009 return ThreadIDTemp;
2014 const llvm::Twine &
Name) {
2016 llvm::raw_svector_ostream Out(Buffer);
2018 auto RuntimeName = Out.str();
2019 auto &Elem = *InternalVars.insert(std::make_pair(RuntimeName,
nullptr)).first;
2021 assert(Elem.second->getType()->getPointerElementType() == Ty &&
2022 "OMP internal variable has different type than requested");
2023 return &*Elem.second;
2026 return Elem.second =
new llvm::GlobalVariable(
2027 CGM.getModule(), Ty,
false,
2028 llvm::GlobalValue::CommonLinkage, llvm::Constant::getNullValue(Ty),
2033 llvm::Twine
Name(
".gomp_critical_user_", CriticalName);
2045 llvm::BasicBlock *ContBlock =
nullptr;
2051 : EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),
2060 CGF.
Builder.CreateCondBr(CallBool, ThenBlock, ContBlock);
2076 StringRef CriticalName,
2090 EnterArgs.push_back(CGF.
Builder.CreateIntCast(
2127 llvm::ConstantInt::get(CGM.IntTy, 0,
true)};
2129 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
2130 Region->emitUntiedSwitch(CGF);
2153 unsigned Index,
const VarDecl *Var) {
2167 ArrayRef<const Expr *> CopyprivateVars, ArrayRef<const Expr *> DestExprs,
2168 ArrayRef<const Expr *> SrcExprs, ArrayRef<const Expr *> AssignmentOps) {
2176 Args.push_back(&LHSArg);
2177 Args.push_back(&RHSArg);
2181 ".omp.copyprivate.copy_func", &CGM.
getModule());
2197 for (
unsigned I = 0,
E = AssignmentOps.size();
I <
E; ++
I) {
2198 auto DestVar = cast<VarDecl>(cast<DeclRefExpr>(DestExprs[
I])->getDecl());
2201 auto SrcVar = cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[
I])->getDecl());
2204 auto *VD = cast<DeclRefExpr>(CopyprivateVars[
I])->getDecl();
2206 CGF.
EmitOMPCopy(Type, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[
I]);
2215 ArrayRef<const Expr *> CopyprivateVars,
2216 ArrayRef<const Expr *> SrcExprs,
2217 ArrayRef<const Expr *> DstExprs,
2218 ArrayRef<const Expr *> AssignmentOps) {
2221 assert(CopyprivateVars.size() == SrcExprs.size() &&
2222 CopyprivateVars.size() == DstExprs.size() &&
2223 CopyprivateVars.size() == AssignmentOps.size());
2224 auto &
C = CGM.getContext();
2234 Address DidIt = Address::invalid();
2235 if (!CopyprivateVars.empty()) {
2237 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
2238 DidIt = CGF.
CreateMemTemp(KmpInt32Ty,
".omp.copyprivate.did_it");
2256 llvm::APInt ArraySize(32, CopyprivateVars.size());
2257 auto CopyprivateArrayTy =
2262 CGF.
CreateMemTemp(CopyprivateArrayTy,
".omp.copyprivate.cpr_list");
2263 for (
unsigned I = 0,
E = CopyprivateVars.size();
I <
E; ++
I) {
2275 CopyprivateVars, SrcExprs, DstExprs, AssignmentOps);
2276 auto *BufSize = CGF.
getTypeSize(CopyprivateArrayTy);
2316 bool ForceSimpleCall) {
2322 if (Kind == OMPD_for)
2323 Flags = OMP_IDENT_BARRIER_IMPL_FOR;
2324 else if (Kind == OMPD_sections)
2325 Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;
2326 else if (Kind == OMPD_single)
2327 Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;
2328 else if (Kind == OMPD_barrier)
2329 Flags = OMP_IDENT_BARRIER_EXPL;
2331 Flags = OMP_IDENT_BARRIER_IMPL;
2336 if (
auto *OMPRegionInfo =
2338 if (!ForceSimpleCall && OMPRegionInfo->hasCancel()) {
2348 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
2351 auto CancelDestination =
2364 bool Chunked,
bool Ordered) {
2365 switch (ScheduleKind) {
2366 case OMPC_SCHEDULE_static:
2369 case OMPC_SCHEDULE_dynamic:
2371 case OMPC_SCHEDULE_guided:
2373 case OMPC_SCHEDULE_runtime:
2375 case OMPC_SCHEDULE_auto:
2378 assert(!Chunked &&
"chunk was specified but schedule kind not known");
2381 llvm_unreachable(
"Unexpected runtime schedule");
2392 bool Chunked)
const {
2416 case OMPC_SCHEDULE_MODIFIER_monotonic:
2419 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2422 case OMPC_SCHEDULE_MODIFIER_simd:
2431 case OMPC_SCHEDULE_MODIFIER_monotonic:
2434 case OMPC_SCHEDULE_MODIFIER_nonmonotonic:
2437 case OMPC_SCHEDULE_MODIFIER_simd:
2451 unsigned IVSize,
bool IVSigned,
2468 if (Chunk ==
nullptr)
2469 Chunk = CGF.
Builder.getIntN(IVSize, 1);
2473 Schedule, ScheduleKind.
M1, ScheduleKind.
M2)),
2474 CGF.
Builder.getIntN(IVSize, 0),
2476 CGF.
Builder.getIntN(IVSize, 1),
2503 if (Chunk ==
nullptr) {
2506 "expected static non-chunked schedule");
2508 Chunk = CGF.
Builder.getIntN(IVSize, 1);
2514 "expected static chunked schedule");
2523 CGF.
Builder.getIntN(IVSize, 1),
2532 unsigned IVSize,
bool IVSigned,
2542 ScheduleNum, ScheduleKind.
M1, ScheduleKind.
M2, IVSize,
2543 Ordered, IL, LB, UB, ST, Chunk);
2635 case OMPC_PROC_BIND_master:
2636 RuntimeProcBind = ProcBindMaster;
2638 case OMPC_PROC_BIND_close:
2639 RuntimeProcBind = ProcBindClose;
2641 case OMPC_PROC_BIND_spread:
2642 RuntimeProcBind = ProcBindSpread;
2645 llvm_unreachable(
"Unsupported proc_bind value.");
2650 llvm::ConstantInt::get(CGM.IntTy, RuntimeProcBind,
true)};
2687 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::empty()
const {
2689 return OffloadEntriesTargetRegion.empty();
2693 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2694 initializeTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
2695 StringRef ParentName,
unsigned LineNum,
2697 assert(CGM.getLangOpts().OpenMPIsDevice &&
"Initialization of entries is "
2698 "only required for the device "
2699 "code generation.");
2700 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] =
2701 OffloadEntryInfoTargetRegion(Order,
nullptr,
nullptr);
2702 ++OffloadingEntriesNum;
2705 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::
2706 registerTargetRegionEntryInfo(
unsigned DeviceID,
unsigned FileID,
2707 StringRef ParentName,
unsigned LineNum,
2708 llvm::Constant *Addr, llvm::Constant *
ID) {
2711 if (CGM.getLangOpts().OpenMPIsDevice) {
2712 assert(hasTargetRegionEntryInfo(DeviceID, FileID, ParentName, LineNum) &&
2713 "Entry must exist.");
2715 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum];
2716 assert(Entry.isValid() &&
"Entry not initialized!");
2717 Entry.setAddress(Addr);
2721 OffloadEntryInfoTargetRegion Entry(OffloadingEntriesNum++, Addr, ID);
2722 OffloadEntriesTargetRegion[DeviceID][FileID][ParentName][LineNum] = Entry;
2726 bool CGOpenMPRuntime::OffloadEntriesInfoManagerTy::hasTargetRegionEntryInfo(
2727 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
2728 unsigned LineNum)
const {
2729 auto PerDevice = OffloadEntriesTargetRegion.find(DeviceID);
2730 if (PerDevice == OffloadEntriesTargetRegion.end())
2732 auto PerFile = PerDevice->second.find(FileID);
2733 if (PerFile == PerDevice->second.end())
2735 auto PerParentName = PerFile->second.find(ParentName);
2736 if (PerParentName == PerFile->second.end())
2738 auto PerLine = PerParentName->second.find(LineNum);
2739 if (PerLine == PerParentName->second.end())
2742 if (PerLine->second.getAddress() || PerLine->second.getID())
2747 void CGOpenMPRuntime::OffloadEntriesInfoManagerTy::actOnTargetRegionEntriesInfo(
2748 const OffloadTargetRegionEntryInfoActTy &
Action) {
2750 for (
auto &D : OffloadEntriesTargetRegion)
2751 for (
auto &F : D.second)
2752 for (
auto &
P : F.second)
2753 for (
auto &L :
P.second)
2754 Action(D.first, F.first,
P.first(), L.first, L.second);
2760 static llvm::Function *
2766 nullptr, C.VoidPtrTy);
2767 Args.push_back(&DummyPtr);
2789 auto &M = CGM.getModule();
2790 auto &C = CGM.getContext();
2793 auto &Devices = CGM.getLangOpts().OMPTargetTriples;
2797 assert(!Devices.empty() &&
"No OpenMP offloading devices??");
2801 auto *OffloadEntryTy =
2803 llvm::GlobalVariable *HostEntriesBegin =
new llvm::GlobalVariable(
2804 M, OffloadEntryTy,
true,
2806 ".omp_offloading.entries_begin");
2807 llvm::GlobalVariable *HostEntriesEnd =
new llvm::GlobalVariable(
2808 M, OffloadEntryTy,
true,
2810 ".omp_offloading.entries_end");
2814 auto *DeviceImageTy = cast<llvm::StructType>(
2817 for (
unsigned i = 0; i < Devices.size(); ++i) {
2818 StringRef T = Devices[i].getTriple();
2819 auto *ImgBegin =
new llvm::GlobalVariable(
2822 Twine(
".omp_offloading.img_start.") + Twine(T));
2823 auto *ImgEnd =
new llvm::GlobalVariable(
2825 nullptr, Twine(
".omp_offloading.img_end.") + Twine(T));
2827 llvm::Constant *Dev =
2828 llvm::ConstantStruct::get(DeviceImageTy, ImgBegin, ImgEnd,
2829 HostEntriesBegin, HostEntriesEnd,
nullptr);
2830 DeviceImagesEntires.push_back(Dev);
2834 llvm::ArrayType *DeviceImagesInitTy =
2835 llvm::ArrayType::get(DeviceImageTy, DeviceImagesEntires.size());
2836 llvm::Constant *DeviceImagesInit =
2837 llvm::ConstantArray::get(DeviceImagesInitTy, DeviceImagesEntires);
2839 llvm::GlobalVariable *DeviceImages =
new llvm::GlobalVariable(
2840 M, DeviceImagesInitTy,
true,
2842 ".omp_offloading.device_images");
2843 DeviceImages->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2846 llvm::Constant *Index[] = {llvm::Constant::getNullValue(CGM.Int32Ty),
2847 llvm::Constant::getNullValue(CGM.Int32Ty)};
2850 auto *BinaryDescriptorTy = cast<llvm::StructType>(
2852 llvm::Constant *TargetRegionsDescriptorInit = llvm::ConstantStruct::get(
2853 BinaryDescriptorTy, llvm::ConstantInt::get(CGM.Int32Ty, Devices.size()),
2854 llvm::ConstantExpr::getGetElementPtr(DeviceImagesInitTy, DeviceImages,
2856 HostEntriesBegin, HostEntriesEnd,
nullptr);
2858 auto *Desc =
new llvm::GlobalVariable(
2859 M, BinaryDescriptorTy,
true,
2861 ".omp_offloading.descriptor");
2868 auto *IdentInfo = &C.Idents.get(
".omp_offloading.reg_unreg_var");
2870 IdentInfo, C.CharTy);
2873 CGM,
".omp_offloading.descriptor_unreg",
2879 CGM,
".omp_offloading.descriptor_reg",
2883 CGM.getCXXABI().registerGlobalDtor(CGF, RegUnregVar, UnRegFn, Desc);
2888 void CGOpenMPRuntime::createOffloadEntry(llvm::Constant *ID,
2889 llvm::Constant *Addr, uint64_t Size) {
2891 auto *TgtOffloadEntryType = cast<llvm::StructType>(
2893 llvm::LLVMContext &C = CGM.getModule().getContext();
2894 llvm::Module &M = CGM.getModule();
2897 llvm::Constant *AddrPtr = llvm::ConstantExpr::getBitCast(ID, CGM.VoidPtrTy);
2900 llvm::Constant *StrPtrInit = llvm::ConstantDataArray::getString(C, Name);
2902 llvm::GlobalVariable *Str =
2903 new llvm::GlobalVariable(M, StrPtrInit->getType(),
true,
2905 ".omp_offloading.entry_name");
2906 Str->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
2907 llvm::Constant *StrPtr = llvm::ConstantExpr::getBitCast(Str, CGM.Int8PtrTy);
2910 llvm::Constant *EntryInit = llvm::ConstantStruct::get(
2911 TgtOffloadEntryType, AddrPtr, StrPtr,
2912 llvm::ConstantInt::get(CGM.SizeTy, Size),
nullptr);
2913 llvm::GlobalVariable *Entry =
new llvm::GlobalVariable(
2915 EntryInit,
".omp_offloading.entry");
2918 Entry->setSection(
".omp_offloading.entries");
2921 Entry->setAlignment(1);
2938 llvm::Module &M = CGM.getModule();
2939 llvm::LLVMContext &C = M.getContext();
2944 llvm::NamedMDNode *MD = M.getOrInsertNamedMetadata(
"omp_offload.info");
2947 auto getMDInt = [&](
unsigned v) {
2948 return llvm::ConstantAsMetadata::get(
2949 llvm::ConstantInt::get(llvm::Type::getInt32Ty(C),
v));
2952 auto getMDString = [&](StringRef
v) {
return llvm::MDString::get(C,
v); };
2955 auto &&TargetRegionMetadataEmitter = [&](
2956 unsigned DeviceID,
unsigned FileID, StringRef ParentName,
unsigned Line,
2968 Ops.push_back(getMDInt(E.getKind()));
2969 Ops.push_back(getMDInt(DeviceID));
2970 Ops.push_back(getMDInt(FileID));
2971 Ops.push_back(getMDString(ParentName));
2972 Ops.push_back(getMDInt(Line));
2973 Ops.push_back(getMDInt(E.getOrder()));
2976 OrderedEntries[E.getOrder()] = &
E;
2979 MD->addOperand(llvm::MDNode::get(C, Ops));
2983 TargetRegionMetadataEmitter);
2985 for (
auto *E : OrderedEntries) {
2986 assert(E &&
"All ordered entries must exist!");
2988 dyn_cast<OffloadEntriesInfoManagerTy::OffloadEntryInfoTargetRegion>(
2990 assert(CE->getID() && CE->getAddress() &&
2991 "Entry ID and Addr are invalid!");
2992 createOffloadEntry(CE->getID(), CE->getAddress(), 0);
2994 llvm_unreachable(
"Unsupported entry kind.");
3004 if (!CGM.getLangOpts().OpenMPIsDevice)
3007 if (CGM.getLangOpts().OMPHostIRFile.empty())
3010 auto Buf = llvm::MemoryBuffer::getFile(CGM.getLangOpts().OMPHostIRFile);
3014 llvm::LLVMContext C;
3015 auto ME = llvm::parseBitcodeFile(Buf.get()->getMemBufferRef(), C);
3020 llvm::NamedMDNode *MD = ME.get()->getNamedMetadata(
"omp_offload.info");
3024 for (
auto I : MD->operands()) {
3025 llvm::MDNode *MN = cast<llvm::MDNode>(
I);
3027 auto getMDInt = [&](
unsigned Idx) {
3028 llvm::ConstantAsMetadata *V =
3029 cast<llvm::ConstantAsMetadata>(MN->getOperand(Idx));
3030 return cast<llvm::ConstantInt>(V->getValue())->getZExtValue();
3033 auto getMDString = [&](
unsigned Idx) {
3034 llvm::MDString *V = cast<llvm::MDString>(MN->getOperand(Idx));
3035 return V->getString();
3038 switch (getMDInt(0)) {
3040 llvm_unreachable(
"Unexpected metadata!");
3042 case OffloadEntriesInfoManagerTy::OffloadEntryInfo::
3043 OFFLOAD_ENTRY_INFO_TARGET_REGION:
3045 getMDInt(1), getMDInt(2),
3046 getMDString(3), getMDInt(4),
3054 if (!KmpRoutineEntryPtrTy) {
3056 auto &C = CGM.getContext();
3057 QualType KmpRoutineEntryTyArgs[] = {KmpInt32Ty, C.VoidPtrTy};
3059 KmpRoutineEntryPtrQTy = C.getPointerType(
3060 C.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));
3061 KmpRoutineEntryPtrTy = CGM.getTypes().ConvertType(KmpRoutineEntryPtrQTy);
3093 RD->completeDefinition();
3119 RD->completeDefinition();
3144 RD->completeDefinition();
3151 struct PrivateHelpersTy {
3152 PrivateHelpersTy(
const VarDecl *Original,
const VarDecl *PrivateCopy,
3153 const VarDecl *PrivateElemInit)
3154 : Original(Original), PrivateCopy(PrivateCopy),
3155 PrivateElemInit(PrivateElemInit) {}
3158 const VarDecl *PrivateElemInit;
3160 typedef std::pair<
CharUnits , PrivateHelpersTy> PrivateDataTy;
3165 if (!Privates.empty()) {
3172 for (
auto &&Pair : Privates) {
3173 auto *VD = Pair.second.Original;
3177 if (VD->hasAttrs()) {
3179 E(VD->getAttrs().end());
3184 RD->completeDefinition();
3193 QualType KmpRoutineEntryPointerQTy) {
3211 UD->completeDefinition();
3212 QualType KmpCmplrdataTy = C.getRecordType(UD);
3213 auto *RD = C.buildImplicitRecord(
"kmp_task_t");
3214 RD->startDefinition();
3230 RD->completeDefinition();
3236 ArrayRef<PrivateDataTy>
Privates) {
3248 RD->completeDefinition();
3266 QualType KmpTaskTWithPrivatesPtrQTy,
3276 Args.push_back(&GtidArg);
3277 Args.push_back(&TaskTypeArg);
3278 auto &TaskEntryFnInfo =
3299 auto *KmpTaskTWithPrivatesQTyRD =
3300 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3303 auto *KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->
getAsTagDecl());
3304 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3308 auto SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);
3314 auto PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);
3316 if (PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {
3319 PrivatesLVal.getPointer(), CGF.
VoidPtrTy);
3321 PrivatesParam = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3323 llvm::Value *CommonArgs[] = {GtidParam, PartidParam, PrivatesParam,
3332 auto LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);
3335 auto UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);
3338 auto StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);
3341 auto LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);
3344 CallArgs.push_back(LBParam);
3345 CallArgs.push_back(UBParam);
3346 CallArgs.push_back(StParam);
3347 CallArgs.push_back(LIParam);
3349 CallArgs.push_back(SharedsParam);
3353 RValue::get(CGF.
Builder.getInt32(0)),
3362 QualType KmpTaskTWithPrivatesPtrQTy,
3363 QualType KmpTaskTWithPrivatesQTy) {
3370 Args.push_back(&GtidArg);
3371 Args.push_back(&TaskTypeArg);
3373 auto &DestructorFnInfo =
3376 auto *DestructorFn =
3378 ".omp_task_destructor.", &CGM.
getModule());
3389 auto *KmpTaskTWithPrivatesQTyRD =
3390 cast<RecordDecl>(KmpTaskTWithPrivatesQTy->
getAsTagDecl());
3391 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3394 cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {
3395 if (
auto DtorKind = Field->getType().isDestructedType()) {
3397 CGF.
pushDestroy(DtorKind, FieldLValue.getAddress(), Field->getType());
3401 return DestructorFn;
3416 ArrayRef<const Expr *> PrivateVars,
3417 ArrayRef<const Expr *> FirstprivateVars,
3418 ArrayRef<const Expr *> LastprivateVars,
3420 ArrayRef<PrivateDataTy>
Privates) {
3424 C,
nullptr, Loc,
nullptr,
3425 C.getPointerType(PrivatesQTy).withConst().withRestrict());
3426 Args.push_back(&TaskPrivatesArg);
3427 llvm::DenseMap<const VarDecl *, unsigned> PrivateVarsPos;
3428 unsigned Counter = 1;
3429 for (
auto *E: PrivateVars) {
3432 nullptr, C.getPointerType(C.getPointerType(E->getType()))
3435 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3436 PrivateVarsPos[VD] = Counter;
3439 for (
auto *E : FirstprivateVars) {
3442 nullptr, C.getPointerType(C.getPointerType(E->getType()))
3445 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3446 PrivateVarsPos[VD] = Counter;
3449 for (
auto *E: LastprivateVars) {
3452 nullptr, C.getPointerType(C.getPointerType(E->getType()))
3455 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3456 PrivateVarsPos[VD] = Counter;
3459 auto &TaskPrivatesMapFnInfo =
3461 auto *TaskPrivatesMapTy =
3465 ".omp_task_privates_map.", &CGM.
getModule());
3467 TaskPrivatesMapFnInfo);
3468 TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);
3472 TaskPrivatesMapFnInfo, Args);
3478 auto *PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->
getAsTagDecl());
3480 for (
auto *Field : PrivatesQTyRD->fields()) {
3482 auto *VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];
3485 RefLVal.getAddress(), RefLVal.getType()->castAs<
PointerType>());
3490 return TaskPrivatesMap;
3494 const PrivateDataTy *P2) {
3495 return P1->first < P2->first ? 1 : (P2->first < P1->first ? -1 : 0);
3505 ArrayRef<PrivateDataTy>
Privates,
bool ForDup) {
3507 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3518 FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();
3519 for (
auto &&Pair : Privates) {
3520 auto *VD = Pair.second.PrivateCopy;
3522 if (Init && (!ForDup || (isa<CXXConstructExpr>(Init) &&
3525 if (
auto *Elem = Pair.second.PrivateElemInit) {
3526 auto *OriginalVD = Pair.second.Original;
3527 auto *SharedField = CapturesInfo.
lookup(OriginalVD);
3530 Address(SharedRefLValue.getPointer(), C.getDeclAlign(OriginalVD)),
3531 SharedRefLValue.
getType(), AlignmentSource::Decl);
3538 SharedRefLValue.getAddress(), Type);
3543 PrivateLValue.
getAddress(), SharedRefLValue.getAddress(), Type,
3544 [&CGF, Elem, Init, &CapturesInfo](
Address DestElement,
3549 Elem, [SrcElement]() ->
Address {
return SrcElement; });
3553 CGF, &CapturesInfo);
3554 CGF.EmitAnyExprToMem(Init, DestElement,
3555 Init->getType().getQualifiers(),
3562 return SharedRefLValue.getAddress();
3578 ArrayRef<PrivateDataTy>
Privates) {
3579 bool InitRequired =
false;
3580 for (
auto &&Pair : Privates) {
3581 auto *VD = Pair.second.PrivateCopy;
3583 InitRequired = InitRequired || (Init && isa<CXXConstructExpr>(Init) &&
3586 return InitRequired;
3603 QualType KmpTaskTWithPrivatesPtrQTy,
3607 ArrayRef<PrivateDataTy>
Privates,
bool WithLastIter) {
3611 nullptr, KmpTaskTWithPrivatesPtrQTy);
3613 nullptr, KmpTaskTWithPrivatesPtrQTy);
3616 Args.push_back(&DstArg);
3617 Args.push_back(&SrcArg);
3618 Args.push_back(&LastprivArg);
3619 auto &TaskDupFnInfo =
3635 auto LIFI = std::next(KmpTaskTQTyRD->
field_begin(), KmpTaskTLastIter);
3637 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3645 assert(!Privates.empty());
3646 Address KmpTaskSharedsPtr = Address::invalid();
3652 TDBase, *KmpTaskTWithPrivatesQTyRD->
field_begin());
3660 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,
3661 SharedsTy, SharedsPtrTy, Data, Privates,
true);
3670 bool NeedsCleanup =
false;
3671 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->
field_begin());
3672 auto *PrivateRD = cast<RecordDecl>(FI->getType()->getAsTagDecl());
3673 for (
auto *FD : PrivateRD->fields()) {
3674 NeedsCleanup = NeedsCleanup || FD->getType().isDestructedType();
3678 return NeedsCleanup;
3681 CGOpenMPRuntime::TaskResultTy
3686 auto &C = CGM.getContext();
3691 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3692 Privates.push_back(std::make_pair(
3694 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3701 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3702 Privates.push_back(std::make_pair(
3705 VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3706 cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl()))));
3712 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
3713 Privates.push_back(std::make_pair(
3715 PrivateHelpersTy(VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),
3719 llvm::array_pod_sort(Privates.begin(), Privates.end(),
3721 auto KmpInt32Ty = C.getIntTypeForBitwidth(32, 1);
3731 auto *KmpTaskTWithPrivatesQTyRD =
3733 auto KmpTaskTWithPrivatesQTy = C.getRecordType(KmpTaskTWithPrivatesQTyRD);
3734 QualType KmpTaskTWithPrivatesPtrQTy =
3735 C.getPointerType(KmpTaskTWithPrivatesQTy);
3736 auto *KmpTaskTWithPrivatesTy = CGF.
ConvertType(KmpTaskTWithPrivatesQTy);
3737 auto *KmpTaskTWithPrivatesPtrTy = KmpTaskTWithPrivatesTy->getPointerTo();
3738 auto *KmpTaskTWithPrivatesTySize = CGF.
getTypeSize(KmpTaskTWithPrivatesQTy);
3739 QualType SharedsPtrTy = C.getPointerType(SharedsTy);
3743 auto *TaskPrivatesMapTy =
3744 std::next(cast<llvm::Function>(TaskFunction)->getArgumentList().
begin(),
3747 if (!Privates.empty()) {
3748 auto FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());
3753 TaskPrivatesMap, TaskPrivatesMapTy);
3755 TaskPrivatesMap = llvm::ConstantPointerNull::get(
3756 cast<llvm::PointerType>(TaskPrivatesMapTy));
3762 KmpTaskTWithPrivatesQTy,
KmpTaskTQTy, SharedsPtrTy, TaskFunction,
3774 DestructorsFlag = 0x8,
3777 unsigned Flags = Data.
Tied ? TiedFlag : 0;
3778 bool NeedsCleanup =
false;
3779 if (!Privates.empty()) {
3782 Flags = Flags | DestructorsFlag;
3785 Flags = Flags | PriorityFlag;
3787 Data.
Final.getPointer()
3789 CGF.
Builder.getInt32(FinalFlag),
3791 : CGF.
Builder.getInt32(Data.
Final.getInt() ? FinalFlag : 0);
3792 TaskFlags = CGF.
Builder.CreateOr(TaskFlags, CGF.
Builder.getInt32(Flags));
3793 auto *SharedsSize = CGM.
getSize(C.getTypeSizeInChars(SharedsTy));
3796 KmpTaskTWithPrivatesTySize, SharedsSize,
3798 TaskEntry, KmpRoutineEntryPtrTy)};
3802 NewTask, KmpTaskTWithPrivatesPtrTy);
3804 KmpTaskTWithPrivatesQTy);
3809 Address KmpTaskSharedsPtr = Address::invalid();
3814 TDBase, *std::next(KmpTaskTQTyRD->field_begin(),
3822 if (!Privates.empty()) {
3823 emitPrivatesInit(CGF, D, KmpTaskSharedsPtr, Base, KmpTaskTWithPrivatesQTyRD,
3824 SharedsTy, SharedsPtrTy, Data, Privates,
3829 CGM, Loc, D, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,
3830 KmpTaskTQTyRD, SharedsTy, SharedsPtrTy, Data, Privates,
3835 enum { Priority = 0, Destructors = 1 };
3837 auto FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);
3838 auto *KmpCmplrdataUD = (*FI)->getType()->getAsUnionType()->getDecl();
3841 CGM, Loc, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,
3842 KmpTaskTWithPrivatesQTy);
3845 Data1LV, *std::next(KmpCmplrdataUD->field_begin(), Destructors));
3847 DestructorFn, KmpRoutineEntryPtrTy),
3853 TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));
3855 Data2LV, *std::next(KmpCmplrdataUD->field_begin(), Priority));
3876 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
3882 auto &C = CGM.getContext();
3884 Address DependenciesArray = Address::invalid();
3885 unsigned NumDependencies = Data.
Dependences.size();
3886 if (NumDependencies) {
3888 enum RTLDependenceKindTy { DepIn = 0x01, DepInOut = 0x3 };
3889 enum RTLDependInfoFieldsTy { BaseAddr, Len, Flags };
3892 C.getIntTypeForBitwidth(C.getTypeSize(C.BoolTy),
false);
3895 KmpDependInfoRD = C.buildImplicitRecord(
"kmp_depend_info");
3906 QualType KmpDependInfoArrayTy = C.getConstantArrayType(
3912 for (
unsigned i = 0; i < NumDependencies; ++i) {
3923 CGF.
Builder.CreatePtrToInt(Addr.getPointer(), CGM.SizeTy);
3925 Size = CGF.
Builder.CreateNUWSub(UpIntPtr, LowIntPtr);
3942 RTLDependenceKindTy DepKind;
3944 case OMPC_DEPEND_in:
3948 case OMPC_DEPEND_out:
3949 case OMPC_DEPEND_inout:
3952 case OMPC_DEPEND_source:
3953 case OMPC_DEPEND_sink:
3955 llvm_unreachable(
"Unknown task dependence type");
3975 llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };
3977 if (NumDependencies) {
3978 DepTaskArgs[0] = UpLoc;
3979 DepTaskArgs[1] = ThreadID;
3980 DepTaskArgs[2] = NewTask;
3981 DepTaskArgs[3] = CGF.
Builder.getInt32(NumDependencies);
3982 DepTaskArgs[4] = DependenciesArray.
getPointer();
3983 DepTaskArgs[5] = CGF.
Builder.getInt32(0);
3984 DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.
VoidPtrTy);
3986 auto &&ThenCodeGen = [
this, Loc, &Data, TDBase, KmpTaskTQTyRD,
3987 NumDependencies, &TaskArgs,
3990 auto PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);
3991 auto PartIdLVal = CGF.EmitLValueForField(TDBase, *PartIdFI);
3992 CGF.EmitStoreOfScalar(CGF.Builder.getInt32(0), PartIdLVal);
3994 if (NumDependencies) {
3995 CGF.EmitRuntimeCall(
4003 dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.CapturedStmtInfo))
4004 Region->emitUntiedSwitch(CGF);
4008 if (NumDependencies) {
4009 DepWaitTaskArgs[0] = UpLoc;
4010 DepWaitTaskArgs[1] = ThreadID;
4011 DepWaitTaskArgs[2] = CGF.Builder.getInt32(NumDependencies);
4012 DepWaitTaskArgs[3] = DependenciesArray.
getPointer();
4013 DepWaitTaskArgs[4] = CGF.Builder.getInt32(0);
4014 DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);
4016 auto &&ElseCodeGen = [&TaskArgs, ThreadID, NewTaskNewTaskTTy, TaskEntry,
4019 auto &RT = CGF.CGM.getOpenMPRuntime();
4025 if (NumDependencies)
4029 auto &&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy](
4032 llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};
4033 CGF.EmitCallOrInvoke(TaskEntry, OutlinedFnArgs);
4065 emitTaskInit(CGF, Loc, D, TaskFunction, SharedsTy, Shareds, Data);
4078 IfVal = llvm::ConstantInt::getSigned(CGF.
IntTy, 1);
4101 enum { NoSchedule = 0, Grainsize = 1, NumTasks = 2 };
4105 llvm::ConstantInt::getSigned(CGF.
IntTy, Data.
Nogroup ? 1 : 0),
4106 llvm::ConstantInt::getSigned(
4108 ? Data.
Schedule.getInt() ? NumTasks : Grainsize
4113 : llvm::ConstantInt::get(CGF.
Int64Ty, 0),
4117 : llvm::ConstantPointerNull::get(CGF.
VoidPtrTy)};
4134 const Expr *,
const Expr *)> &RedOpGen,
4135 const Expr *XExpr =
nullptr,
const Expr *EExpr =
nullptr,
4136 const Expr *UpExpr =
nullptr) {
4149 auto LHSEnd = CGF.
Builder.CreateGEP(LHSBegin, NumElements);
4154 CGF.
Builder.CreateICmpEQ(LHSBegin, LHSEnd,
"omp.arraycpy.isempty");
4155 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
4158 auto EntryBB = CGF.
Builder.GetInsertBlock();
4163 llvm::PHINode *RHSElementPHI = CGF.
Builder.CreatePHI(
4164 RHSBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
4165 RHSElementPHI->addIncoming(RHSBegin, EntryBB);
4170 llvm::PHINode *LHSElementPHI = CGF.
Builder.CreatePHI(
4171 LHSBegin->getType(), 2,
"omp.arraycpy.destElementPast");
4172 LHSElementPHI->addIncoming(LHSBegin, EntryBB);
4179 Scope.addPrivate(LHSVar, [=]() ->
Address {
return LHSElementCurrent; });
4180 Scope.addPrivate(RHSVar, [=]() ->
Address {
return RHSElementCurrent; });
4182 RedOpGen(CGF, XExpr, EExpr, UpExpr);
4183 Scope.ForceCleanup();
4186 auto LHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4187 LHSElementPHI, 1,
"omp.arraycpy.dest.element");
4188 auto RHSElementNext = CGF.
Builder.CreateConstGEP1_32(
4189 RHSElementPHI, 1,
"omp.arraycpy.src.element");
4192 CGF.
Builder.CreateICmpEQ(LHSElementNext, LHSEnd,
"omp.arraycpy.done");
4193 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
4194 LHSElementPHI->addIncoming(LHSElementNext, CGF.
Builder.GetInsertBlock());
4195 RHSElementPHI->addIncoming(RHSElementNext, CGF.
Builder.GetInsertBlock());
4205 const Expr *ReductionOp) {
4206 if (
auto *CE = dyn_cast<CallExpr>(ReductionOp))
4207 if (
auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
4209 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
4210 if (
auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {
4211 std::pair<llvm::Function *, llvm::Function *> Reduction =
4213 RValue Func = RValue::get(Reduction.first);
4222 llvm::Type *ArgsType,
4224 ArrayRef<const Expr *> LHSExprs,
4225 ArrayRef<const Expr *> RHSExprs,
4226 ArrayRef<const Expr *> ReductionOps) {
4235 Args.push_back(&LHSArg);
4236 Args.push_back(&RHSArg);
4240 ".omp.reduction.reduction_func", &CGM.
getModule());
4258 auto IPriv = Privates.begin();
4260 for (
unsigned I = 0, E = ReductionOps.size(); I <
E; ++
I, ++IPriv, ++Idx) {
4261 auto RHSVar = cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[
I])->getDecl());
4262 Scope.addPrivate(RHSVar, [&]() ->
Address {
4265 auto LHSVar = cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[
I])->getDecl());
4266 Scope.addPrivate(LHSVar, [&]() ->
Address {
4269 QualType PrivTy = (*IPriv)->getType();
4277 auto *OVE = cast<OpaqueValueExpr>(VLA->getSizeExpr());
4279 CGF, OVE, RValue::get(CGF.
Builder.CreatePtrToInt(Ptr, CGF.
SizeTy)));
4284 IPriv = Privates.begin();
4285 auto ILHS = LHSExprs.begin();
4286 auto IRHS = RHSExprs.begin();
4287 for (
auto *E : ReductionOps) {
4288 if ((*IPriv)->getType()->isArrayType()) {
4290 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4291 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4293 CGF, (*IPriv)->getType(), LHSVar, RHSVar,
4304 Scope.ForceCleanup();
4310 const Expr *ReductionOp,
4311 const Expr *PrivateRef,
4316 auto *LHSVar = cast<VarDecl>(LHS->
getDecl());
4317 auto *RHSVar = cast<VarDecl>(RHS->
getDecl());
4319 CGF, PrivateRef->
getType(), LHSVar, RHSVar,
4330 ArrayRef<const Expr *> LHSExprs,
4331 ArrayRef<const Expr *> RHSExprs,
4332 ArrayRef<const Expr *> ReductionOps,
4333 bool WithNowait,
bool SimpleReduction) {
4371 auto &C = CGM.getContext();
4373 if (SimpleReduction) {
4375 auto IPriv = Privates.begin();
4376 auto ILHS = LHSExprs.begin();
4377 auto IRHS = RHSExprs.begin();
4378 for (
auto *E : ReductionOps) {
4380 cast<DeclRefExpr>(*IRHS));
4390 auto Size = RHSExprs.size();
4391 for (
auto *E : Privates) {
4392 if (E->getType()->isVariablyModifiedType())
4396 llvm::APInt ArraySize(32, Size);
4401 CGF.
CreateMemTemp(ReductionArrayTy,
".omp.reduction.red_list");
4402 auto IPriv = Privates.begin();
4404 for (
unsigned I = 0, E = RHSExprs.size(); I <
E; ++
I, ++IPriv, ++Idx) {
4411 if ((*IPriv)->getType()->isVariablyModifiedType()) {
4429 LHSExprs, RHSExprs, ReductionOps);
4438 auto *ReductionArrayTySize = CGF.
getTypeSize(ReductionArrayTy);
4445 CGF.
Builder.getInt32(RHSExprs.size()),
4446 ReductionArrayTySize,
4458 auto *SwInst = CGF.
Builder.CreateSwitch(Res, DefaultBB, 2);
4467 SwInst->addCase(CGF.
Builder.getInt32(1), Case1BB);
4476 auto &&CodeGen = [&
Privates, &LHSExprs, &RHSExprs, &ReductionOps](
4478 auto IPriv = Privates.begin();
4479 auto ILHS = LHSExprs.begin();
4480 auto IRHS = RHSExprs.begin();
4481 for (
auto *E : ReductionOps) {
4483 cast<DeclRefExpr>(*IRHS));
4495 RCG.setAction(Action);
4498 CGF.EmitBranch(DefaultBB);
4505 auto *Case2BB = CGF.createBasicBlock(
".omp.reduction.case2");
4506 SwInst->addCase(CGF.Builder.getInt32(2), Case2BB);
4507 CGF.EmitBlock(Case2BB);
4509 auto &&AtomicCodeGen = [Loc, &
Privates, &LHSExprs, &RHSExprs, &ReductionOps](
4511 auto ILHS = LHSExprs.begin();
4512 auto IRHS = RHSExprs.begin();
4513 auto IPriv = Privates.begin();
4514 for (
auto *E : ReductionOps) {
4515 const Expr *XExpr =
nullptr;
4516 const Expr *EExpr =
nullptr;
4517 const Expr *UpExpr =
nullptr;
4519 if (
auto *BO = dyn_cast<BinaryOperator>(E)) {
4520 if (BO->getOpcode() == BO_Assign) {
4521 XExpr = BO->getLHS();
4522 UpExpr = BO->getRHS();
4526 auto *RHSExpr = UpExpr;
4529 if (
auto *ACO = dyn_cast<AbstractConditionalOperator>(
4530 RHSExpr->IgnoreParenImpCasts())) {
4533 RHSExpr = ACO->getCond();
4536 dyn_cast<BinaryOperator>(RHSExpr->IgnoreParenImpCasts())) {
4537 EExpr = BORHS->getRHS();
4538 BO = BORHS->getOpcode();
4542 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4543 auto &&AtomicRedGen = [BO, VD, IPriv,
4545 const Expr *EExpr,
const Expr *UpExpr) {
4546 LValue X = CGF.EmitLValue(XExpr);
4549 E = CGF.EmitAnyExpr(EExpr);
4550 CGF.EmitOMPAtomicSimpleUpdateExpr(
4552 llvm::AtomicOrdering::Monotonic, Loc,
4553 [&CGF, UpExpr, VD, IPriv, Loc](
RValue XRValue) {
4555 PrivateScope.addPrivate(
4556 VD, [&CGF, VD, XRValue, Loc]() ->
Address {
4557 Address LHSTemp = CGF.CreateMemTemp(VD->getType());
4558 CGF.emitOMPSimpleStore(
4559 CGF.MakeAddrLValue(LHSTemp, VD->
getType()), XRValue,
4560 VD->getType().getNonReferenceType(), Loc);
4563 (void)PrivateScope.Privatize();
4564 return CGF.EmitAnyExpr(UpExpr);
4567 if ((*IPriv)->getType()->isArrayType()) {
4569 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4571 AtomicRedGen, XExpr, EExpr, UpExpr);
4574 AtomicRedGen(CGF, XExpr, EExpr, UpExpr);
4578 const Expr *,
const Expr *) {
4579 auto &RT = CGF.CGM.getOpenMPRuntime();
4580 RT.emitCriticalRegion(
4581 CGF,
".atomic_reduction",
4588 if ((*IPriv)->getType()->isArrayType()) {
4589 auto *LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
4590 auto *RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
4594 CritRedGen(CGF,
nullptr,
nullptr,
nullptr);
4612 AtomicRCG.setAction(Action);
4630 if (
auto *Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.
CapturedStmtInfo))
4631 Region->emitUntiedSwitch(CGF);
4640 InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel);
4656 if (CancelRegion == OMPD_parallel)
4657 CancelKind = CancelParallel;
4658 else if (CancelRegion == OMPD_for)
4659 CancelKind = CancelLoop;
4660 else if (CancelRegion == OMPD_sections)
4661 CancelKind = CancelSections;
4663 assert(CancelRegion == OMPD_taskgroup);
4664 CancelKind = CancelTaskgroup;
4676 if (
auto *OMPRegionInfo =
4678 if (OMPRegionInfo->hasCancel()) {
4692 CGF.
Builder.CreateCondBr(Cmp, ExitBB, ContBB);
4712 if (
auto *OMPRegionInfo =
4714 auto &&ThenGen = [Loc, CancelRegion, OMPRegionInfo](
CodeGenFunction &CGF,
4716 auto &RT = CGF.CGM.getOpenMPRuntime();
4718 RT.emitUpdateLocation(CGF, Loc), RT.getThreadID(CGF, Loc),
4721 auto *
Result = CGF.EmitRuntimeCall(
4727 auto *ExitBB = CGF.createBasicBlock(
".cancel.exit");
4728 auto *ContBB = CGF.createBasicBlock(
".cancel.continue");
4729 auto *Cmp = CGF.Builder.CreateIsNotNull(
Result);
4730 CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);
4731 CGF.EmitBlock(ExitBB);
4736 CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());
4737 CGF.EmitBranchThroughCleanup(CancelDest);
4738 CGF.EmitBlock(ContBB,
true);
4754 unsigned &DeviceID,
unsigned &FileID,
4755 unsigned &LineNum) {
4762 assert(Loc.
isValid() &&
"Source location is expected to be always valid.");
4763 assert(Loc.
isFileID() &&
"Source location is expected to refer to a file.");
4766 assert(PLoc.isValid() &&
"Source location is expected to be always valid.");
4768 llvm::sys::fs::UniqueID
ID;
4769 if (llvm::sys::fs::getUniqueID(PLoc.getFilename(),
ID))
4770 llvm_unreachable(
"Source file with target region no longer exists!");
4772 DeviceID = ID.getDevice();
4773 FileID = ID.getFile();
4774 LineNum = PLoc.getLine();
4779 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
4781 assert(!ParentName.empty() &&
"Invalid target region parent name!");
4783 emitTargetOutlinedFunctionHelper(D, ParentName, OutlinedFn, OutlinedFnID,
4784 IsOffloadEntry, CodeGen);
4787 void CGOpenMPRuntime::emitTargetOutlinedFunctionHelper(
4789 llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,
4807 llvm::raw_svector_ostream OS(EntryFnName);
4808 OS <<
"__omp_offloading" << llvm::format(
"_%x", DeviceID)
4809 << llvm::format(
"_%x_", FileID) << ParentName <<
"_l" <<
Line;
4815 CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);
4818 OutlinedFn = CGF.GenerateOpenMPCapturedStmtFunction(CS);
4822 if (!IsOffloadEntry)
4836 if (CGM.getLangOpts().OpenMPIsDevice) {
4837 OutlinedFnID = llvm::ConstantExpr::getBitCast(OutlinedFn, CGM.Int8PtrTy);
4840 OutlinedFnID =
new llvm::GlobalVariable(
4841 CGM.getModule(), CGM.Int8Ty,
true,
4842 llvm::GlobalValue::PrivateLinkage,
4843 llvm::Constant::getNullValue(CGM.Int8Ty),
".omp_offload.region_id");
4847 DeviceID, FileID, ParentName, Line, OutlinedFn, OutlinedFnID);
4852 while (
auto *CS = dyn_cast_or_null<CompoundStmt>(Body))
4853 Body = CS->body_front();
4867 assert(!CGF.
getLangOpts().OpenMPIsDevice &&
"Clauses associated with the "
4868 "teams directive expected to be "
4869 "emitted only for the host!");
4876 "Not expecting clause in directive.");
4888 if (
auto *TeamsDir = dyn_cast_or_null<OMPTeamsDirective>(
4891 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
4900 return CGF.
Builder.getInt32(0);
4916 assert(!CGF.
getLangOpts().OpenMPIsDevice &&
"Clauses associated with the "
4917 "teams directive expected to be "
4918 "emitted only for the host!");
4925 "Not expecting clause in directive.");
4937 if (
auto *TeamsDir = dyn_cast_or_null<OMPTeamsDirective>(
4940 CGOpenMPInnerExprInfo CGInfo(CGF, CS);
4949 return CGF.
Builder.getInt32(0);
4961 class MappableExprsHandler {
4965 enum OpenMPOffloadMappingFlags {
4969 OMP_MAP_FROM = 0x02,
4972 OMP_MAP_ALWAYS = 0x04,
4975 OMP_MAP_DELETE = 0x08,
4978 OMP_MAP_IS_PTR = 0x10,
4983 OMP_MAP_FIRST_REF = 0x20,
4986 OMP_MAP_PRIVATE_PTR = 0x80,
4988 OMP_MAP_PRIVATE_VAL = 0x100,
5002 llvm::SmallPtrSet<const VarDecl *, 8> FirstPrivateDecls;
5004 llvm::Value *getExprTypeSize(
const Expr *E)
const {
5014 if (
const auto *OAE = dyn_cast<OMPArraySectionExpr>(E)) {
5016 OAE->getBase()->IgnoreParenImpCasts())
5017 .getCanonicalType();
5021 if (!OAE->getLength() && OAE->getColonLoc().isValid())
5026 ElemSize = CGF.
getTypeSize(PTy->getPointeeType().getCanonicalType());
5028 auto *ATy = cast<ArrayType>(BaseTy.
getTypePtr());
5029 assert(ATy &&
"Expecting array type if not a pointer type.");
5030 ElemSize = CGF.
getTypeSize(ATy->getElementType().getCanonicalType());
5035 if (!OAE->getLength())
5041 return CGF.
Builder.CreateNUWMul(LengthVal, ElemSize);
5052 bool AddIsFirstFlag)
const {
5055 case OMPC_MAP_alloc:
5056 case OMPC_MAP_release:
5066 Bits = OMP_MAP_FROM;
5068 case OMPC_MAP_tofrom:
5069 Bits = OMP_MAP_TO | OMP_MAP_FROM;
5071 case OMPC_MAP_delete:
5072 Bits = OMP_MAP_DELETE;
5075 llvm_unreachable(
"Unexpected map type!");
5079 Bits |= OMP_MAP_IS_PTR;
5081 Bits |= OMP_MAP_FIRST_REF;
5082 if (MapTypeModifier == OMPC_MAP_always)
5083 Bits |= OMP_MAP_ALWAYS;
5089 bool isFinalArraySectionExpression(
const Expr *E)
const {
5097 if (OASE->getColonLoc().isInvalid())
5100 auto *
Length = OASE->getLength();
5107 OASE->getBase()->IgnoreParenImpCasts())
5108 .getCanonicalType();
5109 if (
auto *ATy = dyn_cast<ConstantArrayType>(BaseQTy.getTypePtr()))
5110 return ATy->getSize().getSExtValue() != 1;
5118 llvm::APSInt ConstLength;
5122 return ConstLength.getSExtValue() != 1;
5129 void generateInfoForComponentList(
5132 MapValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,
5133 MapValuesArrayTy &Sizes, MapFlagsArrayTy &Types,
5134 bool IsFirstComponentList)
const {
5250 bool IsCaptureFirstInfo = IsFirstComponentList;
5253 auto CI = Components.rbegin();
5254 auto CE = Components.rend();
5259 bool IsExpressionFirstInfo =
true;
5262 if (
auto *ME = dyn_cast<MemberExpr>(I->getAssociatedExpression())) {
5269 BP = CGF.
EmitLValue(cast<DeclRefExpr>(I->getAssociatedExpression()))
5275 if (I->getAssociatedDeclaration()->getType()->isAnyPointerType() &&
5276 std::next(I) != CE) {
5278 BP, I->getAssociatedDeclaration()->getType());
5280 I->getAssociatedDeclaration()
5291 for (; I != CE; ++
I) {
5292 auto Next = std::next(I);
5300 bool IsFinalArraySection =
5301 isFinalArraySectionExpression(I->getAssociatedExpression());
5315 if (
Next == CE || IsPointer || IsFinalArraySection) {
5319 assert((
Next == CE ||
5320 isa<MemberExpr>(
Next->getAssociatedExpression()) ||
5321 isa<ArraySubscriptExpr>(
Next->getAssociatedExpression()) ||
5322 isa<OMPArraySectionExpr>(
Next->getAssociatedExpression())) &&
5323 "Unexpected expression");
5326 BasePointers.push_back(BP);
5328 auto *LB = CGF.
EmitLValue(I->getAssociatedExpression()).getPointer();
5329 auto *Size = getExprTypeSize(I->getAssociatedExpression());
5331 Pointers.push_back(LB);
5332 Sizes.push_back(Size);
5337 Types.push_back(getMapTypeBits(MapType, MapTypeModifier,
5338 !IsExpressionFirstInfo,
5339 IsCaptureFirstInfo));
5342 if (IsFinalArraySection)
5349 IsExpressionFirstInfo =
false;
5350 IsCaptureFirstInfo =
false;
5360 unsigned CurrentModifiers) {
5367 return MappableExprsHandler::OMP_MAP_PRIVATE_PTR |
5368 MappableExprsHandler::OMP_MAP_TO;
5371 return CurrentModifiers;
5379 for (
const auto *D : C->varlists())
5380 FirstPrivateDecls.insert(
5386 void generateAllInfo(MapValuesArrayTy &BasePointers,
5387 MapValuesArrayTy &Pointers, MapValuesArrayTy &Sizes,
5388 MapFlagsArrayTy &Types)
const {
5389 BasePointers.clear();
5403 llvm::DenseMap<const ValueDecl *, SmallVector<MapInfo, 8>> Info;
5412 D ? cast<ValueDecl>(D->getCanonicalDecl()) :
nullptr;
5413 Info[VD].push_back({L, MapType, MapModifier});
5417 for (
auto L : C->component_lists())
5418 InfoGen(L.first, L.second, C->getMapType(), C->getMapTypeModifier());
5420 for (
auto L : C->component_lists())
5423 for (
auto L : C->component_lists())
5426 for (
auto &M : Info) {
5429 bool IsFirstComponentList =
true;
5430 for (MapInfo &L : M.second) {
5431 assert(!L.Components.empty() &&
5432 "Not expecting declaration with no component lists.");
5433 generateInfoForComponentList(L.MapType, L.MapTypeModifier, L.Components,
5434 BasePointers, Pointers, Sizes, Types,
5435 IsFirstComponentList);
5436 IsFirstComponentList =
false;
5444 MapValuesArrayTy &BasePointers,
5445 MapValuesArrayTy &Pointers,
5446 MapValuesArrayTy &Sizes,
5447 MapFlagsArrayTy &Types)
const {
5449 "Not expecting to generate map info for a variable array type!");
5451 BasePointers.clear();
5463 bool IsFirstComponentList =
true;
5465 for (
auto L : C->decl_component_lists(VD)) {
5466 assert(L.first == VD &&
5467 "We got information for the wrong declaration??");
5468 assert(!L.second.empty() &&
5469 "Not expecting declaration with no component lists.");
5470 generateInfoForComponentList(C->getMapType(), C->getMapTypeModifier(),
5471 L.second, BasePointers, Pointers, Sizes,
5472 Types, IsFirstComponentList);
5473 IsFirstComponentList =
false;
5481 void generateDefaultMapInfo(
5490 CurBasePointers.push_back(CV);
5491 CurPointers.push_back(CV);
5495 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_TO |
5496 MappableExprsHandler::OMP_MAP_FROM);
5498 CurBasePointers.push_back(CV);
5499 CurPointers.push_back(CV);
5503 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_PRIVATE_VAL);
5508 CurMapTypes.push_back(0u);
5509 CurSizes.push_back(llvm::Constant::getNullValue(CGF.
SizeTy));
5513 CurBasePointers.push_back(CV);
5514 CurPointers.push_back(CV);
5523 CurMapTypes.push_back(ElementType->isAggregateType()
5524 ? (MappableExprsHandler::OMP_MAP_TO |
5525 MappableExprsHandler::OMP_MAP_FROM)
5526 : MappableExprsHandler::OMP_MAP_TO);
5531 CurMapTypes.back() =
5532 adjustMapModifiersForPrivateClauses(CI, CurMapTypes.back());
5536 CurMapTypes.back() |= MappableExprsHandler::OMP_MAP_FIRST_REF;
5543 OMP_DEVICEID_UNDEF = -1,
5558 auto &CGM = CGF.
CGM;
5561 BasePointersArray = PointersArray = SizesArray = MapTypesArray =
nullptr;
5563 if (
unsigned PointerNumVal = BasePointers.size()) {
5566 bool hasRuntimeEvaluationCaptureSize =
false;
5567 for (
auto *
S : Sizes)
5568 if (!isa<llvm::Constant>(
S)) {
5569 hasRuntimeEvaluationCaptureSize =
true;
5573 llvm::APInt PointerNumAP(32, PointerNumVal,
true);
5586 if (hasRuntimeEvaluationCaptureSize) {
5587 QualType SizeArrayType = Ctx.getConstantArrayType(
5596 for (
auto S : Sizes)
5597 ConstSizes.push_back(cast<llvm::Constant>(
S));
5599 auto *SizesArrayInit = llvm::ConstantArray::get(
5600 llvm::ArrayType::get(CGM.SizeTy, ConstSizes.size()), ConstSizes);
5601 auto *SizesArrayGbl =
new llvm::GlobalVariable(
5602 CGM.getModule(), SizesArrayInit->getType(),
5603 true, llvm::GlobalValue::PrivateLinkage,
5604 SizesArrayInit,
".offload_sizes");
5605 SizesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
5606 SizesArray = SizesArrayGbl;
5611 llvm::Constant *MapTypesArrayInit =
5612 llvm::ConstantDataArray::get(CGF.
Builder.getContext(), MapTypes);
5613 auto *MapTypesArrayGbl =
new llvm::GlobalVariable(
5614 CGM.getModule(), MapTypesArrayInit->getType(),
5615 true, llvm::GlobalValue::PrivateLinkage,
5616 MapTypesArrayInit,
".offload_maptypes");
5617 MapTypesArrayGbl->setUnnamedAddr(llvm::GlobalValue::UnnamedAddr::Global);
5618 MapTypesArray = MapTypesArrayGbl;
5620 for (
unsigned i = 0; i < PointerNumVal; ++i) {
5622 if (BPVal->getType()->isPointerTy())
5625 assert(BPVal->getType()->isIntegerTy() &&
5626 "If not a pointer, the value type must be an integer.");
5627 BPVal = CGF.
Builder.CreateIntToPtr(BPVal, CGM.VoidPtrTy);
5630 llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), BasePointersArray,
5632 Address BPAddr(BP, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
5636 if (PVal->getType()->isPointerTy())
5639 assert(PVal->getType()->isIntegerTy() &&
5640 "If not a pointer, the value type must be an integer.");
5641 PVal = CGF.
Builder.CreateIntToPtr(PVal, CGM.VoidPtrTy);
5644 llvm::ArrayType::get(CGM.VoidPtrTy, PointerNumVal), PointersArray, 0,
5646 Address PAddr(P, Ctx.getTypeAlignInChars(Ctx.VoidPtrTy));
5649 if (hasRuntimeEvaluationCaptureSize) {
5651 llvm::ArrayType::get(CGM.SizeTy, PointerNumVal), SizesArray,
5654 Address SAddr(S, Ctx.getTypeAlignInChars(Ctx.getSizeType()));
5656 CGF.
Builder.CreateIntCast(Sizes[i], CGM.SizeTy,
true),
5670 auto &CGM = CGF.
CGM;
5672 BasePointersArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
5673 llvm::ArrayType::get(CGM.VoidPtrTy, NumElems), BasePointersArray,
5675 PointersArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
5676 llvm::ArrayType::get(CGM.VoidPtrTy, NumElems), PointersArray,
5679 SizesArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
5680 llvm::ArrayType::get(CGM.SizeTy, NumElems), SizesArray,
5682 MapTypesArrayArg = CGF.
Builder.CreateConstInBoundsGEP2_32(
5683 llvm::ArrayType::get(CGM.Int32Ty, NumElems), MapTypesArray,
5687 BasePointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
5688 PointersArrayArg = llvm::ConstantPointerNull::get(CGM.VoidPtrPtrTy);
5689 SizesArrayArg = llvm::ConstantPointerNull::get(CGM.SizeTy->getPointerTo());
5691 llvm::ConstantPointerNull::get(CGM.Int32Ty->getPointerTo());
5699 const Expr *IfCond,
const Expr *Device,
5700 ArrayRef<llvm::Value *> CapturedVars) {
5704 assert(OutlinedFn &&
"Invalid outlined function!");
5721 MappableExprsHandler MEHandler(D, CGF);
5725 auto CV = CapturedVars.begin();
5728 CI != CE; ++CI, ++RI, ++CV) {
5732 CurBasePointers.clear();
5733 CurPointers.clear();
5735 CurMapTypes.clear();
5740 CurBasePointers.push_back(*CV);
5741 CurPointers.push_back(*CV);
5742 CurSizes.push_back(CGF.
getTypeSize(RI->getType()));
5744 CurMapTypes.push_back(MappableExprsHandler::OMP_MAP_PRIVATE_VAL |
5745 MappableExprsHandler::OMP_MAP_FIRST_REF);
5749 MEHandler.generateInfoForCapture(CI, CurBasePointers, CurPointers,
5750 CurSizes, CurMapTypes);
5751 if (CurBasePointers.empty())
5752 MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurBasePointers,
5753 CurPointers, CurSizes, CurMapTypes);
5756 assert(!CurBasePointers.empty() &&
"Non-existing map pointer for capture!");
5757 assert(CurBasePointers.size() == CurPointers.size() &&
5758 CurBasePointers.size() == CurSizes.size() &&
5759 CurBasePointers.size() == CurMapTypes.size() &&
5760 "Inconsistent map information sizes!");
5764 KernelArgs.push_back(CurBasePointers.front());
5766 BasePointers.append(CurBasePointers.begin(), CurBasePointers.end());
5767 Pointers.append(CurPointers.begin(), CurPointers.end());
5768 Sizes.append(CurSizes.begin(), CurSizes.end());
5769 MapTypes.append(CurMapTypes.begin(), CurMapTypes.end());
5773 auto OffloadErrorQType =
5774 Ctx.getIntTypeForBitwidth(32,
true);
5782 auto &&ThenGen = [&Ctx, &BasePointers, &Pointers, &Sizes, &MapTypes, Device,
5783 OutlinedFnID, OffloadError, OffloadErrorQType,
5785 auto &RT = CGF.CGM.getOpenMPRuntime();
5792 MapTypesArray, BasePointers, Pointers, Sizes,
5795 SizesArray, MapTypesArray, BasePointersArray,
5796 PointersArray, SizesArray, MapTypesArray,
5797 BasePointers.size());
5809 assert(OutlinedFnID &&
"Invalid outlined function ID!");
5814 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
5817 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
5820 llvm::Value *PointerNum = CGF.Builder.getInt32(BasePointers.size());
5835 assert(ThreadLimit &&
"Thread limit expression should be available along "
5836 "with number of teams.");
5838 DeviceID, OutlinedFnID, PointerNum,
5839 BasePointersArray, PointersArray, SizesArray,
5840 MapTypesArray, NumTeams, ThreadLimit};
5841 Return = CGF.EmitRuntimeCall(
5845 DeviceID, OutlinedFnID, PointerNum, BasePointersArray,
5846 PointersArray, SizesArray, MapTypesArray};
5851 CGF.EmitStoreOfScalar(Return, OffloadError);
5880 auto Failed = CGF.
Builder.CreateIsNotNull(OffloadErrorVal);
5881 CGF.
Builder.CreateCondBr(Failed, OffloadFailedBlock, OffloadContBlock);
5884 CGF.
Builder.CreateCall(OutlinedFn, KernelArgs);
5891 StringRef ParentName) {
5898 bool isTargetDirective = isa<OMPTargetDirective>(
S);
5900 if (isTargetDirective) {
5901 auto *E = cast<OMPExecutableDirective>(
S);
5915 llvm::Constant *Addr;
5916 std::tie(Fn, Addr) =
5917 CodeGenFunction::EmitOMPTargetDirectiveOutlinedFunction(
5918 CGM, cast<OMPTargetDirective>(*E), ParentName,
5920 assert(Fn && Addr &&
"Target region emission failed.");
5925 if (!E->hasAssociatedStmt())
5929 cast<CapturedStmt>(E->getAssociatedStmt())->getCapturedStmt(),
5935 if (
auto *L = dyn_cast<LambdaExpr>(S))
5939 for (
auto *II : S->children())
5944 auto &FD = *cast<FunctionDecl>(GD.
getDecl());
5948 if (!CGM.getLangOpts().OpenMPIsDevice)
5961 if (!CGM.getLangOpts().OpenMPIsDevice)
5969 for (
auto *Ctor : RD->ctors()) {
5970 StringRef ParentName =
5974 auto *Dtor = RD->getDestructor();
5976 StringRef ParentName =
5989 if (isa<FunctionDecl>(VD))
6020 CGF.
Builder.getInt32(CapturedVars.size()),
6024 RealArgs.append(CapturedVars.begin(), CapturedVars.end());
6031 const Expr *NumTeams,
6032 const Expr *ThreadLimit,
6071 unsigned NumOfPtrs = 0;
6076 auto &&BeginThenGen = [&D, &CGF, &BasePointersArray, &PointersArray,
6077 &SizesArray, &MapTypesArray, Device,
6086 MappableExprsHandler MCHandler(D, CGF);
6087 MCHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
6088 NumOfPtrs = BasePointers.size();
6092 MapTypesArray, BasePointers, Pointers, Sizes,
6100 SizesArrayArg, MapTypesArrayArg,
6101 BasePointersArray, PointersArray, SizesArray,
6102 MapTypesArray, NumOfPtrs);
6107 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6110 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6113 auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs);
6116 DeviceID, PointerNum, BasePointersArrayArg,
6117 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
6118 auto &RT = CGF.CGM.getOpenMPRuntime();
6124 auto &&EndThenGen = [&CGF, &BasePointersArray, &PointersArray, &SizesArray,
6125 &MapTypesArray, Device,
6127 assert(BasePointersArray && PointersArray && SizesArray && MapTypesArray &&
6128 NumOfPtrs &&
"Invalid data environment closing arguments.");
6135 SizesArrayArg, MapTypesArrayArg,
6136 BasePointersArray, PointersArray, SizesArray,
6137 MapTypesArray, NumOfPtrs);
6142 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6145 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6148 auto *PointerNum = CGF.Builder.getInt32(NumOfPtrs);
6151 DeviceID, PointerNum, BasePointersArrayArg,
6152 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
6153 auto &RT = CGF.CGM.getOpenMPRuntime();
6169 CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data, CodeGen);
6181 const Expr *Device) {
6185 assert((isa<OMPTargetEnterDataDirective>(D) ||
6186 isa<OMPTargetExitDataDirective>(D) ||
6187 isa<OMPTargetUpdateDirective>(D)) &&
6188 "Expecting either target enter, exit data, or update directives.");
6199 MappableExprsHandler MEHandler(D, CGF);
6200 MEHandler.generateAllInfo(BasePointers, Pointers, Sizes, MapTypes);
6209 SizesArrayArg, MapTypesArrayArg, BasePointers,
6210 Pointers, Sizes, MapTypes);
6212 CGF, BasePointersArrayArg, PointersArrayArg, SizesArrayArg,
6213 MapTypesArrayArg, BasePointersArrayArg, PointersArrayArg, SizesArrayArg,
6214 MapTypesArrayArg, BasePointers.size());
6219 DeviceID = CGF.Builder.CreateIntCast(CGF.EmitScalarExpr(Device),
6222 DeviceID = CGF.Builder.getInt32(OMP_DEVICEID_UNDEF);
6225 auto *PointerNum = CGF.Builder.getInt32(BasePointers.size());
6228 DeviceID, PointerNum, BasePointersArrayArg,
6229 PointersArrayArg, SizesArrayArg, MapTypesArrayArg};
6231 auto &RT = CGF.CGM.getOpenMPRuntime();
6237 llvm_unreachable(
"Unexpected standalone target data directive.");
6239 case OMPD_target_enter_data:
6242 case OMPD_target_exit_data:
6245 case OMPD_target_update:
6249 CGF.EmitRuntimeCall(RT.createRuntimeFunction(RTLFn), OffloadingArgs);
6268 struct ParamAttrTy {
6270 llvm::APSInt StrideOrArg;
6271 llvm::APSInt Alignment;
6276 ArrayRef<ParamAttrTy> ParamAttrs) {
6305 if (
auto *MD = dyn_cast<CXXMethodDecl>(FD)) {
6306 if (ParamAttrs[Offset].
Kind == Vector)
6312 if (ParamAttrs[I + Offset].
Kind == Vector) {
6329 llvm::APSInt VLENVal,
6330 ArrayRef<ParamAttrTy> ParamAttrs,
6331 OMPDeclareSimdDeclAttr::BranchStateTy
State) {
6334 unsigned VecRegSize;
6336 ISADataTy ISAData[] = {
6352 case OMPDeclareSimdDeclAttr::BS_Undefined:
6353 Masked.push_back(
'N');
6354 Masked.push_back(
'M');
6356 case OMPDeclareSimdDeclAttr::BS_Notinbranch:
6357 Masked.push_back(
'N');
6359 case OMPDeclareSimdDeclAttr::BS_Inbranch:
6360 Masked.push_back(
'M');
6363 for (
auto Mask : Masked) {
6364 for (
auto &Data : ISAData) {
6366 llvm::raw_svector_ostream Out(Buffer);
6367 Out <<
"_ZGV" << Data.ISA << Mask;
6369 Out << llvm::APSInt::getUnsigned(Data.VecRegSize /
6373 for (
auto &ParamAttr : ParamAttrs) {
6374 switch (ParamAttr.Kind){
6375 case LinearWithVarStride:
6376 Out <<
's' << ParamAttr.StrideOrArg;
6380 if (!!ParamAttr.StrideOrArg)
6381 Out << ParamAttr.StrideOrArg;
6390 if (!!ParamAttr.Alignment)
6391 Out <<
'a' << ParamAttr.Alignment;
6393 Out <<
'_' << Fn->getName();
6394 Fn->addFnAttr(Out.str());
6400 llvm::Function *Fn) {
6404 llvm::DenseMap<const Decl *, unsigned> ParamPositions;
6405 if (isa<CXXMethodDecl>(FD))
6406 ParamPositions.insert({FD, 0});
6407 unsigned ParamPos = ParamPositions.size();
6409 ParamPositions.insert({
P->getCanonicalDecl(), ParamPos});
6412 for (
auto *
Attr : FD->specific_attrs<OMPDeclareSimdDeclAttr>()) {
6415 for (
auto *E :
Attr->uniforms()) {
6418 if (isa<CXXThisExpr>(E))
6419 Pos = ParamPositions[FD];
6421 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
6423 Pos = ParamPositions[PVD];
6425 ParamAttrs[Pos].Kind = Uniform;
6428 auto NI =
Attr->alignments_begin();
6429 for (
auto *E :
Attr->aligneds()) {
6433 if (isa<CXXThisExpr>(E)) {
6434 Pos = ParamPositions[FD];
6437 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
6439 Pos = ParamPositions[PVD];
6440 ParmTy = PVD->getType();
6442 ParamAttrs[Pos].Alignment =
6443 (*NI) ? (*NI)->EvaluateKnownConstInt(C)
6444 : llvm::APSInt::getUnsigned(
6450 auto SI =
Attr->steps_begin();
6451 auto MI =
Attr->modifiers_begin();
6452 for (
auto *E :
Attr->linears()) {
6455 if (isa<CXXThisExpr>(E))
6456 Pos = ParamPositions[FD];
6458 auto *PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(
E)->getDecl())
6460 Pos = ParamPositions[PVD];
6462 auto &ParamAttr = ParamAttrs[Pos];
6463 ParamAttr.Kind = Linear;
6465 if (!(*SI)->EvaluateAsInt(ParamAttr.StrideOrArg, C,
6467 if (
auto *DRE = cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {
6468 if (
auto *StridePVD = cast<ParmVarDecl>(DRE->getDecl())) {
6469 ParamAttr.Kind = LinearWithVarStride;
6470 ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(
6471 ParamPositions[StridePVD->getCanonicalDecl()]);
6479 llvm::APSInt VLENVal;
6480 if (
const Expr *VLEN =
Attr->getSimdlen())
6481 VLENVal = VLEN->EvaluateKnownConstInt(C);
6482 OMPDeclareSimdDeclAttr::BranchStateTy
State =
Attr->getBranchState();
6483 if (CGM.getTriple().getArch() == llvm::Triple::x86 ||
6484 CGM.getTriple().getArch() == llvm::Triple::x86_64)
6493 static const int DoacrossFinArgs = 2;
6502 assert(CallArgs.size() == DoacrossFinArgs);
6503 std::copy(CallArgs.begin(), CallArgs.end(),
std::begin(Args));
6539 enum { LowerFD = 0, UpperFD, StrideFD };
6559 llvm::ConstantInt::getSigned(CGM.Int32Ty, 1),
6561 DimsAddr.getPointer(), CGM.VoidPtrTy)};
6565 llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {
6569 llvm::makeArrayRef(FiniArgs));
6575 CGM.getContext().getIntTypeForBitwidth(64, 1);
6579 CounterVal->
getType(), Int64Ty,
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
void pushTerminate()
Push a terminate handler on the stack.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
FunctionDecl - An instance of this class is created to represent a function declaration or definition...
virtual void emitForDispatchInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, llvm::Value *UB, llvm::Value *Chunk=nullptr)
llvm::IntegerType * IntTy
int
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
External linkage, which indicates that the entity can be referred to from other translation units...
QualType TgtDeviceImageQTy
struct __tgt_device_image{ void *ImageStart; // Pointer to the target code start. ...
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
Destroyer * getDestroyer(QualType::DestructionKind destructionKind)
A (possibly-)qualified type.
llvm::Value * getPointer() const
CodeGenTypes & getTypes()
unsigned getColumn() const
Return the presumed column number of this location.
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
Expr * getNumIterations() const
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
virtual void emitDistributeStaticInit(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDistScheduleClauseKind SchedKind, unsigned IVSize, bool IVSigned, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk=nullptr)
DestructionKind isDestructedType() const
Returns a nonzero value if objects of this type require non-trivial work to clean up after...
llvm::Module & getModule() const
RecordDecl * buildImplicitRecord(StringRef Name, RecordDecl::TagKind TK=TTK_Struct) const
Create a new implicit TU-level CXXRecordDecl or RecordDecl declaration.
virtual bool emitTargetGlobalVariable(GlobalDecl GD)
Emit the global variable if it is a valid device global variable.
llvm::SmallPtrSet< const VarDecl *, 4 > ThreadPrivateWithDefinition
Set of threadprivate variables with the generated initializer.
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
llvm::ConstantInt * getSize(CharUnits N)
CanQualType getSizeType() const
Return the unique type for "size_t" (C99 7.17), defined in <stddef.h>.
virtual void emitUserDefinedReduction(CodeGenFunction *CGF, const OMPDeclareReductionDecl *D)
Emit code for the specified user defined reduction construct.
llvm::AllocaInst * CreateTempAlloca(llvm::Type *Ty, const Twine &Name="tmp")
CreateTempAlloca - This creates a alloca and inserts it into the entry block.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
QualType getTgtBinaryDescriptorQTy()
Returns __tgt_bin_desc type.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
virtual bool emitTargetGlobal(GlobalDecl GD)
Emit the global GD if it is meaningful for the target.
bool isRecordType() const
SmallVector< const Expr *, 4 > LastprivateCopies
specific_attr_iterator - Iterates over a subrange of an AttrVec, only providing attributes that are o...
Address getAddress() const
llvm::Constant * getOrCreateInternalVariable(llvm::Type *Ty, const llvm::Twine &Name)
Gets (if variable with the given name already exist) or creates internal global variable with the spe...
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
VarDecl * getDefinition(ASTContext &)
Get the real (not just tentative) definition for this declaration.
static Address createIdentFieldGEP(CodeGenFunction &CGF, Address Addr, IdentFieldIndex Field, const llvm::Twine &Name="")
Call to void __kmpc_threadprivate_register( ident_t *, void *data, kmpc_ctor ctor, kmpc_cctor cctor, kmpc_dtor dtor);.
static RecordDecl * createPrivatesRecordDecl(CodeGenModule &CGM, ArrayRef< PrivateDataTy > Privates)
RecordDecl * KmpTaskTQTyRD
The base class of the type hierarchy.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
QualType getRecordType(const RecordDecl *Decl) const
std::unique_ptr< llvm::MemoryBuffer > Buffer
virtual void completeDefinition()
completeDefinition - Notes that the definition of this type is now complete.
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
const LangOptions & getLangOpts() const
unsigned getRawEncoding() const
When a SourceLocation itself cannot be used, this returns an (opaque) 32-bit integer encoding for it...
void scanForTargetRegionsFunctions(const Stmt *S, StringRef ParentName)
Start scanning from statement S and and emit all target regions found along the way.
static CharUnits getOffsetOfIdentField(IdentFieldIndex Field)
static void EmitOMPAggregateReduction(CodeGenFunction &CGF, QualType Type, const VarDecl *LHSVar, const VarDecl *RHSVar, const llvm::function_ref< void(CodeGenFunction &CGF, const Expr *, const Expr *, const Expr *)> &RedOpGen, const Expr *XExpr=nullptr, const Expr *EExpr=nullptr, const Expr *UpExpr=nullptr)
Emit reduction operation for each element of array (required for array sections) LHS op = RHS...
llvm::Value * getCriticalRegionLock(StringRef CriticalName)
Returns corresponding lock object for the specified critical region name.
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
OpenMPSchedType
Schedule types for 'omp for' loops (these enumerators are taken from the enum sched_type in kmp...
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
VarDecl - An instance of this class is created to represent a variable declaration or definition...
llvm::Type * getElementType() const
Return the type of the values stored in this address.
Extra information about a function prototype.
field_iterator field_begin() const
VarDecl * getCapturedVar() const
Retrieve the declaration of the variable being captured.
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
CharUnits getNaturalTypeAlignment(QualType T, AlignmentSource *Source=nullptr, bool forPointeeType=false)
llvm::Value * getThreadID(CodeGenFunction &CGF, SourceLocation Loc)
Gets thread id value for the current thread.
virtual bool isDynamic(OpenMPScheduleClauseKind ScheduleKind) const
Check if the specified ScheduleKind is dynamic.
capture_iterator capture_begin()
Retrieve an iterator pointing to the first capture.
void createOffloadEntriesAndInfoMetadata()
Creates all the offload entries in the current compilation unit along with the associated metadata...
static unsigned evaluateCDTSize(const FunctionDecl *FD, ArrayRef< ParamAttrTy > ParamAttrs)
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
OpenMPDirectiveKind getDirectiveKind() const
QualType getTgtOffloadEntryQTy()
Returns __tgt_offload_entry type.
SmallVector< const Expr *, 4 > PrivateVars
RecordDecl - Represents a struct/union/class.
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
Source[4] in Fortran, do not use for C++.
static llvm::Value * emitDestructorsFunction(CodeGenModule &CGM, SourceLocation Loc, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy)
void emitDestroy(Address addr, QualType type, Destroyer *destroyer, bool useEHCleanupForArray)
emitDestroy - Immediately perform the destruction of the given object.
TagDecl * getAsTagDecl() const
Retrieves the TagDecl that this type refers to, either because the type is a TagType or because it is...
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
Call to void *__kmpc_threadprivate_cached(ident_t *loc, kmp_int32 global_tid, void *data...
llvm::IntegerType * Int64Ty
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
virtual void emitFlush(CodeGenFunction &CGF, ArrayRef< const Expr * > Vars, SourceLocation Loc)
Emit flush of the variables specified in 'omp flush' directive.
SmallVector< const Expr *, 4 > LastprivateVars
QualType getReturnType() const
FieldDecl - An instance of this class is created by Sema::ActOnField to represent a member of a struc...
bool isAnyPointerType() const
llvm::IntegerType * SizeTy
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
virtual void emitCancellationPointCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancellation point' construct.
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
llvm::CallInst * EmitRuntimeCall(llvm::Value *callee, const Twine &name="")
void startDefinition()
Starts the definition of this tag declaration.
This represents clause 'map' in the '#pragma omp ...' directives.
static Address emitAddrOfVarFromArray(CodeGenFunction &CGF, Address Array, unsigned Index, const VarDecl *Var)
Given an array of pointers to variables, project the address of a given variable. ...
This represents clause 'to' in the '#pragma omp ...' directives.
static CharUnits Zero()
Zero - Construct a CharUnits quantity of zero.
llvm::CallSite EmitCallOrInvoke(llvm::Value *Callee, ArrayRef< llvm::Value * > Args, const Twine &Name="")
Emits a call or invoke instruction to the given function, depending on the current state of the EH st...
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
llvm::Type * getKmpc_MicroPointerTy()
Returns pointer to kmpc_micro type.
static void emitOMPIfClause(CodeGenFunction &CGF, const Expr *Cond, const RegionCodeGenTy &ThenGen, const RegionCodeGenTy &ElseGen)
Emits code for OpenMP 'if' clause using specified CodeGen function.
Expr * getCounterValue()
Get the loop counter value.
virtual bool emitTargetFunctions(GlobalDecl GD)
Emit the target regions enclosed in GD function definition or the function itself in case it is a val...
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
const Decl * getDecl() const
unsigned size() const
Return number of entries defined so far.
VerifyDiagnosticConsumer::Directive Directive
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
virtual void emitForStaticInit(CodeGenFunction &CGF, SourceLocation Loc, const OpenMPScheduleTy &ScheduleKind, unsigned IVSize, bool IVSigned, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk=nullptr)
Call the appropriate runtime routine to initialize it before start of loop.
static void emitSingleReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp, const Expr *PrivateRef, const DeclRefExpr *LHS, const DeclRefExpr *RHS)
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
QualType TgtOffloadEntryQTy
Type struct __tgt_offload_entry{ void *addr; // Pointer to the offload entry info.
static CharUnits getIdentAlign(CodeGenModule &CGM)
OpenMPScheduleClauseKind Schedule
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
const Qualifiers & getQuals() const
bool isLocalVarDeclOrParm() const
Similar to isLocalVarDecl but also includes parameters.
RAII for correct setting/restoring of CapturedStmtInfo.
llvm::PointerType * VoidPtrTy
String describing the source location.
virtual std::pair< llvm::Function *, llvm::Function * > getUserDefinedReduction(const OMPDeclareReductionDecl *D)
Get combiner/initializer for the specified user-defined reduction, if any.
void actOnTargetRegionEntriesInfo(const OffloadTargetRegionEntryInfoActTy &Action)
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
OpenMPScheduleClauseModifier M2
bool needsEHCleanup(QualType::DestructionKind kind)
Determines whether an EH cleanup is required to destroy a type with the given destruction kind...
SmallVector< const Expr *, 4 > PrivateCopies
RecordDecl * getDecl() const
CharUnits getPointerSize() const
llvm::CallInst * CreateMemCpy(Address Dest, Address Src, llvm::Value *Size, bool IsVolatile=false)
void EmitAggregateAssign(Address DestPtr, Address SrcPtr, QualType EltTy)
EmitAggregateCopy - Emit an aggregate assignment.
llvm::Type * getIdentTyPointerTy()
Returns pointer to ident_t type.
Scope - A scope is a transient data structure that is used while parsing the program.
virtual void emitDeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn)
Marks function Fn with properly mangled versions of vector functions.
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
SmallVector< const Expr *, 4 > FirstprivateCopies
static void emitPrivatesInit(CodeGenFunction &CGF, const OMPExecutableDirective &D, Address KmpTaskSharedsPtr, LValue TDBase, const RecordDecl *KmpTaskTWithPrivatesQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool ForDup)
Emit initialization for private variables in task-based directives.
static int addMonoNonMonoModifier(OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2)
static llvm::Function * emitCombinerOrInitializer(CodeGenModule &CGM, QualType Ty, const Expr *CombinerInitializer, const VarDecl *In, const VarDecl *Out, bool IsCombiner)
Support for OpenMP 4.5 monotonic and nonmonotonic schedule modifiers.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
unsigned getLine() const
Return the presumed line number of this location.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
detail::InMemoryDirectory::const_iterator I
static llvm::Value * emitTaskPrivateMappingFunction(CodeGenModule &CGM, SourceLocation Loc, ArrayRef< const Expr * > PrivateVars, ArrayRef< const Expr * > FirstprivateVars, ArrayRef< const Expr * > LastprivateVars, QualType PrivatesQTy, ArrayRef< PrivateDataTy > Privates)
Emit a privates mapping function for correct handling of private and firstprivate variables...
virtual void emitReduction(CodeGenFunction &CGF, SourceLocation Loc, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps, bool WithNowait, bool SimpleReduction)
Emit a code for reduction clause.
This represents clause 'from' in the '#pragma omp ...' directives.
virtual llvm::Value * emitForNext(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned, Address IL, Address LB, Address UB, Address ST)
Call __kmpc_dispatch_next( ident_t *loc, kmp_int32 tid, kmp_int32 *p_lastiter, kmp_int[32|64] *p_lowe...
LValue EmitLValueForField(LValue Base, const FieldDecl *Field)
virtual llvm::Value * emitTaskOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, const VarDecl *PartIDVar, const VarDecl *TaskTVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool Tied, unsigned &NumberOfParts)
Emits outlined function for the OpenMP task directive D.
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
OpenMPDependClauseKind getDependencyKind() const
Get dependency type.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource AlignSource=AlignmentSource::Type)
OpenMP 4.0 [2.4, Array Sections].
virtual void emitOrderedRegion(CodeGenFunction &CGF, const RegionCodeGenTy &OrderedOpGen, SourceLocation Loc, bool IsThreads)
Emit an ordered region.
CanQualType getCanonicalTypeUnqualified() const
Describes the capture of either a variable, or 'this', or variable-length array type.
llvm::Constant * createForStaticInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_for_static_init_* runtime function for the specified size IVSize and sign IVSigned...
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
OMP_IDENT_xxx flags; OMP_IDENT_KMPC identifies this union member.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
virtual void emitBarrierCall(CodeGenFunction &CGF, SourceLocation Loc, OpenMPDirectiveKind Kind, bool EmitChecks=true, bool ForceSimpleCall=false)
Emit an implicit/explicit barrier for OpenMP threads.
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
llvm::Constant * createDispatchFiniFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_fini_* runtime function for the specified size IVSize and sign IVSigned...
Expr * getCombiner()
Get combiner expression of the declare reduction construct.
static llvm::Value * emitThreadLimitClauseForTargetDirective(CGOpenMPRuntime &OMPRuntime, CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the thread_limit clause of an enclosed teams directive at the target region scope.
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
QualType getTgtDeviceImageQTy()
Returns __tgt_device_image type.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
KmpTaskTFields
Indexes of fields for type kmp_task_t.
llvm::Value * getPointer() const
static llvm::Value * emitNumTeamsClauseForTargetDirective(CGOpenMPRuntime &OMPRuntime, CodeGenFunction &CGF, const OMPExecutableDirective &D)
Emit the num_teams clause of an enclosed teams directive at the target region scope.
ValueDecl - Represent the declaration of a variable (in which case it is an lvalue) a function (in wh...
Expr - This represents one expression.
StringRef getName() const
Return the actual identifier string.
Allow any unmodeled side effect.
virtual void emitDoacrossOrdered(CodeGenFunction &CGF, const OMPDependClause *C)
Emit code for doacross ordered directive with 'depend' clause.
void loadOffloadInfoMetadata()
Loads all the offload entries information from the host IR metadata.
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
void EmitStoreOfScalar(llvm::Value *Value, Address Addr, bool Volatile, QualType Ty, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, bool isInit=false, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitStoreOfScalar - Store a scalar value to an address, taking care to appropriately convert from the...
static llvm::Value * emitProxyTaskFunction(CodeGenModule &CGM, SourceLocation Loc, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpTaskTWithPrivatesPtrQTy, QualType KmpTaskTWithPrivatesQTy, QualType KmpTaskTQTy, QualType SharedsPtrTy, llvm::Value *TaskFunction, llvm::Value *TaskPrivatesMap)
Emit a proxy function which accepts kmp_task_t as the second argument.
static llvm::Function * createOffloadingBinaryDescriptorFunction(CodeGenModule &CGM, StringRef Name, const RegionCodeGenTy &Codegen)
Create a Ctor/Dtor-like function whose body is emitted through Codegen.
static void emitForStaticInitCall(CodeGenFunction &CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId, llvm::Constant *ForStaticInitFunction, OpenMPSchedType Schedule, OpenMPScheduleClauseModifier M1, OpenMPScheduleClauseModifier M2, unsigned IVSize, bool Ordered, Address IL, Address LB, Address UB, Address ST, llvm::Value *Chunk)
void emitKmpRoutineEntryT(QualType KmpInt32Ty)
Build type kmp_routine_entry_t (if not built yet).
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
static bool checkDestructorsRequired(const RecordDecl *KmpTaskTWithPrivatesQTyRD)
Checks if destructor function is required to be generated.
static RTCancelKind getCancellationKind(OpenMPDirectiveKind CancelRegion)
static RecordDecl * createKmpTaskTRecordDecl(CodeGenModule &CGM, OpenMPDirectiveKind Kind, QualType KmpInt32Ty, QualType KmpRoutineEntryPointerQTy)
const ParmVarDecl * getParamDecl(unsigned i) const
ASTContext & getContext() const
llvm::BasicBlock * getBlock() const
static CharUnits fromQuantity(QuantityType Quantity)
fromQuantity - Construct a CharUnits quantity from a raw integer type.
virtual void emitCriticalRegion(CodeGenFunction &CGF, StringRef CriticalName, const RegionCodeGenTy &CriticalOpGen, SourceLocation Loc, const Expr *Hint=nullptr)
Emits a critical region.
static OpenMPSchedType getRuntimeSchedule(OpenMPScheduleClauseKind ScheduleKind, bool Chunked, bool Ordered)
Map the OpenMP loop schedule to the runtime enumeration.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
bool capturesVariableArrayType() const
Determine whether this capture handles a variable-length array type.
llvm::IntegerType * Int32Ty
static void emitOffloadingArrays(CodeGenFunction &CGF, llvm::Value *&BasePointersArray, llvm::Value *&PointersArray, llvm::Value *&SizesArray, llvm::Value *&MapTypesArray, MappableExprsHandler::MapValuesArrayTy &BasePointers, MappableExprsHandler::MapValuesArrayTy &Pointers, MappableExprsHandler::MapValuesArrayTy &Sizes, MappableExprsHandler::MapFlagsArrayTy &MapTypes)
Emit the arrays used to pass the captures and map information to the offloading runtime library...
static llvm::Value * emitReductionFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > Privates, ArrayRef< const Expr * > LHSExprs, ArrayRef< const Expr * > RHSExprs, ArrayRef< const Expr * > ReductionOps)
static OMPLinearClause * CreateEmpty(const ASTContext &C, unsigned NumVars)
Creates an empty clause with the place for NumVars variables.
SmallVector< const Expr *, 4 > FirstprivateVars
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Represents an unpacked "presumed" location which can be presented to the user.
void Emit(CodeGenFunction &CGF, Flags) override
Emit the cleanup.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
CGOpenMPRuntime(CodeGenModule &CGM)
The result type of a method or function.
OpenMPProcBindClauseKind
OpenMP attributes for 'proc_bind' clause.
virtual llvm::Function * emitRegistrationFunction()
Creates the offloading descriptor in the event any target region was emitted in the current module an...
llvm::Constant * createDispatchInitFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_init_* runtime function for the specified size IVSize and sign IVSigned...
virtual void emitTaskwaitCall(CodeGenFunction &CGF, SourceLocation Loc)
Emit code for 'taskwait' directive.
VarDecl * getCanonicalDecl() override
GlobalDecl - represents a global declaration.
const Expr * getAnyInitializer() const
getAnyInitializer - Get the initializer for this variable, no matter which declaration it is attached...
bool hasTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum) const
Return true if a target region entry with the provided information exists.
SourceLocation getLocStart() const
Returns starting location of directive kind.
QualType KmpDependInfoTy
Type typedef struct kmp_depend_info { kmp_intptr_t base_addr; size_t len; struct { bool in:1; bool ou...
virtual void emitTargetDataCalls(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device, const RegionCodeGenTy &CodeGen)
Emit the target data mapping code associated with D.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
QualType TgtBinaryDescriptorQTy
struct __tgt_bin_desc{ int32_t NumDevices; // Number of devices supported.
static void emitReductionCombiner(CodeGenFunction &CGF, const Expr *ReductionOp)
Emit reduction combiner.
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
Address CreateBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
static llvm::Value * emitCopyprivateCopyFunction(CodeGenModule &CGM, llvm::Type *ArgsType, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
This captures a statement into a function.
llvm::Value * emitUpdateLocation(CodeGenFunction &CGF, SourceLocation Loc, unsigned Flags=0)
Emits object of ident_t type with info for source location.
const char * getFilename() const
Return the presumed filename of this location.
static with chunk adjustment (e.g., simd)
ASTContext & getContext() const
void pushDestroy(QualType::DestructionKind dtorKind, Address addr, QualType type)
pushDestroy - Push the standard destructor for the given type as at least a normal cleanup...
void functionFinished(CodeGenFunction &CGF)
Cleans up references to the objects in finished function.
Encodes a location in the source.
CharUnits getPointerAlign() const
unsigned getNumParams() const
getNumParams - Return the number of parameters this function must have based on its FunctionType...
const Type * getTypePtr() const
Retrieves a pointer to the underlying (unqualified) type.
This represents '#pragma omp declare reduction ...' directive.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
virtual void emitNumTeamsClause(CodeGenFunction &CGF, const Expr *NumTeams, const Expr *ThreadLimit, SourceLocation Loc)
Emits call to void __kmpc_push_num_teams(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_teams...
This is a basic class for representing single OpenMP executable directive.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param IfCond Not a nullptr if if clause was nullptr *otherwise *param Data Additional data for task generation like final list of privates etc *virtual void emitTaskLoopCall(CodeGenFunction &CGF, SourceLocation Loc, const OMPLoopDirective &D, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const Expr *IfCond, const OMPTaskDataTy &Data)
bool isValid() const
Return true if this is a valid SourceLocation object.
Expr * getLowerBoundVariable() const
Lower bound for 'ordered' versions.
TypeSourceInfo * getTrivialTypeSourceInfo(QualType T, SourceLocation Loc=SourceLocation()) const
Allocate a TypeSourceInfo where all locations have been initialized to a given location, which defaults to the empty location.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
bool capturesVariableByCopy() const
Determine whether this capture handles a variable by copy.
OpenMPDirectiveKind
OpenMP directives.
Set if the nonmonotonic schedule modifier was present.
virtual void emitTaskyieldCall(CodeGenFunction &CGF, SourceLocation Loc)
Emits code for a taskyield directive.
OpenMPLinearClauseKind Modifier
Modifier of 'linear' clause.
ArrayRef< ParmVarDecl * > parameters() const
Target region entries related.
bool capturesVariable() const
Determine whether this capture handles a variable (by reference).
static bool classof(const EHScope *Scope)
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
virtual void emitTargetDataStandAloneCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, const Expr *IfCond, const Expr *Device)
Emit the data mapping/movement code associated with the directive D that should be of the form 'targe...
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
const Type * getBaseElementTypeUnsafe() const
Get the base element type of this type, potentially discarding type qualifiers.
virtual void Enter(CodeGenFunction &CGF)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
capture_iterator capture_end() const
Retrieve an iterator pointing past the end of the sequence of captures.
Stmt * getCapturedStmt()
Retrieve the statement being captured.
void registerTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, llvm::Constant *Addr, llvm::Constant *ID)
Register target region entry.
void setAction(PrePostActionTy &Action) const
const T * castAs() const
Member-template castAs<specific type>.
lookup_result lookup(DeclarationName Name) const
lookup - Find the declarations (if any) with the given Name in this context.
Expr * getUpperBoundVariable() const
void emitThreadPrivateVarInit(CodeGenFunction &CGF, Address VDAddr, llvm::Value *Ctor, llvm::Value *CopyCtor, llvm::Value *Dtor, SourceLocation Loc)
Emits initialization code for the threadprivate variables.
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
llvm::Value * NewTaskNewTaskTTy
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
QualType getPointeeType() const
i32 captured_struct **param SharedsTy A type which contains references the shared variables *param Shareds Context with the list of shared variables from the p *TaskFunction *param Data Additional data for task generation like final list of privates etc *TaskResultTy emitTaskInit(CodeGenFunction &CGF, SourceLocation Loc, const OMPExecutableDirective &D, llvm::Value *TaskFunction, QualType SharedsTy, Address Shareds, const OMPTaskDataTy &Data)
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual Address emitThreadIDAddress(CodeGenFunction &CGF, SourceLocation Loc)
Emits address of the word in a memory where current thread id is stored.
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...
CharUnits getAlignment() const
Return the alignment of this pointer.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
DeclContext - This is used only as base class of specific decl types that can act as declaration cont...
virtual void emitForStaticFinish(CodeGenFunction &CGF, SourceLocation Loc)
Call the appropriate runtime routine to notify that we finished all the work with current loop...
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
OpenMPLocationFlags
Values for bit flags used in the ident_t to describe the fields.
Address CreateStructGEP(Address Addr, unsigned Index, CharUnits Offset, const llvm::Twine &Name="")
OpenMPScheduleClauseModifier M1
bool capturesThis() const
Determine whether this capture handles the C++ 'this' pointer.
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
virtual void emitMasterRegion(CodeGenFunction &CGF, const RegionCodeGenTy &MasterOpGen, SourceLocation Loc)
Emits a master region.
void initializeTargetRegionEntryInfo(unsigned DeviceID, unsigned FileID, StringRef ParentName, unsigned LineNum, unsigned Order)
Initialize target region entry.
std::unique_ptr< DiagnosticConsumer > create(StringRef OutputFile, DiagnosticOptions *Diags, bool MergeChildRecords=false)
Returns a DiagnosticConsumer that serializes diagnostics to a bitcode file.
llvm::Function * createOffloadingBinaryDescriptorRegistration()
Creates and registers offloading binary descriptor for the current compilation unit.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::IntegerType * IntPtrTy
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
virtual void emitTargetCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, llvm::Value *OutlinedFn, llvm::Value *OutlinedFnID, const Expr *IfCond, const Expr *Device, ArrayRef< llvm::Value * > CapturedVars)
Emit the target offloading code associated with D.
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
bool empty() const
Return true if a there are no entries defined.
detail::InMemoryDirectory::const_iterator E
void EmitAggregateCopy(Address DestPtr, Address SrcPtr, QualType EltTy, bool isVolatile=false, bool isAssignment=false)
EmitAggregateCopy - Emit an aggregate copy.
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
virtual llvm::Function * emitThreadPrivateVarDefinition(const VarDecl *VD, Address VDAddr, SourceLocation Loc, bool PerformInit, CodeGenFunction *CGF=nullptr)
Emit a code for initialization of threadprivate variable.
Call to void __kmpc_fork_call(ident_t *loc, kmp_int32 argc, kmpc_micro microtask, ...
Not really used in Fortran any more.
virtual void emitTargetOutlinedFunction(const OMPExecutableDirective &D, StringRef ParentName, llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID, bool IsOffloadEntry, const RegionCodeGenTy &CodeGen)
Emit outilined function for 'target' directive.
unsigned Map[Count]
The type of a lookup table which maps from language-specific address spaces to target-specific ones...
const RecordType * getAsStructureType() const
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
const VariableArrayType * getAsVariableArrayType(QualType T) const
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
static CharUnits getIdentSize(CodeGenModule &CGM)
JumpDest ReturnBlock
ReturnBlock - Unified return block.
OffloadEntriesInfoManagerTy OffloadEntriesInfoManager
API for captured statement code generation.
llvm::PointerType * getType() const
Return the type of the pointer value.
static bool classof(const OMPClause *T)
const T * getAs() const
Member-template getAs<specific type>'.
QualType getCanonicalType() const
This file defines OpenMP AST classes for executable directives and clauses.
static FieldDecl * addFieldToRecordDecl(ASTContext &C, DeclContext *DC, QualType FieldTy)
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T)
Address CreateConstArrayGEP(Address Addr, uint64_t Index, CharUnits EltSize, const llvm::Twine &Name="")
Given addr = [n x T]* ...
CleanupTy(PrePostActionTy *Action)
static bool checkInitIsRequired(CodeGenFunction &CGF, ArrayRef< PrivateDataTy > Privates)
Check if duplication function is required for taskloops.
static void getTargetEntryUniqueInfo(ASTContext &C, SourceLocation Loc, unsigned &DeviceID, unsigned &FileID, unsigned &LineNum)
Obtain information that uniquely identifies a target entry.
llvm::PointerType * Int8PtrTy
llvm::AssertingVH< llvm::Instruction > AllocaInsertPt
AllocaInsertPoint - This is an instruction in the entry block before which we prefer to insert alloca...
virtual llvm::Value * emitParallelOrTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
virtual void emitInlinedDirective(CodeGenFunction &CGF, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen, bool HasCancel=false)
Emit code for the directive that does not require outlining.
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
Base for LValueReferenceType and RValueReferenceType.
CXXRecordDecl * getAsCXXRecordDecl() const
Retrieves the CXXRecordDecl that this type refers to, either because the type is a RecordType or beca...
QualType withRestrict() const
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void addDecl(Decl *D)
Add the declaration D into this context.
FieldDecl * LambdaThisCaptureField
virtual bool isStaticNonchunked(OpenMPScheduleClauseKind ScheduleKind, bool Chunked) const
Check if the specified ScheduleKind is static non-chunked.
llvm::Constant * createDispatchNextFunction(unsigned IVSize, bool IVSigned)
Returns __kmpc_dispatch_next_* runtime function for the specified size IVSize and sign IVSigned...
static void emitOffloadingArraysArgument(CodeGenFunction &CGF, llvm::Value *&BasePointersArrayArg, llvm::Value *&PointersArrayArg, llvm::Value *&SizesArrayArg, llvm::Value *&MapTypesArrayArg, llvm::Value *BasePointersArray, llvm::Value *PointersArray, llvm::Value *SizesArray, llvm::Value *MapTypesArray, unsigned NumElems)
Emit the arguments to be passed to the runtime library based on the arrays of pointers, sizes and map types.
QualType getPointeeType() const
SourceManager & getSourceManager()
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
virtual void emitCancelCall(CodeGenFunction &CGF, SourceLocation Loc, const Expr *IfCond, OpenMPDirectiveKind CancelRegion)
Emit code for 'cancel' construct.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
Lower bound for default (unordered) versions.
virtual void emitTeamsCall(CodeGenFunction &CGF, const OMPExecutableDirective &D, SourceLocation Loc, llvm::Value *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars)
Emits code for teams call of the OutlinedFn with variables captured in a record which address is stor...
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
llvm::Function * CreateGlobalInitOrDestructFunction(llvm::FunctionType *ty, const Twine &name, const CGFunctionInfo &FI, SourceLocation Loc=SourceLocation(), bool TLS=false)
static QualType getBaseOriginalType(const Expr *Base)
Return original type of the base expression for array section.
OpenMPOffloadingReservedDeviceIDs
llvm::Type * ConvertType(QualType T)
Privates[]
Gets the list of initial values for linear variables.
OpenMPMapClauseKind
OpenMP mapping kind for 'map' clause.
static const Stmt * ignoreCompoundStmts(const Stmt *Body)
discard all CompoundStmts intervening between two constructs
virtual void emitForOrderedIterationEnd(CodeGenFunction &CGF, SourceLocation Loc, unsigned IVSize, bool IVSigned)
Call the appropriate runtime routine to notify that we finished iteration of the ordered loop with th...
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
Address ReturnValue
ReturnValue - The temporary alloca to hold the return value.
void popTerminate()
Pops a terminate handler off the stack.
virtual void emitNumThreadsClause(CodeGenFunction &CGF, llvm::Value *NumThreads, SourceLocation Loc)
Emits call to void __kmpc_push_num_threads(ident_t *loc, kmp_int32 global_tid, kmp_int32 num_threads)...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
static llvm::Value * emitTaskDupFunction(CodeGenModule &CGM, SourceLocation Loc, const OMPExecutableDirective &D, QualType KmpTaskTWithPrivatesPtrQTy, const RecordDecl *KmpTaskTWithPrivatesQTyRD, const RecordDecl *KmpTaskTQTyRD, QualType SharedsTy, QualType SharedsPtrTy, const OMPTaskDataTy &Data, ArrayRef< PrivateDataTy > Privates, bool WithLastIter)
Emit task_dup function (for initialization of private/firstprivate/lastprivate vars and last_iter fla...
std::pair< llvm::Value *, QualType > getVLASize(const VariableArrayType *vla)
getVLASize - Returns an LLVM value that corresponds to the size, in non-variably-sized elements...
virtual Address getAddrOfThreadPrivate(CodeGenFunction &CGF, const VarDecl *VD, Address VDAddr, SourceLocation Loc)
Returns address of the threadprivate variable for the current thread.
static RecordDecl * createKmpTaskTWithPrivatesRecordDecl(CodeGenModule &CGM, QualType KmpTaskTQTy, ArrayRef< PrivateDataTy > Privates)
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
A reference to a declared variable, function, enum, etc.
virtual void emitProcBindClause(CodeGenFunction &CGF, OpenMPProcBindClauseKind ProcBind, SourceLocation Loc)
Emit call to void __kmpc_push_proc_bind(ident_t *loc, kmp_int32 global_tid, int proc_bind) to generat...
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
ParamKindTy
Kind of parameter in a function with 'declare simd' directive.
virtual void emitDoacrossInit(CodeGenFunction &CGF, const OMPLoopDirective &D)
Emit initialization for doacross loop nesting support.
QualType KmpDimTy
struct kmp_dim { // loop bounds info casted to kmp_int64 kmp_int64 lo; // lower kmp_int64 up; // uppe...
static int array_pod_sort_comparator(const PrivateDataTy *P1, const PrivateDataTy *P2)
An l-value expression is a reference to an object with independent storage.
LValue - This represents an lvalue references.
FunctionDecl * getCanonicalDecl() override
Information for lazily generating a cleanup.
Directive - Abstract class representing a parsed verify directive.
llvm::Constant * createRuntimeFunction(unsigned Function)
Returns specified OpenMP runtime function.
SourceLocation getLocEnd() const
Returns ending location of directive.
bool isNull() const
Return true if this QualType doesn't point to a type yet.
static FieldDecl * Create(const ASTContext &C, DeclContext *DC, SourceLocation StartLoc, SourceLocation IdLoc, IdentifierInfo *Id, QualType T, TypeSourceInfo *TInfo, Expr *BW, bool Mutable, InClassInitStyle InitStyle)
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
virtual void emitTaskgroupRegion(CodeGenFunction &CGF, const RegionCodeGenTy &TaskgroupOpGen, SourceLocation Loc)
Emit a taskgroup region.
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
void operator()(CodeGenFunction &CGF) const
static void emitX86DeclareSimdFunction(const FunctionDecl *FD, llvm::Function *Fn, llvm::APSInt VLENVal, ArrayRef< ParamAttrTy > ParamAttrs, OMPDeclareSimdDeclAttr::BranchStateTy State)
virtual void emitSingleRegion(CodeGenFunction &CGF, const RegionCodeGenTy &SingleOpGen, SourceLocation Loc, ArrayRef< const Expr * > CopyprivateVars, ArrayRef< const Expr * > DestExprs, ArrayRef< const Expr * > SrcExprs, ArrayRef< const Expr * > AssignmentOps)
Emits a single region.
A class which abstracts out some details necessary for making a call.
Attr - This represents one attribute.
SmallVector< const Expr *, 4 > FirstprivateInits
void EmitNullInitialization(Address DestPtr, QualType Ty)
EmitNullInitialization - Generate code to set a value of the given type to null, If the type contains...
llvm::Constant * getOrCreateThreadPrivateCache(const VarDecl *VD)
If the specified mangled name is not in the module, create and return threadprivate cache object...
Expr * getStrideVariable() const
bool Privatize()
Privatizes local variables previously registered as private.
bool isPointerType() const
virtual void emitParallelCall(CodeGenFunction &CGF, SourceLocation Loc, llvm::Value *OutlinedFn, ArrayRef< llvm::Value * > CapturedVars, const Expr *IfCond)
Emits code for parallel or serial call of the OutlinedFn with variables captured in a record which ad...
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
PresumedLoc getPresumedLoc(SourceLocation Loc, bool UseLineDirectives=true) const
Returns the "presumed" location of a SourceLocation specifies.