22 #include "llvm/IR/CallSite.h"
23 using namespace clang;
24 using namespace CodeGen;
31 for (
const auto *C : S.
clauses()) {
33 if (
auto *PreInit = cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
34 for (
const auto *
I : PreInit->decls()) {
35 if (!
I->hasAttr<OMPCaptureNoInitAttr>())
57 bool AsInlined =
false)
60 emitPreInitStmt(CGF, S);
64 for (
auto &C : CS->captures()) {
65 if (
C.capturesVariable() ||
C.capturesVariableByCopy()) {
66 auto *VD =
C.getCapturedVar();
68 isCapturedVar(CGF, VD) ||
70 InlinedShareds.isGlobalVarCaptured(VD)),
73 InlinedShareds.addPrivate(VD, [&CGF, &DRE]() ->
Address {
78 (void)InlinedShareds.Privatize();
88 if (
auto *LD = dyn_cast<OMPLoopDirective>(&S)) {
89 if (
auto *PreInits = cast_or_null<DeclStmt>(LD->getPreInits())) {
90 for (
const auto *
I : PreInits->decls())
99 emitPreInitStmt(CGF, S);
108 auto SizeInChars = C.getTypeSizeInChars(Ty);
109 if (SizeInChars.isZero()) {
111 while (
auto *VAT = C.getAsVariableArrayType(Ty)) {
114 Size = Size ?
Builder.CreateNUWMul(Size, ArraySize) : ArraySize;
116 SizeInChars = C.getTypeSizeInChars(Ty);
117 if (SizeInChars.isZero())
118 return llvm::ConstantInt::get(
SizeTy, 0);
132 I !=
E; ++
I, ++CurField, ++CurCap) {
133 if (CurField->hasCapturedVLAType()) {
134 auto VAT = CurField->getCapturedVLAType();
135 auto *Val = VLASizeMap[VAT->getSizeExpr()];
136 CapturedVars.push_back(Val);
137 }
else if (CurCap->capturesThis())
138 CapturedVars.push_back(CXXThisValue);
139 else if (CurCap->capturesVariableByCopy()) {
145 if (!CurField->getType()->isAnyPointerType()) {
148 Ctx.getUIntPtrType(),
149 Twine(CurCap->getCapturedVar()->getName()) +
".casted");
153 DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
164 CapturedVars.push_back(CV);
166 assert(CurCap->capturesVariable() &&
"Expected capture by reference.");
174 bool isReferenceType =
false) {
186 if (isReferenceType) {
188 auto *RefVal = TmpAddr.getPointer();
201 "CapturedStmtInfo should be set when generating the captured function");
204 assert(CD->hasBody() &&
"missing CapturedDecl body");
212 for (
auto *FD : RD->
fields()) {
223 I->capturesVariableArrayType())
224 ArgType = Ctx.getUIntPtrType();
226 if (
I->capturesVariable() ||
I->capturesVariableByCopy()) {
227 CapVar =
I->getCapturedVar();
229 }
else if (
I->capturesThis())
232 assert(
I->capturesVariableArrayType());
238 FD->getLocation(), II, ArgType));
256 F->addFnAttr(llvm::Attribute::NoUnwind);
259 StartFunction(CD, Ctx.VoidTy, F, FuncInfo, Args, CD->getLocation(),
263 for (
auto *FD : RD->
fields()) {
266 if (
I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
276 if (FD->hasCapturedVLAType()) {
279 Args[Cnt]->getName(), ArgLVal),
283 auto VAT = FD->getCapturedVLAType();
284 VLASizeMap[VAT->getSizeExpr()] = ExprArg;
285 }
else if (
I->capturesVariable()) {
286 auto *Var =
I->getCapturedVar();
295 }
else if (
I->capturesVariableByCopy()) {
296 assert(!FD->getType()->isAnyPointerType() &&
297 "Not expecting a captured pointer.");
298 auto *Var =
I->getCapturedVar();
301 Args[Cnt]->getName(), ArgLVal,
305 assert(
I->capturesThis());
337 auto DestEnd =
Builder.CreateGEP(DestBegin, NumElements);
342 Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arraycpy.isempty");
343 Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
346 auto EntryBB =
Builder.GetInsertBlock();
351 llvm::PHINode *SrcElementPHI =
352 Builder.CreatePHI(SrcBegin->getType(), 2,
"omp.arraycpy.srcElementPast");
353 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
358 llvm::PHINode *DestElementPHI =
359 Builder.CreatePHI(DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
360 DestElementPHI->addIncoming(DestBegin, EntryBB);
366 CopyGen(DestElementCurrent, SrcElementCurrent);
369 auto DestElementNext =
Builder.CreateConstGEP1_32(
370 DestElementPHI, 1,
"omp.arraycpy.dest.element");
371 auto SrcElementNext =
Builder.CreateConstGEP1_32(
372 SrcElementPHI, 1,
"omp.arraycpy.src.element");
375 Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
376 Builder.CreateCondBr(Done, DoneBB, BodyBB);
377 DestElementPHI->addIncoming(DestElementNext,
Builder.GetInsertBlock());
378 SrcElementPHI->addIncoming(SrcElementNext,
Builder.GetInsertBlock());
388 if (
auto *CE = dyn_cast<CallExpr>(ReductionOp))
389 if (
auto *OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))
391 dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))
392 if (
auto *DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))
403 std::pair<llvm::Function *, llvm::Function *> Reduction =
405 auto *CE = cast<CallExpr>(InitOp);
406 auto *OVE = cast<OpaqueValueExpr>(CE->getCallee());
409 auto *LHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());
410 auto *RHSDRE = cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());
412 PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),
413 [=]() ->
Address {
return Private; });
414 PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()),
415 [=]() ->
Address {
return Original; });
416 (void)PrivateScope.Privatize();
422 auto *GV =
new llvm::GlobalVariable(
424 llvm::GlobalValue::PrivateLinkage, Init,
".init");
472 auto DestEnd = CGF.
Builder.CreateGEP(DestBegin, NumElements);
477 CGF.
Builder.CreateICmpEQ(DestBegin, DestEnd,
"omp.arrayinit.isempty");
478 CGF.
Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
481 auto EntryBB = CGF.
Builder.GetInsertBlock();
486 llvm::PHINode *SrcElementPHI =
nullptr;
489 SrcElementPHI = CGF.
Builder.CreatePHI(SrcBegin->getType(), 2,
490 "omp.arraycpy.srcElementPast");
491 SrcElementPHI->addIncoming(SrcBegin, EntryBB);
496 llvm::PHINode *DestElementPHI = CGF.
Builder.CreatePHI(
497 DestBegin->getType(), 2,
"omp.arraycpy.destElementPast");
498 DestElementPHI->addIncoming(DestBegin, EntryBB);
506 if (DRD && (DRD->getInitializer() || !Init)) {
508 SrcElementCurrent, ElementTy);
516 auto SrcElementNext = CGF.
Builder.CreateConstGEP1_32(
517 SrcElementPHI, 1,
"omp.arraycpy.dest.element");
518 SrcElementPHI->addIncoming(SrcElementNext, CGF.
Builder.GetInsertBlock());
522 auto DestElementNext = CGF.
Builder.CreateConstGEP1_32(
523 DestElementPHI, 1,
"omp.arraycpy.dest.element");
526 CGF.
Builder.CreateICmpEQ(DestElementNext, DestEnd,
"omp.arraycpy.done");
527 CGF.
Builder.CreateCondBr(Done, DoneBB, BodyBB);
528 DestElementPHI->addIncoming(DestElementNext, CGF.
Builder.GetInsertBlock());
539 if (BO && BO->getOpcode() == BO_Assign) {
546 DestAddr, SrcAddr, OriginalType,
547 [
this, Copy, SrcVD, DestVD](
Address DestElement,
Address SrcElement) {
556 SrcVD, [SrcElement]() ->
Address {
return SrcElement; });
576 bool FirstprivateIsLastprivate =
false;
579 for (
const auto *D : C->varlists())
586 auto IRef = C->varlist_begin();
587 auto InitsRef = C->inits().begin();
588 for (
auto IInit : C->private_copies()) {
589 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
590 bool ThisFirstprivateIsLastprivate =
591 Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
592 auto *CapFD = CapturesInfo.
lookup(OrigVD);
594 if (!ThisFirstprivateIsLastprivate && FD && (FD == CapFD) &&
595 !FD->getType()->isReferenceType()) {
596 EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
601 FirstprivateIsLastprivate =
602 FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
603 if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
604 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
605 auto *VDInit = cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
609 (*IRef)->getType(),
VK_LValue, (*IRef)->getExprLoc());
625 Emission.getAllocatedAddress(), OriginalAddr,
Type,
626 [
this, VDInit, Init](
Address DestElement,
631 setAddrOfLocalVar(VDInit, SrcElement);
635 LocalDeclMap.erase(VDInit);
639 return Emission.getAllocatedAddress();
647 setAddrOfLocalVar(VDInit, OriginalAddr);
649 LocalDeclMap.erase(VDInit);
653 assert(IsRegistered &&
654 "firstprivate var already registered as private");
662 return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
672 auto IRef = C->varlist_begin();
673 for (
auto IInit : C->private_copies()) {
674 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
675 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
676 auto VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
683 assert(IsRegistered &&
"private var already registered as private");
700 llvm::BasicBlock *CopyBegin =
nullptr, *CopyEnd =
nullptr;
702 auto IRef = C->varlist_begin();
703 auto ISrcRef = C->source_exprs().begin();
704 auto IDestRef = C->destination_exprs().begin();
705 for (
auto *AssignOp : C->assignment_ops()) {
706 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
714 getContext().getTargetInfo().isTLSSupported()) {
716 "Copyin threadprivates should have been captured!");
717 DeclRefExpr DRE(const_cast<VarDecl *>(VD),
true, (*IRef)->getType(),
720 LocalDeclMap.erase(VD);
729 if (CopiedVars.size() == 1) {
741 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
742 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
743 EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
762 bool HasAtLeastOneLastprivate =
false;
765 auto *LoopDirective = cast<OMPLoopDirective>(&D);
766 for (
auto *C : LoopDirective->counters()) {
773 HasAtLeastOneLastprivate =
true;
776 auto IRef = C->varlist_begin();
777 auto IDestRef = C->destination_exprs().begin();
778 for (
auto *IInit : C->private_copies()) {
781 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
784 if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
785 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
788 const_cast<VarDecl *>(OrigVD),
797 if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
798 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
804 assert(IsRegistered &&
805 "lastprivate var already registered as private");
813 return HasAtLeastOneLastprivate;
827 llvm::BasicBlock *ThenBB =
nullptr;
828 llvm::BasicBlock *DoneBB =
nullptr;
829 if (IsLastIterCond) {
832 Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
836 llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
837 if (
auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
838 auto IC = LoopDirective->counters().begin();
839 for (
auto F : LoopDirective->finals()) {
843 AlreadyEmittedVars.insert(D);
845 LoopCountersAndUpdates[D] = F;
850 auto IRef = C->varlist_begin();
851 auto ISrcRef = C->source_exprs().begin();
852 auto IDestRef = C->destination_exprs().begin();
853 for (
auto *AssignOp : C->assignment_ops()) {
854 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
855 QualType Type = PrivateVD->getType();
856 auto *CanonicalVD = PrivateVD->getCanonicalDecl();
857 if (AlreadyEmittedVars.insert(CanonicalVD).second) {
861 if (
auto *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
863 auto *SrcVD = cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
864 auto *DestVD = cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
869 if (
auto RefTy = PrivateVD->getType()->getAs<
ReferenceType>())
873 EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
879 if (
auto *PostUpdate = C->getPostUpdateExpr())
940 auto ILHS = C->lhs_exprs().begin();
941 auto IRHS = C->rhs_exprs().begin();
942 auto IPriv = C->privates().begin();
943 auto IRed = C->reduction_ops().begin();
944 for (
auto IRef : C->varlists()) {
945 auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
946 auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
947 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
949 if (
auto *OASE = dyn_cast<OMPArraySectionExpr>(IRef)) {
950 auto *
Base = OASE->getBase()->IgnoreParenImpCasts();
951 while (
auto *TempOASE = dyn_cast<OMPArraySectionExpr>(
Base))
952 Base = TempOASE->getBase()->IgnoreParenImpCasts();
953 while (
auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
954 Base = TempASE->getBase()->IgnoreParenImpCasts();
955 auto *DE = cast<DeclRefExpr>(
Base);
956 auto *OrigVD = cast<VarDecl>(DE->getDecl());
962 loadToBegin(*
this, OrigVD->getType(), OASELValueLB.getType(),
967 return OASELValueLB.getAddress();
971 OrigVD, [
this, OrigVD, PrivateVD, BaseLValue, OASELValueLB,
972 OASELValueUB, OriginalBaseLValue, DRD, IRed]() ->
Address {
976 auto *Size =
Builder.CreatePtrDiff(OASELValueUB.getPointer(),
977 OASELValueLB.getPointer());
979 Size, llvm::ConstantInt::get(Size->getType(), 1));
981 *
this, cast<OpaqueValueExpr>(
983 .getAsVariableArrayType(PrivateVD->getType())
988 auto Addr = Emission.getAllocatedAddress();
989 auto *Init = PrivateVD->getInit();
992 OASELValueLB.getAddress());
996 OASELValueLB.getPointer());
999 OASELValueLB.getType(), OriginalBaseLValue,
1002 assert(IsRegistered &&
"private var already registered as private");
1008 }
else if (
auto *ASE = dyn_cast<ArraySubscriptExpr>(IRef)) {
1009 auto *
Base = ASE->getBase()->IgnoreParenImpCasts();
1010 while (
auto *TempASE = dyn_cast<ArraySubscriptExpr>(
Base))
1011 Base = TempASE->getBase()->IgnoreParenImpCasts();
1012 auto *DE = cast<DeclRefExpr>(
Base);
1013 auto *OrigVD = cast<VarDecl>(DE->getDecl());
1017 *
this, OrigVD->getType(), ASELValue.getType(), OriginalBaseLValue);
1021 return ASELValue.getAddress();
1025 OrigVD, [
this, OrigVD, PrivateVD, BaseLValue, ASELValue,
1026 OriginalBaseLValue, DRD, IRed]() ->
Address {
1030 if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1032 ASELValue.getAddress(),
1033 ASELValue.getType());
1038 ASELValue.getPointer());
1040 return castToBase(*
this, OrigVD->getType(), ASELValue.getType(),
1041 OriginalBaseLValue, Ptr);
1043 assert(IsRegistered &&
"private var already registered as private");
1052 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());
1053 QualType Type = PrivateVD->getType();
1061 PrivateScope.
addPrivate(LHSVD, [
this, &OriginalAddr,
1065 return OriginalAddr;
1068 if (Type->isVariablyModifiedType()) {
1070 *
this, cast<OpaqueValueExpr>(
1072 .getAsVariableArrayType(PrivateVD->getType())
1075 getTypeSize(OrigVD->getType().getNonReferenceType())));
1079 auto Addr = Emission.getAllocatedAddress();
1080 auto *Init = PrivateVD->getInit();
1082 DRD ? *IRed : Init, OriginalAddr);
1084 return Emission.getAllocatedAddress();
1086 assert(IsRegistered &&
"private var already registered as private");
1098 PrivateScope.
addPrivate(LHSVD, [
this, OrigVD, IRef,
1104 return OriginalAddr;
1108 OrigVD, [
this, PrivateVD, OriginalAddr, DRD, IRed]() ->
Address {
1112 if (DRD && (DRD->getInitializer() || !PrivateVD->hasInit())) {
1121 assert(IsRegistered &&
"private var already registered as private");
1145 bool HasAtLeastOneReduction =
false;
1147 HasAtLeastOneReduction =
true;
1148 Privates.append(C->privates().begin(), C->privates().end());
1149 LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
1150 RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
1151 ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
1153 if (HasAtLeastOneReduction) {
1170 llvm::BasicBlock *DoneBB =
nullptr;
1172 if (
auto *PostUpdate = C->getPostUpdateExpr()) {
1174 if (
auto *Cond = CondGen(CGF)) {
1179 CGF.
Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1197 *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
1200 auto NumThreads = CGF.
EmitScalarExpr(NumThreadsClause->getNumThreads(),
1203 CGF, NumThreads, NumThreadsClause->getLocStart());
1208 CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getLocStart());
1210 const Expr *IfCond =
nullptr;
1213 C->getNameModifier() == OMPD_parallel) {
1214 IfCond = C->getCondition();
1219 OMPLexicalScope
Scope(CGF, S);
1223 CapturedVars, IfCond);
1252 JumpDest LoopExit) {
1253 RunCleanupsScope BodyScope(*
this);
1260 for (
auto *U : C->updates())
1266 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1271 BreakContinueStack.pop_back();
1275 const Stmt &S,
bool RequiresCleanup,
const Expr *LoopCond,
1276 const Expr *IncExpr,
1288 auto ExitBlock = LoopExit.getBlock();
1289 if (RequiresCleanup)
1296 if (ExitBlock != LoopExit.getBlock()) {
1306 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1314 BreakContinueStack.pop_back();
1326 for (
auto *Init : C->inits()) {
1327 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
1330 auto *OrigVD = cast<VarDecl>(Ref->getDecl());
1344 if (
auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
1345 if (
auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
1358 llvm::BasicBlock *DoneBB =
nullptr;
1361 auto IC = C->varlist_begin();
1362 for (
auto *F : C->finals()) {
1364 if (
auto *Cond = CondGen(*
this)) {
1369 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1373 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
1384 if (
auto *PostUpdate = C->getPostUpdateExpr())
1396 unsigned ClauseAlignment = 0;
1397 if (
auto AlignmentExpr = Clause->getAlignment()) {
1400 ClauseAlignment =
static_cast<unsigned>(AlignmentCI->getZExtValue());
1402 for (
auto E : Clause->varlists()) {
1403 unsigned Alignment = ClauseAlignment;
1404 if (Alignment == 0) {
1411 E->getType()->getPointeeType()))
1414 assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
1415 "alignment is not power of 2");
1416 if (Alignment != 0) {
1430 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
1431 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
1434 if (!LocalDeclMap.count(PrivateVD)) {
1440 (*I)->getType(),
VK_LValue, (*I)->getExprLoc());
1457 const Expr *Cond, llvm::BasicBlock *TrueBlock,
1458 llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
1466 for (
auto I : S.
inits()) {
1474 void CodeGenFunction::EmitOMPLinearClause(
1480 auto *LoopDirective = cast<OMPLoopDirective>(&D);
1481 for (
auto *C : LoopDirective->counters()) {
1487 auto CurPrivate = C->privates().begin();
1488 for (
auto *
E : C->varlists()) {
1489 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
1491 cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
1498 assert(IsRegistered &&
"linear var already registered as private");
1516 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1526 llvm::ConstantInt *Val = cast<llvm::ConstantInt>(Len.
getScalarVal());
1543 void CodeGenFunction::EmitOMPSimdFinal(
1548 llvm::BasicBlock *DoneBB =
nullptr;
1551 for (
auto F : D.
finals()) {
1552 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
1553 auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
1556 OrigVD->hasGlobalStorage() || CED) {
1558 if (
auto *Cond = CondGen(*
this)) {
1563 Builder.CreateCondBr(Cond, ThenBB, DoneBB);
1569 OrigAddr =
EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
1573 (*IPC)->getType(),
VK_LValue, (*IPC)->getExprLoc());
1576 OMPPrivateScope VarScope(*
this);
1577 VarScope.addPrivate(OrigVD,
1578 [OrigAddr]() ->
Address {
return OrigAddr; });
1579 (void)VarScope.Privatize();
1591 OMPLoopScope PreInitScope(CGF, S);
1602 llvm::BasicBlock *ContBlock =
nullptr;
1617 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
1625 CGF.
EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
1630 CGF.EmitOMPSimdInit(S);
1640 bool HasLastprivateClause =
1646 CGF.EmitOMPLoopBody(S,
JumpDest());
1650 CGF.EmitOMPSimdFinal(
1653 if (HasLastprivateClause)
1667 OMPLexicalScope
Scope(*
this, S,
true);
1671 void CodeGenFunction::EmitOMPOuterLoop(
bool DynamicOrOrdered,
bool IsMonotonic,
1688 if (!DynamicOrOrdered) {
1696 BoolCondVal = RT.emitForNext(*
this, S.
getLocStart(), IVSize, IVSigned, IL,
1702 auto ExitBlock = LoopExit.getBlock();
1703 if (LoopScope.requiresCleanups())
1707 Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
1708 if (ExitBlock != LoopExit.getBlock()) {
1716 if (DynamicOrOrdered)
1721 BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
1728 EmitOMPSimdInit(S, IsMonotonic);
1733 CGF.EmitOMPLoopBody(S, LoopExit);
1739 CGF, Loc, IVSize, IVSigned);
1744 BreakContinueStack.pop_back();
1745 if (!DynamicOrOrdered) {
1757 if (!DynamicOrOrdered)
1758 RT.emitForStaticFinish(*
this, S.
getLocEnd());
1762 void CodeGenFunction::EmitOMPForOuterLoop(
1769 const bool DynamicOrOrdered =
1770 Ordered || RT.isDynamic(ScheduleKind.
Schedule);
1773 !RT.isStaticNonchunked(ScheduleKind.
Schedule,
1774 Chunk !=
nullptr)) &&
1775 "static non-chunked schedule does not need outer loop");
1831 if (DynamicOrOrdered) {
1833 RT.emitForDispatchInit(*
this, S.
getLocStart(), ScheduleKind, IVSize,
1834 IVSigned, Ordered, UBVal, Chunk);
1836 RT.emitForStaticInit(*
this, S.
getLocStart(), ScheduleKind, IVSize, IVSigned,
1837 Ordered, IL, LB, UB, ST, Chunk);
1840 EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, Ordered, LB, UB,
1844 void CodeGenFunction::EmitOMPDistributeOuterLoop(
1860 RT.emitDistributeStaticInit(*
this, S.
getLocStart(), ScheduleKind,
1861 IVSize, IVSigned,
false,
1862 IL, LB, UB, ST, Chunk);
1864 EmitOMPOuterLoop(
false,
false,
1865 S, LoopScope,
false, LB, UB, ST, IL, Chunk);
1870 OMPLexicalScope
Scope(*
this, S,
true);
1872 *
this, OMPD_distribute_parallel_for,
1874 OMPLoopScope PreInitScope(CGF, S);
1882 OMPLexicalScope
Scope(*
this, S,
true);
1884 *
this, OMPD_distribute_parallel_for_simd,
1886 OMPLoopScope PreInitScope(CGF, S);
1894 OMPLexicalScope
Scope(*
this, S,
true);
1896 *
this, OMPD_distribute_simd,
1898 OMPLoopScope PreInitScope(CGF, S);
1906 OMPLexicalScope
Scope(*
this, S,
true);
1908 *
this, OMPD_target_parallel_for_simd,
1910 OMPLoopScope PreInitScope(CGF, S);
1919 auto VDecl = cast<VarDecl>(Helper->
getDecl());
1925 struct ScheduleKindModifiersTy {
1932 : Kind(Kind), M1(M1), M2(M2) {}
1939 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
1953 bool HasLastprivateClause;
1956 OMPLoopScope PreInitScope(*
this, S);
1961 llvm::BasicBlock *ContBlock =
nullptr;
1974 bool Ordered =
false;
1976 if (OrderedClause->getNumForLoops())
1977 RT.emitDoacrossInit(*
this, S);
1997 OMPPrivateScope LoopScope(*
this);
2011 (void)LoopScope.Privatize();
2017 ScheduleKind.
Schedule =
C->getScheduleKind();
2018 ScheduleKind.
M1 =
C->getFirstScheduleModifier();
2019 ScheduleKind.
M2 =
C->getSecondScheduleModifier();
2020 if (
const auto *Ch =
C->getChunkSize()) {
2033 if (RT.isStaticNonchunked(ScheduleKind.
Schedule,
2034 Chunk !=
nullptr) &&
2037 EmitOMPSimdInit(S,
true);
2043 RT.emitForStaticInit(*
this, S.
getLocStart(), ScheduleKind,
2044 IVSize, IVSigned, Ordered,
2057 CGF.EmitOMPLoopBody(S, LoopExit);
2065 const bool IsMonotonic =
2066 Ordered || ScheduleKind.
Schedule == OMPC_SCHEDULE_static ||
2068 ScheduleKind.
M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
2069 ScheduleKind.
M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
2072 EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
2079 return CGF.
Builder.CreateIsNotNull(
2087 return CGF.
Builder.CreateIsNotNull(
2091 if (HasLastprivateClause)
2097 return CGF.
Builder.CreateIsNotNull(
2106 return HasLastprivateClause;
2110 bool HasLastprivates =
false;
2113 HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
2116 OMPLexicalScope
Scope(*
this, S,
true);
2128 bool HasLastprivates =
false;
2131 HasLastprivates = CGF.EmitOMPWorksharingLoop(S);
2134 OMPLexicalScope
Scope(*
this, S,
true);
2156 bool HasLastprivates =
false;
2159 auto &
C = CGF.CGM.getContext();
2160 auto KmpInt32Ty =
C.getIntTypeForBitwidth(32, 1);
2163 CGF.Builder.getInt32(0));
2164 auto *GlobalUBVal = CS !=
nullptr ? CGF.Builder.getInt32(CS->size() - 1)
2165 : CGF.Builder.getInt32(0);
2169 CGF.Builder.getInt32(1));
2171 CGF.Builder.getInt32(0));
2197 auto *ExitBB = CGF.createBasicBlock(
".omp.sections.exit");
2199 CGF.EmitLoadOfLValue(IV, S.getLocStart()).getScalarVal(), ExitBB,
2200 CS ==
nullptr ? 1 : CS->size());
2202 unsigned CaseNumber = 0;
2203 for (
auto *SubStmt : CS->children()) {
2204 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2205 CGF.EmitBlock(CaseBB);
2206 SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
2207 CGF.EmitStmt(SubStmt);
2208 CGF.EmitBranch(ExitBB);
2212 auto CaseBB = CGF.createBasicBlock(
".omp.sections.case");
2213 CGF.EmitBlock(CaseBB);
2214 SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
2216 CGF.EmitBranch(ExitBB);
2218 CGF.EmitBlock(ExitBB,
true);
2222 if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
2226 CGF.CGM.getOpenMPRuntime().emitBarrierCall(
2230 CGF.EmitOMPPrivateClause(S, LoopScope);
2231 HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
2232 CGF.EmitOMPReductionClauseInit(S, LoopScope);
2233 (void)LoopScope.Privatize();
2237 ScheduleKind.
Schedule = OMPC_SCHEDULE_static;
2238 CGF.CGM.getOpenMPRuntime().emitForStaticInit(
2239 CGF, S.getLocStart(), ScheduleKind, 32,
2240 true,
false, IL.getAddress(), LB.
getAddress(),
2243 auto *UBVal = CGF.EmitLoadOfScalar(UB, S.getLocStart());
2244 auto *MinUBGlobalUB = CGF.Builder.CreateSelect(
2245 CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
2246 CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
2248 CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getLocStart()), IV);
2250 CGF.EmitOMPInnerLoop(S,
false, &Cond, &Inc, BodyGen,
2253 CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getLocStart());
2254 CGF.EmitOMPReductionClauseFinal(S);
2258 return CGF.
Builder.CreateIsNotNull(
2263 if (HasLastprivates)
2270 bool HasCancel =
false;
2271 if (
auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
2272 HasCancel = OSD->hasCancel();
2273 else if (
auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
2274 HasCancel = OPSD->hasCancel();
2290 OMPLexicalScope
Scope(*
this, S,
true);
2304 OMPLexicalScope
Scope(*
this, S,
true);
2319 CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
2320 DestExprs.append(C->destination_exprs().begin(),
2321 C->destination_exprs().end());
2322 SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
2323 AssignmentOps.append(C->assignment_ops().begin(),
2324 C->assignment_ops().end());
2336 OMPLexicalScope
Scope(*
this, S,
true);
2338 CopyprivateVars, DestExprs,
2339 SrcExprs, AssignmentOps);
2355 OMPLexicalScope
Scope(*
this, S,
true);
2364 Expr *Hint =
nullptr;
2366 Hint = HintClause->getHint();
2367 OMPLexicalScope
Scope(*
this, S,
true);
2378 CGF.EmitOMPWorksharingLoop(S);
2388 CGF.EmitOMPWorksharingLoop(S);
2398 CGF.EmitSections(S);
2409 auto *
I = CS->getCapturedDecl()->param_begin();
2410 auto *PartId = std::next(I);
2411 auto *TaskT = std::next(I, 4);
2416 auto *Cond = Clause->getCondition();
2419 Data.
Final.setInt(CondConstant);
2424 Data.
Final.setInt(
false);
2430 auto *Prio = Clause->getPriority();
2435 Prio->getExprLoc()));
2442 auto IRef = C->varlist_begin();
2443 for (
auto *IInit : C->private_copies()) {
2444 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2445 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2452 EmittedAsPrivate.clear();
2455 auto IRef = C->varlist_begin();
2456 auto IElemInitRef = C->inits().begin();
2457 for (
auto *IInit : C->private_copies()) {
2458 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2459 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2469 llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
2471 auto IRef = C->varlist_begin();
2472 auto ID = C->destination_exprs().begin();
2473 for (
auto *IInit : C->private_copies()) {
2474 auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
2475 if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
2479 LastprivateDstsOrigs.insert(
2480 {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
2481 cast<DeclRefExpr>(*IRef)});
2488 for (
auto *IRef : C->varlists())
2489 Data.
Dependences.push_back(std::make_pair(C->getDependencyKind(), IRef));
2490 auto &&CodeGen = [PartId, &
S, &Data, CS, &BodyGen, &LastprivateDstsOrigs](
2494 if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
2495 !Data.LastprivateVars.empty()) {
2496 auto *CopyFn = CGF.Builder.CreateLoad(
2497 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(3)));
2498 auto *PrivatesPtr = CGF.Builder.CreateLoad(
2499 CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(2)));
2503 CallArgs.push_back(PrivatesPtr);
2504 for (
auto *
E : Data.PrivateVars) {
2505 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2506 Address PrivatePtr = CGF.CreateMemTemp(
2507 CGF.getContext().getPointerType(
E->getType()),
".priv.ptr.addr");
2508 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2509 CallArgs.push_back(PrivatePtr.getPointer());
2511 for (
auto *
E : Data.FirstprivateVars) {
2512 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2514 CGF.CreateMemTemp(CGF.getContext().getPointerType(
E->getType()),
2515 ".firstpriv.ptr.addr");
2516 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2517 CallArgs.push_back(PrivatePtr.getPointer());
2519 for (
auto *
E : Data.LastprivateVars) {
2520 auto *VD = cast<VarDecl>(cast<DeclRefExpr>(
E)->getDecl());
2522 CGF.CreateMemTemp(CGF.getContext().getPointerType(
E->getType()),
2523 ".lastpriv.ptr.addr");
2524 PrivatePtrs.push_back(std::make_pair(VD, PrivatePtr));
2525 CallArgs.push_back(PrivatePtr.getPointer());
2527 CGF.EmitRuntimeCall(CopyFn, CallArgs);
2528 for (
auto &&Pair : LastprivateDstsOrigs) {
2529 auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
2531 const_cast<VarDecl *>(OrigVD),
2532 CGF.CapturedStmtInfo->lookup(
2534 Pair.second->getType(),
VK_LValue, Pair.second->getExprLoc());
2535 Scope.
addPrivate(Pair.first, [&CGF, &DRE]() {
2536 return CGF.EmitLValue(&DRE).getAddress();
2539 for (
auto &&Pair : PrivatePtrs) {
2541 CGF.getContext().getDeclAlign(Pair.first));
2545 (void)
Scope.Privatize();
2550 auto *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
2551 S, *
I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
2552 Data.NumberOfParts);
2553 OMPLexicalScope
Scope(*
this, S);
2554 TaskGen(*
this, OutlinedFn, Data);
2562 const Expr *IfCond =
nullptr;
2565 C->getNameModifier() == OMPD_task) {
2566 IfCond = C->getCondition();
2575 CGF.
EmitStmt(CS->getCapturedStmt());
2577 auto &&TaskGen = [&
S, SharedsTy, CapturedStruct,
2580 CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.
getLocStart(),
S, OutlinedFn,
2581 SharedsTy, CapturedStruct, IfCond,
2606 OMPLexicalScope
Scope(*
this, S,
true);
2613 return llvm::makeArrayRef(FlushClause->varlist_begin(),
2614 FlushClause->varlist_end());
2617 }(), S.getLocStart());
2623 auto IVDecl = cast<VarDecl>(IVExpr->getDecl());
2639 OMPLoopScope PreInitScope(*
this, S);
2644 llvm::BasicBlock *ContBlock =
nullptr;
2677 ScheduleKind = C->getDistScheduleKind();
2678 if (
const auto *Ch = C->getChunkSize()) {
2696 if (RT.isStaticNonchunked(ScheduleKind,
2697 Chunk !=
nullptr)) {
2698 RT.emitDistributeStaticInit(*
this, S.
getLocStart(), ScheduleKind,
2699 IVSize, IVSigned,
false,
2712 CGF.EmitOMPLoopBody(S, LoopExit);
2722 EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope,
2741 OMPLexicalScope
Scope(*
this, S,
true);
2752 Fn->addFnAttr(llvm::Attribute::NoInline);
2768 CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
2770 CGF.EmitNounwindRuntimeCall(OutlinedFn, CapturedVars);
2777 OMPLexicalScope
Scope(*
this, S,
true);
2785 "DestType must have scalar evaluation kind.");
2786 assert(!Val.
isAggregate() &&
"Must be a scalar or complex.");
2798 "DestType must have complex evaluation kind.");
2804 DestElementType, Loc);
2806 ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
2808 assert(Val.
isComplex() &&
"Must be a scalar or complex.");
2812 Val.
getComplexVal().first, SrcElementType, DestElementType, Loc);
2814 Val.
getComplexVal().second, SrcElementType, DestElementType, Loc);
2825 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
2826 : llvm::AtomicOrdering::Monotonic,
2836 *
this, RVal, RValTy, LVal.
getType(), Loc)),
2845 llvm_unreachable(
"Must be a scalar or complex.");
2853 assert(V->
isLValue() &&
"V of 'omp atomic read' is not lvalue");
2854 assert(X->
isLValue() &&
"X of 'omp atomic read' is not lvalue");
2857 RValue Res = XLValue.isGlobalReg()
2861 IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
2862 : llvm::AtomicOrdering::Monotonic,
2863 XLValue.isVolatile());
2877 assert(X->
isLValue() &&
"X of 'omp atomic write' is not lvalue");
2890 llvm::AtomicOrdering AO,
2891 bool IsXLHSInRHSPart) {
2896 if (BO == BO_Comma || !Update.
isScalar() ||
2904 return std::make_pair(
false,
RValue::get(
nullptr));
2906 llvm::AtomicRMWInst::BinOp RMWOp;
2909 RMWOp = llvm::AtomicRMWInst::Add;
2912 if (!IsXLHSInRHSPart)
2913 return std::make_pair(
false,
RValue::get(
nullptr));
2914 RMWOp = llvm::AtomicRMWInst::Sub;
2920 RMWOp = llvm::AtomicRMWInst::Or;
2923 RMWOp = llvm::AtomicRMWInst::Xor;
2927 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
2928 : llvm::AtomicRMWInst::Max)
2929 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
2930 : llvm::AtomicRMWInst::UMax);
2934 ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
2935 : llvm::AtomicRMWInst::Min)
2936 : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
2937 : llvm::AtomicRMWInst::UMin);
2940 RMWOp = llvm::AtomicRMWInst::Xchg;
2949 return std::make_pair(
false,
RValue::get(
nullptr));
2967 llvm_unreachable(
"Unsupported atomic update operation");
2970 if (
auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
2971 UpdateVal = CGF.
Builder.CreateIntCast(
3005 const Expr *UE,
bool IsXLHSInRHSPart,
3008 "Update expr in 'atomic update' must be a binary operator.");
3016 assert(X->
isLValue() &&
"X of 'omp atomic update' is not lvalue");
3019 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3020 : llvm::AtomicOrdering::Monotonic;
3021 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3022 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3023 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3024 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3026 [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](
RValue XRValue) ->
RValue {
3032 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3055 llvm_unreachable(
"Must be a scalar or complex.");
3059 bool IsPostfixUpdate,
const Expr *V,
3061 const Expr *UE,
bool IsXLHSInRHSPart,
3063 assert(X->
isLValue() &&
"X of 'omp atomic capture' is not lvalue");
3064 assert(V->
isLValue() &&
"V of 'omp atomic capture' is not lvalue");
3069 auto AO = IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
3070 : llvm::AtomicOrdering::Monotonic;
3075 "Update expr in 'atomic capture' must be a binary operator.");
3083 auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
3084 auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
3085 auto *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
3086 NewVValType = XRValExpr->getType();
3087 auto *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
3088 auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
3093 NewVVal = IsPostfixUpdate ? XRValue : Res;
3097 XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
3100 if (IsPostfixUpdate) {
3102 NewVVal = Res.second;
3116 auto &&Gen = [&CGF, &NewVVal, ExprRValue](
RValue XRValue) ->
RValue {
3122 XLValue, ExprRValue, BO_Assign,
false, AO,
3126 NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
3140 bool IsSeqCst,
bool IsPostfixUpdate,
3142 const Expr *UE,
bool IsXLHSInRHSPart,
3157 IsXLHSInRHSPart, Loc);
3161 case OMPC_num_threads:
3163 case OMPC_firstprivate:
3164 case OMPC_lastprivate:
3165 case OMPC_reduction:
3175 case OMPC_copyprivate:
3177 case OMPC_proc_bind:
3184 case OMPC_mergeable:
3189 case OMPC_num_teams:
3190 case OMPC_thread_limit:
3192 case OMPC_grainsize:
3194 case OMPC_num_tasks:
3196 case OMPC_dist_schedule:
3197 case OMPC_defaultmap:
3201 case OMPC_use_device_ptr:
3202 case OMPC_is_device_ptr:
3203 llvm_unreachable(
"Clause is not allowed in 'omp atomic'.");
3212 if (C->getClauseKind() != OMPC_seq_cst) {
3213 Kind = C->getClauseKind();
3220 if (
const auto *EWC = dyn_cast<ExprWithCleanups>(CS)) {
3224 if (
const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
3225 for (
const auto *C : Compound->body()) {
3226 if (
const auto *EWC = dyn_cast<ExprWithCleanups>(C)) {
3234 CGF.EmitStopPoint(CS);
3239 OMPLexicalScope
Scope(*
this, S,
true);
3243 std::pair<llvm::Function * , llvm::Constant * >
3246 bool IsOffloadEntry) {
3247 llvm::Function *OutlinedFn =
nullptr;
3248 llvm::Constant *OutlinedFnID =
nullptr;
3260 S, ParentName, OutlinedFn, OutlinedFnID, IsOffloadEntry, CodeGen);
3261 return std::make_pair(OutlinedFn, OutlinedFnID);
3270 llvm::Function *Fn =
nullptr;
3271 llvm::Constant *FnID =
nullptr;
3274 const Expr *IfCond =
nullptr;
3277 IfCond = C->getCondition();
3281 const Expr *Device =
nullptr;
3283 Device = C->getDevice();
3289 bool IsOffloadEntry =
true;
3293 IsOffloadEntry =
false;
3296 IsOffloadEntry =
false;
3298 assert(
CurFuncDecl &&
"No parent declaration for target region!");
3299 StringRef ParentName;
3302 if (
auto *D = dyn_cast<CXXConstructorDecl>(
CurFuncDecl))
3304 else if (
auto *D = dyn_cast<CXXDestructorDecl>(
CurFuncDecl))
3311 CGM, S, ParentName, IsOffloadEntry);
3312 OMPLexicalScope
Scope(*
this, S);
3324 *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
3330 Expr *NumTeams = (NT) ? NT->getNumTeams() :
nullptr;
3331 Expr *ThreadLimit = (TL) ? TL->getThreadLimit() :
nullptr;
3337 OMPLexicalScope
Scope(CGF, S);
3363 const Expr *IfCond =
nullptr;
3366 C->getNameModifier() == OMPD_cancel) {
3367 IfCond = C->getCondition();
3377 if (Kind == OMPD_parallel || Kind == OMPD_task)
3379 assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
3380 Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for);
3381 return BreakContinueStack.back().BreakBlock;
3396 OMPLexicalScope
Scope(*
this, S,
true);
3404 const Expr *IfCond =
nullptr;
3406 IfCond = C->getCondition();
3409 const Expr *Device =
nullptr;
3411 Device = C->getDevice();
3424 const Expr *IfCond =
nullptr;
3426 IfCond = C->getCondition();
3429 const Expr *Device =
nullptr;
3431 Device = C->getDevice();
3444 const Expr *IfCond =
nullptr;
3446 IfCond = C->getCondition();
3449 const Expr *Device =
nullptr;
3451 Device = C->getDevice();
3470 auto *VDecl = cast<VarDecl>(Helper->
getDecl());
3481 const Expr *IfCond =
nullptr;
3484 C->getNameModifier() == OMPD_taskloop) {
3485 IfCond = C->getCondition();
3517 llvm::BasicBlock *ContBlock =
nullptr;
3518 OMPLoopScope PreInitScope(CGF, S);
3519 if (CGF.ConstantFoldsToSimpleInteger(S.
getPreCond(), CondConstant)) {
3523 auto *ThenBlock = CGF.createBasicBlock(
"taskloop.if.then");
3524 ContBlock = CGF.createBasicBlock(
"taskloop.if.end");
3526 CGF.getProfileCount(&S));
3527 CGF.EmitBlock(ThenBlock);
3528 CGF.incrementProfileCounter(&S);
3532 CGF.EmitOMPSimdInit(S);
3536 enum { LowerBound = 5, UpperBound, Stride, LastIter };
3537 auto *
I = CS->getCapturedDecl()->param_begin();
3538 auto *LBP = std::next(
I, LowerBound);
3539 auto *UBP = std::next(
I, UpperBound);
3540 auto *STP = std::next(
I, Stride);
3541 auto *LIP = std::next(
I, LastIter);
3549 CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
3550 bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
3554 const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
3555 CGF.EmitVarDecl(*IVDecl);
3556 CGF.EmitIgnoredExpr(S.
getInit());
3562 CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
3570 CGF.EmitOMPLoopBody(S,
JumpDest());
3571 CGF.EmitStopPoint(&S);
3576 CGF.EmitBranch(ContBlock);
3577 CGF.EmitBlock(ContBlock,
true);
3580 if (HasLastprivateClause) {
3581 CGF.EmitOMPLastprivateClauseFinal(
3583 CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
3584 CGF.GetAddrOfLocalVar(*LIP),
false,
3588 auto &&TaskGen = [&
S, SharedsTy, CapturedStruct,
3592 OMPLoopScope PreInitScope(CGF, S);
3594 OutlinedFn, SharedsTy,
3595 CapturedStruct, IfCond, Data);
3621 const Expr *IfCond =
nullptr;
3623 IfCond = C->getCondition();
3626 const Expr *Device =
nullptr;
3628 Device = C->getDevice();
const CGFunctionInfo & arrangeBuiltinFunctionDeclaration(QualType resultType, const FunctionArgList &args)
A builtin function is a freestanding function using the default C conventions.
This represents '#pragma omp distribute simd' composite directive.
static LValue loadToBegin(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV)
This represents '#pragma omp master' directive.
virtual void EmitBody(CodeGenFunction &CGF, const Stmt *S)
Emit the captured statement body.
Address EmitLoadOfReference(Address Ref, const ReferenceType *RefTy, AlignmentSource *Source=nullptr)
static Address castValueFromUintptr(CodeGenFunction &CGF, QualType DstType, StringRef Name, LValue AddrLV, bool isReferenceType=false)
This represents '#pragma omp task' directive.
static const Decl * getCanonicalDecl(const Decl *D)
This represents 'thread_limit' clause in the '#pragma omp ...' directive.
bool isXLHSInRHSPart() const
Return true if helper update expression has form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' and...
void EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S)
This represents clause 'copyin' in the '#pragma omp ...' directives.
PointerType - C99 6.7.5.1 - Pointer Declarators.
Scheduling data for loop-based OpenMP directives.
A (possibly-)qualified type.
capture_init_iterator capture_init_begin()
Retrieve the first initialization argument.
llvm::Value * getPointer() const
CodeGenTypes & getTypes()
void EmitOMPReductionClauseFinal(const OMPExecutableDirective &D)
Emit final update of reduction values to original variables at the end of the directive.
ArrayRef< OMPClause * > clauses()
llvm::Type * ConvertTypeForMem(QualType T)
void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
EmitBranchOnBoolExpr - Emit a branch on a boolean condition (e.g.
void EmitVarDecl(const VarDecl &D)
EmitVarDecl - Emit a local variable declaration.
CharUnits getDeclAlign(const Decl *D, bool ForAlignof=false) const
Return a conservative estimate of the alignment of the specified decl D.
Address getAllocatedAddress() const
Returns the raw, allocated address, which is not necessarily the address of the object itself...
llvm::Module & getModule() const
void push(llvm::BasicBlock *Header, llvm::DebugLoc Location=llvm::DebugLoc())
Begin a new structured loop.
void EmitOMPDistributeDirective(const OMPDistributeDirective &S)
IdentifierInfo * getIdentifier() const
getIdentifier - Get the identifier that names this declaration, if there is one.
void EmitOMPAggregateAssign(Address DestAddr, Address SrcAddr, QualType OriginalType, const llvm::function_ref< void(Address, Address)> &CopyGen)
Perform element by element copying of arrays with type OriginalType from SrcAddr to DestAddr using co...
static OMPClauseWithPreInit * get(OMPClause *C)
void EmitStoreThroughGlobalRegLValue(RValue Src, LValue Dst)
Store of global named registers are always calls to intrinsics.
std::pair< llvm::Value *, llvm::Value * > getComplexVal() const
getComplexVal - Return the real/imag components of this complex value.
QualType getIntTypeForBitwidth(unsigned DestWidth, unsigned Signed) const
getIntTypeForBitwidth - sets integer QualTy according to specified details: bitwidth, signed/unsigned.
SmallVector< std::pair< OpenMPDependClauseKind, const Expr * >, 4 > Dependences
This represents '#pragma omp for simd' directive.
QuantityType getQuantity() const
getQuantity - Get the raw integer representation of this quantity.
SmallVector< const Expr *, 4 > LastprivateCopies
This represents 'grainsize' clause in the '#pragma omp ...' directive.
Address getAddress() const
const Decl * CurCodeDecl
CurCodeDecl - This is the inner-most code context, which includes blocks.
llvm::Value * getTypeSize(QualType Ty)
Returns calculated size of the specified type.
This represents 'if' clause in the '#pragma omp ...' directive.
void EmitOMPOrderedDirective(const OMPOrderedDirective &S)
QualType getLValueReferenceType(QualType T, bool SpelledAsLValue=true) const
Return the uniqued reference to the type for an lvalue reference to the specified type...
This represents 'priority' clause in the '#pragma omp ...' directive.
static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper, const ImplicitParamDecl *PVD, CodeGenFunction::OMPPrivateScope &Privates)
Emit a helper variable and return corresponding lvalue.
The base class of the type hierarchy.
void EmitStoreThroughLValue(RValue Src, LValue Dst, bool isInit=false)
EmitStoreThroughLValue - Store the specified rvalue into the specified lvalue, where both are guarant...
static void emitInitWithReductionInitializer(CodeGenFunction &CGF, const OMPDeclareReductionDecl *DRD, const Expr *InitOp, Address Private, Address Original, QualType Ty)
Address GenerateCapturedStmtArgument(const CapturedStmt &S)
QualType getRecordType(const RecordDecl *Decl) const
const Expr * getInit() const
The l-value was an access to a declared entity or something equivalently strong, like the address of ...
This represents '#pragma omp parallel for' directive.
const LangOptions & getLangOpts() const
void EmitAtomicStore(RValue rvalue, LValue lvalue, bool isInit)
bool ConstantFoldsToSimpleInteger(const Expr *Cond, bool &Result, bool AllowLabels=false)
ConstantFoldsToSimpleInteger - If the specified expression does not fold to a constant, or if it does but contains a label, return false.
void EmitOMPCopy(QualType OriginalType, Address DestAddr, Address SrcAddr, const VarDecl *DestVD, const VarDecl *SrcVD, const Expr *Copy)
Emit proper copying of data from one variable to another.
void EmitOMPParallelForSimdDirective(const OMPParallelForSimdDirective &S)
static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty, const Twine &Name, llvm::Value *Init=nullptr)
This represents '#pragma omp target exit data' directive.
Address GetAddrOfLocalVar(const VarDecl *VD)
GetAddrOfLocalVar - Return the address of a local variable.
llvm::Function * GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S)
QualType getVariableArrayDecayedType(QualType Ty) const
Returns a vla type where known sizes are replaced with [*].
VarDecl - An instance of this class is created to represent a variable declaration or definition...
This represents clause 'private' in the '#pragma omp ...' directives.
void EmitOMPCriticalDirective(const OMPCriticalDirective &S)
llvm::Type * getElementType() const
Return the type of the values stored in this address.
This represents 'num_threads' clause in the '#pragma omp ...' directive.
uint64_t getProfileCount(const Stmt *S)
Get the profiler's count for the given statement.
field_iterator field_begin() const
uint64_t getTypeSize(QualType T) const
Return the size of the specified (complete) type T, in bits.
bool hasGlobalStorage() const
Returns true for all variables that do not have local storage.
void EmitVariablyModifiedType(QualType Ty)
EmitVLASize - Capture all the sizes for the VLA expressions in the given variably-modified type and s...
CharUnits getNaturalTypeAlignment(QualType T, AlignmentSource *Source=nullptr, bool forPointeeType=false)
This represents implicit clause 'flush' for the '#pragma omp flush' directive.
void EmitOMPLastprivateClauseFinal(const OMPExecutableDirective &D, bool NoFinals, llvm::Value *IsLastIterCond=nullptr)
Emit final copying of lastprivate values to original variables at the end of the worksharing or simd ...
static void EmitOMPAggregateInit(CodeGenFunction &CGF, Address DestAddr, QualType Type, const Expr *Init, Address SrcAddr=Address::invalid())
Emit initialization of arrays of complex types.
This represents 'nogroup' clause in the '#pragma omp ...' directive.
A jump destination is an abstract label, branching to which may require a jump out through normal cle...
This represents 'safelen' clause in the '#pragma omp ...' directive.
OpenMPDirectiveKind getDirectiveKind() const
Expr * IgnoreImpCasts() LLVM_READONLY
IgnoreImpCasts - Skip past any implicit casts which might surround this expression.
void EmitOMPTaskLoopSimdDirective(const OMPTaskLoopSimdDirective &S)
SmallVector< const Expr *, 4 > PrivateVars
RecordDecl - Represents a struct/union/class.
llvm::DenseMap< const VarDecl *, FieldDecl * > LambdaCaptureFields
One of these records is kept for each identifier that is lexed.
void EmitOMPSimdDirective(const OMPSimdDirective &S)
This represents '#pragma omp parallel' directive.
class LLVM_ALIGNAS(8) DependentTemplateSpecializationType const IdentifierInfo * Name
Represents a template specialization type whose template cannot be resolved, e.g. ...
static void emitCommonOMPParallelDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
CodeGenFunction - This class organizes the per-function state that is used while generating LLVM code...
bool isVariablyModifiedType() const
Whether this type is a variably-modified type (C99 6.7.5).
Holds long-lived AST nodes (such as types and decls) that can be referred to throughout the semantic ...
This represents 'simd' clause in the '#pragma omp ...' directive.
void EmitExprAsInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
EmitExprAsInit - Emits the code necessary to initialize a location in memory with the given initializ...
void EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S)
The scope used to remap some variables as private in the OpenMP loop body (or other captured region e...
bool isReferenceType() const
SmallVector< const Expr *, 4 > LastprivateVars
This represents clause 'lastprivate' in the '#pragma omp ...' directives.
bool isAnyPointerType() const
llvm::IntegerType * SizeTy
An RAII object to set (and then clear) a mapping for an OpaqueValueExpr.
LValue EmitOMPArraySectionExpr(const OMPArraySectionExpr *E, bool IsLowerBound=true)
static void emitSimdlenSafelenClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, bool IsMonotonic)
ArrayRef< Expr * > updates()
void EmitAtomicUpdate(LValue LVal, llvm::AtomicOrdering AO, const llvm::function_ref< RValue(RValue)> &UpdateOp, bool IsVolatile)
void EmitOMPPrivateLoopCounters(const OMPLoopDirective &S, OMPPrivateScope &LoopScope)
Emit initial code for loop counters of loop-based directives.
bool hasSameType(QualType T1, QualType T2) const
Determine whether the given types T1 and T2 are equivalent.
virtual const FieldDecl * lookup(const VarDecl *VD) const
Lookup the captured field decl for a variable.
This represents '#pragma omp barrier' directive.
This is a common base class for loop directives ('omp simd', 'omp for', 'omp for simd' etc...
This represents '#pragma omp critical' directive.
static std::pair< bool, RValue > emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X, RValue Update, BinaryOperatorKind BO, llvm::AtomicOrdering AO, bool IsXLHSInRHSPart)
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
An r-value expression (a pr-value in the C++11 taxonomy) produces a temporary value.
ComplexPairTy EmitLoadOfComplex(LValue src, SourceLocation loc)
EmitLoadOfComplex - Load a complex number from the specified l-value.
void EmitOMPDistributeParallelForSimdDirective(const OMPDistributeParallelForSimdDirective &S)
bool EmitOMPLastprivateClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for lastprivate variables.
static void EmitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind, bool IsSeqCst, bool IsPostfixUpdate, const Expr *X, const Expr *V, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAnyExpr(const Expr *E, AggValueSlot aggSlot=AggValueSlot::ignored(), bool ignoreResult=false)
EmitAnyExpr - Emit code to compute the specified expression which can have any type.
This represents clause 'copyprivate' in the '#pragma omp ...' directives.
OpenMPDistScheduleClauseKind
OpenMP attributes for 'dist_schedule' clause.
This represents '#pragma omp distribute parallel for' composite directive.
Stmt * getBody() const override
Expr * getInitializer()
Get initializer expression (if specified) of the declare reduction construct.
static bool hasScalarEvaluationKind(QualType T)
ArrayRef< Expr * > finals()
CharUnits getAlignment() const
const TargetInfo & getTargetInfo() const
void emitOMPSimpleStore(LValue LVal, RValue RVal, QualType RValTy, SourceLocation Loc)
OpenMPScheduleClauseKind Schedule
Address CreateElementBitCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
Cast the element type of the given address to a different type, preserving information like the align...
CharUnits - This is an opaque type for sizes expressed in character units.
static const OMPDeclareReductionDecl * getReductionInit(const Expr *ReductionOp)
Check if the combiner is a call to UDR combiner and if it is so return the UDR decl used for reductio...
void pop()
End the current loop.
Expr * getX()
Get 'x' part of the associated expression/statement.
void EmitOMPCancellationPointDirective(const OMPCancellationPointDirective &S)
field_range fields() const
void EmitOMPTargetDirective(const OMPTargetDirective &S)
A builtin binary operation expression such as "x + y" or "x <= y".
unsigned getOpenMPDefaultSimdAlign(QualType T) const
Get default simd alignment of the specified complete type in bits.
OpenMPScheduleClauseModifier M2
void setVectorizeWidth(unsigned W)
Set the vectorize width for the next loop pushed.
SmallVector< const Expr *, 4 > PrivateCopies
static llvm::Function * emitOutlinedOrderedFunction(CodeGenModule &CGM, const CapturedStmt *S)
virtual bool hasBuiltinAtomic(uint64_t AtomicSizeInBits, uint64_t AlignmentInBits) const
Returns true if the given target supports lock-free atomic operations at the specified width and alig...
void EmitOMPTargetUpdateDirective(const OMPTargetUpdateDirective &S)
This represents '#pragma omp cancellation point' directive.
void EmitAggregateAssign(Address DestPtr, Address SrcPtr, QualType EltTy)
EmitAggregateCopy - Emit an aggregate assignment.
static LValue EmitOMPHelperVar(CodeGenFunction &CGF, const DeclRefExpr *Helper)
Emit a helper variable and return corresponding lvalue.
Scope - A scope is a transient data structure that is used while parsing the program.
void incrementProfileCounter(const Stmt *S)
Increment the profiler's counter for the given statement.
This represents 'final' clause in the '#pragma omp ...' directive.
void EmitStmt(const Stmt *S)
EmitStmt - Emit the code for the statement.
void EmitOMPParallelDirective(const OMPParallelDirective &S)
llvm::BasicBlock * createBasicBlock(const Twine &name="", llvm::Function *parent=nullptr, llvm::BasicBlock *before=nullptr)
createBasicBlock - Create an LLVM basic block.
This represents '#pragma omp teams' directive.
void EmitIgnoredExpr(const Expr *E)
EmitIgnoredExpr - Emit an expression in a context which ignores the result.
void assignRegionCounters(GlobalDecl GD, llvm::Function *Fn)
Assign counters to regions and configure them for PGO of a given function.
SmallVector< const Expr *, 4 > FirstprivateCopies
llvm::Constant * GetAddrOfGlobal(GlobalDecl GD, bool IsForDefinition=false)
This represents clause 'reduction' in the '#pragma omp ...' directives.
bool requiresCleanups() const
Determine whether this scope requires any cleanups.
bool isStaticLocal() const
isStaticLocal - Returns true if a variable with function scope is a static local variable.
CharUnits getTypeSizeInChars(QualType T) const
Return the size of the specified (complete) type T, in characters.
bool hasCancel() const
Return true if current directive has inner cancel directive.
const RecordDecl * getCapturedRecordDecl() const
Retrieve the record declaration for captured variables.
An ordinary object is located at an address in memory.
This represents the body of a CapturedStmt, and serves as its DeclContext.
detail::InMemoryDirectory::const_iterator I
static void EmitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, SourceLocation Loc)
static llvm::Value * convertToScalarValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
const ArrayType * getAsArrayTypeUnsafe() const
A variant of getAs<> for array types which silently discards qualifiers from the outermost type...
static CodeGenFunction::ComplexPairTy convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType, QualType DestType, SourceLocation Loc)
This represents '#pragma omp target parallel for simd' directive.
LValue MakeAddrLValue(Address Addr, QualType T, AlignmentSource AlignSource=AlignmentSource::Type)
ArrayRef< Expr * > private_counters()
OpenMPDirectiveKind getCancelRegion() const
Get cancellation region for the current cancellation point.
llvm::Constant * getStaticLocalDeclAddress(const VarDecl *D)
CompoundStmt - This represents a group of statements like { stmt stmt }.
std::pair< llvm::Value *, llvm::Value * > ComplexPairTy
AutoVarEmission EmitAutoVarAlloca(const VarDecl &var)
EmitAutoVarAlloca - Emit the alloca and debug information for a local variable.
void EmitOMPParallelSectionsDirective(const OMPParallelSectionsDirective &S)
This represents '#pragma omp taskgroup' directive.
void EmitAnyExprToMem(const Expr *E, Address Location, Qualifiers Quals, bool IsInitializer)
EmitAnyExprToMem - Emits the code necessary to evaluate an arbitrary expression into the given memory...
This represents clause 'aligned' in the '#pragma omp ...' directives.
RValue - This trivial value class is used to represent the result of an expression that is evaluated...
bool addPrivate(const VarDecl *LocalVD, llvm::function_ref< Address()> PrivateGen)
Registers LocalVD variable as a private and apply PrivateGen function for it to generate correspondin...
static Address castToBase(CodeGenFunction &CGF, QualType BaseTy, QualType ElTy, LValue BaseLV, llvm::Value *Addr)
bool isPostfixUpdate() const
Return true if 'v' expression must be updated to original value of 'x', false if 'v' must be updated ...
QualType getPointeeType() const
If this is a pointer, ObjC object pointer, or block pointer, this returns the respective pointee...
This represents '#pragma omp distribute' directive.
This represents implicit clause 'depend' for the '#pragma omp task' directive.
virtual StringRef getHelperName() const
Get the name of the capture helper.
static TypeEvaluationKind getEvaluationKind(QualType T)
hasAggregateLLVMType - Return true if the specified AST type will map into an aggregate LLVM type or ...
llvm::Value * getPointer() const
This represents 'proc_bind' clause in the '#pragma omp ...' directive.
Expr - This represents one expression.
void EmitAutoVarInit(const AutoVarEmission &emission)
Enters a new scope for capturing cleanups, all of which will be executed once the scope is exited...
This represents 'simdlen' clause in the '#pragma omp ...' directive.
llvm::Value * EmitComplexToScalarConversion(ComplexPairTy Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified complex type to the specified destination type, where the destination type is an LLVM scalar type.
bool isOpenMPParallelDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a parallel-kind directive.
void EmitOMPTeamsDirective(const OMPTeamsDirective &S)
void SetInternalFunctionAttributes(const Decl *D, llvm::Function *F, const CGFunctionInfo &FI)
Set the attributes on the LLVM function for the given decl and function info.
Expr * getIterationVariable() const
static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S, const Expr *Cond, llvm::BasicBlock *TrueBlock, llvm::BasicBlock *FalseBlock, uint64_t TrueCount)
OpenMPClauseKind
OpenMP clauses.
ASTContext & getContext() const
void EmitOMPLinearClauseInit(const OMPLoopDirective &D)
Emit initial code for linear variables.
unsigned getContextParamPosition() const
This represents 'ordered' clause in the '#pragma omp ...' directive.
CharUnits toCharUnitsFromBits(int64_t BitSize) const
Convert a size in bits to a size in characters.
std::pair< bool, RValue > EmitOMPAtomicSimpleUpdateExpr(LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart, llvm::AtomicOrdering AO, SourceLocation Loc, const llvm::function_ref< RValue(RValue)> &CommonGen)
Emit atomic update code for constructs: X = X BO E or X = E BO E.
This represents '#pragma omp for' directive.
SmallVector< const Expr *, 4 > FirstprivateVars
void EmitOMPMasterDirective(const OMPMasterDirective &S)
static void EmitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *V, SourceLocation Loc)
static std::pair< llvm::Function *, llvm::Constant * > EmitOMPTargetDirectiveOutlinedFunction(CodeGenModule &CGM, const OMPTargetDirective &S, StringRef ParentName, bool IsOffloadEntry)
Emit outlined function for the target directive.
void setParallel(bool Enable=true)
Set the next pushed loop as parallel.
LValue MakeNaturalAlignAddrLValue(llvm::Value *V, QualType T)
Expr * getIsLastIterVariable() const
UnaryOperator - This represents the unary-expression's (except sizeof and alignof), the postinc/postdec operators from postfix-expression, and various extensions.
bool isOpenMPTaskLoopDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a taskloop directive.
void EmitOMPBarrierDirective(const OMPBarrierDirective &S)
Expr * getNextUpperBound() const
This represents '#pragma omp cancel' directive.
This represents clause 'firstprivate' in the '#pragma omp ...' directives.
static void EmitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
RValue EmitAtomicLoad(LValue LV, SourceLocation SL, AggValueSlot Slot=AggValueSlot::ignored())
VarDecl * getCanonicalDecl() override
GlobalDecl - represents a global declaration.
This represents '#pragma omp flush' directive.
This represents '#pragma omp parallel for simd' directive.
This represents 'seq_cst' clause in the '#pragma omp atomic' directive.
This represents 'untied' clause in the '#pragma omp ...' directive.
SourceLocation getLocStart() const
Returns starting location of directive kind.
LValue EmitLoadOfPointerLValue(Address Ptr, const PointerType *PtrTy)
static void emitAlignedClause(CodeGenFunction &CGF, const OMPExecutableDirective &D)
The l-value was considered opaque, so the alignment was determined from a type.
void EmitAlignmentAssumption(llvm::Value *PtrValue, unsigned Alignment, llvm::Value *OffsetValue=nullptr)
This represents '#pragma omp target enter data' directive.
void EmitOMPFlushDirective(const OMPFlushDirective &S)
bool HaveInsertPoint() const
HaveInsertPoint - True if an insertion point is defined.
This represents 'num_teams' clause in the '#pragma omp ...' directive.
OpaqueValueExpr - An expression referring to an opaque object of a fixed type and value class...
This captures a statement into a function.
ASTContext & getContext() const
This represents '#pragma omp single' directive.
Encodes a location in the source.
IdentifierInfo & get(StringRef Name)
Return the identifier token info for the specified named identifier.
This represents 'hint' clause in the '#pragma omp ...' directive.
void EmitOMPForDirective(const OMPForDirective &S)
This represents '#pragma omp declare reduction ...' directive.
Pseudo declaration for capturing expressions.
llvm::Value * EvaluateExprAsBool(const Expr *E)
EvaluateExprAsBool - Perform the usual unary conversions on the specified expression and compare the ...
llvm::PointerIntPair< llvm::Value *, 1, bool > Final
This is a basic class for representing single OpenMP executable directive.
llvm::Value * EmitLoadOfScalar(Address Addr, bool Volatile, QualType Ty, SourceLocation Loc, AlignmentSource AlignSource=AlignmentSource::Type, llvm::MDNode *TBAAInfo=nullptr, QualType TBAABaseTy=QualType(), uint64_t TBAAOffset=0, bool isNontemporal=false)
EmitLoadOfScalar - Load a scalar value from an address, taking care to appropriately convert from the...
Expr * getLowerBoundVariable() const
This represents 'schedule' clause in the '#pragma omp ...' directive.
Expr * getExpr()
Get 'expr' part of the associated expression/statement.
static OMPLinearClause * Create(const ASTContext &C, SourceLocation StartLoc, SourceLocation LParenLoc, OpenMPLinearClauseKind Modifier, SourceLocation ModifierLoc, SourceLocation ColonLoc, SourceLocation EndLoc, ArrayRef< Expr * > VL, ArrayRef< Expr * > PL, ArrayRef< Expr * > IL, Expr *Step, Expr *CalcStep, Stmt *PreInit, Expr *PostUpdate)
Creates clause with a list of variables VL and a linear step Step.
OpenMPDirectiveKind
OpenMP directives.
void EmitOMPLinearClauseFinal(const OMPLoopDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
Emit final code for linear clauses.
std::string getAsString() const
getAsString - Retrieve the human-readable string for this name.
param_iterator param_begin() const
Retrieve an iterator pointing to the first parameter decl.
This represents '#pragma omp taskwait' directive.
void EmitOMPTaskgroupDirective(const OMPTaskgroupDirective &S)
This file defines OpenMP nodes for declarative directives.
void EmitOMPTargetParallelForSimdDirective(const OMPTargetParallelForSimdDirective &S)
bool isTrivialInitializer(const Expr *Init)
Determine whether the given initializer is trivial in the sense that it requires no code to be genera...
CharUnits alignmentOfArrayElement(CharUnits elementSize) const
Given that this is the alignment of the first element of an array, return the minimum alignment of an...
void EmitOMPSingleDirective(const OMPSingleDirective &S)
void StartFunction(GlobalDecl GD, QualType RetTy, llvm::Function *Fn, const CGFunctionInfo &FnInfo, const FunctionArgList &Args, SourceLocation Loc=SourceLocation(), SourceLocation StartLoc=SourceLocation())
Emit code for the start of a function.
const LangOptions & getLangOpts() const
This represents '#pragma omp target' directive.
const T * castAs() const
Member-template castAs<specific type>.
void EmitOMPForSimdDirective(const OMPForSimdDirective &S)
Expr * getUpperBoundVariable() const
static void EmitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst, bool IsPostfixUpdate, const Expr *V, const Expr *X, const Expr *E, const Expr *UE, bool IsXLHSInRHSPart, SourceLocation Loc)
Expr * getV()
Get 'v' part of the associated expression/statement.
void EmitOMPAtomicDirective(const OMPAtomicDirective &S)
JumpDest getJumpDestInCurrentScope(llvm::BasicBlock *Target)
The given basic block lies in the current EH scope, but may be a target of a potentially scope-crossi...
void EmitOMPSectionDirective(const OMPSectionDirective &S)
void EmitOMPSectionsDirective(const OMPSectionsDirective &S)
This represents '#pragma omp ordered' directive.
const SpecificClause * getSingleClause() const
Gets a single clause of the specified kind associated with the current directive iff there is only on...
void EmitOMPDistributeLoop(const OMPDistributeDirective &S)
This represents '#pragma omp target update' directive.
bool isVolatileQualified() const
Determine whether this type is volatile-qualified.
void EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S)
void enterFullExpression(const ExprWithCleanups *E)
AlignmentSource getAlignmentSource() const
void EmitDecl(const Decl &D)
EmitDecl - Emit a declaration.
void FinishFunction(SourceLocation EndLoc=SourceLocation())
FinishFunction - Complete IR generation of the current function.
Expr * getPreCond() const
SourceLocation getExprLoc() const LLVM_READONLY
getExprLoc - Return the preferred location for the arrow when diagnosing a problem with a generic exp...
JumpDest getOMPCancelDestination(OpenMPDirectiveKind Kind)
llvm::Value * EmitScalarExpr(const Expr *E, bool IgnoreResultAssign=false)
EmitScalarExpr - Emit the computation of the specified expression of LLVM scalar type, returning the result.
FunctionArgList - Type for representing both the decl and type of parameters to a function...
if(T->getSizeExpr()) TRY_TO(TraverseStmt(T-> getSizeExpr()))
void GenerateOpenMPCapturedVars(const CapturedStmt &S, SmallVectorImpl< llvm::Value * > &CapturedVars)
CGFunctionInfo - Class to encapsulate the information about a function definition.
CharUnits getAlignment() const
Return the alignment of this pointer.
This class organizes the cross-function state that is used while generating LLVM code.
CGOpenMPRuntime & getOpenMPRuntime()
Return a reference to the configured OpenMP runtime.
OpenMPScheduleClauseModifier
OpenMP modifiers for 'schedule' clause.
Class provides a way to call simple version of codegen for OpenMP region, or an advanced with possibl...
This represents 'device' clause in the '#pragma omp ...' directive.
void EmitOMPTaskyieldDirective(const OMPTaskyieldDirective &S)
static RValue getComplex(llvm::Value *V1, llvm::Value *V2)
void EmitOMPTargetParallelDirective(const OMPTargetParallelDirective &S)
Address CreateMemTemp(QualType T, const Twine &Name="tmp")
CreateMemTemp - Create a temporary memory object of the given type, with appropriate alignment...
bool hasCancel() const
Return true if current directive has inner cancel directive.
llvm::Value * getScalarVal() const
getScalarVal() - Return the Value* of this scalar value.
static AggValueSlot ignored()
ignored - Returns an aggregate value slot indicating that the aggregate value is being ignored...
void EmitOMPParallelForDirective(const OMPParallelForDirective &S)
This represents '#pragma omp section' directive.
std::vector< llvm::Triple > OMPTargetTriples
Triples of the OpenMP targets that the host code codegen should take into account in order to generat...
OpenMPScheduleClauseModifier M1
A basic class for pre|post-action for advanced codegen sequence for OpenMP region.
void EmitOMPPrivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPTargetParallelForDirective(const OMPTargetParallelForDirective &S)
const Stmt * getBody() const
This represents '#pragma omp simd' directive.
llvm::LoadInst * CreateLoad(Address Addr, const llvm::Twine &Name="")
llvm::IntegerType * IntPtrTy
QualType getPointerType(QualType T) const
Return the uniqued reference to the type for a pointer to the specified type.
static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF, const OMPExecutableDirective &S, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
bool isOpenMPSimdDirective(OpenMPDirectiveKind DKind)
Checks if the specified directive is a simd directive.
bool isLValue() const
isLValue - True if this expression is an "l-value" according to the rules of the current language...
llvm::PointerIntPair< llvm::Value *, 1, bool > Priority
static RValue convertToType(CodeGenFunction &CGF, RValue Value, QualType SourceType, QualType ResType, SourceLocation Loc)
void EmitOMPLinearClause(const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope)
Emit initial code for linear clauses.
llvm::Constant * EmitNullConstant(QualType T)
Return the result of value-initializing the given type, i.e.
This represents clause 'linear' in the '#pragma omp ...' directives.
DeclarationNameInfo getDirectiveName() const
Return name of the directive.
Expr * getEnsureUpperBound() const
detail::InMemoryDirectory::const_iterator E
bool EmitOMPFirstprivateClause(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
void EmitOMPCancelDirective(const OMPCancelDirective &S)
Expr * getUpdateExpr()
Get helper expression of the form 'OpaqueValueExpr(x) binop OpaqueValueExpr(expr)' or 'OpaqueValueExp...
llvm::StoreInst * CreateStore(llvm::Value *Val, Address Addr, bool IsVolatile=false)
This represents '#pragma omp atomic' directive.
unsigned Map[Count]
The type of a lookup table which maps from language-specific address spaces to target-specific ones...
Expr * getCalcLastIteration() const
Expr * IgnoreParenImpCasts() LLVM_READONLY
IgnoreParenImpCasts - Ignore parentheses and implicit casts.
QualType getNonReferenceType() const
If Type is a reference type (e.g., const int&), returns the type that the reference refers to ("const...
ArrayRef< Expr * > counters()
JumpDest ReturnBlock
ReturnBlock - Unified return block.
SwitchStmt - This represents a 'switch' stmt.
void EmitAutoVarCleanups(const AutoVarEmission &emission)
API for captured statement code generation.
llvm::PointerType * getType() const
Return the type of the pointer value.
Complex values, per C99 6.2.5p11.
static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst, LValue LVal, RValue RVal)
const T * getAs() const
Member-template getAs<specific type>'.
This file defines OpenMP AST classes for executable directives and clauses.
static ImplicitParamDecl * Create(ASTContext &C, DeclContext *DC, SourceLocation IdLoc, IdentifierInfo *Id, QualType T)
void EmitStoreOfComplex(ComplexPairTy V, LValue dest, bool isInit)
EmitStoreOfComplex - Store a complex number into the specified l-value.
ArrayRef< Expr * > inits()
void EmitOMPTaskBasedDirective(const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen, OMPTaskDataTy &Data)
virtual llvm::Value * emitParallelOrTeamsOutlinedFunction(const OMPExecutableDirective &D, const VarDecl *ThreadIDVar, OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen)
Emits outlined function for the specified OpenMP parallel directive D.
LValue EmitLoadOfReferenceLValue(Address Ref, const ReferenceType *RefTy)
OpenMPScheduleClauseKind
OpenMP attributes for 'schedule' clause.
void EmitScalarInit(const Expr *init, const ValueDecl *D, LValue lvalue, bool capturedByInit)
Base for LValueReferenceType and RValueReferenceType.
StringRef getMangledName(GlobalDecl GD)
Internal linkage, which indicates that the entity can be referred to from within the translation unit...
void EmitBlock(llvm::BasicBlock *BB, bool IsFinished=false)
EmitBlock - Emit the given block.
void EmitOMPTargetEnterDataDirective(const OMPTargetEnterDataDirective &S)
void EmitOMPDistributeSimdDirective(const OMPDistributeSimdDirective &S)
llvm::ConstantInt * getSize(CharUnits numChars)
Emit the given number of characters as a value of type size_t.
const Decl * CurFuncDecl
CurFuncDecl - Holds the Decl for the current outermost non-closure context.
bool hasAssociatedStmt() const
Returns true if directive has associated statement.
void EmitStopPoint(const Stmt *S)
EmitStopPoint - Emit a debug stoppoint if we are emitting debug info.
X
Add a minimal nested name specifier fixit hint to allow lookup of a tag name from an outer enclosing ...
param_iterator param_end() const
Retrieve an iterator one past the last parameter decl.
CapturedDecl * getCapturedDecl()
Retrieve the outlined function declaration.
Expr * getNextLowerBound() const
void EmitOMPReductionClauseInit(const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope)
Emit initial code for reduction variables.
This represents '#pragma omp target parallel' directive.
static llvm::iterator_range< specific_clause_iterator< SpecificClause > > getClausesOfKind(ArrayRef< OMPClause * > Clauses)
This represents 'nowait' clause in the '#pragma omp ...' directive.
llvm::PointerIntPair< llvm::Value *, 1, bool > Schedule
void EmitBranch(llvm::BasicBlock *Block)
EmitBranch - Emit a branch to the specified basic block from the current insert block, taking care to avoid creation of branches from dummy blocks.
static void emitPostUpdateForReductionClause(CodeGenFunction &CGF, const OMPExecutableDirective &D, const llvm::function_ref< llvm::Value *(CodeGenFunction &)> &CondGen)
This represents 'num_tasks' clause in the '#pragma omp ...' directive.
Privates[]
Gets the list of initial values for linear variables.
LValue EmitLValue(const Expr *E)
EmitLValue - Emit code to compute a designator that specifies the location of the expression...
RValue EmitLoadOfLValue(LValue V, SourceLocation Loc)
EmitLoadOfLValue - Given an expression that represents a value lvalue, this method emits the address ...
This represents '#pragma omp taskloop simd' directive.
std::pair< llvm::Value *, QualType > getVLASize(const VariableArrayType *vla)
getVLASize - Returns an LLVM value that corresponds to the size, in non-variably-sized elements...
bool EmitOMPCopyinClause(const OMPExecutableDirective &D)
Emit code for copyin clause in D directive.
This represents 'dist_schedule' clause in the '#pragma omp ...' directive.
CGCapturedStmtInfo * CapturedStmtInfo
llvm::Value * EmitScalarConversion(llvm::Value *Src, QualType SrcTy, QualType DstTy, SourceLocation Loc)
Emit a conversion from the specified type to the specified destination type, both of which are LLVM s...
const llvm::function_ref< void(CodeGenFunction &, llvm::Value *, const OMPTaskDataTy &)> TaskGenTy
This represents '#pragma omp sections' directive.
This represents '#pragma omp target data' directive.
void EmitOMPTargetExitDataDirective(const OMPTargetExitDataDirective &S)
void EmitOMPInnerLoop(const Stmt &S, bool RequiresCleanup, const Expr *LoopCond, const Expr *IncExpr, const llvm::function_ref< void(CodeGenFunction &)> &BodyGen, const llvm::function_ref< void(CodeGenFunction &)> &PostIncGen)
Emit inner loop of the worksharing/simd construct.
A reference to a declared variable, function, enum, etc.
static RValue get(llvm::Value *V)
QualType getUIntPtrType() const
Return a type compatible with "uintptr_t" (C99 7.18.1.4), as defined by the target.
Expr * getLastIteration() const
void setVectorizeEnable(bool Enable=true)
Set the next pushed loop 'vectorize.enable'.
void EmitOMPTargetDataDirective(const OMPTargetDataDirective &S)
void EmitBranchThroughCleanup(JumpDest Dest)
EmitBranchThroughCleanup - Emit a branch from the current insert block through the normal cleanup han...
void EmitOMPDistributeParallelForDirective(const OMPDistributeParallelForDirective &S)
An l-value expression is a reference to an object with independent storage.
static RValue getAggregate(Address addr, bool isVolatile=false)
int64_t toBits(CharUnits CharSize) const
Convert a size in characters to a size in bits.
LValue - This represents an lvalue references.
This represents '#pragma omp taskyield' directive.
This represents '#pragma omp distribute parallel for simd' composite directive.
This represents '#pragma omp parallel sections' directive.
bool hasSignedIntegerRepresentation() const
Determine whether this type has an signed integer representation of some sort, e.g., it is an signed integer type or a vector.
SourceLocation getLocEnd() const
Returns ending location of directive.
Address CreatePointerBitCastOrAddrSpaceCast(Address Addr, llvm::Type *Ty, const llvm::Twine &Name="")
llvm::Value * emitArrayLength(const ArrayType *arrayType, QualType &baseType, Address &addr)
emitArrayLength - Compute the length of an array, even if it's a VLA, and drill down to the base elem...
Expr *const * const_capture_init_iterator
Const iterator that walks over the capture initialization arguments.
Stmt * getAssociatedStmt() const
Returns statement associated with the directive.
capture_init_iterator capture_init_end()
Retrieve the iterator pointing one past the last initialization argument.
void EmitOMPTaskDirective(const OMPTaskDirective &S)
A class which abstracts out some details necessary for making a call.
This represents '#pragma omp target parallel for' directive.
SmallVector< const Expr *, 4 > FirstprivateInits
Expr * getStrideVariable() const
bool Privatize()
Privatizes local variables previously registered as private.
Qualifiers getQualifiers() const
Retrieve the set of qualifiers applied to this type.
bool isPointerType() const
llvm::FunctionType * GetFunctionType(const CGFunctionInfo &Info)
GetFunctionType - Get the LLVM function type for.
This represents '#pragma omp taskloop' directive.