LLVM  13.0.0git
OMPIRBuilder.cpp
Go to the documentation of this file.
1 //===- OpenMPIRBuilder.cpp - Builder for LLVM-IR for OpenMP directives ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 ///
10 /// This file implements the OpenMPIRBuilder class, which is used as a
11 /// convenient way to create LLVM instructions for OpenMP directives.
12 ///
13 //===----------------------------------------------------------------------===//
14 
16 
17 #include "llvm/ADT/StringRef.h"
18 #include "llvm/ADT/Triple.h"
19 #include "llvm/IR/CFG.h"
20 #include "llvm/IR/DebugInfo.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/MDBuilder.h"
24 #include "llvm/Support/Error.h"
27 
28 #include <sstream>
29 
30 #define DEBUG_TYPE "openmp-ir-builder"
31 
32 using namespace llvm;
33 using namespace omp;
34 
35 static cl::opt<bool>
36  OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden,
37  cl::desc("Use optimistic attributes describing "
38  "'as-if' properties of runtime calls."),
39  cl::init(false));
40 
42  LLVMContext &Ctx = Fn.getContext();
43 
44  // Get the function's current attributes.
45  auto Attrs = Fn.getAttributes();
46  auto FnAttrs = Attrs.getFnAttributes();
47  auto RetAttrs = Attrs.getRetAttributes();
49  for (size_t ArgNo = 0; ArgNo < Fn.arg_size(); ++ArgNo)
50  ArgAttrs.emplace_back(Attrs.getParamAttributes(ArgNo));
51 
52 #define OMP_ATTRS_SET(VarName, AttrSet) AttributeSet VarName = AttrSet;
53 #include "llvm/Frontend/OpenMP/OMPKinds.def"
54 
55  // Add attributes to the function declaration.
56  switch (FnID) {
57 #define OMP_RTL_ATTRS(Enum, FnAttrSet, RetAttrSet, ArgAttrSets) \
58  case Enum: \
59  FnAttrs = FnAttrs.addAttributes(Ctx, FnAttrSet); \
60  RetAttrs = RetAttrs.addAttributes(Ctx, RetAttrSet); \
61  for (size_t ArgNo = 0; ArgNo < ArgAttrSets.size(); ++ArgNo) \
62  ArgAttrs[ArgNo] = \
63  ArgAttrs[ArgNo].addAttributes(Ctx, ArgAttrSets[ArgNo]); \
64  Fn.setAttributes(AttributeList::get(Ctx, FnAttrs, RetAttrs, ArgAttrs)); \
65  break;
66 #include "llvm/Frontend/OpenMP/OMPKinds.def"
67  default:
68  // Attributes are optional.
69  break;
70  }
71 }
72 
75  FunctionType *FnTy = nullptr;
76  Function *Fn = nullptr;
77 
78  // Try to find the declation in the module first.
79  switch (FnID) {
80 #define OMP_RTL(Enum, Str, IsVarArg, ReturnType, ...) \
81  case Enum: \
82  FnTy = FunctionType::get(ReturnType, ArrayRef<Type *>{__VA_ARGS__}, \
83  IsVarArg); \
84  Fn = M.getFunction(Str); \
85  break;
86 #include "llvm/Frontend/OpenMP/OMPKinds.def"
87  }
88 
89  if (!Fn) {
90  // Create a new declaration if we need one.
91  switch (FnID) {
92 #define OMP_RTL(Enum, Str, ...) \
93  case Enum: \
94  Fn = Function::Create(FnTy, GlobalValue::ExternalLinkage, Str, M); \
95  break;
96 #include "llvm/Frontend/OpenMP/OMPKinds.def"
97  }
98 
99  // Add information if the runtime function takes a callback function
100  if (FnID == OMPRTL___kmpc_fork_call || FnID == OMPRTL___kmpc_fork_teams) {
101  if (!Fn->hasMetadata(LLVMContext::MD_callback)) {
102  LLVMContext &Ctx = Fn->getContext();
103  MDBuilder MDB(Ctx);
104  // Annotate the callback behavior of the runtime function:
105  // - The callback callee is argument number 2 (microtask).
106  // - The first two arguments of the callback callee are unknown (-1).
107  // - All variadic arguments to the runtime function are passed to the
108  // callback callee.
109  Fn->addMetadata(
110  LLVMContext::MD_callback,
112  2, {-1, -1}, /* VarArgsArePassed */ true)}));
113  }
114  }
115 
116  LLVM_DEBUG(dbgs() << "Created OpenMP runtime function " << Fn->getName()
117  << " with type " << *Fn->getFunctionType() << "\n");
118  addAttributes(FnID, *Fn);
119 
120  } else {
121  LLVM_DEBUG(dbgs() << "Found OpenMP runtime function " << Fn->getName()
122  << " with type " << *Fn->getFunctionType() << "\n");
123  }
124 
125  assert(Fn && "Failed to create OpenMP runtime function");
126 
127  // Cast the function to the expected type if necessary
129  return {FnTy, C};
130 }
131 
133  FunctionCallee RTLFn = getOrCreateRuntimeFunction(M, FnID);
134  auto *Fn = dyn_cast<llvm::Function>(RTLFn.getCallee());
135  assert(Fn && "Failed to create OpenMP runtime function pointer");
136  return Fn;
137 }
138 
139 void OpenMPIRBuilder::initialize() { initializeTypes(M); }
140 
141 void OpenMPIRBuilder::finalize(Function *Fn, bool AllowExtractorSinking) {
142  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
144  SmallVector<OutlineInfo, 16> DeferredOutlines;
145  for (OutlineInfo &OI : OutlineInfos) {
146  // Skip functions that have not finalized yet; may happen with nested
147  // function generation.
148  if (Fn && OI.getFunction() != Fn) {
149  DeferredOutlines.push_back(OI);
150  continue;
151  }
152 
153  ParallelRegionBlockSet.clear();
154  Blocks.clear();
155  OI.collectBlocks(ParallelRegionBlockSet, Blocks);
156 
157  Function *OuterFn = OI.getFunction();
158  CodeExtractorAnalysisCache CEAC(*OuterFn);
159  CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
160  /* AggregateArgs */ false,
161  /* BlockFrequencyInfo */ nullptr,
162  /* BranchProbabilityInfo */ nullptr,
163  /* AssumptionCache */ nullptr,
164  /* AllowVarArgs */ true,
165  /* AllowAlloca */ true,
166  /* Suffix */ ".omp_par");
167 
168  LLVM_DEBUG(dbgs() << "Before outlining: " << *OuterFn << "\n");
169  LLVM_DEBUG(dbgs() << "Entry " << OI.EntryBB->getName()
170  << " Exit: " << OI.ExitBB->getName() << "\n");
171  assert(Extractor.isEligible() &&
172  "Expected OpenMP outlining to be possible!");
173 
174  Function *OutlinedFn = Extractor.extractCodeRegion(CEAC);
175 
176  LLVM_DEBUG(dbgs() << "After outlining: " << *OuterFn << "\n");
177  LLVM_DEBUG(dbgs() << " Outlined function: " << *OutlinedFn << "\n");
178  assert(OutlinedFn->getReturnType()->isVoidTy() &&
179  "OpenMP outlined functions should not return a value!");
180 
181  // For compability with the clang CG we move the outlined function after the
182  // one with the parallel region.
183  OutlinedFn->removeFromParent();
184  M.getFunctionList().insertAfter(OuterFn->getIterator(), OutlinedFn);
185 
186  // Remove the artificial entry introduced by the extractor right away, we
187  // made our own entry block after all.
188  {
189  BasicBlock &ArtificialEntry = OutlinedFn->getEntryBlock();
190  assert(ArtificialEntry.getUniqueSuccessor() == OI.EntryBB);
191  assert(OI.EntryBB->getUniquePredecessor() == &ArtificialEntry);
192  if (AllowExtractorSinking) {
193  // Move instructions from the to-be-deleted ArtificialEntry to the entry
194  // basic block of the parallel region. CodeExtractor may have sunk
195  // allocas/bitcasts for values that are solely used in the outlined
196  // region and do not escape.
197  assert(!ArtificialEntry.empty() &&
198  "Expected instructions to sink in the outlined region");
199  for (BasicBlock::iterator It = ArtificialEntry.begin(),
200  End = ArtificialEntry.end();
201  It != End;) {
202  Instruction &I = *It;
203  It++;
204 
205  if (I.isTerminator())
206  continue;
207 
208  I.moveBefore(*OI.EntryBB, OI.EntryBB->getFirstInsertionPt());
209  }
210  }
211  OI.EntryBB->moveBefore(&ArtificialEntry);
212  ArtificialEntry.eraseFromParent();
213  }
214  assert(&OutlinedFn->getEntryBlock() == OI.EntryBB);
215  assert(OutlinedFn && OutlinedFn->getNumUses() == 1);
216 
217  // Run a user callback, e.g. to add attributes.
218  if (OI.PostOutlineCB)
219  OI.PostOutlineCB(*OutlinedFn);
220  }
221 
222  // Remove work items that have been completed.
223  OutlineInfos = std::move(DeferredOutlines);
224 }
225 
227  assert(OutlineInfos.empty() && "There must be no outstanding outlinings");
228 }
229 
231  IdentFlag LocFlags,
232  unsigned Reserve2Flags) {
233  // Enable "C-mode".
234  LocFlags |= OMP_IDENT_FLAG_KMPC;
235 
236  Value *&Ident =
237  IdentMap[{SrcLocStr, uint64_t(LocFlags) << 31 | Reserve2Flags}];
238  if (!Ident) {
239  Constant *I32Null = ConstantInt::getNullValue(Int32);
240  Constant *IdentData[] = {
241  I32Null, ConstantInt::get(Int32, uint32_t(LocFlags)),
242  ConstantInt::get(Int32, Reserve2Flags), I32Null, SrcLocStr};
243  Constant *Initializer = ConstantStruct::get(
244  cast<StructType>(IdentPtr->getPointerElementType()), IdentData);
245 
246  // Look for existing encoding of the location + flags, not needed but
247  // minimizes the difference to the existing solution while we transition.
248  for (GlobalVariable &GV : M.getGlobalList())
249  if (GV.getType() == IdentPtr && GV.hasInitializer())
250  if (GV.getInitializer() == Initializer)
251  return Ident = &GV;
252 
253  auto *GV = new GlobalVariable(M, IdentPtr->getPointerElementType(),
254  /* isConstant = */ true,
255  GlobalValue::PrivateLinkage, Initializer);
256  GV->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
257  GV->setAlignment(Align(8));
258  Ident = GV;
259  }
260  return Builder.CreatePointerCast(Ident, IdentPtr);
261 }
262 
264  LLVMContext &Ctx = M.getContext();
265  Triple triple(M.getTargetTriple());
266 
267  // This test is adequate until deviceRTL has finer grained lane widths
268  return triple.isAMDGCN() ? Type::getInt64Ty(Ctx) : Type::getInt32Ty(Ctx);
269 }
270 
272  Constant *&SrcLocStr = SrcLocStrMap[LocStr];
273  if (!SrcLocStr) {
274  Constant *Initializer =
275  ConstantDataArray::getString(M.getContext(), LocStr);
276 
277  // Look for existing encoding of the location, not needed but minimizes the
278  // difference to the existing solution while we transition.
279  for (GlobalVariable &GV : M.getGlobalList())
280  if (GV.isConstant() && GV.hasInitializer() &&
281  GV.getInitializer() == Initializer)
282  return SrcLocStr = ConstantExpr::getPointerCast(&GV, Int8Ptr);
283 
284  SrcLocStr = Builder.CreateGlobalStringPtr(LocStr, /* Name */ "",
285  /* AddressSpace */ 0, &M);
286  }
287  return SrcLocStr;
288 }
289 
291  StringRef FileName,
292  unsigned Line,
293  unsigned Column) {
294  SmallString<128> Buffer;
295  Buffer.push_back(';');
296  Buffer.append(FileName);
297  Buffer.push_back(';');
298  Buffer.append(FunctionName);
299  Buffer.push_back(';');
300  Buffer.append(std::to_string(Line));
301  Buffer.push_back(';');
302  Buffer.append(std::to_string(Column));
303  Buffer.push_back(';');
304  Buffer.push_back(';');
305  return getOrCreateSrcLocStr(Buffer.str());
306 }
307 
309  return getOrCreateSrcLocStr(";unknown;unknown;0;0;;");
310 }
311 
312 Constant *
314  DILocation *DIL = Loc.DL.get();
315  if (!DIL)
316  return getOrCreateDefaultSrcLocStr();
317  StringRef FileName = M.getName();
318  if (DIFile *DIF = DIL->getFile())
319  if (Optional<StringRef> Source = DIF->getSource())
320  FileName = *Source;
321  StringRef Function = DIL->getScope()->getSubprogram()->getName();
322  Function =
323  !Function.empty() ? Function : Loc.IP.getBlock()->getParent()->getName();
324  return getOrCreateSrcLocStr(Function, FileName, DIL->getLine(),
325  DIL->getColumn());
326 }
327 
329  return Builder.CreateCall(
330  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num), Ident,
331  "omp_global_thread_num");
332 }
333 
336  bool ForceSimpleCall, bool CheckCancelFlag) {
337  if (!updateToLocation(Loc))
338  return Loc.IP;
339  return emitBarrierImpl(Loc, DK, ForceSimpleCall, CheckCancelFlag);
340 }
341 
344  bool ForceSimpleCall, bool CheckCancelFlag) {
345  // Build call __kmpc_cancel_barrier(loc, thread_id) or
346  // __kmpc_barrier(loc, thread_id);
347 
348  IdentFlag BarrierLocFlags;
349  switch (Kind) {
350  case OMPD_for:
351  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_FOR;
352  break;
353  case OMPD_sections:
354  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SECTIONS;
355  break;
356  case OMPD_single:
357  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL_SINGLE;
358  break;
359  case OMPD_barrier:
360  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_EXPL;
361  break;
362  default:
363  BarrierLocFlags = OMP_IDENT_FLAG_BARRIER_IMPL;
364  break;
365  }
366 
367  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
368  Value *Args[] = {getOrCreateIdent(SrcLocStr, BarrierLocFlags),
369  getOrCreateThreadID(getOrCreateIdent(SrcLocStr))};
370 
371  // If we are in a cancellable parallel region, barriers are cancellation
372  // points.
373  // TODO: Check why we would force simple calls or to ignore the cancel flag.
374  bool UseCancelBarrier =
375  !ForceSimpleCall && isLastFinalizationInfoCancellable(OMPD_parallel);
376 
377  Value *Result =
378  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(
379  UseCancelBarrier ? OMPRTL___kmpc_cancel_barrier
380  : OMPRTL___kmpc_barrier),
381  Args);
382 
383  if (UseCancelBarrier && CheckCancelFlag)
384  emitCancelationCheckImpl(Result, OMPD_parallel);
385 
386  return Builder.saveIP();
387 }
388 
391  Value *IfCondition,
392  omp::Directive CanceledDirective) {
393  if (!updateToLocation(Loc))
394  return Loc.IP;
395 
396  // LLVM utilities like blocks with terminators.
397  auto *UI = Builder.CreateUnreachable();
398 
399  Instruction *ThenTI = UI, *ElseTI = nullptr;
400  if (IfCondition)
401  SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
402  Builder.SetInsertPoint(ThenTI);
403 
404  Value *CancelKind = nullptr;
405  switch (CanceledDirective) {
406 #define OMP_CANCEL_KIND(Enum, Str, DirectiveEnum, Value) \
407  case DirectiveEnum: \
408  CancelKind = Builder.getInt32(Value); \
409  break;
410 #include "llvm/Frontend/OpenMP/OMPKinds.def"
411  default:
412  llvm_unreachable("Unknown cancel kind!");
413  }
414 
415  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
416  Value *Ident = getOrCreateIdent(SrcLocStr);
417  Value *Args[] = {Ident, getOrCreateThreadID(Ident), CancelKind};
418  Value *Result = Builder.CreateCall(
419  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_cancel), Args);
420 
421  // The actual cancel logic is shared with others, e.g., cancel_barriers.
422  emitCancelationCheckImpl(Result, CanceledDirective);
423 
424  // Update the insertion point and remove the terminator we introduced.
425  Builder.SetInsertPoint(UI->getParent());
426  UI->eraseFromParent();
427 
428  return Builder.saveIP();
429 }
430 
432  Value *CancelFlag, omp::Directive CanceledDirective) {
433  assert(isLastFinalizationInfoCancellable(CanceledDirective) &&
434  "Unexpected cancellation!");
435 
436  // For a cancel barrier we create two new blocks.
437  BasicBlock *BB = Builder.GetInsertBlock();
438  BasicBlock *NonCancellationBlock;
439  if (Builder.GetInsertPoint() == BB->end()) {
440  // TODO: This branch will not be needed once we moved to the
441  // OpenMPIRBuilder codegen completely.
442  NonCancellationBlock = BasicBlock::Create(
443  BB->getContext(), BB->getName() + ".cont", BB->getParent());
444  } else {
445  NonCancellationBlock = SplitBlock(BB, &*Builder.GetInsertPoint());
446  BB->getTerminator()->eraseFromParent();
447  Builder.SetInsertPoint(BB);
448  }
449  BasicBlock *CancellationBlock = BasicBlock::Create(
450  BB->getContext(), BB->getName() + ".cncl", BB->getParent());
451 
452  // Jump to them based on the return value.
453  Value *Cmp = Builder.CreateIsNull(CancelFlag);
454  Builder.CreateCondBr(Cmp, NonCancellationBlock, CancellationBlock,
455  /* TODO weight */ nullptr, nullptr);
456 
457  // From the cancellation block we finalize all variables and go to the
458  // post finalization block that is known to the FiniCB callback.
459  Builder.SetInsertPoint(CancellationBlock);
460  auto &FI = FinalizationStack.back();
461  FI.FiniCB(Builder.saveIP());
462 
463  // The continuation block is where code generation continues.
464  Builder.SetInsertPoint(NonCancellationBlock, NonCancellationBlock->begin());
465 }
466 
468  const LocationDescription &Loc, InsertPointTy OuterAllocaIP,
469  BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB,
470  FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads,
471  omp::ProcBindKind ProcBind, bool IsCancellable) {
472  if (!updateToLocation(Loc))
473  return Loc.IP;
474 
475  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
476  Value *Ident = getOrCreateIdent(SrcLocStr);
477  Value *ThreadID = getOrCreateThreadID(Ident);
478 
479  if (NumThreads) {
480  // Build call __kmpc_push_num_threads(&Ident, global_tid, num_threads)
481  Value *Args[] = {
482  Ident, ThreadID,
483  Builder.CreateIntCast(NumThreads, Int32, /*isSigned*/ false)};
484  Builder.CreateCall(
485  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_num_threads), Args);
486  }
487 
488  if (ProcBind != OMP_PROC_BIND_default) {
489  // Build call __kmpc_push_proc_bind(&Ident, global_tid, proc_bind)
490  Value *Args[] = {
491  Ident, ThreadID,
492  ConstantInt::get(Int32, unsigned(ProcBind), /*isSigned=*/true)};
493  Builder.CreateCall(
494  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_push_proc_bind), Args);
495  }
496 
497  BasicBlock *InsertBB = Builder.GetInsertBlock();
498  Function *OuterFn = InsertBB->getParent();
499 
500  // Save the outer alloca block because the insertion iterator may get
501  // invalidated and we still need this later.
502  BasicBlock *OuterAllocaBlock = OuterAllocaIP.getBlock();
503 
504  // Vector to remember instructions we used only during the modeling but which
505  // we want to delete at the end.
506  SmallVector<Instruction *, 4> ToBeDeleted;
507 
508  // Change the location to the outer alloca insertion point to create and
509  // initialize the allocas we pass into the parallel region.
510  Builder.restoreIP(OuterAllocaIP);
511  AllocaInst *TIDAddr = Builder.CreateAlloca(Int32, nullptr, "tid.addr");
512  AllocaInst *ZeroAddr = Builder.CreateAlloca(Int32, nullptr, "zero.addr");
513 
514  // If there is an if condition we actually use the TIDAddr and ZeroAddr in the
515  // program, otherwise we only need them for modeling purposes to get the
516  // associated arguments in the outlined function. In the former case,
517  // initialize the allocas properly, in the latter case, delete them later.
518  if (IfCondition) {
519  Builder.CreateStore(Constant::getNullValue(Int32), TIDAddr);
520  Builder.CreateStore(Constant::getNullValue(Int32), ZeroAddr);
521  } else {
522  ToBeDeleted.push_back(TIDAddr);
523  ToBeDeleted.push_back(ZeroAddr);
524  }
525 
526  // Create an artificial insertion point that will also ensure the blocks we
527  // are about to split are not degenerated.
528  auto *UI = new UnreachableInst(Builder.getContext(), InsertBB);
529 
530  Instruction *ThenTI = UI, *ElseTI = nullptr;
531  if (IfCondition)
532  SplitBlockAndInsertIfThenElse(IfCondition, UI, &ThenTI, &ElseTI);
533 
534  BasicBlock *ThenBB = ThenTI->getParent();
535  BasicBlock *PRegEntryBB = ThenBB->splitBasicBlock(ThenTI, "omp.par.entry");
536  BasicBlock *PRegBodyBB =
537  PRegEntryBB->splitBasicBlock(ThenTI, "omp.par.region");
538  BasicBlock *PRegPreFiniBB =
539  PRegBodyBB->splitBasicBlock(ThenTI, "omp.par.pre_finalize");
540  BasicBlock *PRegExitBB =
541  PRegPreFiniBB->splitBasicBlock(ThenTI, "omp.par.exit");
542 
543  auto FiniCBWrapper = [&](InsertPointTy IP) {
544  // Hide "open-ended" blocks from the given FiniCB by setting the right jump
545  // target to the region exit block.
546  if (IP.getBlock()->end() == IP.getPoint()) {
548  Builder.restoreIP(IP);
549  Instruction *I = Builder.CreateBr(PRegExitBB);
550  IP = InsertPointTy(I->getParent(), I->getIterator());
551  }
552  assert(IP.getBlock()->getTerminator()->getNumSuccessors() == 1 &&
553  IP.getBlock()->getTerminator()->getSuccessor(0) == PRegExitBB &&
554  "Unexpected insertion point for finalization call!");
555  return FiniCB(IP);
556  };
557 
558  FinalizationStack.push_back({FiniCBWrapper, OMPD_parallel, IsCancellable});
559 
560  // Generate the privatization allocas in the block that will become the entry
561  // of the outlined function.
562  Builder.SetInsertPoint(PRegEntryBB->getTerminator());
563  InsertPointTy InnerAllocaIP = Builder.saveIP();
564 
565  AllocaInst *PrivTIDAddr =
566  Builder.CreateAlloca(Int32, nullptr, "tid.addr.local");
567  Instruction *PrivTID = Builder.CreateLoad(Int32, PrivTIDAddr, "tid");
568 
569  // Add some fake uses for OpenMP provided arguments.
570  ToBeDeleted.push_back(Builder.CreateLoad(Int32, TIDAddr, "tid.addr.use"));
571  Instruction *ZeroAddrUse = Builder.CreateLoad(Int32, ZeroAddr,
572  "zero.addr.use");
573  ToBeDeleted.push_back(ZeroAddrUse);
574 
575  // ThenBB
576  // |
577  // V
578  // PRegionEntryBB <- Privatization allocas are placed here.
579  // |
580  // V
581  // PRegionBodyBB <- BodeGen is invoked here.
582  // |
583  // V
584  // PRegPreFiniBB <- The block we will start finalization from.
585  // |
586  // V
587  // PRegionExitBB <- A common exit to simplify block collection.
588  //
589 
590  LLVM_DEBUG(dbgs() << "Before body codegen: " << *OuterFn << "\n");
591 
592  // Let the caller create the body.
593  assert(BodyGenCB && "Expected body generation callback!");
594  InsertPointTy CodeGenIP(PRegBodyBB, PRegBodyBB->begin());
595  BodyGenCB(InnerAllocaIP, CodeGenIP, *PRegPreFiniBB);
596 
597  LLVM_DEBUG(dbgs() << "After body codegen: " << *OuterFn << "\n");
598 
599  FunctionCallee RTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_fork_call);
600  if (auto *F = dyn_cast<llvm::Function>(RTLFn.getCallee())) {
601  if (!F->hasMetadata(llvm::LLVMContext::MD_callback)) {
602  llvm::LLVMContext &Ctx = F->getContext();
603  MDBuilder MDB(Ctx);
604  // Annotate the callback behavior of the __kmpc_fork_call:
605  // - The callback callee is argument number 2 (microtask).
606  // - The first two arguments of the callback callee are unknown (-1).
607  // - All variadic arguments to the __kmpc_fork_call are passed to the
608  // callback callee.
609  F->addMetadata(
610  llvm::LLVMContext::MD_callback,
612  Ctx, {MDB.createCallbackEncoding(2, {-1, -1},
613  /* VarArgsArePassed */ true)}));
614  }
615  }
616 
617  OutlineInfo OI;
618  OI.PostOutlineCB = [=](Function &OutlinedFn) {
619  // Add some known attributes.
620  OutlinedFn.addParamAttr(0, Attribute::NoAlias);
621  OutlinedFn.addParamAttr(1, Attribute::NoAlias);
622  OutlinedFn.addFnAttr(Attribute::NoUnwind);
623  OutlinedFn.addFnAttr(Attribute::NoRecurse);
624 
625  assert(OutlinedFn.arg_size() >= 2 &&
626  "Expected at least tid and bounded tid as arguments");
627  unsigned NumCapturedVars =
628  OutlinedFn.arg_size() - /* tid & bounded tid */ 2;
629 
630  CallInst *CI = cast<CallInst>(OutlinedFn.user_back());
631  CI->getParent()->setName("omp_parallel");
632  Builder.SetInsertPoint(CI);
633 
634  // Build call __kmpc_fork_call(Ident, n, microtask, var1, .., varn);
635  Value *ForkCallArgs[] = {
636  Ident, Builder.getInt32(NumCapturedVars),
637  Builder.CreateBitCast(&OutlinedFn, ParallelTaskPtr)};
638 
639  SmallVector<Value *, 16> RealArgs;
640  RealArgs.append(std::begin(ForkCallArgs), std::end(ForkCallArgs));
641  RealArgs.append(CI->arg_begin() + /* tid & bound tid */ 2, CI->arg_end());
642 
643  Builder.CreateCall(RTLFn, RealArgs);
644 
645  LLVM_DEBUG(dbgs() << "With fork_call placed: "
646  << *Builder.GetInsertBlock()->getParent() << "\n");
647 
648  InsertPointTy ExitIP(PRegExitBB, PRegExitBB->end());
649 
650  // Initialize the local TID stack location with the argument value.
651  Builder.SetInsertPoint(PrivTID);
652  Function::arg_iterator OutlinedAI = OutlinedFn.arg_begin();
653  Builder.CreateStore(Builder.CreateLoad(Int32, OutlinedAI), PrivTIDAddr);
654 
655  // If no "if" clause was present we do not need the call created during
656  // outlining, otherwise we reuse it in the serialized parallel region.
657  if (!ElseTI) {
658  CI->eraseFromParent();
659  } else {
660 
661  // If an "if" clause was present we are now generating the serialized
662  // version into the "else" branch.
663  Builder.SetInsertPoint(ElseTI);
664 
665  // Build calls __kmpc_serialized_parallel(&Ident, GTid);
666  Value *SerializedParallelCallArgs[] = {Ident, ThreadID};
667  Builder.CreateCall(
668  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_serialized_parallel),
669  SerializedParallelCallArgs);
670 
671  // OutlinedFn(&GTid, &zero, CapturedStruct);
672  CI->removeFromParent();
673  Builder.Insert(CI);
674 
675  // __kmpc_end_serialized_parallel(&Ident, GTid);
676  Value *EndArgs[] = {Ident, ThreadID};
677  Builder.CreateCall(
678  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_serialized_parallel),
679  EndArgs);
680 
681  LLVM_DEBUG(dbgs() << "With serialized parallel region: "
682  << *Builder.GetInsertBlock()->getParent() << "\n");
683  }
684 
685  for (Instruction *I : ToBeDeleted)
686  I->eraseFromParent();
687  };
688 
689  // Adjust the finalization stack, verify the adjustment, and call the
690  // finalize function a last time to finalize values between the pre-fini
691  // block and the exit block if we left the parallel "the normal way".
692  auto FiniInfo = FinalizationStack.pop_back_val();
693  (void)FiniInfo;
694  assert(FiniInfo.DK == OMPD_parallel &&
695  "Unexpected finalization stack state!");
696 
697  Instruction *PRegPreFiniTI = PRegPreFiniBB->getTerminator();
698 
699  InsertPointTy PreFiniIP(PRegPreFiniBB, PRegPreFiniTI->getIterator());
700  FiniCB(PreFiniIP);
701 
702  OI.EntryBB = PRegEntryBB;
703  OI.ExitBB = PRegExitBB;
704 
705  SmallPtrSet<BasicBlock *, 32> ParallelRegionBlockSet;
707  OI.collectBlocks(ParallelRegionBlockSet, Blocks);
708 
709  // Ensure a single exit node for the outlined region by creating one.
710  // We might have multiple incoming edges to the exit now due to finalizations,
711  // e.g., cancel calls that cause the control flow to leave the region.
712  BasicBlock *PRegOutlinedExitBB = PRegExitBB;
713  PRegExitBB = SplitBlock(PRegExitBB, &*PRegExitBB->getFirstInsertionPt());
714  PRegOutlinedExitBB->setName("omp.par.outlined.exit");
715  Blocks.push_back(PRegOutlinedExitBB);
716 
717  CodeExtractorAnalysisCache CEAC(*OuterFn);
718  CodeExtractor Extractor(Blocks, /* DominatorTree */ nullptr,
719  /* AggregateArgs */ false,
720  /* BlockFrequencyInfo */ nullptr,
721  /* BranchProbabilityInfo */ nullptr,
722  /* AssumptionCache */ nullptr,
723  /* AllowVarArgs */ true,
724  /* AllowAlloca */ true,
725  /* Suffix */ ".omp_par");
726 
727  // Find inputs to, outputs from the code region.
728  BasicBlock *CommonExit = nullptr;
729  SetVector<Value *> Inputs, Outputs, SinkingCands, HoistingCands;
730  Extractor.findAllocas(CEAC, SinkingCands, HoistingCands, CommonExit);
731  Extractor.findInputsOutputs(Inputs, Outputs, SinkingCands);
732 
733  LLVM_DEBUG(dbgs() << "Before privatization: " << *OuterFn << "\n");
734 
735  FunctionCallee TIDRTLFn =
736  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_global_thread_num);
737 
738  auto PrivHelper = [&](Value &V) {
739  if (&V == TIDAddr || &V == ZeroAddr)
740  return;
741 
743  for (Use &U : V.uses())
744  if (auto *UserI = dyn_cast<Instruction>(U.getUser()))
745  if (ParallelRegionBlockSet.count(UserI->getParent()))
746  Uses.insert(&U);
747 
748  // __kmpc_fork_call expects extra arguments as pointers. If the input
749  // already has a pointer type, everything is fine. Otherwise, store the
750  // value onto stack and load it back inside the to-be-outlined region. This
751  // will ensure only the pointer will be passed to the function.
752  // FIXME: if there are more than 15 trailing arguments, they must be
753  // additionally packed in a struct.
754  Value *Inner = &V;
755  if (!V.getType()->isPointerTy()) {
757  LLVM_DEBUG(llvm::dbgs() << "Forwarding input as pointer: " << V << "\n");
758 
759  Builder.restoreIP(OuterAllocaIP);
760  Value *Ptr =
761  Builder.CreateAlloca(V.getType(), nullptr, V.getName() + ".reloaded");
762 
763  // Store to stack at end of the block that currently branches to the entry
764  // block of the to-be-outlined region.
765  Builder.SetInsertPoint(InsertBB,
766  InsertBB->getTerminator()->getIterator());
767  Builder.CreateStore(&V, Ptr);
768 
769  // Load back next to allocations in the to-be-outlined region.
770  Builder.restoreIP(InnerAllocaIP);
771  Inner = Builder.CreateLoad(V.getType(), Ptr);
772  }
773 
774  Value *ReplacementValue = nullptr;
775  CallInst *CI = dyn_cast<CallInst>(&V);
776  if (CI && CI->getCalledFunction() == TIDRTLFn.getCallee()) {
777  ReplacementValue = PrivTID;
778  } else {
779  Builder.restoreIP(
780  PrivCB(InnerAllocaIP, Builder.saveIP(), V, *Inner, ReplacementValue));
781  assert(ReplacementValue &&
782  "Expected copy/create callback to set replacement value!");
783  if (ReplacementValue == &V)
784  return;
785  }
786 
787  for (Use *UPtr : Uses)
788  UPtr->set(ReplacementValue);
789  };
790 
791  // Reset the inner alloca insertion as it will be used for loading the values
792  // wrapped into pointers before passing them into the to-be-outlined region.
793  // Configure it to insert immediately after the fake use of zero address so
794  // that they are available in the generated body and so that the
795  // OpenMP-related values (thread ID and zero address pointers) remain leading
796  // in the argument list.
797  InnerAllocaIP = IRBuilder<>::InsertPoint(
798  ZeroAddrUse->getParent(), ZeroAddrUse->getNextNode()->getIterator());
799 
800  // Reset the outer alloca insertion point to the entry of the relevant block
801  // in case it was invalidated.
802  OuterAllocaIP = IRBuilder<>::InsertPoint(
803  OuterAllocaBlock, OuterAllocaBlock->getFirstInsertionPt());
804 
805  for (Value *Input : Inputs) {
806  LLVM_DEBUG(dbgs() << "Captured input: " << *Input << "\n");
807  PrivHelper(*Input);
808  }
809  LLVM_DEBUG({
810  for (Value *Output : Outputs)
811  LLVM_DEBUG(dbgs() << "Captured output: " << *Output << "\n");
812  });
813  assert(Outputs.empty() &&
814  "OpenMP outlining should not produce live-out values!");
815 
816  LLVM_DEBUG(dbgs() << "After privatization: " << *OuterFn << "\n");
817  LLVM_DEBUG({
818  for (auto *BB : Blocks)
819  dbgs() << " PBR: " << BB->getName() << "\n";
820  });
821 
822  // Register the outlined info.
823  addOutlineInfo(std::move(OI));
824 
825  InsertPointTy AfterIP(UI->getParent(), UI->getParent()->end());
826  UI->eraseFromParent();
827 
828  return AfterIP;
829 }
830 
832  // Build call void __kmpc_flush(ident_t *loc)
833  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
834  Value *Args[] = {getOrCreateIdent(SrcLocStr)};
835 
836  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_flush), Args);
837 }
838 
840  if (!updateToLocation(Loc))
841  return;
842  emitFlush(Loc);
843 }
844 
846  // Build call kmp_int32 __kmpc_omp_taskwait(ident_t *loc, kmp_int32
847  // global_tid);
848  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
849  Value *Ident = getOrCreateIdent(SrcLocStr);
850  Value *Args[] = {Ident, getOrCreateThreadID(Ident)};
851 
852  // Ignore return result until untied tasks are supported.
853  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskwait),
854  Args);
855 }
856 
858  if (!updateToLocation(Loc))
859  return;
860  emitTaskwaitImpl(Loc);
861 }
862 
864  // Build call __kmpc_omp_taskyield(loc, thread_id, 0);
865  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
866  Value *Ident = getOrCreateIdent(SrcLocStr);
867  Constant *I32Null = ConstantInt::getNullValue(Int32);
868  Value *Args[] = {Ident, getOrCreateThreadID(Ident), I32Null};
869 
870  Builder.CreateCall(getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_omp_taskyield),
871  Args);
872 }
873 
875  if (!updateToLocation(Loc))
876  return;
877  emitTaskyieldImpl(Loc);
878 }
879 
882  BodyGenCallbackTy BodyGenCB,
883  FinalizeCallbackTy FiniCB) {
884 
885  if (!updateToLocation(Loc))
886  return Loc.IP;
887 
888  Directive OMPD = Directive::OMPD_master;
889  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
890  Value *Ident = getOrCreateIdent(SrcLocStr);
891  Value *ThreadId = getOrCreateThreadID(Ident);
892  Value *Args[] = {Ident, ThreadId};
893 
894  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_master);
895  Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
896 
897  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_master);
898  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
899 
900  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
901  /*Conditional*/ true, /*hasFinalize*/ true);
902 }
903 
904 CanonicalLoopInfo *OpenMPIRBuilder::createLoopSkeleton(
905  DebugLoc DL, Value *TripCount, Function *F, BasicBlock *PreInsertBefore,
906  BasicBlock *PostInsertBefore, const Twine &Name) {
907  Module *M = F->getParent();
908  LLVMContext &Ctx = M->getContext();
909  Type *IndVarTy = TripCount->getType();
910 
911  // Create the basic block structure.
912  BasicBlock *Preheader =
913  BasicBlock::Create(Ctx, "omp_" + Name + ".preheader", F, PreInsertBefore);
914  BasicBlock *Header =
915  BasicBlock::Create(Ctx, "omp_" + Name + ".header", F, PreInsertBefore);
916  BasicBlock *Cond =
917  BasicBlock::Create(Ctx, "omp_" + Name + ".cond", F, PreInsertBefore);
918  BasicBlock *Body =
919  BasicBlock::Create(Ctx, "omp_" + Name + ".body", F, PreInsertBefore);
920  BasicBlock *Latch =
921  BasicBlock::Create(Ctx, "omp_" + Name + ".inc", F, PostInsertBefore);
922  BasicBlock *Exit =
923  BasicBlock::Create(Ctx, "omp_" + Name + ".exit", F, PostInsertBefore);
924  BasicBlock *After =
925  BasicBlock::Create(Ctx, "omp_" + Name + ".after", F, PostInsertBefore);
926 
927  // Use specified DebugLoc for new instructions.
928  Builder.SetCurrentDebugLocation(DL);
929 
930  Builder.SetInsertPoint(Preheader);
931  Builder.CreateBr(Header);
932 
933  Builder.SetInsertPoint(Header);
934  PHINode *IndVarPHI = Builder.CreatePHI(IndVarTy, 2, "omp_" + Name + ".iv");
935  IndVarPHI->addIncoming(ConstantInt::get(IndVarTy, 0), Preheader);
936  Builder.CreateBr(Cond);
937 
938  Builder.SetInsertPoint(Cond);
939  Value *Cmp =
940  Builder.CreateICmpULT(IndVarPHI, TripCount, "omp_" + Name + ".cmp");
941  Builder.CreateCondBr(Cmp, Body, Exit);
942 
943  Builder.SetInsertPoint(Body);
944  Builder.CreateBr(Latch);
945 
946  Builder.SetInsertPoint(Latch);
947  Value *Next = Builder.CreateAdd(IndVarPHI, ConstantInt::get(IndVarTy, 1),
948  "omp_" + Name + ".next", /*HasNUW=*/true);
949  Builder.CreateBr(Header);
950  IndVarPHI->addIncoming(Next, Latch);
951 
952  Builder.SetInsertPoint(Exit);
953  Builder.CreateBr(After);
954 
955  // Remember and return the canonical control flow.
956  LoopInfos.emplace_front();
957  CanonicalLoopInfo *CL = &LoopInfos.front();
958 
959  CL->Preheader = Preheader;
960  CL->Header = Header;
961  CL->Cond = Cond;
962  CL->Body = Body;
963  CL->Latch = Latch;
964  CL->Exit = Exit;
965  CL->After = After;
966 
967  CL->IsValid = true;
968 
969 #ifndef NDEBUG
970  CL->assertOK();
971 #endif
972  return CL;
973 }
974 
977  LoopBodyGenCallbackTy BodyGenCB,
978  Value *TripCount, const Twine &Name) {
979  BasicBlock *BB = Loc.IP.getBlock();
980  BasicBlock *NextBB = BB->getNextNode();
981 
982  CanonicalLoopInfo *CL = createLoopSkeleton(Loc.DL, TripCount, BB->getParent(),
983  NextBB, NextBB, Name);
984  BasicBlock *After = CL->getAfter();
985 
986  // If location is not set, don't connect the loop.
987  if (updateToLocation(Loc)) {
988  // Split the loop at the insertion point: Branch to the preheader and move
989  // every following instruction to after the loop (the After BB). Also, the
990  // new successor is the loop's after block.
991  Builder.CreateBr(CL->Preheader);
992  After->getInstList().splice(After->begin(), BB->getInstList(),
993  Builder.GetInsertPoint(), BB->end());
994  After->replaceSuccessorsPhiUsesWith(BB, After);
995  }
996 
997  // Emit the body content. We do it after connecting the loop to the CFG to
998  // avoid that the callback encounters degenerate BBs.
999  BodyGenCB(CL->getBodyIP(), CL->getIndVar());
1000 
1001 #ifndef NDEBUG
1002  CL->assertOK();
1003 #endif
1004  return CL;
1005 }
1006 
1008  const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB,
1009  Value *Start, Value *Stop, Value *Step, bool IsSigned, bool InclusiveStop,
1010  InsertPointTy ComputeIP, const Twine &Name) {
1011 
1012  // Consider the following difficulties (assuming 8-bit signed integers):
1013  // * Adding \p Step to the loop counter which passes \p Stop may overflow:
1014  // DO I = 1, 100, 50
1015  /// * A \p Step of INT_MIN cannot not be normalized to a positive direction:
1016  // DO I = 100, 0, -128
1017 
1018  // Start, Stop and Step must be of the same integer type.
1019  auto *IndVarTy = cast<IntegerType>(Start->getType());
1020  assert(IndVarTy == Stop->getType() && "Stop type mismatch");
1021  assert(IndVarTy == Step->getType() && "Step type mismatch");
1022 
1023  LocationDescription ComputeLoc =
1024  ComputeIP.isSet() ? LocationDescription(ComputeIP, Loc.DL) : Loc;
1025  updateToLocation(ComputeLoc);
1026 
1027  ConstantInt *Zero = ConstantInt::get(IndVarTy, 0);
1028  ConstantInt *One = ConstantInt::get(IndVarTy, 1);
1029 
1030  // Like Step, but always positive.
1031  Value *Incr = Step;
1032 
1033  // Distance between Start and Stop; always positive.
1034  Value *Span;
1035 
1036  // Condition whether there are no iterations are executed at all, e.g. because
1037  // UB < LB.
1038  Value *ZeroCmp;
1039 
1040  if (IsSigned) {
1041  // Ensure that increment is positive. If not, negate and invert LB and UB.
1042  Value *IsNeg = Builder.CreateICmpSLT(Step, Zero);
1043  Incr = Builder.CreateSelect(IsNeg, Builder.CreateNeg(Step), Step);
1044  Value *LB = Builder.CreateSelect(IsNeg, Stop, Start);
1045  Value *UB = Builder.CreateSelect(IsNeg, Start, Stop);
1046  Span = Builder.CreateSub(UB, LB, "", false, true);
1047  ZeroCmp = Builder.CreateICmp(
1048  InclusiveStop ? CmpInst::ICMP_SLT : CmpInst::ICMP_SLE, UB, LB);
1049  } else {
1050  Span = Builder.CreateSub(Stop, Start, "", true);
1051  ZeroCmp = Builder.CreateICmp(
1052  InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Stop, Start);
1053  }
1054 
1055  Value *CountIfLooping;
1056  if (InclusiveStop) {
1057  CountIfLooping = Builder.CreateAdd(Builder.CreateUDiv(Span, Incr), One);
1058  } else {
1059  // Avoid incrementing past stop since it could overflow.
1060  Value *CountIfTwo = Builder.CreateAdd(
1061  Builder.CreateUDiv(Builder.CreateSub(Span, One), Incr), One);
1062  Value *OneCmp = Builder.CreateICmp(
1063  InclusiveStop ? CmpInst::ICMP_ULT : CmpInst::ICMP_ULE, Span, Incr);
1064  CountIfLooping = Builder.CreateSelect(OneCmp, One, CountIfTwo);
1065  }
1066  Value *TripCount = Builder.CreateSelect(ZeroCmp, Zero, CountIfLooping,
1067  "omp_" + Name + ".tripcount");
1068 
1069  auto BodyGen = [=](InsertPointTy CodeGenIP, Value *IV) {
1070  Builder.restoreIP(CodeGenIP);
1071  Value *Span = Builder.CreateMul(IV, Step);
1072  Value *IndVar = Builder.CreateAdd(Span, Start);
1073  BodyGenCB(Builder.saveIP(), IndVar);
1074  };
1075  LocationDescription LoopLoc = ComputeIP.isSet() ? Loc.IP : Builder.saveIP();
1076  return createCanonicalLoop(LoopLoc, BodyGen, TripCount, Name);
1077 }
1078 
1079 // Returns an LLVM function to call for initializing loop bounds using OpenMP
1080 // static scheduling depending on `type`. Only i32 and i64 are supported by the
1081 // runtime. Always interpret integers as unsigned similarly to
1082 // CanonicalLoopInfo.
1084  OpenMPIRBuilder &OMPBuilder) {
1085  unsigned Bitwidth = Ty->getIntegerBitWidth();
1086  if (Bitwidth == 32)
1087  return OMPBuilder.getOrCreateRuntimeFunction(
1088  M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_4u);
1089  if (Bitwidth == 64)
1090  return OMPBuilder.getOrCreateRuntimeFunction(
1091  M, omp::RuntimeFunction::OMPRTL___kmpc_for_static_init_8u);
1092  llvm_unreachable("unknown OpenMP loop iterator bitwidth");
1093 }
1094 
1095 // Sets the number of loop iterations to the given value. This value must be
1096 // valid in the condition block (i.e., defined in the preheader) and is
1097 // interpreted as an unsigned integer.
1099  Instruction *CmpI = &CLI->getCond()->front();
1100  assert(isa<CmpInst>(CmpI) && "First inst must compare IV with TripCount");
1101  CmpI->setOperand(1, TripCount);
1102  CLI->assertOK();
1103 }
1104 
1106  const LocationDescription &Loc, CanonicalLoopInfo *CLI,
1107  InsertPointTy AllocaIP, bool NeedsBarrier, Value *Chunk) {
1108  // Set up the source location value for OpenMP runtime.
1109  if (!updateToLocation(Loc))
1110  return nullptr;
1111 
1112  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1113  Value *SrcLoc = getOrCreateIdent(SrcLocStr);
1114 
1115  // Declare useful OpenMP runtime functions.
1116  Value *IV = CLI->getIndVar();
1117  Type *IVTy = IV->getType();
1118  FunctionCallee StaticInit = getKmpcForStaticInitForType(IVTy, M, *this);
1119  FunctionCallee StaticFini =
1120  getOrCreateRuntimeFunction(M, omp::OMPRTL___kmpc_for_static_fini);
1121 
1122  // Allocate space for computed loop bounds as expected by the "init" function.
1123  Builder.restoreIP(AllocaIP);
1124  Type *I32Type = Type::getInt32Ty(M.getContext());
1125  Value *PLastIter = Builder.CreateAlloca(I32Type, nullptr, "p.lastiter");
1126  Value *PLowerBound = Builder.CreateAlloca(IVTy, nullptr, "p.lowerbound");
1127  Value *PUpperBound = Builder.CreateAlloca(IVTy, nullptr, "p.upperbound");
1128  Value *PStride = Builder.CreateAlloca(IVTy, nullptr, "p.stride");
1129 
1130  // At the end of the preheader, prepare for calling the "init" function by
1131  // storing the current loop bounds into the allocated space. A canonical loop
1132  // always iterates from 0 to trip-count with step 1. Note that "init" expects
1133  // and produces an inclusive upper bound.
1134  Builder.SetInsertPoint(CLI->getPreheader()->getTerminator());
1135  Constant *Zero = ConstantInt::get(IVTy, 0);
1136  Constant *One = ConstantInt::get(IVTy, 1);
1137  Builder.CreateStore(Zero, PLowerBound);
1138  Value *UpperBound = Builder.CreateSub(CLI->getTripCount(), One);
1139  Builder.CreateStore(UpperBound, PUpperBound);
1140  Builder.CreateStore(One, PStride);
1141 
1142  if (!Chunk)
1143  Chunk = One;
1144 
1145  Value *ThreadNum = getOrCreateThreadID(SrcLoc);
1146 
1147  // TODO: extract scheduling type and map it to OMP constant. This is curently
1148  // happening in kmp.h and its ilk and needs to be moved to OpenMP.td first.
1149  constexpr int StaticSchedType = 34;
1150  Constant *SchedulingType = ConstantInt::get(I32Type, StaticSchedType);
1151 
1152  // Call the "init" function and update the trip count of the loop with the
1153  // value it produced.
1154  Builder.CreateCall(StaticInit,
1155  {SrcLoc, ThreadNum, SchedulingType, PLastIter, PLowerBound,
1156  PUpperBound, PStride, One, Chunk});
1157  Value *LowerBound = Builder.CreateLoad(IVTy, PLowerBound);
1158  Value *InclusiveUpperBound = Builder.CreateLoad(IVTy, PUpperBound);
1159  Value *TripCountMinusOne = Builder.CreateSub(InclusiveUpperBound, LowerBound);
1160  Value *TripCount = Builder.CreateAdd(TripCountMinusOne, One);
1161  setCanonicalLoopTripCount(CLI, TripCount);
1162 
1163  // Update all uses of the induction variable except the one in the condition
1164  // block that compares it with the actual upper bound, and the increment in
1165  // the latch block.
1166  // TODO: this can eventually move to CanonicalLoopInfo or to a new
1167  // CanonicalLoopInfoUpdater interface.
1168  Builder.SetInsertPoint(CLI->getBody(), CLI->getBody()->getFirstInsertionPt());
1169  Value *UpdatedIV = Builder.CreateAdd(IV, LowerBound);
1170  IV->replaceUsesWithIf(UpdatedIV, [&](Use &U) {
1171  auto *Instr = dyn_cast<Instruction>(U.getUser());
1172  return !Instr ||
1173  (Instr->getParent() != CLI->getCond() &&
1174  Instr->getParent() != CLI->getLatch() && Instr != UpdatedIV);
1175  });
1176 
1177  // In the "exit" block, call the "fini" function.
1178  Builder.SetInsertPoint(CLI->getExit(),
1179  CLI->getExit()->getTerminator()->getIterator());
1180  Builder.CreateCall(StaticFini, {SrcLoc, ThreadNum});
1181 
1182  // Add the barrier if requested.
1183  if (NeedsBarrier)
1184  createBarrier(LocationDescription(Builder.saveIP(), Loc.DL),
1185  omp::Directive::OMPD_for, /* ForceSimpleCall */ false,
1186  /* CheckCancelFlag */ false);
1187 
1188  CLI->assertOK();
1189  return CLI;
1190 }
1191 
1193  const LocationDescription &Loc, CanonicalLoopInfo *CLI,
1194  InsertPointTy AllocaIP, bool NeedsBarrier) {
1195  // Currently only supports static schedules.
1196  return createStaticWorkshareLoop(Loc, CLI, AllocaIP, NeedsBarrier);
1197 }
1198 
1199 /// Make \p Source branch to \p Target.
1200 ///
1201 /// Handles two situations:
1202 /// * \p Source already has an unconditional branch.
1203 /// * \p Source is a degenerate block (no terminator because the BB is
1204 /// the current head of the IR construction).
1206  if (Instruction *Term = Source->getTerminator()) {
1207  auto *Br = cast<BranchInst>(Term);
1208  assert(!Br->isConditional() &&
1209  "BB's terminator must be an unconditional branch (or degenerate)");
1210  BasicBlock *Succ = Br->getSuccessor(0);
1211  Succ->removePredecessor(Source, /*KeepOneInputPHIs=*/true);
1212  Br->setSuccessor(0, Target);
1213  return;
1214  }
1215 
1216  auto *NewBr = BranchInst::Create(Target, Source);
1217  NewBr->setDebugLoc(DL);
1218 }
1219 
1220 /// Redirect all edges that branch to \p OldTarget to \p NewTarget. That is,
1221 /// after this \p OldTarget will be orphaned.
1222 static void redirectAllPredecessorsTo(BasicBlock *OldTarget,
1223  BasicBlock *NewTarget, DebugLoc DL) {
1224  for (BasicBlock *Pred : make_early_inc_range(predecessors(OldTarget)))
1225  redirectTo(Pred, NewTarget, DL);
1226 }
1227 
1228 /// Determine which blocks in \p BBs are reachable from outside and remove the
1229 /// ones that are not reachable from the function.
1231  SmallPtrSet<BasicBlock *, 6> BBsToErase{BBs.begin(), BBs.end()};
1232  auto HasRemainingUses = [&BBsToErase](BasicBlock *BB) {
1233  for (Use &U : BB->uses()) {
1234  auto *UseInst = dyn_cast<Instruction>(U.getUser());
1235  if (!UseInst)
1236  continue;
1237  if (BBsToErase.count(UseInst->getParent()))
1238  continue;
1239  return true;
1240  }
1241  return false;
1242  };
1243 
1244  while (true) {
1245  bool Changed = false;
1246  for (BasicBlock *BB : make_early_inc_range(BBsToErase)) {
1247  if (HasRemainingUses(BB)) {
1248  BBsToErase.erase(BB);
1249  Changed = true;
1250  }
1251  }
1252  if (!Changed)
1253  break;
1254  }
1255 
1256  SmallVector<BasicBlock *, 7> BBVec(BBsToErase.begin(), BBsToErase.end());
1257  DeleteDeadBlocks(BBVec);
1258 }
1259 
1262  InsertPointTy ComputeIP) {
1263  assert(Loops.size() >= 1 && "At least one loop required");
1264  size_t NumLoops = Loops.size();
1265 
1266  // Nothing to do if there is already just one loop.
1267  if (NumLoops == 1)
1268  return Loops.front();
1269 
1270  CanonicalLoopInfo *Outermost = Loops.front();
1271  CanonicalLoopInfo *Innermost = Loops.back();
1272  BasicBlock *OrigPreheader = Outermost->getPreheader();
1273  BasicBlock *OrigAfter = Outermost->getAfter();
1274  Function *F = OrigPreheader->getParent();
1275 
1276  // Setup the IRBuilder for inserting the trip count computation.
1277  Builder.SetCurrentDebugLocation(DL);
1278  if (ComputeIP.isSet())
1279  Builder.restoreIP(ComputeIP);
1280  else
1281  Builder.restoreIP(Outermost->getPreheaderIP());
1282 
1283  // Derive the collapsed' loop trip count.
1284  // TODO: Find common/largest indvar type.
1285  Value *CollapsedTripCount = nullptr;
1286  for (CanonicalLoopInfo *L : Loops) {
1287  Value *OrigTripCount = L->getTripCount();
1288  if (!CollapsedTripCount) {
1289  CollapsedTripCount = OrigTripCount;
1290  continue;
1291  }
1292 
1293  // TODO: Enable UndefinedSanitizer to diagnose an overflow here.
1294  CollapsedTripCount = Builder.CreateMul(CollapsedTripCount, OrigTripCount,
1295  {}, /*HasNUW=*/true);
1296  }
1297 
1298  // Create the collapsed loop control flow.
1299  CanonicalLoopInfo *Result =
1300  createLoopSkeleton(DL, CollapsedTripCount, F,
1301  OrigPreheader->getNextNode(), OrigAfter, "collapsed");
1302 
1303  // Build the collapsed loop body code.
1304  // Start with deriving the input loop induction variables from the collapsed
1305  // one, using a divmod scheme. To preserve the original loops' order, the
1306  // innermost loop use the least significant bits.
1307  Builder.restoreIP(Result->getBodyIP());
1308 
1309  Value *Leftover = Result->getIndVar();
1310  SmallVector<Value *> NewIndVars;
1311  NewIndVars.set_size(NumLoops);
1312  for (int i = NumLoops - 1; i >= 1; --i) {
1313  Value *OrigTripCount = Loops[i]->getTripCount();
1314 
1315  Value *NewIndVar = Builder.CreateURem(Leftover, OrigTripCount);
1316  NewIndVars[i] = NewIndVar;
1317 
1318  Leftover = Builder.CreateUDiv(Leftover, OrigTripCount);
1319  }
1320  // Outermost loop gets all the remaining bits.
1321  NewIndVars[0] = Leftover;
1322 
1323  // Construct the loop body control flow.
1324  // We progressively construct the branch structure following in direction of
1325  // the control flow, from the leading in-between code, the loop nest body, the
1326  // trailing in-between code, and rejoining the collapsed loop's latch.
1327  // ContinueBlock and ContinuePred keep track of the source(s) of next edge. If
1328  // the ContinueBlock is set, continue with that block. If ContinuePred, use
1329  // its predecessors as sources.
1330  BasicBlock *ContinueBlock = Result->getBody();
1331  BasicBlock *ContinuePred = nullptr;
1332  auto ContinueWith = [&ContinueBlock, &ContinuePred, DL](BasicBlock *Dest,
1333  BasicBlock *NextSrc) {
1334  if (ContinueBlock)
1335  redirectTo(ContinueBlock, Dest, DL);
1336  else
1337  redirectAllPredecessorsTo(ContinuePred, Dest, DL);
1338 
1339  ContinueBlock = nullptr;
1340  ContinuePred = NextSrc;
1341  };
1342 
1343  // The code before the nested loop of each level.
1344  // Because we are sinking it into the nest, it will be executed more often
1345  // that the original loop. More sophisticated schemes could keep track of what
1346  // the in-between code is and instantiate it only once per thread.
1347  for (size_t i = 0; i < NumLoops - 1; ++i)
1348  ContinueWith(Loops[i]->getBody(), Loops[i + 1]->getHeader());
1349 
1350  // Connect the loop nest body.
1351  ContinueWith(Innermost->getBody(), Innermost->getLatch());
1352 
1353  // The code after the nested loop at each level.
1354  for (size_t i = NumLoops - 1; i > 0; --i)
1355  ContinueWith(Loops[i]->getAfter(), Loops[i - 1]->getLatch());
1356 
1357  // Connect the finished loop to the collapsed loop latch.
1358  ContinueWith(Result->getLatch(), nullptr);
1359 
1360  // Replace the input loops with the new collapsed loop.
1361  redirectTo(Outermost->getPreheader(), Result->getPreheader(), DL);
1362  redirectTo(Result->getAfter(), Outermost->getAfter(), DL);
1363 
1364  // Replace the input loop indvars with the derived ones.
1365  for (size_t i = 0; i < NumLoops; ++i)
1366  Loops[i]->getIndVar()->replaceAllUsesWith(NewIndVars[i]);
1367 
1368  // Remove unused parts of the input loops.
1369  SmallVector<BasicBlock *, 12> OldControlBBs;
1370  OldControlBBs.reserve(6 * Loops.size());
1371  for (CanonicalLoopInfo *Loop : Loops)
1372  Loop->collectControlBlocks(OldControlBBs);
1373  removeUnusedBlocksFromParent(OldControlBBs);
1374 
1375 #ifndef NDEBUG
1376  Result->assertOK();
1377 #endif
1378  return Result;
1379 }
1380 
1381 std::vector<CanonicalLoopInfo *>
1383  ArrayRef<Value *> TileSizes) {
1384  assert(TileSizes.size() == Loops.size() &&
1385  "Must pass as many tile sizes as there are loops");
1386  int NumLoops = Loops.size();
1387  assert(NumLoops >= 1 && "At least one loop to tile required");
1388 
1389  CanonicalLoopInfo *OutermostLoop = Loops.front();
1390  CanonicalLoopInfo *InnermostLoop = Loops.back();
1391  Function *F = OutermostLoop->getBody()->getParent();
1392  BasicBlock *InnerEnter = InnermostLoop->getBody();
1393  BasicBlock *InnerLatch = InnermostLoop->getLatch();
1394 
1395  // Collect original trip counts and induction variable to be accessible by
1396  // index. Also, the structure of the original loops is not preserved during
1397  // the construction of the tiled loops, so do it before we scavenge the BBs of
1398  // any original CanonicalLoopInfo.
1399  SmallVector<Value *, 4> OrigTripCounts, OrigIndVars;
1400  for (CanonicalLoopInfo *L : Loops) {
1401  OrigTripCounts.push_back(L->getTripCount());
1402  OrigIndVars.push_back(L->getIndVar());
1403  }
1404 
1405  // Collect the code between loop headers. These may contain SSA definitions
1406  // that are used in the loop nest body. To be usable with in the innermost
1407  // body, these BasicBlocks will be sunk into the loop nest body. That is,
1408  // these instructions may be executed more often than before the tiling.
1409  // TODO: It would be sufficient to only sink them into body of the
1410  // corresponding tile loop.
1412  for (int i = 0; i < NumLoops - 1; ++i) {
1413  CanonicalLoopInfo *Surrounding = Loops[i];
1414  CanonicalLoopInfo *Nested = Loops[i + 1];
1415 
1416  BasicBlock *EnterBB = Surrounding->getBody();
1417  BasicBlock *ExitBB = Nested->getHeader();
1418  InbetweenCode.emplace_back(EnterBB, ExitBB);
1419  }
1420 
1421  // Compute the trip counts of the floor loops.
1422  Builder.SetCurrentDebugLocation(DL);
1423  Builder.restoreIP(OutermostLoop->getPreheaderIP());
1424  SmallVector<Value *, 4> FloorCount, FloorRems;
1425  for (int i = 0; i < NumLoops; ++i) {
1426  Value *TileSize = TileSizes[i];
1427  Value *OrigTripCount = OrigTripCounts[i];
1428  Type *IVType = OrigTripCount->getType();
1429 
1430  Value *FloorTripCount = Builder.CreateUDiv(OrigTripCount, TileSize);
1431  Value *FloorTripRem = Builder.CreateURem(OrigTripCount, TileSize);
1432 
1433  // 0 if tripcount divides the tilesize, 1 otherwise.
1434  // 1 means we need an additional iteration for a partial tile.
1435  //
1436  // Unfortunately we cannot just use the roundup-formula
1437  // (tripcount + tilesize - 1)/tilesize
1438  // because the summation might overflow. We do not want introduce undefined
1439  // behavior when the untiled loop nest did not.
1440  Value *FloorTripOverflow =
1441  Builder.CreateICmpNE(FloorTripRem, ConstantInt::get(IVType, 0));
1442 
1443  FloorTripOverflow = Builder.CreateZExt(FloorTripOverflow, IVType);
1444  FloorTripCount =
1445  Builder.CreateAdd(FloorTripCount, FloorTripOverflow,
1446  "omp_floor" + Twine(i) + ".tripcount", true);
1447 
1448  // Remember some values for later use.
1449  FloorCount.push_back(FloorTripCount);
1450  FloorRems.push_back(FloorTripRem);
1451  }
1452 
1453  // Generate the new loop nest, from the outermost to the innermost.
1454  std::vector<CanonicalLoopInfo *> Result;
1455  Result.reserve(NumLoops * 2);
1456 
1457  // The basic block of the surrounding loop that enters the nest generated
1458  // loop.
1459  BasicBlock *Enter = OutermostLoop->getPreheader();
1460 
1461  // The basic block of the surrounding loop where the inner code should
1462  // continue.
1463  BasicBlock *Continue = OutermostLoop->getAfter();
1464 
1465  // Where the next loop basic block should be inserted.
1466  BasicBlock *OutroInsertBefore = InnermostLoop->getExit();
1467 
1468  auto EmbeddNewLoop =
1469  [this, DL, F, InnerEnter, &Enter, &Continue, &OutroInsertBefore](
1470  Value *TripCount, const Twine &Name) -> CanonicalLoopInfo * {
1471  CanonicalLoopInfo *EmbeddedLoop = createLoopSkeleton(
1472  DL, TripCount, F, InnerEnter, OutroInsertBefore, Name);
1473  redirectTo(Enter, EmbeddedLoop->getPreheader(), DL);
1474  redirectTo(EmbeddedLoop->getAfter(), Continue, DL);
1475 
1476  // Setup the position where the next embedded loop connects to this loop.
1477  Enter = EmbeddedLoop->getBody();
1478  Continue = EmbeddedLoop->getLatch();
1479  OutroInsertBefore = EmbeddedLoop->getLatch();
1480  return EmbeddedLoop;
1481  };
1482 
1483  auto EmbeddNewLoops = [&Result, &EmbeddNewLoop](ArrayRef<Value *> TripCounts,
1484  const Twine &NameBase) {
1485  for (auto P : enumerate(TripCounts)) {
1486  CanonicalLoopInfo *EmbeddedLoop =
1487  EmbeddNewLoop(P.value(), NameBase + Twine(P.index()));
1488  Result.push_back(EmbeddedLoop);
1489  }
1490  };
1491 
1492  EmbeddNewLoops(FloorCount, "floor");
1493 
1494  // Within the innermost floor loop, emit the code that computes the tile
1495  // sizes.
1496  Builder.SetInsertPoint(Enter->getTerminator());
1497  SmallVector<Value *, 4> TileCounts;
1498  for (int i = 0; i < NumLoops; ++i) {
1499  CanonicalLoopInfo *FloorLoop = Result[i];
1500  Value *TileSize = TileSizes[i];
1501 
1502  Value *FloorIsEpilogue =
1503  Builder.CreateICmpEQ(FloorLoop->getIndVar(), FloorCount[i]);
1504  Value *TileTripCount =
1505  Builder.CreateSelect(FloorIsEpilogue, FloorRems[i], TileSize);
1506 
1507  TileCounts.push_back(TileTripCount);
1508  }
1509 
1510  // Create the tile loops.
1511  EmbeddNewLoops(TileCounts, "tile");
1512 
1513  // Insert the inbetween code into the body.
1514  BasicBlock *BodyEnter = Enter;
1515  BasicBlock *BodyEntered = nullptr;
1516  for (std::pair<BasicBlock *, BasicBlock *> P : InbetweenCode) {
1517  BasicBlock *EnterBB = P.first;
1518  BasicBlock *ExitBB = P.second;
1519 
1520  if (BodyEnter)
1521  redirectTo(BodyEnter, EnterBB, DL);
1522  else
1523  redirectAllPredecessorsTo(BodyEntered, EnterBB, DL);
1524 
1525  BodyEnter = nullptr;
1526  BodyEntered = ExitBB;
1527  }
1528 
1529  // Append the original loop nest body into the generated loop nest body.
1530  if (BodyEnter)
1531  redirectTo(BodyEnter, InnerEnter, DL);
1532  else
1533  redirectAllPredecessorsTo(BodyEntered, InnerEnter, DL);
1534  redirectAllPredecessorsTo(InnerLatch, Continue, DL);
1535 
1536  // Replace the original induction variable with an induction variable computed
1537  // from the tile and floor induction variables.
1538  Builder.restoreIP(Result.back()->getBodyIP());
1539  for (int i = 0; i < NumLoops; ++i) {
1540  CanonicalLoopInfo *FloorLoop = Result[i];
1541  CanonicalLoopInfo *TileLoop = Result[NumLoops + i];
1542  Value *OrigIndVar = OrigIndVars[i];
1543  Value *Size = TileSizes[i];
1544 
1545  Value *Scale =
1546  Builder.CreateMul(Size, FloorLoop->getIndVar(), {}, /*HasNUW=*/true);
1547  Value *Shift =
1548  Builder.CreateAdd(Scale, TileLoop->getIndVar(), {}, /*HasNUW=*/true);
1549  OrigIndVar->replaceAllUsesWith(Shift);
1550  }
1551 
1552  // Remove unused parts of the original loops.
1553  SmallVector<BasicBlock *, 12> OldControlBBs;
1554  OldControlBBs.reserve(6 * Loops.size());
1555  for (CanonicalLoopInfo *Loop : Loops)
1556  Loop->collectControlBlocks(OldControlBBs);
1557  removeUnusedBlocksFromParent(OldControlBBs);
1558 
1559 #ifndef NDEBUG
1560  for (CanonicalLoopInfo *GenL : Result)
1561  GenL->assertOK();
1562 #endif
1563  return Result;
1564 }
1565 
1568  llvm::Value *BufSize, llvm::Value *CpyBuf,
1569  llvm::Value *CpyFn, llvm::Value *DidIt) {
1570  if (!updateToLocation(Loc))
1571  return Loc.IP;
1572 
1573  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1574  Value *Ident = getOrCreateIdent(SrcLocStr);
1575  Value *ThreadId = getOrCreateThreadID(Ident);
1576 
1577  llvm::Value *DidItLD = Builder.CreateLoad(Builder.getInt32Ty(), DidIt);
1578 
1579  Value *Args[] = {Ident, ThreadId, BufSize, CpyBuf, CpyFn, DidItLD};
1580 
1581  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_copyprivate);
1582  Builder.CreateCall(Fn, Args);
1583 
1584  return Builder.saveIP();
1585 }
1586 
1589  BodyGenCallbackTy BodyGenCB,
1590  FinalizeCallbackTy FiniCB, llvm::Value *DidIt) {
1591 
1592  if (!updateToLocation(Loc))
1593  return Loc.IP;
1594 
1595  // If needed (i.e. not null), initialize `DidIt` with 0
1596  if (DidIt) {
1597  Builder.CreateStore(Builder.getInt32(0), DidIt);
1598  }
1599 
1600  Directive OMPD = Directive::OMPD_single;
1601  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1602  Value *Ident = getOrCreateIdent(SrcLocStr);
1603  Value *ThreadId = getOrCreateThreadID(Ident);
1604  Value *Args[] = {Ident, ThreadId};
1605 
1606  Function *EntryRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_single);
1607  Instruction *EntryCall = Builder.CreateCall(EntryRTLFn, Args);
1608 
1609  Function *ExitRTLFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_single);
1610  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1611 
1612  // generates the following:
1613  // if (__kmpc_single()) {
1614  // .... single region ...
1615  // __kmpc_end_single
1616  // }
1617 
1618  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1619  /*Conditional*/ true, /*hasFinalize*/ true);
1620 }
1621 
1623  const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB,
1624  FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst) {
1625 
1626  if (!updateToLocation(Loc))
1627  return Loc.IP;
1628 
1629  Directive OMPD = Directive::OMPD_critical;
1630  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1631  Value *Ident = getOrCreateIdent(SrcLocStr);
1632  Value *ThreadId = getOrCreateThreadID(Ident);
1633  Value *LockVar = getOMPCriticalRegionLock(CriticalName);
1634  Value *Args[] = {Ident, ThreadId, LockVar};
1635 
1637  Function *RTFn = nullptr;
1638  if (HintInst) {
1639  // Add Hint to entry Args and create call
1640  EnterArgs.push_back(HintInst);
1641  RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical_with_hint);
1642  } else {
1643  RTFn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_critical);
1644  }
1645  Instruction *EntryCall = Builder.CreateCall(RTFn, EnterArgs);
1646 
1647  Function *ExitRTLFn =
1648  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_end_critical);
1649  Instruction *ExitCall = Builder.CreateCall(ExitRTLFn, Args);
1650 
1651  return EmitOMPInlinedRegion(OMPD, EntryCall, ExitCall, BodyGenCB, FiniCB,
1652  /*Conditional*/ false, /*hasFinalize*/ true);
1653 }
1654 
1655 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::EmitOMPInlinedRegion(
1656  Directive OMPD, Instruction *EntryCall, Instruction *ExitCall,
1657  BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, bool Conditional,
1658  bool HasFinalize) {
1659 
1660  if (HasFinalize)
1661  FinalizationStack.push_back({FiniCB, OMPD, /*IsCancellable*/ false});
1662 
1663  // Create inlined region's entry and body blocks, in preparation
1664  // for conditional creation
1665  BasicBlock *EntryBB = Builder.GetInsertBlock();
1666  Instruction *SplitPos = EntryBB->getTerminator();
1667  if (!isa_and_nonnull<BranchInst>(SplitPos))
1668  SplitPos = new UnreachableInst(Builder.getContext(), EntryBB);
1669  BasicBlock *ExitBB = EntryBB->splitBasicBlock(SplitPos, "omp_region.end");
1670  BasicBlock *FiniBB =
1671  EntryBB->splitBasicBlock(EntryBB->getTerminator(), "omp_region.finalize");
1672 
1673  Builder.SetInsertPoint(EntryBB->getTerminator());
1674  emitCommonDirectiveEntry(OMPD, EntryCall, ExitBB, Conditional);
1675 
1676  // generate body
1677  BodyGenCB(/* AllocaIP */ InsertPointTy(),
1678  /* CodeGenIP */ Builder.saveIP(), *FiniBB);
1679 
1680  // If we didn't emit a branch to FiniBB during body generation, it means
1681  // FiniBB is unreachable (e.g. while(1);). stop generating all the
1682  // unreachable blocks, and remove anything we are not going to use.
1683  auto SkipEmittingRegion = FiniBB->hasNPredecessors(0);
1684  if (SkipEmittingRegion) {
1685  FiniBB->eraseFromParent();
1686  ExitCall->eraseFromParent();
1687  // Discard finalization if we have it.
1688  if (HasFinalize) {
1689  assert(!FinalizationStack.empty() &&
1690  "Unexpected finalization stack state!");
1691  FinalizationStack.pop_back();
1692  }
1693  } else {
1694  // emit exit call and do any needed finalization.
1695  auto FinIP = InsertPointTy(FiniBB, FiniBB->getFirstInsertionPt());
1696  assert(FiniBB->getTerminator()->getNumSuccessors() == 1 &&
1697  FiniBB->getTerminator()->getSuccessor(0) == ExitBB &&
1698  "Unexpected control flow graph state!!");
1699  emitCommonDirectiveExit(OMPD, FinIP, ExitCall, HasFinalize);
1700  assert(FiniBB->getUniquePredecessor()->getUniqueSuccessor() == FiniBB &&
1701  "Unexpected Control Flow State!");
1702  MergeBlockIntoPredecessor(FiniBB);
1703  }
1704 
1705  // If we are skipping the region of a non conditional, remove the exit
1706  // block, and clear the builder's insertion point.
1707  assert(SplitPos->getParent() == ExitBB &&
1708  "Unexpected Insertion point location!");
1709  if (!Conditional && SkipEmittingRegion) {
1710  ExitBB->eraseFromParent();
1711  Builder.ClearInsertionPoint();
1712  } else {
1713  auto merged = MergeBlockIntoPredecessor(ExitBB);
1714  BasicBlock *ExitPredBB = SplitPos->getParent();
1715  auto InsertBB = merged ? ExitPredBB : ExitBB;
1716  if (!isa_and_nonnull<BranchInst>(SplitPos))
1717  SplitPos->eraseFromParent();
1718  Builder.SetInsertPoint(InsertBB);
1719  }
1720 
1721  return Builder.saveIP();
1722 }
1723 
1724 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveEntry(
1725  Directive OMPD, Value *EntryCall, BasicBlock *ExitBB, bool Conditional) {
1726 
1727  // if nothing to do, Return current insertion point.
1728  if (!Conditional)
1729  return Builder.saveIP();
1730 
1731  BasicBlock *EntryBB = Builder.GetInsertBlock();
1732  Value *CallBool = Builder.CreateIsNotNull(EntryCall);
1733  auto *ThenBB = BasicBlock::Create(M.getContext(), "omp_region.body");
1734  auto *UI = new UnreachableInst(Builder.getContext(), ThenBB);
1735 
1736  // Emit thenBB and set the Builder's insertion point there for
1737  // body generation next. Place the block after the current block.
1738  Function *CurFn = EntryBB->getParent();
1739  CurFn->getBasicBlockList().insertAfter(EntryBB->getIterator(), ThenBB);
1740 
1741  // Move Entry branch to end of ThenBB, and replace with conditional
1742  // branch (If-stmt)
1743  Instruction *EntryBBTI = EntryBB->getTerminator();
1744  Builder.CreateCondBr(CallBool, ThenBB, ExitBB);
1745  EntryBBTI->removeFromParent();
1746  Builder.SetInsertPoint(UI);
1747  Builder.Insert(EntryBBTI);
1748  UI->eraseFromParent();
1749  Builder.SetInsertPoint(ThenBB->getTerminator());
1750 
1751  // return an insertion point to ExitBB.
1752  return IRBuilder<>::InsertPoint(ExitBB, ExitBB->getFirstInsertionPt());
1753 }
1754 
1755 OpenMPIRBuilder::InsertPointTy OpenMPIRBuilder::emitCommonDirectiveExit(
1756  omp::Directive OMPD, InsertPointTy FinIP, Instruction *ExitCall,
1757  bool HasFinalize) {
1758 
1759  Builder.restoreIP(FinIP);
1760 
1761  // If there is finalization to do, emit it before the exit call
1762  if (HasFinalize) {
1763  assert(!FinalizationStack.empty() &&
1764  "Unexpected finalization stack state!");
1765 
1766  FinalizationInfo Fi = FinalizationStack.pop_back_val();
1767  assert(Fi.DK == OMPD && "Unexpected Directive for Finalization call!");
1768 
1769  Fi.FiniCB(FinIP);
1770 
1771  BasicBlock *FiniBB = FinIP.getBlock();
1772  Instruction *FiniBBTI = FiniBB->getTerminator();
1773 
1774  // set Builder IP for call creation
1775  Builder.SetInsertPoint(FiniBBTI);
1776  }
1777 
1778  // place the Exitcall as last instruction before Finalization block terminator
1779  ExitCall->removeFromParent();
1780  Builder.Insert(ExitCall);
1781 
1782  return IRBuilder<>::InsertPoint(ExitCall->getParent(),
1783  ExitCall->getIterator());
1784 }
1785 
1787  InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr,
1788  llvm::IntegerType *IntPtrTy, bool BranchtoEnd) {
1789  if (!IP.isSet())
1790  return IP;
1791 
1793 
1794  // creates the following CFG structure
1795  // OMP_Entry : (MasterAddr != PrivateAddr)?
1796  // F T
1797  // | \
1798  // | copin.not.master
1799  // | /
1800  // v /
1801  // copyin.not.master.end
1802  // |
1803  // v
1804  // OMP.Entry.Next
1805 
1806  BasicBlock *OMP_Entry = IP.getBlock();
1807  Function *CurFn = OMP_Entry->getParent();
1808  BasicBlock *CopyBegin =
1809  BasicBlock::Create(M.getContext(), "copyin.not.master", CurFn);
1810  BasicBlock *CopyEnd = nullptr;
1811 
1812  // If entry block is terminated, split to preserve the branch to following
1813  // basic block (i.e. OMP.Entry.Next), otherwise, leave everything as is.
1814  if (isa_and_nonnull<BranchInst>(OMP_Entry->getTerminator())) {
1815  CopyEnd = OMP_Entry->splitBasicBlock(OMP_Entry->getTerminator(),
1816  "copyin.not.master.end");
1817  OMP_Entry->getTerminator()->eraseFromParent();
1818  } else {
1819  CopyEnd =
1820  BasicBlock::Create(M.getContext(), "copyin.not.master.end", CurFn);
1821  }
1822 
1823  Builder.SetInsertPoint(OMP_Entry);
1824  Value *MasterPtr = Builder.CreatePtrToInt(MasterAddr, IntPtrTy);
1825  Value *PrivatePtr = Builder.CreatePtrToInt(PrivateAddr, IntPtrTy);
1826  Value *cmp = Builder.CreateICmpNE(MasterPtr, PrivatePtr);
1827  Builder.CreateCondBr(cmp, CopyBegin, CopyEnd);
1828 
1829  Builder.SetInsertPoint(CopyBegin);
1830  if (BranchtoEnd)
1831  Builder.SetInsertPoint(Builder.CreateBr(CopyEnd));
1832 
1833  return Builder.saveIP();
1834 }
1835 
1838  std::string Name) {
1840  Builder.restoreIP(Loc.IP);
1841 
1842  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1843  Value *Ident = getOrCreateIdent(SrcLocStr);
1844  Value *ThreadId = getOrCreateThreadID(Ident);
1845  Value *Args[] = {ThreadId, Size, Allocator};
1846 
1847  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_alloc);
1848 
1849  return Builder.CreateCall(Fn, Args, Name);
1850 }
1851 
1854  std::string Name) {
1856  Builder.restoreIP(Loc.IP);
1857 
1858  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1859  Value *Ident = getOrCreateIdent(SrcLocStr);
1860  Value *ThreadId = getOrCreateThreadID(Ident);
1861  Value *Args[] = {ThreadId, Addr, Allocator};
1862  Function *Fn = getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_free);
1863  return Builder.CreateCall(Fn, Args, Name);
1864 }
1865 
1867  const LocationDescription &Loc, llvm::Value *Pointer,
1870  Builder.restoreIP(Loc.IP);
1871 
1872  Constant *SrcLocStr = getOrCreateSrcLocStr(Loc);
1873  Value *Ident = getOrCreateIdent(SrcLocStr);
1874  Value *ThreadId = getOrCreateThreadID(Ident);
1875  Constant *ThreadPrivateCache =
1876  getOrCreateOMPInternalVariable(Int8PtrPtr, Name);
1877  llvm::Value *Args[] = {Ident, ThreadId, Pointer, Size, ThreadPrivateCache};
1878 
1879  Function *Fn =
1880  getOrCreateRuntimeFunctionPtr(OMPRTL___kmpc_threadprivate_cached);
1881 
1882  return Builder.CreateCall(Fn, Args);
1883 }
1884 
1885 std::string OpenMPIRBuilder::getNameWithSeparators(ArrayRef<StringRef> Parts,
1886  StringRef FirstSeparator,
1887  StringRef Separator) {
1888  SmallString<128> Buffer;
1889  llvm::raw_svector_ostream OS(Buffer);
1890  StringRef Sep = FirstSeparator;
1891  for (StringRef Part : Parts) {
1892  OS << Sep << Part;
1893  Sep = Separator;
1894  }
1895  return OS.str().str();
1896 }
1897 
1898 Constant *OpenMPIRBuilder::getOrCreateOMPInternalVariable(
1899  llvm::Type *Ty, const llvm::Twine &Name, unsigned AddressSpace) {
1900  // TODO: Replace the twine arg with stringref to get rid of the conversion
1901  // logic. However This is taken from current implementation in clang as is.
1902  // Since this method is used in many places exclusively for OMP internal use
1903  // we will keep it as is for temporarily until we move all users to the
1904  // builder and then, if possible, fix it everywhere in one go.
1905  SmallString<256> Buffer;
1906  llvm::raw_svector_ostream Out(Buffer);
1907  Out << Name;
1908  StringRef RuntimeName = Out.str();
1909  auto &Elem = *InternalVars.try_emplace(RuntimeName, nullptr).first;
1910  if (Elem.second) {
1911  assert(Elem.second->getType()->getPointerElementType() == Ty &&
1912  "OMP internal variable has different type than requested");
1913  } else {
1914  // TODO: investigate the appropriate linkage type used for the global
1915  // variable for possibly changing that to internal or private, or maybe
1916  // create different versions of the function for different OMP internal
1917  // variables.
1918  Elem.second = new llvm::GlobalVariable(
1919  M, Ty, /*IsConstant*/ false, llvm::GlobalValue::CommonLinkage,
1920  llvm::Constant::getNullValue(Ty), Elem.first(),
1921  /*InsertBefore=*/nullptr, llvm::GlobalValue::NotThreadLocal,
1922  AddressSpace);
1923  }
1924 
1925  return Elem.second;
1926 }
1927 
1928 Value *OpenMPIRBuilder::getOMPCriticalRegionLock(StringRef CriticalName) {
1929  std::string Prefix = Twine("gomp_critical_user_", CriticalName).str();
1930  std::string Name = getNameWithSeparators({Prefix, "var"}, ".", ".");
1931  return getOrCreateOMPInternalVariable(KmpCriticalNameTy, Name);
1932 }
1933 
1934 // Create all simple and struct types exposed by the runtime and remember
1935 // the llvm::PointerTypes of them for easy access later.
1936 void OpenMPIRBuilder::initializeTypes(Module &M) {
1937  LLVMContext &Ctx = M.getContext();
1938  StructType *T;
1939 #define OMP_TYPE(VarName, InitValue) VarName = InitValue;
1940 #define OMP_ARRAY_TYPE(VarName, ElemTy, ArraySize) \
1941  VarName##Ty = ArrayType::get(ElemTy, ArraySize); \
1942  VarName##PtrTy = PointerType::getUnqual(VarName##Ty);
1943 #define OMP_FUNCTION_TYPE(VarName, IsVarArg, ReturnType, ...) \
1944  VarName = FunctionType::get(ReturnType, {__VA_ARGS__}, IsVarArg); \
1945  VarName##Ptr = PointerType::getUnqual(VarName);
1946 #define OMP_STRUCT_TYPE(VarName, StructName, ...) \
1947  T = StructType::getTypeByName(Ctx, StructName); \
1948  if (!T) \
1949  T = StructType::create(Ctx, {__VA_ARGS__}, StructName); \
1950  VarName = T; \
1951  VarName##Ptr = PointerType::getUnqual(T);
1952 #include "llvm/Frontend/OpenMP/OMPKinds.def"
1953 }
1954 
1957  SmallVectorImpl<BasicBlock *> &BlockVector) {
1959  BlockSet.insert(EntryBB);
1960  BlockSet.insert(ExitBB);
1961 
1962  Worklist.push_back(EntryBB);
1963  while (!Worklist.empty()) {
1964  BasicBlock *BB = Worklist.pop_back_val();
1965  BlockVector.push_back(BB);
1966  for (BasicBlock *SuccBB : successors(BB))
1967  if (BlockSet.insert(SuccBB).second)
1968  Worklist.push_back(SuccBB);
1969  }
1970 }
1971 
1972 void CanonicalLoopInfo::collectControlBlocks(
1974  // We only count those BBs as control block for which we do not need to
1975  // reverse the CFG, i.e. not the loop body which can contain arbitrary control
1976  // flow. For consistency, this also means we do not add the Body block, which
1977  // is just the entry to the body code.
1978  BBs.reserve(BBs.size() + 6);
1979  BBs.append({Preheader, Header, Cond, Latch, Exit, After});
1980 }
1981 
1983 #ifndef NDEBUG
1984  if (!IsValid)
1985  return;
1986 
1987  // Verify standard control-flow we use for OpenMP loops.
1988  assert(Preheader);
1989  assert(isa<BranchInst>(Preheader->getTerminator()) &&
1990  "Preheader must terminate with unconditional branch");
1991  assert(Preheader->getSingleSuccessor() == Header &&
1992  "Preheader must jump to header");
1993 
1994  assert(Header);
1995  assert(isa<BranchInst>(Header->getTerminator()) &&
1996  "Header must terminate with unconditional branch");
1997  assert(Header->getSingleSuccessor() == Cond &&
1998  "Header must jump to exiting block");
1999 
2000  assert(Cond);
2001  assert(Cond->getSinglePredecessor() == Header &&
2002  "Exiting block only reachable from header");
2003 
2004  assert(isa<BranchInst>(Cond->getTerminator()) &&
2005  "Exiting block must terminate with conditional branch");
2006  assert(size(successors(Cond)) == 2 &&
2007  "Exiting block must have two successors");
2008  assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(0) == Body &&
2009  "Exiting block's first successor jump to the body");
2010  assert(cast<BranchInst>(Cond->getTerminator())->getSuccessor(1) == Exit &&
2011  "Exiting block's second successor must exit the loop");
2012 
2013  assert(Body);
2014  assert(Body->getSinglePredecessor() == Cond &&
2015  "Body only reachable from exiting block");
2016  assert(!isa<PHINode>(Body->front()));
2017 
2018  assert(Latch);
2019  assert(isa<BranchInst>(Latch->getTerminator()) &&
2020  "Latch must terminate with unconditional branch");
2021  assert(Latch->getSingleSuccessor() == Header && "Latch must jump to header");
2022  // TODO: To support simple redirecting of the end of the body code that has
2023  // multiple; introduce another auxiliary basic block like preheader and after.
2024  assert(Latch->getSinglePredecessor() != nullptr);
2025  assert(!isa<PHINode>(Latch->front()));
2026 
2027  assert(Exit);
2028  assert(isa<BranchInst>(Exit->getTerminator()) &&
2029  "Exit block must terminate with unconditional branch");
2030  assert(Exit->getSingleSuccessor() == After &&
2031  "Exit block must jump to after block");
2032 
2033  assert(After);
2034  assert(After->getSinglePredecessor() == Exit &&
2035  "After block only reachable from exit block");
2036  assert(After->empty() || !isa<PHINode>(After->front()));
2037 
2038  Instruction *IndVar = getIndVar();
2039  assert(IndVar && "Canonical induction variable not found?");
2040  assert(isa<IntegerType>(IndVar->getType()) &&
2041  "Induction variable must be an integer");
2042  assert(cast<PHINode>(IndVar)->getParent() == Header &&
2043  "Induction variable must be a PHI in the loop header");
2044  assert(cast<PHINode>(IndVar)->getIncomingBlock(0) == Preheader);
2045  assert(
2046  cast<ConstantInt>(cast<PHINode>(IndVar)->getIncomingValue(0))->isZero());
2047  assert(cast<PHINode>(IndVar)->getIncomingBlock(1) == Latch);
2048 
2049  auto *NextIndVar = cast<PHINode>(IndVar)->getIncomingValue(1);
2050  assert(cast<Instruction>(NextIndVar)->getParent() == Latch);
2051  assert(cast<BinaryOperator>(NextIndVar)->getOpcode() == BinaryOperator::Add);
2052  assert(cast<BinaryOperator>(NextIndVar)->getOperand(0) == IndVar);
2053  assert(cast<ConstantInt>(cast<BinaryOperator>(NextIndVar)->getOperand(1))
2054  ->isOne());
2055 
2056  Value *TripCount = getTripCount();
2057  assert(TripCount && "Loop trip count not found?");
2058  assert(IndVar->getType() == TripCount->getType() &&
2059  "Trip count and induction variable must have the same type");
2060 
2061  auto *CmpI = cast<CmpInst>(&Cond->front());
2062  assert(CmpI->getPredicate() == CmpInst::ICMP_ULT &&
2063  "Exit condition must be a signed less-than comparison");
2064  assert(CmpI->getOperand(0) == IndVar &&
2065  "Exit condition must compare the induction variable");
2066  assert(CmpI->getOperand(1) == TripCount &&
2067  "Exit condition must compare with the trip count");
2068 #endif
2069 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
i
i
Definition: README.txt:29
llvm::EngineKind::Kind
Kind
Definition: ExecutionEngine.h:524
llvm::CanonicalLoopInfo::getPreheaderIP
OpenMPIRBuilder::InsertPointTy getPreheaderIP() const
Return the insertion point for user code before the loop.
Definition: OMPIRBuilder.h:934
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
llvm::OpenMPIRBuilder::createCachedThreadPrivate
CallInst * createCachedThreadPrivate(const LocationDescription &Loc, llvm::Value *Pointer, llvm::ConstantInt *Size, const llvm::Twine &Name=Twine(""))
Create a runtime call for kmpc_threadprivate_cached.
Definition: OMPIRBuilder.cpp:1866
llvm::BasicBlock::end
iterator end()
Definition: BasicBlock.h:298
Attrs
Function Attrs
Definition: README_ALTIVEC.txt:215
llvm::predecessors
pred_range predecessors(BasicBlock *BB)
Definition: CFG.h:127
llvm::OpenMPIRBuilder::LocationDescription
Description of a LLVM-IR insertion point (IP) and a debug/source location (filename,...
Definition: OMPIRBuilder.h:142
llvm::OpenMPIRBuilder::OutlineInfo::ExitBB
BasicBlock * ExitBB
Definition: OMPIRBuilder.h:566
llvm
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::OpenMPIRBuilder::createCritical
InsertPointTy createCritical(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, StringRef CriticalName, Value *HintInst)
Generator for '#omp critical'.
Definition: OMPIRBuilder.cpp:1622
llvm::CanonicalLoopInfo::getAfter
BasicBlock * getAfter() const
The after block is intended for clean-up code such as lifetime end markers.
Definition: OMPIRBuilder.h:911
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
llvm::cl::Prefix
@ Prefix
Definition: CommandLine.h:161
llvm::Function::empty
bool empty() const
Definition: Function.h:770
llvm::Function::getBasicBlockList
const BasicBlockListType & getBasicBlockList() const
Get the underlying elements of the Function...
Definition: Function.h:741
llvm::CodeExtractor::findAllocas
void findAllocas(const CodeExtractorAnalysisCache &CEAC, ValueSet &SinkCands, ValueSet &HoistCands, BasicBlock *&ExitBlock) const
Find the set of allocas whose life ranges are contained within the outlined region.
Definition: CodeExtractor.cpp:494
llvm::Function
Definition: Function.h:61
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:529
StringRef.h
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1312
llvm::CodeExtractor::extractCodeRegion
Function * extractCodeRegion(const CodeExtractorAnalysisCache &CEAC)
Perform the extraction, returning the new function.
Definition: CodeExtractor.cpp:1567
Loops
Hexagon Hardware Loops
Definition: HexagonHardwareLoops.cpp:372
llvm::Target
Target - Wrapper for Target specific information.
Definition: TargetRegistry.h:124
llvm::GlobalValue::NotThreadLocal
@ NotThreadLocal
Definition: GlobalValue.h:179
llvm::ilist_node_with_parent::getNextNode
NodeTy * getNextNode()
Get the next node, or nullptr for the list tail.
Definition: ilist_node.h:288
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::BasicBlock::empty
bool empty() const
Definition: BasicBlock.h:307
llvm::Value::replaceUsesWithIf
void replaceUsesWithIf(Value *New, llvm::function_ref< bool(Use &U)> ShouldReplace)
Go through the uses list for this definition and make each use point to "V" if the callback ShouldRep...
Definition: Value.h:315
llvm::OpenMPIRBuilder::tileLoops
std::vector< CanonicalLoopInfo * > tileLoops(DebugLoc DL, ArrayRef< CanonicalLoopInfo * > Loops, ArrayRef< Value * > TileSizes)
Tile a loop nest.
Definition: OMPIRBuilder.cpp:1382
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:1923
llvm::OpenMPIRBuilder::createWorkshareLoop
CanonicalLoopInfo * createWorkshareLoop(const LocationDescription &Loc, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier)
Modifies the canonical loop to be a workshare loop.
Definition: OMPIRBuilder.cpp:1192
llvm::OpenMPIRBuilder::getLanemaskType
Type * getLanemaskType()
Definition: OMPIRBuilder.cpp:263
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:748
llvm::OpenMPIRBuilder::InsertPointTy
IRBuilder<>::InsertPoint InsertPointTy
Type used throughout for insertion points.
Definition: OMPIRBuilder.h:53
llvm::BasicBlock::replaceSuccessorsPhiUsesWith
void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New)
Update all phi nodes in this basic block's successors to refer to basic block New instead of basic bl...
Definition: BasicBlock.cpp:451
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2586
llvm::OpenMPIRBuilder::getOrCreateThreadID
Value * getOrCreateThreadID(Value *Ident)
Return the current thread ID.
Definition: OMPIRBuilder.cpp:328
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::OpenMPIRBuilder::emitCancelationCheckImpl
void emitCancelationCheckImpl(Value *CancelFlag, omp::Directive CanceledDirective)
Generate control flow and cleanup for cancellation.
Definition: OMPIRBuilder.cpp:431
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2207
llvm::DeleteDeadBlocks
void DeleteDeadBlocks(ArrayRef< BasicBlock * > BBs, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified blocks from BB.
Definition: BasicBlockUtils.cpp:94
Error.h
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:45
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
llvm::DILocation
Debug location.
Definition: DebugInfoMetadata.h:1558
llvm::Function::getContext
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:312
llvm::Function::arg_size
size_t arg_size() const
Definition: Function.h:812
Shift
bool Shift
Definition: README.txt:468
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::BasicBlock::eraseFromParent
SymbolTableList< BasicBlock >::iterator eraseFromParent()
Unlink 'this' from the containing function and delete it.
Definition: BasicBlock.cpp:129
llvm::sys::path::end
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:233
llvm::sys::path::begin
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:224
llvm::BasicBlock::getSingleSuccessor
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:294
llvm::BasicBlock::splitBasicBlock
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:375
llvm::GlobalValue::UnnamedAddr::Global
@ Global
llvm::Optional
Definition: APInt.h:33
T
#define T
Definition: Mips16ISelLowering.cpp:341
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
CodeExtractor.h
llvm::OpenMPIRBuilder::FinalizeCallbackTy
std::function< void(InsertPointTy CodeGenIP)> FinalizeCallbackTy
Callback type for variable finalization (think destructors).
Definition: OMPIRBuilder.h:63
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:752
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:264
llvm::BasicBlock::hasNPredecessors
bool hasNPredecessors(unsigned N) const
Return true if this block has exactly N predecessors.
Definition: BasicBlock.cpp:286
llvm::CallBase::arg_begin
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1306
llvm::SmallVectorImpl::pop_back_val
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:635
llvm::successors
succ_range successors(Instruction *I)
Definition: CFG.h:262
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:204
llvm::OpenMPIRBuilder::emitTaskyieldImpl
void emitTaskyieldImpl(const LocationDescription &Loc)
Generate a taskyield runtime call.
Definition: OMPIRBuilder.cpp:863
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1198
llvm::OpenMPIRBuilder::collapseLoops
CanonicalLoopInfo * collapseLoops(DebugLoc DL, ArrayRef< CanonicalLoopInfo * > Loops, InsertPointTy ComputeIP)
Collapse a loop nest into a single loop.
Definition: OMPIRBuilder.cpp:1261
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::ConstantExpr::getPointerCast
static Constant * getPointerCast(Constant *C, Type *Ty)
Create a BitCast, AddrSpaceCast, or a PtrToInt cast constant expression.
Definition: Constants.cpp:2019
llvm::BasicBlock::getUniqueSuccessor
const BasicBlock * getUniqueSuccessor() const
Return the successor of this block if it has a unique successor.
Definition: BasicBlock.cpp:302
Uses
SmallPtrSet< MachineInstr *, 2 > Uses
Definition: ARMLowOverheadLoops.cpp:583
llvm::CanonicalLoopInfo::getIndVar
Instruction * getIndVar() const
Returns the instruction representing the current logical induction variable.
Definition: OMPIRBuilder.h:924
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
llvm::OpenMPIRBuilder::finalize
void finalize(Function *Fn=nullptr, bool AllowExtractorSinking=false)
Finalize the underlying module, e.g., by outlining regions.
Definition: OMPIRBuilder.cpp:141
CommandLine.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:77
llvm::Instruction::getNumSuccessors
unsigned getNumSuccessors() const
Return the number of successors that this instruction has.
Definition: Instruction.cpp:736
llvm::OpenMPIRBuilder::getOrCreateDefaultSrcLocStr
Constant * getOrCreateDefaultSrcLocStr()
Return the (LLVM-IR) string describing the default source location.
Definition: OMPIRBuilder.cpp:308
llvm::OpenMPIRBuilder::getOrCreateIdent
Value * getOrCreateIdent(Constant *SrcLocStr, omp::IdentFlag Flags=omp::IdentFlag(0), unsigned Reserve2Flags=0)
Return an ident_t* encoding the source location SrcLocStr and Flags.
Definition: OMPIRBuilder.cpp:230
OMPIRBuilder.h
isZero
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:519
llvm::SmallVectorImpl::append
void append(in_iter in_start, in_iter in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:648
llvm::CanonicalLoopInfo::assertOK
void assertOK() const
Consistency self-check.
Definition: OMPIRBuilder.cpp:1982
llvm::CanonicalLoopInfo::getCond
BasicBlock * getCond() const
The condition block computes whether there is another loop iteration.
Definition: OMPIRBuilder.h:891
llvm::OpenMPIRBuilder
An interface to create LLVM-IR for OpenMP directives.
Definition: OMPIRBuilder.h:29
llvm::OpenMPIRBuilder::createOMPFree
CallInst * createOMPFree(const LocationDescription &Loc, Value *Addr, Value *Allocator, std::string Name="")
Create a runtime call for kmpc_free.
Definition: OMPIRBuilder.cpp:1852
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
cmp
< i32 >< i32 > cmp
Definition: README.txt:1447
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:748
TileSize
static cl::opt< unsigned > TileSize("fuse-matrix-tile-size", cl::init(4), cl::Hidden, cl::desc("Tile size for matrix instruction fusion using square-shaped tiles."))
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1396
llvm::BasicBlock::begin
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:296
redirectAllPredecessorsTo
static void redirectAllPredecessorsTo(BasicBlock *OldTarget, BasicBlock *NewTarget, DebugLoc DL)
Redirect all edges that branch to OldTarget to NewTarget.
Definition: OMPIRBuilder.cpp:1222
llvm::OpenMPIRBuilder::createBarrier
InsertPointTy createBarrier(const LocationDescription &Loc, omp::Directive DK, bool ForceSimpleCall=false, bool CheckCancelFlag=true)
Emitter methods for OpenMP directives.
Definition: OMPIRBuilder.cpp:335
getOpcode
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:199
IP
Definition: NVPTXLowerArgs.cpp:166
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:249
llvm::OpenMPIRBuilder::~OpenMPIRBuilder
~OpenMPIRBuilder()
Definition: OMPIRBuilder.cpp:226
llvm::IntegerType
Class to represent integer types.
Definition: DerivedTypes.h:40
llvm::CodeExtractor::findInputsOutputs
void findInputsOutputs(ValueSet &Inputs, ValueSet &Outputs, const ValueSet &Allocas) const
Compute the set of input values and output values for the code.
Definition: CodeExtractor.cpp:646
llvm::Instruction
Definition: Instruction.h:45
MDBuilder.h
setCanonicalLoopTripCount
void setCanonicalLoopTripCount(CanonicalLoopInfo *CLI, Value *TripCount)
Definition: OMPIRBuilder.cpp:1098
llvm::GlobalObject::addMetadata
void addMetadata(unsigned KindID, MDNode &MD)
Add a metadata attachment.
Definition: Metadata.cpp:1262
llvm::Value::setName
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:370
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:885
llvm::Use::getUser
User * getUser() const
Returns the User that contains this Use.
Definition: Use.h:73
llvm::CanonicalLoopInfo::getBodyIP
OpenMPIRBuilder::InsertPointTy getBodyIP() const
Return the insertion point for user code in the body.
Definition: OMPIRBuilder.h:939
llvm::Instruction::getSuccessor
BasicBlock * getSuccessor(unsigned Idx) const
Return the specified successor. This instruction must be a terminator.
Definition: Instruction.cpp:748
removeUnusedBlocksFromParent
static void removeUnusedBlocksFromParent(ArrayRef< BasicBlock * > BBs)
Determine which blocks in BBs are reachable from outside and remove the ones that are not reachable f...
Definition: OMPIRBuilder.cpp:1230
llvm::Instruction::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: Instruction.cpp:74
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::omp::IdentFlag
IdentFlag
IDs for all omp runtime library ident_t flag encodings (see their defintion in openmp/runtime/src/kmp...
Definition: OMPConstants.h:74
llvm::OpenMPIRBuilder::emitTaskwaitImpl
void emitTaskwaitImpl(const LocationDescription &Loc)
Generate a taskwait runtime call.
Definition: OMPIRBuilder.cpp:845
llvm::AddressSpace
AddressSpace
Definition: NVPTXBaseInfo.h:21
llvm::StringRef::str
LLVM_NODISCARD std::string str() const
str - Get the contents as an std::string.
Definition: StringRef.h:248
llvm::OpenMPIRBuilder::createOMPAlloc
CallInst * createOMPAlloc(const LocationDescription &Loc, Value *Size, Value *Allocator, std::string Name="")
Create a runtime call for kmpc_Alloc.
Definition: OMPIRBuilder.cpp:1836
llvm::SetVector::empty
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
llvm::OpenMPIRBuilder::addAttributes
void addAttributes(omp::RuntimeFunction FnID, Function &Fn)
Add attributes known for FnID to Fn.
Definition: OMPIRBuilder.cpp:41
llvm::Type::getIntegerBitWidth
unsigned getIntegerBitWidth() const
Definition: DerivedTypes.h:96
llvm::SmallString< 128 >
CFG.h
llvm::Twine::str
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
llvm::OpenMPIRBuilder::OutlineInfo::EntryBB
BasicBlock * EntryBB
Definition: OMPIRBuilder.h:566
llvm::Function::getAttributes
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:239
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLExtras.h:176
llvm::omp::RuntimeFunction
RuntimeFunction
IDs for all omp runtime library (RTL) functions.
Definition: OMPConstants.h:54
llvm::SmallString::append
void append(StringRef RHS)
Append from a StringRef.
Definition: SmallString.h:67
llvm::OpenMPIRBuilder::createTaskyield
void createTaskyield(const LocationDescription &Loc)
Generator for '#omp taskyield'.
Definition: OMPIRBuilder.cpp:874
llvm::cl::opt< bool >
llvm::CanonicalLoopInfo::getPreheader
BasicBlock * getPreheader() const
The preheader ensures that there is only a single edge entering the loop.
Definition: OMPIRBuilder.h:883
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:170
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:80
llvm::PHINode::addIncoming
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Definition: Instructions.h:2720
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3061
DebugInfo.h
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::DebugLoc::get
DILocation * get() const
Get the underlying DILocation.
Definition: DebugLoc.cpp:21
llvm::OpenMPIRBuilder::createCanonicalLoop
CanonicalLoopInfo * createCanonicalLoop(const LocationDescription &Loc, LoopBodyGenCallbackTy BodyGenCB, Value *TripCount, const Twine &Name="loop")
Generator for the control flow structure of an OpenMP canonical loop.
Definition: OMPIRBuilder.cpp:976
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:440
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:581
llvm::OpenMPIRBuilder::createParallel
IRBuilder ::InsertPoint createParallel(const LocationDescription &Loc, InsertPointTy AllocaIP, BodyGenCallbackTy BodyGenCB, PrivatizeCallbackTy PrivCB, FinalizeCallbackTy FiniCB, Value *IfCondition, Value *NumThreads, omp::ProcBindKind ProcBind, bool IsCancellable)
Generator for '#omp parallel'.
Definition: OMPIRBuilder.cpp:467
llvm::Triple::isAMDGCN
bool isAMDGCN() const
Tests whether the target is AMDGCN.
Definition: Triple.h:707
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::MDBuilder::createCallbackEncoding
MDNode * createCallbackEncoding(unsigned CalleeArgNo, ArrayRef< int > Arguments, bool VarArgsArePassed)
Return metadata describing a callback (see llvm::AbstractCallSite).
Definition: MDBuilder.cpp:107
llvm::OpenMPIRBuilder::LocationDescription::DL
DebugLoc DL
Definition: OMPIRBuilder.h:150
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::OpenMPIRBuilder::getOrCreateRuntimeFunctionPtr
Function * getOrCreateRuntimeFunctionPtr(omp::RuntimeFunction FnID)
Definition: OMPIRBuilder.cpp:132
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::OpenMPIRBuilder::createMaster
InsertPointTy createMaster(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB)
Generator for '#omp master'.
Definition: OMPIRBuilder.cpp:881
llvm::SplitBlockAndInsertIfThenElse
void SplitBlockAndInsertIfThenElse(Value *Cond, Instruction *SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
Definition: BasicBlockUtils.cpp:1445
llvm::OpenMPIRBuilder::createFlush
void createFlush(const LocationDescription &Loc)
Generator for '#omp flush'.
Definition: OMPIRBuilder.cpp:839
llvm::CallBase::arg_end
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1312
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
llvm::Function::getFunction
const Function & getFunction() const
Definition: Function.h:135
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:649
llvm::OpenMPIRBuilder::OutlineInfo
Helper that contains information about regions we need to outline during finalization.
Definition: OMPIRBuilder.h:563
llvm::CanonicalLoopInfo
Class to represented the control flow structure of an OpenMP canonical loop.
Definition: OMPIRBuilder.h:857
llvm::User::setOperand
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
Triple.h
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:751
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1486
llvm::SmallPtrSetImplBase::clear
void clear()
Definition: SmallPtrSet.h:94
llvm::Sched::Source
@ Source
Definition: TargetLowering.h:98
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::MergeBlockIntoPredecessor
bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false)
Attempts to merge a block into its predecessor, if possible.
Definition: BasicBlockUtils.cpp:173
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:212
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:167
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:747
llvm::M68kBeads::Term
@ Term
Definition: M68kBaseInfo.h:71
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::Value::getNumUses
unsigned getNumUses() const
This method computes the number of uses of this Value.
Definition: Value.cpp:244
llvm::BasicBlock::getUniquePredecessor
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:272
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:526
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
getParent
static const Function * getParent(const Value *V)
Definition: BasicAliasAnalysis.cpp:759
uint32_t
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3040
llvm::GlobalObject::hasMetadata
bool hasMetadata() const
Return true if this value has any metadata attached to it.
Definition: Value.h:594
llvm::CodeExtractor
Utility class for extracting code into a new function.
Definition: CodeExtractor.h:85
llvm::OpenMPIRBuilder::OutlineInfo::PostOutlineCB
PostOutlineCBTy PostOutlineCB
Definition: OMPIRBuilder.h:565
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:298
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:148
llvm::GlobalValue::CommonLinkage
@ CommonLinkage
Tentative definitions.
Definition: GlobalValue.h:58
llvm::FunctionCallee::getCallee
Value * getCallee()
Definition: DerivedTypes.h:183
llvm::BasicBlock::front
const Instruction & front() const
Definition: BasicBlock.h:308
llvm::CanonicalLoopInfo::getExit
BasicBlock * getExit() const
Reaching the exit indicates no more iterations are being executed.
Definition: OMPIRBuilder.h:904
llvm::OpenMPIRBuilder::createStaticWorkshareLoop
CanonicalLoopInfo * createStaticWorkshareLoop(const LocationDescription &Loc, CanonicalLoopInfo *CLI, InsertPointTy AllocaIP, bool NeedsBarrier, Value *Chunk=nullptr)
Modifies the canonical loop to be a statically-scheduled workshare loop.
Definition: OMPIRBuilder.cpp:1105
llvm::OpenMPIRBuilder::createTaskwait
void createTaskwait(const LocationDescription &Loc)
Generator for '#omp taskwait'.
Definition: OMPIRBuilder.cpp:857
redirectTo
static void redirectTo(BasicBlock *Source, BasicBlock *Target, DebugLoc DL)
Make Source branch to Target.
Definition: OMPIRBuilder.cpp:1205
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:205
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:347
llvm::ArrayRef::begin
iterator begin() const
Definition: ArrayRef.h:151
llvm::OpenMPIRBuilder::createCancel
InsertPointTy createCancel(const LocationDescription &Loc, Value *IfCondition, omp::Directive CanceledDirective)
Generator for '#omp cancel'.
Definition: OMPIRBuilder.cpp:390
OptimisticAttributes
static cl::opt< bool > OptimisticAttributes("openmp-ir-builder-optimistic-attributes", cl::Hidden, cl::desc("Use optimistic attributes describing " "'as-if' properties of runtime calls."), cl::init(false))
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:715
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:165
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:585
llvm::OpenMPIRBuilder::getOrCreateSrcLocStr
Constant * getOrCreateSrcLocStr(StringRef LocStr)
Return the (LLVM-IR) string describing the source location LocStr.
Definition: OMPIRBuilder.cpp:271
llvm::CodeExtractorAnalysisCache
A cache for the CodeExtractor analysis.
Definition: CodeExtractor.h:46
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:184
llvm::SmallString::str
StringRef str() const
Explicit conversion to StringRef.
Definition: SmallString.h:259
llvm::MDBuilder
Definition: MDBuilder.h:35
llvm::OpenMPIRBuilder::initialize
void initialize()
Initialize the internal state, this will put structures types and potentially other helpers into the ...
Definition: OMPIRBuilder.cpp:139
llvm::BasicBlock::getInstList
const InstListType & getInstList() const
Return the underlying instruction list container.
Definition: BasicBlock.h:363
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
llvm::CanonicalLoopInfo::getLatch
BasicBlock * getLatch() const
Reaching the latch indicates the end of the loop body code.
Definition: OMPIRBuilder.h:901
Allocator
Basic Register Allocator
Definition: RegAllocBasic.cpp:146
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::to_string
std::string to_string(const T &Value)
Definition: ScopedPrinter.h:62
llvm::ArrayRef::size
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:163
llvm::raw_svector_ostream
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:647
llvm::OpenMPIRBuilder::emitBarrierImpl
InsertPointTy emitBarrierImpl(const LocationDescription &Loc, omp::Directive DK, bool ForceSimpleCall, bool CheckCancelFlag)
Generate a barrier runtime call.
Definition: OMPIRBuilder.cpp:343
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:164
llvm::PHINode
Definition: Instructions.h:2572
llvm::BasicBlock::removePredecessor
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:321
llvm::Function::removeFromParent
void removeFromParent()
removeFromParent - This method unlinks 'this' from the containing module, but does not delete it.
Definition: Function.cpp:329
llvm::SmallVectorImpl< BasicBlock * >
llvm::CodeExtractor::isEligible
bool isEligible() const
Test whether this code extractor is eligible.
Definition: CodeExtractor.cpp:619
llvm::CanonicalLoopInfo::getTripCount
Value * getTripCount() const
Returns the llvm::Value containing the number of loop iterations.
Definition: OMPIRBuilder.h:916
llvm::SmallPtrSetImpl
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:343
llvm::OpenMPIRBuilder::emitFlush
void emitFlush(const LocationDescription &Loc)
Generate a flush runtime call.
Definition: OMPIRBuilder.cpp:831
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1450
llvm::OpenMPIRBuilder::OutlineInfo::collectBlocks
void collectBlocks(SmallPtrSetImpl< BasicBlock * > &BlockSet, SmallVectorImpl< BasicBlock * > &BlockVector)
Collect all blocks in between EntryBB and ExitBB in both the given vector and set.
Definition: OMPIRBuilder.cpp:1955
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::UnreachableInst
This function has undefined behavior.
Definition: Instructions.h:4650
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:389
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:61
llvm::cl::desc
Definition: CommandLine.h:411
llvm::OpenMPIRBuilder::createSingle
InsertPointTy createSingle(const LocationDescription &Loc, BodyGenCallbackTy BodyGenCB, FinalizeCallbackTy FiniCB, llvm::Value *DidIt)
Generator for '#omp single'.
Definition: OMPIRBuilder.cpp:1588
llvm::OpenMPIRBuilder::createCopyinClauseBlocks
InsertPointTy createCopyinClauseBlocks(InsertPointTy IP, Value *MasterAddr, Value *PrivateAddr, llvm::IntegerType *IntPtrTy, bool BranchtoEnd=true)
Generate conditional branch and relevant BasicBlocks through which private threads copy the 'copyin' ...
Definition: OMPIRBuilder.cpp:1786
llvm::OpenMPIRBuilder::createCopyPrivate
InsertPointTy createCopyPrivate(const LocationDescription &Loc, llvm::Value *BufSize, llvm::Value *CpyBuf, llvm::Value *CpyFn, llvm::Value *DidIt)
Generator for __kmpc_copyprivate.
Definition: OMPIRBuilder.cpp:1567
llvm::SetVector< Value * >
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:624
llvm::OpenMPIRBuilder::getOrCreateRuntimeFunction
FunctionCallee getOrCreateRuntimeFunction(Module &M, omp::RuntimeFunction FnID)
Return the function declaration for the runtime function with FnID.
Definition: OMPIRBuilder.cpp:74
BasicBlockUtils.h
llvm::SplitBlock
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
Definition: BasicBlockUtils.cpp:820
llvm::CanonicalLoopInfo::getBody
BasicBlock * getBody() const
The body block is the single entry for a loop iteration and not controlled by CanonicalLoopInfo.
Definition: OMPIRBuilder.h:896
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
getKmpcForStaticInitForType
static FunctionCallee getKmpcForStaticInitForType(Type *Ty, Module &M, OpenMPIRBuilder &OMPBuilder)
Definition: OMPIRBuilder.cpp:1083
llvm::Directive
Definition: DirectiveEmitter.h:100
llvm::ArrayRef::end
iterator end() const
Definition: ArrayRef.h:152
llvm::DIFile
File.
Definition: DebugInfoMetadata.h:526
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:102
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:44
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:908
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::OpenMPIRBuilder::LocationDescription::IP
InsertPointTy IP
Definition: OMPIRBuilder.h:149