LLVM  15.0.0git
ArgumentPromotion.cpp
Go to the documentation of this file.
1 //===- ArgumentPromotion.cpp - Promote by-reference arguments -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass promotes "by reference" arguments to be "by value" arguments. In
10 // practice, this means looking for internal functions that have pointer
11 // arguments. If it can prove, through the use of alias analysis, that an
12 // argument is *only* loaded, then it can pass the value into the function
13 // instead of the address of the value. This can cause recursive simplification
14 // of code and lead to the elimination of allocas (especially in C++ template
15 // code like the STL).
16 //
17 // This pass also handles aggregate arguments that are passed into a function,
18 // scalarizing them if the elements of the aggregate are only loaded. Note that
19 // by default it refuses to scalarize aggregates which would require passing in
20 // more than three operands to the function, because passing thousands of
21 // operands for a large array or structure is unprofitable! This limit can be
22 // configured or disabled, however.
23 //
24 // Note that this transformation could also be done for arguments that are only
25 // stored to (returning the value instead), but does not currently. This case
26 // would be best handled when and if LLVM begins supporting multiple return
27 // values from functions.
28 //
29 //===----------------------------------------------------------------------===//
30 
33 #include "llvm/ADT/None.h"
34 #include "llvm/ADT/Optional.h"
35 #include "llvm/ADT/STLExtras.h"
36 #include "llvm/ADT/ScopeExit.h"
37 #include "llvm/ADT/SmallPtrSet.h"
38 #include "llvm/ADT/SmallVector.h"
39 #include "llvm/ADT/Statistic.h"
40 #include "llvm/ADT/Twine.h"
47 #include "llvm/Analysis/Loads.h"
52 #include "llvm/IR/Argument.h"
53 #include "llvm/IR/Attributes.h"
54 #include "llvm/IR/BasicBlock.h"
55 #include "llvm/IR/CFG.h"
56 #include "llvm/IR/Constants.h"
57 #include "llvm/IR/DataLayout.h"
58 #include "llvm/IR/DerivedTypes.h"
59 #include "llvm/IR/Function.h"
60 #include "llvm/IR/IRBuilder.h"
61 #include "llvm/IR/InstrTypes.h"
62 #include "llvm/IR/Instruction.h"
63 #include "llvm/IR/Instructions.h"
64 #include "llvm/IR/Metadata.h"
65 #include "llvm/IR/Module.h"
66 #include "llvm/IR/NoFolder.h"
67 #include "llvm/IR/PassManager.h"
68 #include "llvm/IR/Type.h"
69 #include "llvm/IR/Use.h"
70 #include "llvm/IR/User.h"
71 #include "llvm/IR/Value.h"
72 #include "llvm/InitializePasses.h"
73 #include "llvm/Pass.h"
74 #include "llvm/Support/Casting.h"
75 #include "llvm/Support/Debug.h"
77 #include "llvm/Transforms/IPO.h"
78 #include <algorithm>
79 #include <cassert>
80 #include <cstdint>
81 #include <utility>
82 #include <vector>
83 
84 using namespace llvm;
85 
86 #define DEBUG_TYPE "argpromotion"
87 
88 STATISTIC(NumArgumentsPromoted, "Number of pointer arguments promoted");
89 STATISTIC(NumByValArgsPromoted, "Number of byval arguments promoted");
90 STATISTIC(NumArgumentsDead, "Number of dead pointer args eliminated");
91 
92 namespace {
93 
94 struct ArgPart {
95  Type *Ty;
96  Align Alignment;
97  /// A representative guaranteed-executed load instruction for use by
98  /// metadata transfer.
99  LoadInst *MustExecLoad;
100 };
101 
102 using OffsetAndArgPart = std::pair<int64_t, ArgPart>;
103 
104 } // end anonymous namespace
105 
107  Value *Ptr, Type *ResElemTy, int64_t Offset) {
108  // For non-opaque pointers, try to create a "nice" GEP if possible, otherwise
109  // fall back to an i8 GEP to a specific offset.
110  unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
111  APInt OrigOffset(DL.getIndexTypeSizeInBits(Ptr->getType()), Offset);
112  if (!Ptr->getType()->isOpaquePointerTy()) {
113  Type *OrigElemTy = Ptr->getType()->getNonOpaquePointerElementType();
114  if (OrigOffset == 0 && OrigElemTy == ResElemTy)
115  return Ptr;
116 
117  if (OrigElemTy->isSized()) {
118  APInt TmpOffset = OrigOffset;
119  Type *TmpTy = OrigElemTy;
120  SmallVector<APInt> IntIndices =
121  DL.getGEPIndicesForOffset(TmpTy, TmpOffset);
122  if (TmpOffset == 0) {
123  // Try to add trailing zero indices to reach the right type.
124  while (TmpTy != ResElemTy) {
125  Type *NextTy = GetElementPtrInst::getTypeAtIndex(TmpTy, (uint64_t)0);
126  if (!NextTy)
127  break;
128 
129  IntIndices.push_back(APInt::getZero(
130  isa<StructType>(TmpTy) ? 32 : OrigOffset.getBitWidth()));
131  TmpTy = NextTy;
132  }
133 
134  SmallVector<Value *> Indices;
135  for (const APInt &Index : IntIndices)
136  Indices.push_back(IRB.getInt(Index));
137 
138  if (OrigOffset != 0 || TmpTy == ResElemTy) {
139  Ptr = IRB.CreateGEP(OrigElemTy, Ptr, Indices);
140  return IRB.CreateBitCast(Ptr, ResElemTy->getPointerTo(AddrSpace));
141  }
142  }
143  }
144  }
145 
146  if (OrigOffset != 0) {
147  Ptr = IRB.CreateBitCast(Ptr, IRB.getInt8PtrTy(AddrSpace));
148  Ptr = IRB.CreateGEP(IRB.getInt8Ty(), Ptr, IRB.getInt(OrigOffset));
149  }
150  return IRB.CreateBitCast(Ptr, ResElemTy->getPointerTo(AddrSpace));
151 }
152 
153 /// DoPromotion - This method actually performs the promotion of the specified
154 /// arguments, and returns the new function. At this point, we know that it's
155 /// safe to do so.
157  Function *F,
158  const DenseMap<Argument *, SmallVector<OffsetAndArgPart, 4>> &ArgsToPromote,
159  SmallPtrSetImpl<Argument *> &ByValArgsToTransform,
160  Optional<function_ref<void(CallBase &OldCS, CallBase &NewCS)>>
161  ReplaceCallSite) {
162  // Start by computing a new prototype for the function, which is the same as
163  // the old function, but has modified arguments.
164  FunctionType *FTy = F->getFunctionType();
165  std::vector<Type *> Params;
166 
167  // Attribute - Keep track of the parameter attributes for the arguments
168  // that we are *not* promoting. For the ones that we do promote, the parameter
169  // attributes are lost
170  SmallVector<AttributeSet, 8> ArgAttrVec;
171  AttributeList PAL = F->getAttributes();
172 
173  // First, determine the new argument list
174  unsigned ArgNo = 0;
175  for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E;
176  ++I, ++ArgNo) {
177  if (ByValArgsToTransform.count(&*I)) {
178  // Simple byval argument? Just add all the struct element types.
179  Type *AgTy = I->getParamByValType();
180  StructType *STy = cast<StructType>(AgTy);
181  llvm::append_range(Params, STy->elements());
182  ArgAttrVec.insert(ArgAttrVec.end(), STy->getNumElements(),
183  AttributeSet());
184  ++NumByValArgsPromoted;
185  } else if (!ArgsToPromote.count(&*I)) {
186  // Unchanged argument
187  Params.push_back(I->getType());
188  ArgAttrVec.push_back(PAL.getParamAttrs(ArgNo));
189  } else if (I->use_empty()) {
190  // Dead argument (which are always marked as promotable)
191  ++NumArgumentsDead;
192  } else {
193  const auto &ArgParts = ArgsToPromote.find(&*I)->second;
194  for (const auto &Pair : ArgParts) {
195  Params.push_back(Pair.second.Ty);
196  ArgAttrVec.push_back(AttributeSet());
197  }
198  ++NumArgumentsPromoted;
199  }
200  }
201 
202  Type *RetTy = FTy->getReturnType();
203 
204  // Construct the new function type using the new arguments.
205  FunctionType *NFTy = FunctionType::get(RetTy, Params, FTy->isVarArg());
206 
207  // Create the new function body and insert it into the module.
208  Function *NF = Function::Create(NFTy, F->getLinkage(), F->getAddressSpace(),
209  F->getName());
210  NF->copyAttributesFrom(F);
211  NF->copyMetadata(F, 0);
212 
213  // The new function will have the !dbg metadata copied from the original
214  // function. The original function may not be deleted, and dbg metadata need
215  // to be unique, so we need to drop it.
216  F->setSubprogram(nullptr);
217 
218  LLVM_DEBUG(dbgs() << "ARG PROMOTION: Promoting to:" << *NF << "\n"
219  << "From: " << *F);
220 
221  uint64_t LargestVectorWidth = 0;
222  for (auto *I : Params)
223  if (auto *VT = dyn_cast<llvm::VectorType>(I))
224  LargestVectorWidth = std::max(
225  LargestVectorWidth, VT->getPrimitiveSizeInBits().getKnownMinSize());
226 
227  // Recompute the parameter attributes list based on the new arguments for
228  // the function.
229  NF->setAttributes(AttributeList::get(F->getContext(), PAL.getFnAttrs(),
230  PAL.getRetAttrs(), ArgAttrVec));
231  AttributeFuncs::updateMinLegalVectorWidthAttr(*NF, LargestVectorWidth);
232  ArgAttrVec.clear();
233 
234  F->getParent()->getFunctionList().insert(F->getIterator(), NF);
235  NF->takeName(F);
236 
237  // Loop over all of the callers of the function, transforming the call sites
238  // to pass in the loaded pointers.
239  //
241  const DataLayout &DL = F->getParent()->getDataLayout();
242  while (!F->use_empty()) {
243  CallBase &CB = cast<CallBase>(*F->user_back());
244  assert(CB.getCalledFunction() == F);
245  const AttributeList &CallPAL = CB.getAttributes();
246  IRBuilder<NoFolder> IRB(&CB);
247 
248  // Loop over the operands, inserting GEP and loads in the caller as
249  // appropriate.
250  auto *AI = CB.arg_begin();
251  ArgNo = 0;
252  for (Function::arg_iterator I = F->arg_begin(), E = F->arg_end(); I != E;
253  ++I, ++AI, ++ArgNo)
254  if (!ArgsToPromote.count(&*I) && !ByValArgsToTransform.count(&*I)) {
255  Args.push_back(*AI); // Unmodified argument
256  ArgAttrVec.push_back(CallPAL.getParamAttrs(ArgNo));
257  } else if (ByValArgsToTransform.count(&*I)) {
258  // Emit a GEP and load for each element of the struct.
259  Type *AgTy = I->getParamByValType();
260  StructType *STy = cast<StructType>(AgTy);
261  Value *Idxs[2] = {
262  ConstantInt::get(Type::getInt32Ty(F->getContext()), 0), nullptr};
263  const StructLayout *SL = DL.getStructLayout(STy);
264  Align StructAlign = *I->getParamAlign();
265  for (unsigned J = 0, Elems = STy->getNumElements(); J != Elems; ++J) {
266  Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), J);
267  auto *Idx =
268  IRB.CreateGEP(STy, *AI, Idxs, (*AI)->getName() + "." + Twine(J));
269  // TODO: Tell AA about the new values?
270  Align Alignment =
271  commonAlignment(StructAlign, SL->getElementOffset(J));
272  Args.push_back(IRB.CreateAlignedLoad(
273  STy->getElementType(J), Idx, Alignment, Idx->getName() + ".val"));
274  ArgAttrVec.push_back(AttributeSet());
275  }
276  } else if (!I->use_empty()) {
277  Value *V = *AI;
278  const auto &ArgParts = ArgsToPromote.find(&*I)->second;
279  for (const auto &Pair : ArgParts) {
280  LoadInst *LI = IRB.CreateAlignedLoad(
281  Pair.second.Ty,
282  createByteGEP(IRB, DL, V, Pair.second.Ty, Pair.first),
283  Pair.second.Alignment, V->getName() + ".val");
284  if (Pair.second.MustExecLoad) {
285  LI->setAAMetadata(Pair.second.MustExecLoad->getAAMetadata());
286  LI->copyMetadata(*Pair.second.MustExecLoad,
287  {LLVMContext::MD_range, LLVMContext::MD_nonnull,
288  LLVMContext::MD_dereferenceable,
289  LLVMContext::MD_dereferenceable_or_null,
290  LLVMContext::MD_align, LLVMContext::MD_noundef});
291  }
292  Args.push_back(LI);
293  ArgAttrVec.push_back(AttributeSet());
294  }
295  }
296 
297  // Push any varargs arguments on the list.
298  for (; AI != CB.arg_end(); ++AI, ++ArgNo) {
299  Args.push_back(*AI);
300  ArgAttrVec.push_back(CallPAL.getParamAttrs(ArgNo));
301  }
302 
304  CB.getOperandBundlesAsDefs(OpBundles);
305 
306  CallBase *NewCS = nullptr;
307  if (InvokeInst *II = dyn_cast<InvokeInst>(&CB)) {
308  NewCS = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(),
309  Args, OpBundles, "", &CB);
310  } else {
311  auto *NewCall = CallInst::Create(NF, Args, OpBundles, "", &CB);
312  NewCall->setTailCallKind(cast<CallInst>(&CB)->getTailCallKind());
313  NewCS = NewCall;
314  }
315  NewCS->setCallingConv(CB.getCallingConv());
316  NewCS->setAttributes(AttributeList::get(F->getContext(),
317  CallPAL.getFnAttrs(),
318  CallPAL.getRetAttrs(), ArgAttrVec));
319  NewCS->copyMetadata(CB, {LLVMContext::MD_prof, LLVMContext::MD_dbg});
320  Args.clear();
321  ArgAttrVec.clear();
322 
324  LargestVectorWidth);
325 
326  // Update the callgraph to know that the callsite has been transformed.
327  if (ReplaceCallSite)
328  (*ReplaceCallSite)(CB, *NewCS);
329 
330  if (!CB.use_empty()) {
331  CB.replaceAllUsesWith(NewCS);
332  NewCS->takeName(&CB);
333  }
334 
335  // Finally, remove the old call from the program, reducing the use-count of
336  // F.
337  CB.eraseFromParent();
338  }
339 
340  // Since we have now created the new function, splice the body of the old
341  // function right into the new function, leaving the old rotting hulk of the
342  // function empty.
343  NF->getBasicBlockList().splice(NF->begin(), F->getBasicBlockList());
344 
345  // Loop over the argument list, transferring uses of the old arguments over to
346  // the new arguments, also transferring over the names as well.
348  for (Argument &Arg : F->args()) {
349  if (!ArgsToPromote.count(&Arg) && !ByValArgsToTransform.count(&Arg)) {
350  // If this is an unmodified argument, move the name and users over to the
351  // new version.
352  Arg.replaceAllUsesWith(&*I2);
353  I2->takeName(&Arg);
354  ++I2;
355  continue;
356  }
357 
358  if (ByValArgsToTransform.count(&Arg)) {
359  // In the callee, we create an alloca, and store each of the new incoming
360  // arguments into the alloca.
361  Instruction *InsertPt = &NF->begin()->front();
362 
363  // Just add all the struct element types.
364  Type *AgTy = Arg.getParamByValType();
365  Align StructAlign = *Arg.getParamAlign();
366  Value *TheAlloca = new AllocaInst(AgTy, DL.getAllocaAddrSpace(), nullptr,
367  StructAlign, "", InsertPt);
368  StructType *STy = cast<StructType>(AgTy);
369  Value *Idxs[2] = {ConstantInt::get(Type::getInt32Ty(F->getContext()), 0),
370  nullptr};
371  const StructLayout *SL = DL.getStructLayout(STy);
372 
373  for (unsigned J = 0, Elems = STy->getNumElements(); J != Elems; ++J) {
374  Idxs[1] = ConstantInt::get(Type::getInt32Ty(F->getContext()), J);
376  AgTy, TheAlloca, Idxs, TheAlloca->getName() + "." + Twine(J),
377  InsertPt);
378  I2->setName(Arg.getName() + "." + Twine(J));
379  Align Alignment = commonAlignment(StructAlign, SL->getElementOffset(J));
380  new StoreInst(&*I2++, Idx, false, Alignment, InsertPt);
381  }
382 
383  // Anything that used the arg should now use the alloca.
384  Arg.replaceAllUsesWith(TheAlloca);
385  TheAlloca->takeName(&Arg);
386  continue;
387  }
388 
389  // There potentially are metadata uses for things like llvm.dbg.value.
390  // Replace them with undef, after handling the other regular uses.
391  auto RauwUndefMetadata = make_scope_exit(
392  [&]() { Arg.replaceAllUsesWith(UndefValue::get(Arg.getType())); });
393 
394  if (Arg.use_empty())
395  continue;
396 
398  for (const auto &Pair : ArgsToPromote.find(&Arg)->second) {
399  Argument &NewArg = *I2++;
400  NewArg.setName(Arg.getName() + "." + Twine(Pair.first) + ".val");
401  OffsetToArg.insert({Pair.first, &NewArg});
402  }
403 
404  // Otherwise, if we promoted this argument, then all users are load
405  // instructions (with possible casts and GEPs in between).
406 
407  SmallVector<Value *, 16> Worklist;
409  append_range(Worklist, Arg.users());
410  while (!Worklist.empty()) {
411  Value *V = Worklist.pop_back_val();
412  if (isa<BitCastInst>(V) || isa<GetElementPtrInst>(V)) {
413  DeadInsts.push_back(cast<Instruction>(V));
414  append_range(Worklist, V->users());
415  continue;
416  }
417 
418  if (auto *LI = dyn_cast<LoadInst>(V)) {
419  Value *Ptr = LI->getPointerOperand();
420  APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
421  Ptr =
423  /* AllowNonInbounds */ true);
424  assert(Ptr == &Arg && "Not constant offset from arg?");
425  LI->replaceAllUsesWith(OffsetToArg[Offset.getSExtValue()]);
426  DeadInsts.push_back(LI);
427  continue;
428  }
429 
430  llvm_unreachable("Unexpected user");
431  }
432 
433  for (Instruction *I : DeadInsts) {
434  I->replaceAllUsesWith(PoisonValue::get(I->getType()));
435  I->eraseFromParent();
436  }
437  }
438 
439  return NF;
440 }
441 
442 /// Return true if we can prove that all callees pass in a valid pointer for the
443 /// specified function argument.
445  Align NeededAlign,
446  uint64_t NeededDerefBytes) {
447  Function *Callee = Arg->getParent();
448  const DataLayout &DL = Callee->getParent()->getDataLayout();
449  APInt Bytes(64, NeededDerefBytes);
450 
451  // Check if the argument itself is marked dereferenceable and aligned.
452  if (isDereferenceableAndAlignedPointer(Arg, NeededAlign, Bytes, DL))
453  return true;
454 
455  // Look at all call sites of the function. At this point we know we only have
456  // direct callees.
457  return all_of(Callee->users(), [&](User *U) {
458  CallBase &CB = cast<CallBase>(*U);
459  return isDereferenceableAndAlignedPointer(
460  CB.getArgOperand(Arg->getArgNo()), NeededAlign, Bytes, DL);
461  });
462 }
463 
464 /// Determine that this argument is safe to promote, and find the argument
465 /// parts it can be promoted into.
466 static bool findArgParts(Argument *Arg, const DataLayout &DL, AAResults &AAR,
467  unsigned MaxElements, bool IsRecursive,
468  SmallVectorImpl<OffsetAndArgPart> &ArgPartsVec) {
469  // Quick exit for unused arguments
470  if (Arg->use_empty())
471  return true;
472 
473  // We can only promote this argument if all of the uses are loads at known
474  // offsets.
475  //
476  // Promoting the argument causes it to be loaded in the caller
477  // unconditionally. This is only safe if we can prove that either the load
478  // would have happened in the callee anyway (ie, there is a load in the entry
479  // block) or the pointer passed in at every call site is guaranteed to be
480  // valid.
481  // In the former case, invalid loads can happen, but would have happened
482  // anyway, in the latter case, invalid loads won't happen. This prevents us
483  // from introducing an invalid load that wouldn't have happened in the
484  // original code.
485 
487  Align NeededAlign(1);
488  uint64_t NeededDerefBytes = 0;
489 
490  // Returns None if this load is not based on the argument. Return true if
491  // we can promote the load, false otherwise.
492  auto HandleLoad = [&](LoadInst *LI,
493  bool GuaranteedToExecute) -> Optional<bool> {
494  // Don't promote volatile or atomic loads.
495  if (!LI->isSimple())
496  return false;
497 
498  Value *Ptr = LI->getPointerOperand();
499  APInt Offset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0);
500  Ptr = Ptr->stripAndAccumulateConstantOffsets(DL, Offset,
501  /* AllowNonInbounds */ true);
502  if (Ptr != Arg)
503  return None;
504 
505  if (Offset.getSignificantBits() >= 64)
506  return false;
507 
508  Type *Ty = LI->getType();
509  TypeSize Size = DL.getTypeStoreSize(Ty);
510  // Don't try to promote scalable types.
511  if (Size.isScalable())
512  return false;
513 
514  // If this is a recursive function and one of the types is a pointer,
515  // then promoting it might lead to recursive promotion.
516  if (IsRecursive && Ty->isPointerTy())
517  return false;
518 
519  int64_t Off = Offset.getSExtValue();
520  auto Pair = ArgParts.try_emplace(
521  Off, ArgPart{Ty, LI->getAlign(), GuaranteedToExecute ? LI : nullptr});
522  ArgPart &Part = Pair.first->second;
523  bool OffsetNotSeenBefore = Pair.second;
524 
525  // We limit promotion to only promoting up to a fixed number of elements of
526  // the aggregate.
527  if (MaxElements > 0 && ArgParts.size() > MaxElements) {
528  LLVM_DEBUG(dbgs() << "ArgPromotion of " << *Arg << " failed: "
529  << "more than " << MaxElements << " parts\n");
530  return false;
531  }
532 
533  // For now, we only support loading one specific type at a given offset.
534  if (Part.Ty != Ty) {
535  LLVM_DEBUG(dbgs() << "ArgPromotion of " << *Arg << " failed: "
536  << "loaded via both " << *Part.Ty << " and " << *Ty
537  << " at offset " << Off << "\n");
538  return false;
539  }
540 
541  // If this load is not guaranteed to execute, and we haven't seen a load at
542  // this offset before (or it had lower alignment), then we need to remember
543  // that requirement.
544  // Note that skipping loads of previously seen offsets is only correct
545  // because we only allow a single type for a given offset, which also means
546  // that the number of accessed bytes will be the same.
547  if (!GuaranteedToExecute &&
548  (OffsetNotSeenBefore || Part.Alignment < LI->getAlign())) {
549  // We won't be able to prove dereferenceability for negative offsets.
550  if (Off < 0)
551  return false;
552 
553  // If the offset is not aligned, an aligned base pointer won't help.
554  if (!isAligned(LI->getAlign(), Off))
555  return false;
556 
557  NeededDerefBytes = std::max(NeededDerefBytes, Off + Size.getFixedValue());
558  NeededAlign = std::max(NeededAlign, LI->getAlign());
559  }
560 
561  Part.Alignment = std::max(Part.Alignment, LI->getAlign());
562  return true;
563  };
564 
565  // Look for loads that are guaranteed to execute on entry.
566  for (Instruction &I : Arg->getParent()->getEntryBlock()) {
567  if (LoadInst *LI = dyn_cast<LoadInst>(&I))
568  if (Optional<bool> Res = HandleLoad(LI, /* GuaranteedToExecute */ true))
569  if (!*Res)
570  return false;
571 
573  break;
574  }
575 
576  // Now look at all loads of the argument. Remember the load instructions
577  // for the aliasing check below.
578  SmallVector<Value *, 16> Worklist;
579  SmallPtrSet<Value *, 16> Visited;
581  auto AppendUsers = [&](Value *V) {
582  for (User *U : V->users())
583  if (Visited.insert(U).second)
584  Worklist.push_back(U);
585  };
586  AppendUsers(Arg);
587  while (!Worklist.empty()) {
588  Value *V = Worklist.pop_back_val();
589  if (isa<BitCastInst>(V)) {
590  AppendUsers(V);
591  continue;
592  }
593 
594  if (auto *GEP = dyn_cast<GetElementPtrInst>(V)) {
595  if (!GEP->hasAllConstantIndices())
596  return false;
597  AppendUsers(V);
598  continue;
599  }
600 
601  if (auto *LI = dyn_cast<LoadInst>(V)) {
602  if (!*HandleLoad(LI, /* GuaranteedToExecute */ false))
603  return false;
604  Loads.push_back(LI);
605  continue;
606  }
607 
608  // Unknown user.
609  LLVM_DEBUG(dbgs() << "ArgPromotion of " << *Arg << " failed: "
610  << "unknown user " << *V << "\n");
611  return false;
612  }
613 
614  if (NeededDerefBytes || NeededAlign > 1) {
615  // Try to prove a required deref / aligned requirement.
616  if (!allCallersPassValidPointerForArgument(Arg, NeededAlign,
617  NeededDerefBytes)) {
618  LLVM_DEBUG(dbgs() << "ArgPromotion of " << *Arg << " failed: "
619  << "not dereferenceable or aligned\n");
620  return false;
621  }
622  }
623 
624  if (ArgParts.empty())
625  return true; // No users, this is a dead argument.
626 
627  // Sort parts by offset.
628  append_range(ArgPartsVec, ArgParts);
629  sort(ArgPartsVec,
630  [](const auto &A, const auto &B) { return A.first < B.first; });
631 
632  // Make sure the parts are non-overlapping.
633  // TODO: As we're doing pure load promotion here, overlap should be fine from
634  // a correctness perspective. Profitability is less obvious though.
635  int64_t Offset = ArgPartsVec[0].first;
636  for (const auto &Pair : ArgPartsVec) {
637  if (Pair.first < Offset)
638  return false; // Overlap with previous part.
639 
640  Offset = Pair.first + DL.getTypeStoreSize(Pair.second.Ty);
641  }
642 
643  // Okay, now we know that the argument is only used by load instructions, and
644  // it is safe to unconditionally perform all of them. Use alias analysis to
645  // check to see if the pointer is guaranteed to not be modified from entry of
646  // the function to each of the load instructions.
647 
648  // Because there could be several/many load instructions, remember which
649  // blocks we know to be transparent to the load.
651 
652  for (LoadInst *Load : Loads) {
653  // Check to see if the load is invalidated from the start of the block to
654  // the load itself.
655  BasicBlock *BB = Load->getParent();
656 
658  if (AAR.canInstructionRangeModRef(BB->front(), *Load, Loc, ModRefInfo::Mod))
659  return false; // Pointer is invalidated!
660 
661  // Now check every path from the entry block to the load for transparency.
662  // To do this, we perform a depth first search on the inverse CFG from the
663  // loading block.
664  for (BasicBlock *P : predecessors(BB)) {
665  for (BasicBlock *TranspBB : inverse_depth_first_ext(P, TranspBlocks))
666  if (AAR.canBasicBlockModify(*TranspBB, Loc))
667  return false;
668  }
669  }
670 
671  // If the path from the entry of the function to each load is free of
672  // instructions that potentially invalidate the load, we can make the
673  // transformation!
674  return true;
675 }
676 
678  // There is no size information, so be conservative.
679  if (!Ty->isSized())
680  return false;
681 
682  // If the alloc size is not equal to the storage size, then there are padding
683  // bytes. For x86_fp80 on x86-64, size: 80 alloc size: 128.
684  if (DL.getTypeSizeInBits(Ty) != DL.getTypeAllocSizeInBits(Ty))
685  return false;
686 
687  // FIXME: This isn't the right way to check for padding in vectors with
688  // non-byte-size elements.
689  if (VectorType *SeqTy = dyn_cast<VectorType>(Ty))
690  return isDenselyPacked(SeqTy->getElementType(), DL);
691 
692  // For array types, check for padding within members.
693  if (ArrayType *SeqTy = dyn_cast<ArrayType>(Ty))
694  return isDenselyPacked(SeqTy->getElementType(), DL);
695 
696  if (!isa<StructType>(Ty))
697  return true;
698 
699  // Check for padding within and between elements of a struct.
700  StructType *StructTy = cast<StructType>(Ty);
701  const StructLayout *Layout = DL.getStructLayout(StructTy);
702  uint64_t StartPos = 0;
703  for (unsigned I = 0, E = StructTy->getNumElements(); I < E; ++I) {
704  Type *ElTy = StructTy->getElementType(I);
705  if (!isDenselyPacked(ElTy, DL))
706  return false;
707  if (StartPos != Layout->getElementOffsetInBits(I))
708  return false;
709  StartPos += DL.getTypeAllocSizeInBits(ElTy);
710  }
711 
712  return true;
713 }
714 
715 /// Checks if the padding bytes of an argument could be accessed.
717  assert(Arg->hasByValAttr());
718 
719  // Track all the pointers to the argument to make sure they are not captured.
720  SmallPtrSet<Value *, 16> PtrValues;
721  PtrValues.insert(Arg);
722 
723  // Track all of the stores.
725 
726  // Scan through the uses recursively to make sure the pointer is always used
727  // sanely.
728  SmallVector<Value *, 16> WorkList(Arg->users());
729  while (!WorkList.empty()) {
730  Value *V = WorkList.pop_back_val();
731  if (isa<GetElementPtrInst>(V) || isa<PHINode>(V)) {
732  if (PtrValues.insert(V).second)
733  append_range(WorkList, V->users());
734  } else if (StoreInst *Store = dyn_cast<StoreInst>(V)) {
735  Stores.push_back(Store);
736  } else if (!isa<LoadInst>(V)) {
737  return true;
738  }
739  }
740 
741  // Check to make sure the pointers aren't captured
742  for (StoreInst *Store : Stores)
743  if (PtrValues.count(Store->getValueOperand()))
744  return true;
745 
746  return false;
747 }
748 
749 /// Check if callers and callee agree on how promoted arguments would be
750 /// passed.
752  const TargetTransformInfo &TTI) {
753  return all_of(F.uses(), [&](const Use &U) {
754  CallBase *CB = dyn_cast<CallBase>(U.getUser());
755  if (!CB)
756  return false;
757 
758  const Function *Caller = CB->getCaller();
759  const Function *Callee = CB->getCalledFunction();
760  return TTI.areTypesABICompatible(Caller, Callee, Types);
761  });
762 }
763 
764 /// PromoteArguments - This method checks the specified function to see if there
765 /// are any promotable arguments and if it is safe to promote the function (for
766 /// example, all callers are direct). If safe to promote some arguments, it
767 /// calls the DoPromotion method.
768 static Function *
770  unsigned MaxElements,
771  Optional<function_ref<void(CallBase &OldCS, CallBase &NewCS)>>
772  ReplaceCallSite,
773  const TargetTransformInfo &TTI, bool IsRecursive) {
774  // Don't perform argument promotion for naked functions; otherwise we can end
775  // up removing parameters that are seemingly 'not used' as they are referred
776  // to in the assembly.
777  if(F->hasFnAttribute(Attribute::Naked))
778  return nullptr;
779 
780  // Make sure that it is local to this module.
781  if (!F->hasLocalLinkage())
782  return nullptr;
783 
784  // Don't promote arguments for variadic functions. Adding, removing, or
785  // changing non-pack parameters can change the classification of pack
786  // parameters. Frontends encode that classification at the call site in the
787  // IR, while in the callee the classification is determined dynamically based
788  // on the number of registers consumed so far.
789  if (F->isVarArg())
790  return nullptr;
791 
792  // Don't transform functions that receive inallocas, as the transformation may
793  // not be safe depending on calling convention.
794  if (F->getAttributes().hasAttrSomewhere(Attribute::InAlloca))
795  return nullptr;
796 
797  // First check: see if there are any pointer arguments! If not, quick exit.
798  SmallVector<Argument *, 16> PointerArgs;
799  for (Argument &I : F->args())
800  if (I.getType()->isPointerTy())
801  PointerArgs.push_back(&I);
802  if (PointerArgs.empty())
803  return nullptr;
804 
805  // Second check: make sure that all callers are direct callers. We can't
806  // transform functions that have indirect callers. Also see if the function
807  // is self-recursive.
808  for (Use &U : F->uses()) {
809  CallBase *CB = dyn_cast<CallBase>(U.getUser());
810  // Must be a direct call.
811  if (CB == nullptr || !CB->isCallee(&U) ||
812  CB->getFunctionType() != F->getFunctionType())
813  return nullptr;
814 
815  // Can't change signature of musttail callee
816  if (CB->isMustTailCall())
817  return nullptr;
818 
819  if (CB->getFunction() == F)
820  IsRecursive = true;
821  }
822 
823  // Can't change signature of musttail caller
824  // FIXME: Support promoting whole chain of musttail functions
825  for (BasicBlock &BB : *F)
826  if (BB.getTerminatingMustTailCall())
827  return nullptr;
828 
829  const DataLayout &DL = F->getParent()->getDataLayout();
830 
831  AAResults &AAR = AARGetter(*F);
832 
833  // Check to see which arguments are promotable. If an argument is promotable,
834  // add it to ArgsToPromote.
836  SmallPtrSet<Argument *, 8> ByValArgsToTransform;
837  for (Argument *PtrArg : PointerArgs) {
838  // Replace sret attribute with noalias. This reduces register pressure by
839  // avoiding a register copy.
840  if (PtrArg->hasStructRetAttr()) {
841  unsigned ArgNo = PtrArg->getArgNo();
842  F->removeParamAttr(ArgNo, Attribute::StructRet);
843  F->addParamAttr(ArgNo, Attribute::NoAlias);
844  for (Use &U : F->uses()) {
845  CallBase &CB = cast<CallBase>(*U.getUser());
846  CB.removeParamAttr(ArgNo, Attribute::StructRet);
847  CB.addParamAttr(ArgNo, Attribute::NoAlias);
848  }
849  }
850 
851  // If we can promote the pointer to its value.
853  if (findArgParts(PtrArg, DL, AAR, MaxElements, IsRecursive, ArgParts)) {
855  for (const auto &Pair : ArgParts)
856  Types.push_back(Pair.second.Ty);
857 
858  if (areTypesABICompatible(Types, *F, TTI)) {
859  ArgsToPromote.insert({PtrArg, std::move(ArgParts)});
860  continue;
861  }
862  }
863 
864  // Otherwise, if this is a byval argument, and if the aggregate type is
865  // small, just pass the elements, which is always safe, if the passed value
866  // is densely packed or if we can prove the padding bytes are never
867  // accessed.
868  //
869  // Only handle arguments with specified alignment; if it's unspecified, the
870  // actual alignment of the argument is target-specific.
871  Type *ByValTy = PtrArg->getParamByValType();
872  bool IsSafeToPromote =
873  ByValTy && PtrArg->getParamAlign() &&
875  !canPaddingBeAccessed(PtrArg));
876  if (!IsSafeToPromote) {
877  LLVM_DEBUG(dbgs() << "ArgPromotion disables passing the elements of"
878  << " the argument '" << PtrArg->getName()
879  << "' because it is not safe.\n");
880  continue;
881  }
882  if (StructType *STy = dyn_cast<StructType>(ByValTy)) {
883  if (MaxElements > 0 && STy->getNumElements() > MaxElements) {
884  LLVM_DEBUG(dbgs() << "ArgPromotion disables passing the elements of"
885  << " the argument '" << PtrArg->getName()
886  << "' because it would require adding more"
887  << " than " << MaxElements
888  << " arguments to the function.\n");
889  continue;
890  }
892  append_range(Types, STy->elements());
893 
894  // If all the elements are single-value types, we can promote it.
895  bool AllSimple =
896  all_of(Types, [](Type *Ty) { return Ty->isSingleValueType(); });
897 
898  // Safe to transform. Passing the elements as a scalar will allow sroa to
899  // hack on the new alloca we introduce.
900  if (AllSimple && areTypesABICompatible(Types, *F, TTI))
901  ByValArgsToTransform.insert(PtrArg);
902  }
903  }
904 
905  // No promotable pointer arguments.
906  if (ArgsToPromote.empty() && ByValArgsToTransform.empty())
907  return nullptr;
908 
909  return doPromotion(F, ArgsToPromote, ByValArgsToTransform, ReplaceCallSite);
910 }
911 
914  LazyCallGraph &CG,
915  CGSCCUpdateResult &UR) {
916  bool Changed = false, LocalChange;
917 
918  // Iterate until we stop promoting from this SCC.
919  do {
920  LocalChange = false;
921 
923  AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager();
924 
925  bool IsRecursive = C.size() > 1;
926  for (LazyCallGraph::Node &N : C) {
927  Function &OldF = N.getFunction();
928 
929  // FIXME: This lambda must only be used with this function. We should
930  // skip the lambda and just get the AA results directly.
931  auto AARGetter = [&](Function &F) -> AAResults & {
932  assert(&F == &OldF && "Called with an unexpected function!");
933  return FAM.getResult<AAManager>(F);
934  };
935 
937  Function *NewF = promoteArguments(&OldF, AARGetter, MaxElements, None,
938  TTI, IsRecursive);
939  if (!NewF)
940  continue;
941  LocalChange = true;
942 
943  // Directly substitute the functions in the call graph. Note that this
944  // requires the old function to be completely dead and completely
945  // replaced by the new function. It does no call graph updates, it merely
946  // swaps out the particular function mapped to a particular node in the
947  // graph.
948  C.getOuterRefSCC().replaceNodeFunction(N, *NewF);
949  FAM.clear(OldF, OldF.getName());
950  OldF.eraseFromParent();
951 
952  PreservedAnalyses FuncPA;
953  FuncPA.preserveSet<CFGAnalyses>();
954  for (auto *U : NewF->users()) {
955  auto *UserF = cast<CallBase>(U)->getFunction();
956  FAM.invalidate(*UserF, FuncPA);
957  }
958  }
959 
960  Changed |= LocalChange;
961  } while (LocalChange);
962 
963  if (!Changed)
964  return PreservedAnalyses::all();
965 
967  // We've cleared out analyses for deleted functions.
969  // We've manually invalidated analyses for functions we've modified.
971  return PA;
972 }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
AssumptionCache.h
llvm::AAManager
A manager for alias analyses.
Definition: AliasAnalysis.h:1303
llvm::Type::isSized
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:264
llvm::TargetIRAnalysis
Analysis pass providing the TargetTransformInfo.
Definition: TargetTransformInfo.h:2479
llvm::isAligned
bool isAligned(Align Lhs, uint64_t SizeInBytes)
Checks that SizeInBytes is a multiple of the alignment.
Definition: Alignment.h:146
llvm::MemoryLocation::get
static MemoryLocation get(const LoadInst *LI)
Return a location with information about the memory reference by the given instruction.
Definition: MemoryLocation.cpp:35
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::CallBase::getOperandBundlesAsDefs
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: Instructions.cpp:396
llvm::InvokeInst::Create
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3807
NoFolder.h
Optional.h
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
Metadata.h
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:218
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:780
llvm::Function::getBasicBlockList
const BasicBlockListType & getBasicBlockList() const
Get the underlying elements of the Function...
Definition: Function.h:703
Loads.h
llvm::Function
Definition: Function.h:60
llvm::AnalysisManager::invalidate
void invalidate(IRUnitT &IR, const PreservedAnalyses &PA)
Invalidate cached analyses for an IR unit.
Definition: PassManagerImpl.h:89
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
Pass.h
llvm::Function::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Function.cpp:364
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
Statistic.h
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:168
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:729
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2495
llvm::AllAnalysesOn
This templated class represents "all analyses that operate over <a particular IR unit>" (e....
Definition: PassManager.h:90
llvm::SmallDenseMap
Definition: DenseMap.h:882
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
ValueTracking.h
llvm::AttributeList::get
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:1050
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
promoteArguments
static Function * promoteArguments(Function *F, function_ref< AAResults &(Function &F)> AARGetter, unsigned MaxElements, Optional< function_ref< void(CallBase &OldCS, CallBase &NewCS)>> ReplaceCallSite, const TargetTransformInfo &TTI, bool IsRecursive)
PromoteArguments - This method checks the specified function to see if there are any promotable argum...
Definition: ArgumentPromotion.cpp:769
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::APInt::getBitWidth
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1411
Module.h
llvm::AttributeList
Definition: Attributes.h:425
llvm::CallBase::getAttributes
AttributeList getAttributes() const
Return the parameter attributes for this call.
Definition: InstrTypes.h:1474
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1254
llvm::CallBase::isCallee
bool isCallee(Value::const_user_iterator UI) const
Determine whether the passed iterator points to the callee operand's Use.
Definition: InstrTypes.h:1407
llvm::AttributeList::getFnAttrs
AttributeSet getFnAttrs() const
The function attributes are returned.
Definition: Attributes.cpp:1388
llvm::CallBase::isMustTailCall
bool isMustTailCall() const
Tests if this call site must be tail call optimized.
Definition: Instructions.cpp:298
llvm::Optional
Definition: APInt.h:33
llvm::Value::stripAndAccumulateConstantOffsets
const Value * stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, bool AllowNonInbounds, bool AllowInvariantGroup=false, function_ref< bool(Value &Value, APInt &Offset)> ExternalAnalysis=nullptr) const
Accumulate the constant offset this value has compared to a base pointer.
llvm::SmallPtrSet< Value *, 16 >
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::Instruction::copyMetadata
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition: Instruction.cpp:841
STLExtras.h
llvm::CallBase::arg_begin
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Definition: InstrTypes.h:1316
llvm::SmallVectorImpl::pop_back_val
LLVM_NODISCARD T pop_back_val()
Definition: SmallVector.h:654
llvm::ArrayType
Class to represent array types.
Definition: DerivedTypes.h:357
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:260
llvm::IRBuilderBase::CreateAlignedLoad
LoadInst * CreateAlignedLoad(Type *Ty, Value *Ptr, MaybeAlign Align, const char *Name)
Definition: IRBuilder.h:1672
llvm::CallBase::addParamAttr
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1526
BasicAliasAnalysis.h
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:216
llvm::Type::getNonOpaquePointerElementType
Type * getNonOpaquePointerElementType() const
Only use this method in code that is not reachable with opaque pointers, or part of deprecated method...
Definition: Type.h:382
Use.h
llvm::APInt::getZero
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:177
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
DepthFirstIterator.h
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::ArgumentPromotionPass::run
PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, LazyCallGraph &CG, CGSCCUpdateResult &UR)
Definition: ArgumentPromotion.cpp:912
llvm::FunctionType::isVarArg
bool isVarArg() const
Definition: DerivedTypes.h:123
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:186
Instruction.h
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::Type::isSingleValueType
bool isSingleValueType() const
Return true if the type is a valid type for a register in codegen.
Definition: Type.h:249
llvm::CallBase::removeParamAttr
void removeParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Removes the attribute from the given argument.
Definition: InstrTypes.h:1568
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1617
llvm::LazyCallGraph::SCC
An SCC of the call graph.
Definition: LazyCallGraph.h:419
Constants.h
llvm::AAResults
Definition: AliasAnalysis.h:511
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1731
llvm::User
Definition: User.h:44
ArgumentPromotion.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
Twine.h
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1396
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1478
llvm::AnalysisManager::clear
void clear(IRUnitT &IR, llvm::StringRef Name)
Clear any cached analysis results for a single unit of IR.
Definition: PassManagerImpl.h:36
llvm::CallInst::Create
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:1504
llvm::StructLayout::getElementOffsetInBits
uint64_t getElementOffsetInBits(unsigned Idx) const
Definition: DataLayout.h:657
TargetLibraryInfo.h
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::make_scope_exit
LLVM_NODISCARD detail::scope_exit< typename std::decay< Callable >::type > make_scope_exit(Callable &&F)
Definition: ScopeExit.h:59
llvm::Instruction
Definition: Instruction.h:42
llvm::IRBuilderBase::getInt8Ty
IntegerType * getInt8Ty()
Fetch the type representing an 8-bit integer.
Definition: IRBuilder.h:491
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::predecessors
auto predecessors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:30
llvm::Function::copyAttributesFrom
void copyAttributesFrom(const Function *Src)
copyAttributesFrom - copy all additional attributes (those not needed to create a Function) from the ...
Definition: Function.cpp:711
llvm::Value::setName
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:372
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1777
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:928
SmallPtrSet.h
LazyCallGraph.h
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1455
llvm::None
const NoneType None
Definition: None.h:24
Type.h
llvm::CallBase::getCaller
Function * getCaller()
Helper to get the caller (the parent function).
Definition: Instructions.cpp:282
CFG.h
llvm::InvokeInst
Invoke instruction.
Definition: Instructions.h:3763
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::VectorType
Base class of all SIMD vector types.
Definition: DerivedTypes.h:389
BasicBlock.h
llvm::IRBuilderBase::CreateBitCast
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1958
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:297
llvm::AttributeList::getRetAttrs
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
Definition: Attributes.cpp:1384
llvm::StructLayout
Used to lazily calculate structure layout information for a target machine, based on the DataLayout s...
Definition: DataLayout.h:622
uint64_t
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:173
IPO.h
llvm::GlobalObject::copyMetadata
void copyMetadata(const GlobalObject *Src, unsigned Offset)
Copy metadata from Src, adjusting offsets by Offset.
Definition: Metadata.cpp:1564
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
MemoryLocation.h
llvm::DenseMap
Definition: DenseMap.h:716
CGSCCPassManager.h
llvm::AAResults::canInstructionRangeModRef
bool canInstructionRangeModRef(const Instruction &I1, const Instruction &I2, const MemoryLocation &Loc, const ModRefInfo Mode)
Check if it is possible for the execution of the specified instructions to mod(according to the mode)...
Definition: AliasAnalysis.cpp:789
I
#define I(x, y, z)
Definition: MD5.cpp:58
createByteGEP
static Value * createByteGEP(IRBuilderBase &IRB, const DataLayout &DL, Value *Ptr, Type *ResElemTy, int64_t Offset)
Definition: ArgumentPromotion.cpp:106
llvm::IRBuilderBase::getInt8PtrTy
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:549
llvm::IRBuilderBase::getInt
ConstantInt * getInt(const APInt &AI)
Get a constant integer value.
Definition: IRBuilder.h:477
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
doPromotion
static Function * doPromotion(Function *F, const DenseMap< Argument *, SmallVector< OffsetAndArgPart, 4 >> &ArgsToPromote, SmallPtrSetImpl< Argument * > &ByValArgsToTransform, Optional< function_ref< void(CallBase &OldCS, CallBase &NewCS)>> ReplaceCallSite)
DoPromotion - This method actually performs the promotion of the specified arguments,...
Definition: ArgumentPromotion.cpp:156
areTypesABICompatible
static bool areTypesABICompatible(ArrayRef< Type * > Types, const Function &F, const TargetTransformInfo &TTI)
Check if callers and callee agree on how promoted arguments would be passed.
Definition: ArgumentPromotion.cpp:751
llvm::CallBase::arg_end
User::op_iterator arg_end()
Return the iterator pointing to the end of the argument list.
Definition: InstrTypes.h:1322
llvm::LazyCallGraph::Node
A node in the call graph.
Definition: LazyCallGraph.h:315
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::df_iterator_default_set
Definition: DepthFirstIterator.h:70
llvm::GetElementPtrInst::Create
static GetElementPtrInst * Create(Type *PointeeType, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:942
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
None.h
DataLayout.h
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:252
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:113
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:69
llvm::append_range
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:1823
llvm::ModRefInfo::Mod
@ Mod
The access may modify the value stored in memory.
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
findArgParts
static bool findArgParts(Argument *Arg, const DataLayout &DL, AAResults &AAR, unsigned MaxElements, bool IsRecursive, SmallVectorImpl< OffsetAndArgPart > &ArgPartsVec)
Determine that this argument is safe to promote, and find the argument parts it can be promoted into.
Definition: ArgumentPromotion.cpp:466
llvm::AttributeFuncs::updateMinLegalVectorWidthAttr
void updateMinLegalVectorWidthAttr(Function &Fn, uint64_t Width)
Update min-legal-vector-width if it is in Attribute and less than Width.
Definition: Attributes.cpp:2068
llvm::ArgumentPromotionPass::isDenselyPacked
static bool isDenselyPacked(Type *Ty, const DataLayout &DL)
Checks if a type could have padding bytes.
Definition: ArgumentPromotion.cpp:677
CallGraphSCCPass.h
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:305
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:209
llvm::Function::begin
iterator begin()
Definition: Function.h:726
llvm::StructType::getNumElements
unsigned getNumElements() const
Random access to the elements.
Definition: DerivedTypes.h:327
Argument.h
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::empty
LLVM_NODISCARD bool empty() const
Definition: DenseMap.h:98
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:186
Attributes.h
llvm::CGSCCUpdateResult
Support structure for SCC passes to communicate updates the call graph back to the CGSCC pass manager...
Definition: CGSCCPassManager.h:232
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:213
llvm::StructType::elements
ArrayRef< Type * > elements() const
Definition: DerivedTypes.h:319
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::StructLayout::getElementOffset
uint64_t getElementOffset(unsigned Idx) const
Definition: DataLayout.h:652
llvm::isGuaranteedToTransferExecutionToSuccessor
bool isGuaranteedToTransferExecutionToSuccessor(const Instruction *I)
Return true if this function can prove that the instruction I will always transfer execution to one o...
Definition: ValueTracking.cpp:5435
llvm::TypeSize
Definition: TypeSize.h:435
Casting.h
llvm::Instruction::setAAMetadata
void setAAMetadata(const AAMDNodes &N)
Sets the AA metadata on this instruction from the AAMDNodes structure.
Definition: Metadata.cpp:1477
llvm::GetElementPtrInst::getTypeAtIndex
static Type * getTypeAtIndex(Type *Ty, Value *Idx)
Return the type of the element at the given index of an indexable type.
Definition: Instructions.cpp:1750
Function.h
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::size
unsigned size() const
Definition: DenseMap.h:101
llvm::sort
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1562
PassManager.h
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:591
llvm::Function::arg_begin
arg_iterator arg_begin()
Definition: Function.h:741
CallGraph.h
llvm::Type::isOpaquePointerTy
bool isOpaquePointerTy() const
True if this is an instance of an opaque PointerType.
Definition: Type.cpp:61
Instructions.h
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:188
llvm::AttributeSet
Definition: Attributes.h:290
SmallVector.h
User.h
llvm::SmallPtrSetImplBase::empty
LLVM_NODISCARD bool empty() const
Definition: SmallPtrSet.h:92
N
#define N
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::try_emplace
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
Definition: DenseMap.h:224
TargetTransformInfo.h
llvm::AAResults::canBasicBlockModify
bool canBasicBlockModify(const BasicBlock &BB, const MemoryLocation &Loc)
Check if it is possible for execution of the specified basic block to modify the location Loc.
Definition: AliasAnalysis.cpp:779
ScopeExit.h
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::StructType::getElementType
Type * getElementType(unsigned N) const
Definition: DerivedTypes.h:328
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1174
DerivedTypes.h
llvm::SmallPtrSetImpl
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:378
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
llvm::AllocaInst
an instruction to allocate memory on the stack
Definition: Instructions.h:58
llvm::inverse_depth_first_ext
iterator_range< idf_ext_iterator< T, SetTy > > inverse_depth_first_ext(const T &G, SetTy &S)
Definition: DepthFirstIterator.h:303
canPaddingBeAccessed
static bool canPaddingBeAccessed(Argument *Arg)
Checks if the padding bytes of an argument could be accessed.
Definition: ArgumentPromotion.cpp:716
llvm::LazyCallGraph
A lazily constructed view of the call graph of a module.
Definition: LazyCallGraph.h:110
raw_ostream.h
llvm::AttributeList::getParamAttrs
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
Definition: Attributes.cpp:1380
llvm::FunctionAnalysisManagerCGSCCProxy
A proxy from a FunctionAnalysisManager to an SCC.
Definition: CGSCCPassManager.h:392
allCallersPassValidPointerForArgument
static bool allCallersPassValidPointerForArgument(Argument *Arg, Align NeededAlign, uint64_t NeededDerefBytes)
Return true if we can prove that all callees pass in a valid pointer for the specified function argum...
Definition: ArgumentPromotion.cpp:444
Value.h
llvm::Function::setAttributes
void setAttributes(AttributeList Attrs)
Set the attribute list for this Function.
Definition: Function.h:317
InitializePasses.h
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition: DerivedTypes.h:124
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::MemoryLocation
Representation for a specific memory location.
Definition: MemoryLocation.h:210
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1459
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
llvm::isDereferenceableAndAlignedPointer
bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, Align Alignment, const DataLayout &DL, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Returns true if V is always a dereferenceable pointer with alignment greater or equal than requested.
Definition: Loads.cpp:205
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
llvm::SmallVectorImpl::insert
iterator insert(iterator I, T &&Elt)
Definition: SmallVector.h:792
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1796