LLVM  13.0.0git
VNCoercion.cpp
Go to the documentation of this file.
4 #include "llvm/IR/IRBuilder.h"
5 #include "llvm/Support/Debug.h"
6 
7 #define DEBUG_TYPE "vncoerce"
8 
9 namespace llvm {
10 namespace VNCoercion {
11 
13  return Ty->isStructTy() || Ty->isArrayTy() || isa<ScalableVectorType>(Ty);
14 }
15 
16 /// Return true if coerceAvailableValueToLoadType will succeed.
17 bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy,
18  const DataLayout &DL) {
19  Type *StoredTy = StoredVal->getType();
20 
21  if (StoredTy == LoadTy)
22  return true;
23 
24  // If the loaded/stored value is a first class array/struct, or scalable type,
25  // don't try to transform them. We need to be able to bitcast to integer.
28  return false;
29 
30  uint64_t StoreSize = DL.getTypeSizeInBits(StoredTy).getFixedSize();
31 
32  // The store size must be byte-aligned to support future type casts.
33  if (llvm::alignTo(StoreSize, 8) != StoreSize)
34  return false;
35 
36  // The store has to be at least as big as the load.
37  if (StoreSize < DL.getTypeSizeInBits(LoadTy).getFixedSize())
38  return false;
39 
40  bool StoredNI = DL.isNonIntegralPointerType(StoredTy->getScalarType());
41  bool LoadNI = DL.isNonIntegralPointerType(LoadTy->getScalarType());
42  // Don't coerce non-integral pointers to integers or vice versa.
43  if (StoredNI != LoadNI) {
44  // As a special case, allow coercion of memset used to initialize
45  // an array w/null. Despite non-integral pointers not generally having a
46  // specific bit pattern, we do assume null is zero.
47  if (auto *CI = dyn_cast<Constant>(StoredVal))
48  return CI->isNullValue();
49  return false;
50  } else if (StoredNI && LoadNI &&
51  StoredTy->getPointerAddressSpace() !=
52  LoadTy->getPointerAddressSpace()) {
53  return false;
54  }
55 
56 
57  // The implementation below uses inttoptr for vectors of unequal size; we
58  // can't allow this for non integral pointers. We could teach it to extract
59  // exact subvectors if desired.
60  if (StoredNI && StoreSize != DL.getTypeSizeInBits(LoadTy).getFixedSize())
61  return false;
62 
63  return true;
64 }
65 
66 template <class T, class HelperClass>
67 static T *coerceAvailableValueToLoadTypeHelper(T *StoredVal, Type *LoadedTy,
68  HelperClass &Helper,
69  const DataLayout &DL) {
70  assert(canCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, DL) &&
71  "precondition violation - materialization can't fail");
72  if (auto *C = dyn_cast<Constant>(StoredVal))
73  StoredVal = ConstantFoldConstant(C, DL);
74 
75  // If this is already the right type, just return it.
76  Type *StoredValTy = StoredVal->getType();
77 
78  uint64_t StoredValSize = DL.getTypeSizeInBits(StoredValTy).getFixedSize();
79  uint64_t LoadedValSize = DL.getTypeSizeInBits(LoadedTy).getFixedSize();
80 
81  // If the store and reload are the same size, we can always reuse it.
82  if (StoredValSize == LoadedValSize) {
83  // Pointer to Pointer -> use bitcast.
84  if (StoredValTy->isPtrOrPtrVectorTy() && LoadedTy->isPtrOrPtrVectorTy()) {
85  StoredVal = Helper.CreateBitCast(StoredVal, LoadedTy);
86  } else {
87  // Convert source pointers to integers, which can be bitcast.
88  if (StoredValTy->isPtrOrPtrVectorTy()) {
89  StoredValTy = DL.getIntPtrType(StoredValTy);
90  StoredVal = Helper.CreatePtrToInt(StoredVal, StoredValTy);
91  }
92 
93  Type *TypeToCastTo = LoadedTy;
94  if (TypeToCastTo->isPtrOrPtrVectorTy())
95  TypeToCastTo = DL.getIntPtrType(TypeToCastTo);
96 
97  if (StoredValTy != TypeToCastTo)
98  StoredVal = Helper.CreateBitCast(StoredVal, TypeToCastTo);
99 
100  // Cast to pointer if the load needs a pointer type.
101  if (LoadedTy->isPtrOrPtrVectorTy())
102  StoredVal = Helper.CreateIntToPtr(StoredVal, LoadedTy);
103  }
104 
105  if (auto *C = dyn_cast<ConstantExpr>(StoredVal))
106  StoredVal = ConstantFoldConstant(C, DL);
107 
108  return StoredVal;
109  }
110  // If the loaded value is smaller than the available value, then we can
111  // extract out a piece from it. If the available value is too small, then we
112  // can't do anything.
113  assert(StoredValSize >= LoadedValSize &&
114  "canCoerceMustAliasedValueToLoad fail");
115 
116  // Convert source pointers to integers, which can be manipulated.
117  if (StoredValTy->isPtrOrPtrVectorTy()) {
118  StoredValTy = DL.getIntPtrType(StoredValTy);
119  StoredVal = Helper.CreatePtrToInt(StoredVal, StoredValTy);
120  }
121 
122  // Convert vectors and fp to integer, which can be manipulated.
123  if (!StoredValTy->isIntegerTy()) {
124  StoredValTy = IntegerType::get(StoredValTy->getContext(), StoredValSize);
125  StoredVal = Helper.CreateBitCast(StoredVal, StoredValTy);
126  }
127 
128  // If this is a big-endian system, we need to shift the value down to the low
129  // bits so that a truncate will work.
130  if (DL.isBigEndian()) {
131  uint64_t ShiftAmt = DL.getTypeStoreSizeInBits(StoredValTy).getFixedSize() -
132  DL.getTypeStoreSizeInBits(LoadedTy).getFixedSize();
133  StoredVal = Helper.CreateLShr(
134  StoredVal, ConstantInt::get(StoredVal->getType(), ShiftAmt));
135  }
136 
137  // Truncate the integer to the right size now.
138  Type *NewIntTy = IntegerType::get(StoredValTy->getContext(), LoadedValSize);
139  StoredVal = Helper.CreateTruncOrBitCast(StoredVal, NewIntTy);
140 
141  if (LoadedTy != NewIntTy) {
142  // If the result is a pointer, inttoptr.
143  if (LoadedTy->isPtrOrPtrVectorTy())
144  StoredVal = Helper.CreateIntToPtr(StoredVal, LoadedTy);
145  else
146  // Otherwise, bitcast.
147  StoredVal = Helper.CreateBitCast(StoredVal, LoadedTy);
148  }
149 
150  if (auto *C = dyn_cast<Constant>(StoredVal))
151  StoredVal = ConstantFoldConstant(C, DL);
152 
153  return StoredVal;
154 }
155 
156 /// If we saw a store of a value to memory, and
157 /// then a load from a must-aliased pointer of a different type, try to coerce
158 /// the stored value. LoadedTy is the type of the load we want to replace.
159 /// IRB is IRBuilder used to insert new instructions.
160 ///
161 /// If we can't do it, return null.
163  IRBuilderBase &IRB,
164  const DataLayout &DL) {
165  return coerceAvailableValueToLoadTypeHelper(StoredVal, LoadedTy, IRB, DL);
166 }
167 
168 /// This function is called when we have a memdep query of a load that ends up
169 /// being a clobbering memory write (store, memset, memcpy, memmove). This
170 /// means that the write *may* provide bits used by the load but we can't be
171 /// sure because the pointers don't must-alias.
172 ///
173 /// Check this case to see if there is anything more we can do before we give
174 /// up. This returns -1 if we have to give up, or a byte number in the stored
175 /// value of the piece that feeds the load.
176 static int analyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr,
177  Value *WritePtr,
178  uint64_t WriteSizeInBits,
179  const DataLayout &DL) {
180  // If the loaded/stored value is a first class array/struct, or scalable type,
181  // don't try to transform them. We need to be able to bitcast to integer.
183  return -1;
184 
185  int64_t StoreOffset = 0, LoadOffset = 0;
186  Value *StoreBase =
187  GetPointerBaseWithConstantOffset(WritePtr, StoreOffset, DL);
188  Value *LoadBase = GetPointerBaseWithConstantOffset(LoadPtr, LoadOffset, DL);
189  if (StoreBase != LoadBase)
190  return -1;
191 
192  uint64_t LoadSize = DL.getTypeSizeInBits(LoadTy).getFixedSize();
193 
194  if ((WriteSizeInBits & 7) | (LoadSize & 7))
195  return -1;
196  uint64_t StoreSize = WriteSizeInBits / 8; // Convert to bytes.
197  LoadSize /= 8;
198 
199  // If the Load isn't completely contained within the stored bits, we don't
200  // have all the bits to feed it. We could do something crazy in the future
201  // (issue a smaller load then merge the bits in) but this seems unlikely to be
202  // valuable.
203  if (StoreOffset > LoadOffset ||
204  StoreOffset + StoreSize < LoadOffset + LoadSize)
205  return -1;
206 
207  // If the load and store are to the exact same address, they should have been
208  // a must alias. AA must have gotten confused.
209  // FIXME: Study to see if/when this happens. One case is forwarding a memset
210  // to a load from the base of the memset.
211 
212  // If the load and store don't overlap at all, the store doesn't provide
213  // anything to the load. In this case, they really don't alias at all, AA
214  // must have gotten confused. The if statement above ensure the condition
215  // that StoreOffset <= LoadOffset.
216  if (StoreOffset + int64_t(StoreSize) <= LoadOffset)
217  return -1;
218 
219  // Okay, we can do this transformation. Return the number of bytes into the
220  // store that the load is.
221  return LoadOffset - StoreOffset;
222 }
223 
224 /// This function is called when we have a
225 /// memdep query of a load that ends up being a clobbering store.
227  StoreInst *DepSI, const DataLayout &DL) {
228  auto *StoredVal = DepSI->getValueOperand();
229 
230  // Cannot handle reading from store of first-class aggregate or scalable type.
231  if (isFirstClassAggregateOrScalableType(StoredVal->getType()))
232  return -1;
233 
234  if (!canCoerceMustAliasedValueToLoad(StoredVal, LoadTy, DL))
235  return -1;
236 
237  Value *StorePtr = DepSI->getPointerOperand();
238  uint64_t StoreSize =
239  DL.getTypeSizeInBits(DepSI->getValueOperand()->getType()).getFixedSize();
240  return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, StorePtr, StoreSize,
241  DL);
242 }
243 
244 /// Looks at a memory location for a load (specified by MemLocBase, Offs, and
245 /// Size) and compares it against a load.
246 ///
247 /// If the specified load could be safely widened to a larger integer load
248 /// that is 1) still efficient, 2) safe for the target, and 3) would provide
249 /// the specified memory location value, then this function returns the size
250 /// in bytes of the load width to use. If not, this returns zero.
251 static unsigned getLoadLoadClobberFullWidthSize(const Value *MemLocBase,
252  int64_t MemLocOffs,
253  unsigned MemLocSize,
254  const LoadInst *LI) {
255  // We can only extend simple integer loads.
256  if (!isa<IntegerType>(LI->getType()) || !LI->isSimple())
257  return 0;
258 
259  // Load widening is hostile to ThreadSanitizer: it may cause false positives
260  // or make the reports more cryptic (access sizes are wrong).
261  if (LI->getParent()->getParent()->hasFnAttribute(Attribute::SanitizeThread))
262  return 0;
263 
264  const DataLayout &DL = LI->getModule()->getDataLayout();
265 
266  // Get the base of this load.
267  int64_t LIOffs = 0;
268  const Value *LIBase =
270 
271  // If the two pointers are not based on the same pointer, we can't tell that
272  // they are related.
273  if (LIBase != MemLocBase)
274  return 0;
275 
276  // Okay, the two values are based on the same pointer, but returned as
277  // no-alias. This happens when we have things like two byte loads at "P+1"
278  // and "P+3". Check to see if increasing the size of the "LI" load up to its
279  // alignment (or the largest native integer type) will allow us to load all
280  // the bits required by MemLoc.
281 
282  // If MemLoc is before LI, then no widening of LI will help us out.
283  if (MemLocOffs < LIOffs)
284  return 0;
285 
286  // Get the alignment of the load in bytes. We assume that it is safe to load
287  // any legal integer up to this size without a problem. For example, if we're
288  // looking at an i8 load on x86-32 that is known 1024 byte aligned, we can
289  // widen it up to an i32 load. If it is known 2-byte aligned, we can widen it
290  // to i16.
291  unsigned LoadAlign = LI->getAlignment();
292 
293  int64_t MemLocEnd = MemLocOffs + MemLocSize;
294 
295  // If no amount of rounding up will let MemLoc fit into LI, then bail out.
296  if (LIOffs + LoadAlign < MemLocEnd)
297  return 0;
298 
299  // This is the size of the load to try. Start with the next larger power of
300  // two.
301  unsigned NewLoadByteSize = LI->getType()->getPrimitiveSizeInBits() / 8U;
302  NewLoadByteSize = NextPowerOf2(NewLoadByteSize);
303 
304  while (true) {
305  // If this load size is bigger than our known alignment or would not fit
306  // into a native integer register, then we fail.
307  if (NewLoadByteSize > LoadAlign ||
308  !DL.fitsInLegalInteger(NewLoadByteSize * 8))
309  return 0;
310 
311  if (LIOffs + NewLoadByteSize > MemLocEnd &&
313  Attribute::SanitizeAddress) ||
315  Attribute::SanitizeHWAddress)))
316  // We will be reading past the location accessed by the original program.
317  // While this is safe in a regular build, Address Safety analysis tools
318  // may start reporting false warnings. So, don't do widening.
319  return 0;
320 
321  // If a load of this width would include all of MemLoc, then we succeed.
322  if (LIOffs + NewLoadByteSize >= MemLocEnd)
323  return NewLoadByteSize;
324 
325  NewLoadByteSize <<= 1;
326  }
327 }
328 
329 /// This function is called when we have a
330 /// memdep query of a load that ends up being clobbered by another load. See if
331 /// the other load can feed into the second load.
332 int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI,
333  const DataLayout &DL) {
334  // Cannot handle reading from store of first-class aggregate yet.
335  if (DepLI->getType()->isStructTy() || DepLI->getType()->isArrayTy())
336  return -1;
337 
338  if (!canCoerceMustAliasedValueToLoad(DepLI, LoadTy, DL))
339  return -1;
340 
341  Value *DepPtr = DepLI->getPointerOperand();
342  uint64_t DepSize = DL.getTypeSizeInBits(DepLI->getType()).getFixedSize();
343  int R = analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, DepPtr, DepSize, DL);
344  if (R != -1)
345  return R;
346 
347  // If we have a load/load clobber an DepLI can be widened to cover this load,
348  // then we should widen it!
349  int64_t LoadOffs = 0;
350  const Value *LoadBase =
351  GetPointerBaseWithConstantOffset(LoadPtr, LoadOffs, DL);
352  unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedSize();
353 
354  unsigned Size =
355  getLoadLoadClobberFullWidthSize(LoadBase, LoadOffs, LoadSize, DepLI);
356  if (Size == 0)
357  return -1;
358 
359  // Check non-obvious conditions enforced by MDA which we rely on for being
360  // able to materialize this potentially available value
361  assert(DepLI->isSimple() && "Cannot widen volatile/atomic load!");
362  assert(DepLI->getType()->isIntegerTy() && "Can't widen non-integer load");
363 
364  return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, DepPtr, Size * 8, DL);
365 }
366 
368  MemIntrinsic *MI, const DataLayout &DL) {
369  // If the mem operation is a non-constant size, we can't handle it.
370  ConstantInt *SizeCst = dyn_cast<ConstantInt>(MI->getLength());
371  if (!SizeCst)
372  return -1;
373  uint64_t MemSizeInBits = SizeCst->getZExtValue() * 8;
374 
375  // If this is memset, we just need to see if the offset is valid in the size
376  // of the memset..
377  if (MI->getIntrinsicID() == Intrinsic::memset) {
378  if (DL.isNonIntegralPointerType(LoadTy->getScalarType())) {
379  auto *CI = dyn_cast<ConstantInt>(cast<MemSetInst>(MI)->getValue());
380  if (!CI || !CI->isZero())
381  return -1;
382  }
383  return analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(),
384  MemSizeInBits, DL);
385  }
386 
387  // If we have a memcpy/memmove, the only case we can handle is if this is a
388  // copy from constant memory. In that case, we can read directly from the
389  // constant memory.
390  MemTransferInst *MTI = cast<MemTransferInst>(MI);
391 
392  Constant *Src = dyn_cast<Constant>(MTI->getSource());
393  if (!Src)
394  return -1;
395 
396  GlobalVariable *GV = dyn_cast<GlobalVariable>(getUnderlyingObject(Src));
397  if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer())
398  return -1;
399 
400  // See if the access is within the bounds of the transfer.
401  int Offset = analyzeLoadFromClobberingWrite(LoadTy, LoadPtr, MI->getDest(),
402  MemSizeInBits, DL);
403  if (Offset == -1)
404  return Offset;
405 
406  unsigned AS = Src->getType()->getPointerAddressSpace();
407  // Otherwise, see if we can constant fold a load from the constant with the
408  // offset applied as appropriate.
409  if (Offset) {
410  Src = ConstantExpr::getBitCast(Src,
411  Type::getInt8PtrTy(Src->getContext(), AS));
412  Constant *OffsetCst =
415  Src, OffsetCst);
416  }
417  Src = ConstantExpr::getBitCast(Src, PointerType::get(LoadTy, AS));
418  if (ConstantFoldLoadFromConstPtr(Src, LoadTy, DL))
419  return Offset;
420  return -1;
421 }
422 
423 template <class T, class HelperClass>
424 static T *getStoreValueForLoadHelper(T *SrcVal, unsigned Offset, Type *LoadTy,
425  HelperClass &Helper,
426  const DataLayout &DL) {
427  LLVMContext &Ctx = SrcVal->getType()->getContext();
428 
429  // If two pointers are in the same address space, they have the same size,
430  // so we don't need to do any truncation, etc. This avoids introducing
431  // ptrtoint instructions for pointers that may be non-integral.
432  if (SrcVal->getType()->isPointerTy() && LoadTy->isPointerTy() &&
433  cast<PointerType>(SrcVal->getType())->getAddressSpace() ==
434  cast<PointerType>(LoadTy)->getAddressSpace()) {
435  return SrcVal;
436  }
437 
438  uint64_t StoreSize =
439  (DL.getTypeSizeInBits(SrcVal->getType()).getFixedSize() + 7) / 8;
440  uint64_t LoadSize = (DL.getTypeSizeInBits(LoadTy).getFixedSize() + 7) / 8;
441  // Compute which bits of the stored value are being used by the load. Convert
442  // to an integer type to start with.
443  if (SrcVal->getType()->isPtrOrPtrVectorTy())
444  SrcVal = Helper.CreatePtrToInt(SrcVal, DL.getIntPtrType(SrcVal->getType()));
445  if (!SrcVal->getType()->isIntegerTy())
446  SrcVal = Helper.CreateBitCast(SrcVal, IntegerType::get(Ctx, StoreSize * 8));
447 
448  // Shift the bits to the least significant depending on endianness.
449  unsigned ShiftAmt;
450  if (DL.isLittleEndian())
451  ShiftAmt = Offset * 8;
452  else
453  ShiftAmt = (StoreSize - LoadSize - Offset) * 8;
454  if (ShiftAmt)
455  SrcVal = Helper.CreateLShr(SrcVal,
456  ConstantInt::get(SrcVal->getType(), ShiftAmt));
457 
458  if (LoadSize != StoreSize)
459  SrcVal = Helper.CreateTruncOrBitCast(SrcVal,
460  IntegerType::get(Ctx, LoadSize * 8));
461  return SrcVal;
462 }
463 
464 /// This function is called when we have a memdep query of a load that ends up
465 /// being a clobbering store. This means that the store provides bits used by
466 /// the load but the pointers don't must-alias. Check this case to see if
467 /// there is anything more we can do before we give up.
468 Value *getStoreValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy,
469  Instruction *InsertPt, const DataLayout &DL) {
470 
471  IRBuilder<> Builder(InsertPt);
472  SrcVal = getStoreValueForLoadHelper(SrcVal, Offset, LoadTy, Builder, DL);
473  return coerceAvailableValueToLoadTypeHelper(SrcVal, LoadTy, Builder, DL);
474 }
475 
477  Type *LoadTy, const DataLayout &DL) {
479  SrcVal = getStoreValueForLoadHelper(SrcVal, Offset, LoadTy, F, DL);
480  return coerceAvailableValueToLoadTypeHelper(SrcVal, LoadTy, F, DL);
481 }
482 
483 /// This function is called when we have a memdep query of a load that ends up
484 /// being a clobbering load. This means that the load *may* provide bits used
485 /// by the load but we can't be sure because the pointers don't must-alias.
486 /// Check this case to see if there is anything more we can do before we give
487 /// up.
488 Value *getLoadValueForLoad(LoadInst *SrcVal, unsigned Offset, Type *LoadTy,
489  Instruction *InsertPt, const DataLayout &DL) {
490  // If Offset+LoadTy exceeds the size of SrcVal, then we must be wanting to
491  // widen SrcVal out to a larger load.
492  unsigned SrcValStoreSize =
493  DL.getTypeStoreSize(SrcVal->getType()).getFixedSize();
494  unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedSize();
495  if (Offset + LoadSize > SrcValStoreSize) {
496  assert(SrcVal->isSimple() && "Cannot widen volatile/atomic load!");
497  assert(SrcVal->getType()->isIntegerTy() && "Can't widen non-integer load");
498  // If we have a load/load clobber an DepLI can be widened to cover this
499  // load, then we should widen it to the next power of 2 size big enough!
500  unsigned NewLoadSize = Offset + LoadSize;
501  if (!isPowerOf2_32(NewLoadSize))
502  NewLoadSize = NextPowerOf2(NewLoadSize);
503 
504  Value *PtrVal = SrcVal->getPointerOperand();
505  // Insert the new load after the old load. This ensures that subsequent
506  // memdep queries will find the new load. We can't easily remove the old
507  // load completely because it is already in the value numbering table.
508  IRBuilder<> Builder(SrcVal->getParent(), ++BasicBlock::iterator(SrcVal));
509  Type *DestTy = IntegerType::get(LoadTy->getContext(), NewLoadSize * 8);
510  Type *DestPTy =
511  PointerType::get(DestTy, PtrVal->getType()->getPointerAddressSpace());
512  Builder.SetCurrentDebugLocation(SrcVal->getDebugLoc());
513  PtrVal = Builder.CreateBitCast(PtrVal, DestPTy);
514  LoadInst *NewLoad = Builder.CreateLoad(DestTy, PtrVal);
515  NewLoad->takeName(SrcVal);
516  NewLoad->setAlignment(SrcVal->getAlign());
517 
518  LLVM_DEBUG(dbgs() << "GVN WIDENED LOAD: " << *SrcVal << "\n");
519  LLVM_DEBUG(dbgs() << "TO: " << *NewLoad << "\n");
520 
521  // Replace uses of the original load with the wider load. On a big endian
522  // system, we need to shift down to get the relevant bits.
523  Value *RV = NewLoad;
524  if (DL.isBigEndian())
525  RV = Builder.CreateLShr(RV, (NewLoadSize - SrcValStoreSize) * 8);
526  RV = Builder.CreateTrunc(RV, SrcVal->getType());
527  SrcVal->replaceAllUsesWith(RV);
528 
529  SrcVal = NewLoad;
530  }
531 
532  return getStoreValueForLoad(SrcVal, Offset, LoadTy, InsertPt, DL);
533 }
534 
536  Type *LoadTy, const DataLayout &DL) {
537  unsigned SrcValStoreSize =
538  DL.getTypeStoreSize(SrcVal->getType()).getFixedSize();
539  unsigned LoadSize = DL.getTypeStoreSize(LoadTy).getFixedSize();
540  if (Offset + LoadSize > SrcValStoreSize)
541  return nullptr;
542  return getConstantStoreValueForLoad(SrcVal, Offset, LoadTy, DL);
543 }
544 
545 template <class T, class HelperClass>
547  Type *LoadTy, HelperClass &Helper,
548  const DataLayout &DL) {
549  LLVMContext &Ctx = LoadTy->getContext();
550  uint64_t LoadSize = DL.getTypeSizeInBits(LoadTy).getFixedSize() / 8;
551 
552  // We know that this method is only called when the mem transfer fully
553  // provides the bits for the load.
554  if (MemSetInst *MSI = dyn_cast<MemSetInst>(SrcInst)) {
555  // memset(P, 'x', 1234) -> splat('x'), even if x is a variable, and
556  // independently of what the offset is.
557  T *Val = cast<T>(MSI->getValue());
558  if (LoadSize != 1)
559  Val =
560  Helper.CreateZExtOrBitCast(Val, IntegerType::get(Ctx, LoadSize * 8));
561  T *OneElt = Val;
562 
563  // Splat the value out to the right number of bits.
564  for (unsigned NumBytesSet = 1; NumBytesSet != LoadSize;) {
565  // If we can double the number of bytes set, do it.
566  if (NumBytesSet * 2 <= LoadSize) {
567  T *ShVal = Helper.CreateShl(
568  Val, ConstantInt::get(Val->getType(), NumBytesSet * 8));
569  Val = Helper.CreateOr(Val, ShVal);
570  NumBytesSet <<= 1;
571  continue;
572  }
573 
574  // Otherwise insert one byte at a time.
575  T *ShVal = Helper.CreateShl(Val, ConstantInt::get(Val->getType(), 1 * 8));
576  Val = Helper.CreateOr(OneElt, ShVal);
577  ++NumBytesSet;
578  }
579 
580  return coerceAvailableValueToLoadTypeHelper(Val, LoadTy, Helper, DL);
581  }
582 
583  // Otherwise, this is a memcpy/memmove from a constant global.
584  MemTransferInst *MTI = cast<MemTransferInst>(SrcInst);
585  Constant *Src = cast<Constant>(MTI->getSource());
586 
587  unsigned AS = Src->getType()->getPointerAddressSpace();
588  // Otherwise, see if we can constant fold a load from the constant with the
589  // offset applied as appropriate.
590  if (Offset) {
591  Src = ConstantExpr::getBitCast(Src,
592  Type::getInt8PtrTy(Src->getContext(), AS));
593  Constant *OffsetCst =
596  Src, OffsetCst);
597  }
598  Src = ConstantExpr::getBitCast(Src, PointerType::get(LoadTy, AS));
599  return ConstantFoldLoadFromConstPtr(Src, LoadTy, DL);
600 }
601 
602 /// This function is called when we have a
603 /// memdep query of a load that ends up being a clobbering mem intrinsic.
605  Type *LoadTy, Instruction *InsertPt,
606  const DataLayout &DL) {
607  IRBuilder<> Builder(InsertPt);
608  return getMemInstValueForLoadHelper<Value, IRBuilder<>>(SrcInst, Offset,
609  LoadTy, Builder, DL);
610 }
611 
613  Type *LoadTy, const DataLayout &DL) {
614  // The only case analyzeLoadFromClobberingMemInst cannot be converted to a
615  // constant is when it's a memset of a non-constant.
616  if (auto *MSI = dyn_cast<MemSetInst>(SrcInst))
617  if (!isa<Constant>(MSI->getValue()))
618  return nullptr;
620  return getMemInstValueForLoadHelper<Constant, ConstantFolder>(SrcInst, Offset,
621  LoadTy, F, DL);
622 }
623 } // namespace VNCoercion
624 } // namespace llvm
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::NextPowerOf2
uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:684
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:158
llvm::VNCoercion::analyzeLoadFromClobberingMemInst
int analyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr, MemIntrinsic *DepMI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the memory int...
Definition: VNCoercion.cpp:367
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
llvm
Definition: AllocatorList.h:23
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::VNCoercion::getStoreValueForLoad
Value * getStoreValueForLoad(Value *SrcVal, unsigned Offset, Type *LoadTy, Instruction *InsertPt, const DataLayout &DL)
If analyzeLoadFromClobberingStore returned an offset, this function can be used to actually perform t...
Definition: VNCoercion.cpp:468
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:256
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:229
llvm::VNCoercion::analyzeLoadFromClobberingWrite
static int analyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr, Value *WritePtr, uint64_t WriteSizeInBits, const DataLayout &DL)
This function is called when we have a memdep query of a load that ends up being a clobbering memory ...
Definition: VNCoercion.cpp:176
llvm::MemTransferInst
This class wraps the llvm.memcpy/memmove intrinsics.
Definition: IntrinsicInst.h:869
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:693
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:317
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:693
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2207
ValueTracking.h
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::MemIntrinsic
This is the common base class for memset/memcpy/memmove.
Definition: IntrinsicInst.h:826
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:345
T
#define T
Definition: Mips16ISelLowering.cpp:341
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::VNCoercion::coerceAvailableValueToLoadType
Value * coerceAvailableValueToLoadType(Value *StoredVal, Type *LoadedTy, IRBuilderBase &IRB, const DataLayout &DL)
If we saw a store of a value to memory, and then a load from a must-aliased pointer of a different ty...
Definition: VNCoercion.cpp:162
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:266
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:492
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:222
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:202
ConstantFolding.h
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:77
llvm::Type::isArrayTy
bool isArrayTy() const
True if this is an instance of ArrayType.
Definition: Type.h:226
llvm::ConstantFolder
ConstantFolder - Create constants with minimum, target independent, folding.
Definition: ConstantFolder.h:28
llvm::StoreInst::getValueOperand
Value * getValueOperand()
Definition: Instructions.h:398
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::Instruction
Definition: Instruction.h:45
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:885
llvm::getUnderlyingObject
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
Definition: ValueTracking.cpp:4296
llvm::LoadInst::getAlignment
unsigned getAlignment() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:219
llvm::MemSetInst
This class wraps the llvm.memset intrinsic.
Definition: IntrinsicInst.h:857
llvm::VNCoercion::coerceAvailableValueToLoadTypeHelper
static T * coerceAvailableValueToLoadTypeHelper(T *StoredVal, Type *LoadedTy, HelperClass &Helper, const DataLayout &DL)
Definition: VNCoercion.cpp:67
llvm::VNCoercion::getMemInstValueForLoad
Value * getMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset, Type *LoadTy, Instruction *InsertPt, const DataLayout &DL)
If analyzeLoadFromClobberingMemInst returned an offset, this function can be used to actually perform...
Definition: VNCoercion.cpp:604
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:202
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:303
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::GlobalVariable::hasDefinitiveInitializer
bool hasDefinitiveInitializer() const
hasDefinitiveInitializer - Whether the global variable has an initializer, and any other instances of...
Definition: GlobalVariable.h:110
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
VNCoercion.h
llvm::LoadInst::setAlignment
void setAlignment(Align Align)
Definition: Instructions.h:226
llvm::VNCoercion::getMemInstValueForLoadHelper
T * getMemInstValueForLoadHelper(MemIntrinsic *SrcInst, unsigned Offset, Type *LoadTy, HelperClass &Helper, const DataLayout &DL)
Definition: VNCoercion.cpp:546
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:649
llvm::VNCoercion::getConstantLoadValueForLoad
Constant * getConstantLoadValueForLoad(Constant *SrcVal, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:535
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:258
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:526
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:937
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:95
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::Type::isPtrOrPtrVectorTy
bool isPtrOrPtrVectorTy() const
Return true if this is a pointer type or a vector of pointer types.
Definition: Type.h:232
llvm::VNCoercion::isFirstClassAggregateOrScalableType
static bool isFirstClassAggregateOrScalableType(Type *Ty)
Definition: VNCoercion.cpp:12
llvm::Type::getContext
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::VNCoercion::getLoadValueForLoad
Value * getLoadValueForLoad(LoadInst *SrcVal, unsigned Offset, Type *LoadTy, Instruction *InsertPt, const DataLayout &DL)
If analyzeLoadFromClobberingLoad returned an offset, this function can be used to actually perform th...
Definition: VNCoercion.cpp:488
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:140
llvm::VNCoercion::getConstantMemInstValueForLoad
Constant * getConstantMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:612
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:205
llvm::VNCoercion::getLoadLoadClobberFullWidthSize
static unsigned getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs, unsigned MemLocSize, const LoadInst *LI)
Looks at a memory location for a load (specified by MemLocBase, Offs, and Size) and compares it again...
Definition: VNCoercion.cpp:251
llvm::VNCoercion::analyzeLoadFromClobberingStore
int analyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr, StoreInst *DepSI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the store at D...
Definition: VNCoercion.cpp:226
llvm::GetPointerBaseWithConstantOffset
Value * GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, const DataLayout &DL, bool AllowNonInbounds=true)
Analyze the specified pointer to see if it can be expressed as a base pointer plus a constant offset.
Definition: ValueTracking.h:279
llvm::ConstantExpr::getGetElementPtr
static Constant * getGetElementPtr(Type *Ty, Constant *C, ArrayRef< Constant * > IdxList, bool InBounds=false, Optional< unsigned > InRangeIndex=None, Type *OnlyIfReducedTy=nullptr)
Getelementptr form.
Definition: Constants.h:1205
llvm::VNCoercion::canCoerceMustAliasedValueToLoad
bool canCoerceMustAliasedValueToLoad(Value *StoredVal, Type *LoadTy, const DataLayout &DL)
Return true if CoerceAvailableValueToLoadType would succeed if it was called.
Definition: VNCoercion.cpp:17
llvm::VNCoercion::getConstantStoreValueForLoad
Constant * getConstantStoreValueForLoad(Constant *SrcVal, unsigned Offset, Type *LoadTy, const DataLayout &DL)
Definition: VNCoercion.cpp:476
llvm::Type::isStructTy
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:223
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:365
llvm::StoreInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:401
llvm::GlobalVariable::isConstant
bool isConstant() const
If the value is a global constant, its value is immutable throughout the runtime execution of the pro...
Definition: GlobalVariable.h:153
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::VNCoercion::analyzeLoadFromClobberingLoad
int analyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr, LoadInst *DepLI, const DataLayout &DL)
This function determines whether a value for the pointer LoadPtr can be extracted from the load at De...
Definition: VNCoercion.cpp:332
llvm::ConstantFoldConstant
Constant * ConstantFoldConstant(const Constant *C, const DataLayout &DL, const TargetLibraryInfo *TLI=nullptr)
ConstantFoldConstant - Fold the constant using the specified DataLayout.
Definition: ConstantFolding.cpp:1233
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:397
llvm::IntegerType::get
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:276
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:376
llvm::VNCoercion::getStoreValueForLoadHelper
static T * getStoreValueForLoadHelper(T *SrcVal, unsigned Offset, Type *LoadTy, HelperClass &Helper, const DataLayout &DL)
Definition: VNCoercion.cpp:424
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
Debug.h
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:129
llvm::MemTransferBase::getSource
Value * getSource() const
This is just like getRawSource, but it strips off any cast instructions that feed it,...
Definition: IntrinsicInst.h:669
llvm::ConstantFoldLoadFromConstPtr
Constant * ConstantFoldLoadFromConstPtr(Constant *C, Type *Ty, const DataLayout &DL)
ConstantFoldLoadFromConstPtr - Return the value that a load from C would produce if it is constant an...
Definition: ConstantFolding.cpp:675