LLVM  14.0.0git
InstCombineCalls.cpp
Go to the documentation of this file.
1 //===- InstCombineCalls.cpp -----------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the visitCall, visitInvoke, and visitCallBr functions.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "InstCombineInternal.h"
14 #include "llvm/ADT/APFloat.h"
15 #include "llvm/ADT/APInt.h"
16 #include "llvm/ADT/APSInt.h"
17 #include "llvm/ADT/ArrayRef.h"
19 #include "llvm/ADT/None.h"
20 #include "llvm/ADT/Optional.h"
21 #include "llvm/ADT/STLExtras.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/Statistic.h"
25 #include "llvm/ADT/Twine.h"
30 #include "llvm/Analysis/Loads.h"
35 #include "llvm/IR/Attributes.h"
36 #include "llvm/IR/BasicBlock.h"
37 #include "llvm/IR/Constant.h"
38 #include "llvm/IR/Constants.h"
39 #include "llvm/IR/DataLayout.h"
40 #include "llvm/IR/DerivedTypes.h"
41 #include "llvm/IR/Function.h"
42 #include "llvm/IR/GlobalVariable.h"
43 #include "llvm/IR/InlineAsm.h"
44 #include "llvm/IR/InstrTypes.h"
45 #include "llvm/IR/Instruction.h"
46 #include "llvm/IR/Instructions.h"
47 #include "llvm/IR/IntrinsicInst.h"
48 #include "llvm/IR/Intrinsics.h"
49 #include "llvm/IR/IntrinsicsAArch64.h"
50 #include "llvm/IR/IntrinsicsAMDGPU.h"
51 #include "llvm/IR/IntrinsicsARM.h"
52 #include "llvm/IR/IntrinsicsHexagon.h"
53 #include "llvm/IR/LLVMContext.h"
54 #include "llvm/IR/Metadata.h"
55 #include "llvm/IR/PatternMatch.h"
56 #include "llvm/IR/Statepoint.h"
57 #include "llvm/IR/Type.h"
58 #include "llvm/IR/User.h"
59 #include "llvm/IR/Value.h"
60 #include "llvm/IR/ValueHandle.h"
62 #include "llvm/Support/Casting.h"
64 #include "llvm/Support/Compiler.h"
65 #include "llvm/Support/Debug.h"
67 #include "llvm/Support/KnownBits.h"
74 #include <algorithm>
75 #include <cassert>
76 #include <cstdint>
77 #include <cstring>
78 #include <utility>
79 #include <vector>
80 
81 #define DEBUG_TYPE "instcombine"
83 
84 using namespace llvm;
85 using namespace PatternMatch;
86 
87 STATISTIC(NumSimplified, "Number of library calls simplified");
88 
90  "instcombine-guard-widening-window",
91  cl::init(3),
92  cl::desc("How wide an instruction window to bypass looking for "
93  "another guard"));
94 
95 namespace llvm {
96 /// enable preservation of attributes in assume like:
97 /// call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]
99 } // namespace llvm
100 
101 /// Return the specified type promoted as it would be to pass though a va_arg
102 /// area.
103 static Type *getPromotedType(Type *Ty) {
104  if (IntegerType* ITy = dyn_cast<IntegerType>(Ty)) {
105  if (ITy->getBitWidth() < 32)
106  return Type::getInt32Ty(Ty->getContext());
107  }
108  return Ty;
109 }
110 
112  Align DstAlign = getKnownAlignment(MI->getRawDest(), DL, MI, &AC, &DT);
113  MaybeAlign CopyDstAlign = MI->getDestAlign();
114  if (!CopyDstAlign || *CopyDstAlign < DstAlign) {
115  MI->setDestAlignment(DstAlign);
116  return MI;
117  }
118 
119  Align SrcAlign = getKnownAlignment(MI->getRawSource(), DL, MI, &AC, &DT);
120  MaybeAlign CopySrcAlign = MI->getSourceAlign();
121  if (!CopySrcAlign || *CopySrcAlign < SrcAlign) {
122  MI->setSourceAlignment(SrcAlign);
123  return MI;
124  }
125 
126  // If we have a store to a location which is known constant, we can conclude
127  // that the store must be storing the constant value (else the memory
128  // wouldn't be constant), and this must be a noop.
129  if (AA->pointsToConstantMemory(MI->getDest())) {
130  // Set the size of the copy to 0, it will be deleted on the next iteration.
131  MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
132  return MI;
133  }
134 
135  // If MemCpyInst length is 1/2/4/8 bytes then replace memcpy with
136  // load/store.
137  ConstantInt *MemOpLength = dyn_cast<ConstantInt>(MI->getLength());
138  if (!MemOpLength) return nullptr;
139 
140  // Source and destination pointer types are always "i8*" for intrinsic. See
141  // if the size is something we can handle with a single primitive load/store.
142  // A single load+store correctly handles overlapping memory in the memmove
143  // case.
144  uint64_t Size = MemOpLength->getLimitedValue();
145  assert(Size && "0-sized memory transferring should be removed already.");
146 
147  if (Size > 8 || (Size&(Size-1)))
148  return nullptr; // If not 1/2/4/8 bytes, exit.
149 
150  // If it is an atomic and alignment is less than the size then we will
151  // introduce the unaligned memory access which will be later transformed
152  // into libcall in CodeGen. This is not evident performance gain so disable
153  // it now.
154  if (isa<AtomicMemTransferInst>(MI))
155  if (*CopyDstAlign < Size || *CopySrcAlign < Size)
156  return nullptr;
157 
158  // Use an integer load+store unless we can find something better.
159  unsigned SrcAddrSp =
160  cast<PointerType>(MI->getArgOperand(1)->getType())->getAddressSpace();
161  unsigned DstAddrSp =
162  cast<PointerType>(MI->getArgOperand(0)->getType())->getAddressSpace();
163 
164  IntegerType* IntType = IntegerType::get(MI->getContext(), Size<<3);
165  Type *NewSrcPtrTy = PointerType::get(IntType, SrcAddrSp);
166  Type *NewDstPtrTy = PointerType::get(IntType, DstAddrSp);
167 
168  // If the memcpy has metadata describing the members, see if we can get the
169  // TBAA tag describing our copy.
170  MDNode *CopyMD = nullptr;
171  if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa)) {
172  CopyMD = M;
173  } else if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) {
174  if (M->getNumOperands() == 3 && M->getOperand(0) &&
175  mdconst::hasa<ConstantInt>(M->getOperand(0)) &&
176  mdconst::extract<ConstantInt>(M->getOperand(0))->isZero() &&
177  M->getOperand(1) &&
178  mdconst::hasa<ConstantInt>(M->getOperand(1)) &&
179  mdconst::extract<ConstantInt>(M->getOperand(1))->getValue() ==
180  Size &&
181  M->getOperand(2) && isa<MDNode>(M->getOperand(2)))
182  CopyMD = cast<MDNode>(M->getOperand(2));
183  }
184 
185  Value *Src = Builder.CreateBitCast(MI->getArgOperand(1), NewSrcPtrTy);
186  Value *Dest = Builder.CreateBitCast(MI->getArgOperand(0), NewDstPtrTy);
187  LoadInst *L = Builder.CreateLoad(IntType, Src);
188  // Alignment from the mem intrinsic will be better, so use it.
189  L->setAlignment(*CopySrcAlign);
190  if (CopyMD)
191  L->setMetadata(LLVMContext::MD_tbaa, CopyMD);
192  MDNode *LoopMemParallelMD =
193  MI->getMetadata(LLVMContext::MD_mem_parallel_loop_access);
194  if (LoopMemParallelMD)
195  L->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
196  MDNode *AccessGroupMD = MI->getMetadata(LLVMContext::MD_access_group);
197  if (AccessGroupMD)
198  L->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
199 
200  StoreInst *S = Builder.CreateStore(L, Dest);
201  // Alignment from the mem intrinsic will be better, so use it.
202  S->setAlignment(*CopyDstAlign);
203  if (CopyMD)
204  S->setMetadata(LLVMContext::MD_tbaa, CopyMD);
205  if (LoopMemParallelMD)
206  S->setMetadata(LLVMContext::MD_mem_parallel_loop_access, LoopMemParallelMD);
207  if (AccessGroupMD)
208  S->setMetadata(LLVMContext::MD_access_group, AccessGroupMD);
209 
210  if (auto *MT = dyn_cast<MemTransferInst>(MI)) {
211  // non-atomics can be volatile
212  L->setVolatile(MT->isVolatile());
213  S->setVolatile(MT->isVolatile());
214  }
215  if (isa<AtomicMemTransferInst>(MI)) {
216  // atomics have to be unordered
218  S->setOrdering(AtomicOrdering::Unordered);
219  }
220 
221  // Set the size of the copy to 0, it will be deleted on the next iteration.
222  MI->setLength(Constant::getNullValue(MemOpLength->getType()));
223  return MI;
224 }
225 
227  const Align KnownAlignment =
228  getKnownAlignment(MI->getDest(), DL, MI, &AC, &DT);
229  MaybeAlign MemSetAlign = MI->getDestAlign();
230  if (!MemSetAlign || *MemSetAlign < KnownAlignment) {
231  MI->setDestAlignment(KnownAlignment);
232  return MI;
233  }
234 
235  // If we have a store to a location which is known constant, we can conclude
236  // that the store must be storing the constant value (else the memory
237  // wouldn't be constant), and this must be a noop.
238  if (AA->pointsToConstantMemory(MI->getDest())) {
239  // Set the size of the copy to 0, it will be deleted on the next iteration.
240  MI->setLength(Constant::getNullValue(MI->getLength()->getType()));
241  return MI;
242  }
243 
244  // Extract the length and alignment and fill if they are constant.
245  ConstantInt *LenC = dyn_cast<ConstantInt>(MI->getLength());
246  ConstantInt *FillC = dyn_cast<ConstantInt>(MI->getValue());
247  if (!LenC || !FillC || !FillC->getType()->isIntegerTy(8))
248  return nullptr;
249  const uint64_t Len = LenC->getLimitedValue();
250  assert(Len && "0-sized memory setting should be removed already.");
251  const Align Alignment = assumeAligned(MI->getDestAlignment());
252 
253  // If it is an atomic and alignment is less than the size then we will
254  // introduce the unaligned memory access which will be later transformed
255  // into libcall in CodeGen. This is not evident performance gain so disable
256  // it now.
257  if (isa<AtomicMemSetInst>(MI))
258  if (Alignment < Len)
259  return nullptr;
260 
261  // memset(s,c,n) -> store s, c (for n=1,2,4,8)
262  if (Len <= 8 && isPowerOf2_32((uint32_t)Len)) {
263  Type *ITy = IntegerType::get(MI->getContext(), Len*8); // n=1 -> i8.
264 
265  Value *Dest = MI->getDest();
266  unsigned DstAddrSp = cast<PointerType>(Dest->getType())->getAddressSpace();
267  Type *NewDstPtrTy = PointerType::get(ITy, DstAddrSp);
268  Dest = Builder.CreateBitCast(Dest, NewDstPtrTy);
269 
270  // Extract the fill value and store.
271  uint64_t Fill = FillC->getZExtValue()*0x0101010101010101ULL;
272  StoreInst *S = Builder.CreateStore(ConstantInt::get(ITy, Fill), Dest,
273  MI->isVolatile());
274  S->setAlignment(Alignment);
275  if (isa<AtomicMemSetInst>(MI))
276  S->setOrdering(AtomicOrdering::Unordered);
277 
278  // Set the size of the copy to 0, it will be deleted on the next iteration.
279  MI->setLength(Constant::getNullValue(LenC->getType()));
280  return MI;
281  }
282 
283  return nullptr;
284 }
285 
286 // TODO, Obvious Missing Transforms:
287 // * Narrow width by halfs excluding zero/undef lanes
288 Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) {
289  Value *LoadPtr = II.getArgOperand(0);
290  const Align Alignment =
291  cast<ConstantInt>(II.getArgOperand(1))->getAlignValue();
292 
293  // If the mask is all ones or undefs, this is a plain vector load of the 1st
294  // argument.
295  if (maskIsAllOneOrUndef(II.getArgOperand(2))) {
296  LoadInst *L = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
297  "unmaskedload");
298  L->copyMetadata(II);
299  return L;
300  }
301 
302  // If we can unconditionally load from this address, replace with a
303  // load/select idiom. TODO: use DT for context sensitive query
304  if (isDereferenceablePointer(LoadPtr, II.getType(),
305  II.getModule()->getDataLayout(), &II, nullptr)) {
306  LoadInst *LI = Builder.CreateAlignedLoad(II.getType(), LoadPtr, Alignment,
307  "unmaskedload");
308  LI->copyMetadata(II);
309  return Builder.CreateSelect(II.getArgOperand(2), LI, II.getArgOperand(3));
310  }
311 
312  return nullptr;
313 }
314 
315 // TODO, Obvious Missing Transforms:
316 // * Single constant active lane -> store
317 // * Narrow width by halfs excluding zero/undef lanes
318 Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) {
319  auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
320  if (!ConstMask)
321  return nullptr;
322 
323  // If the mask is all zeros, this instruction does nothing.
324  if (ConstMask->isNullValue())
325  return eraseInstFromFunction(II);
326 
327  // If the mask is all ones, this is a plain vector store of the 1st argument.
328  if (ConstMask->isAllOnesValue()) {
329  Value *StorePtr = II.getArgOperand(1);
330  Align Alignment = cast<ConstantInt>(II.getArgOperand(2))->getAlignValue();
331  StoreInst *S =
332  new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment);
333  S->copyMetadata(II);
334  return S;
335  }
336 
337  if (isa<ScalableVectorType>(ConstMask->getType()))
338  return nullptr;
339 
340  // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
341  APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
342  APInt UndefElts(DemandedElts.getBitWidth(), 0);
343  if (Value *V =
344  SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, UndefElts))
345  return replaceOperand(II, 0, V);
346 
347  return nullptr;
348 }
349 
350 // TODO, Obvious Missing Transforms:
351 // * Single constant active lane load -> load
352 // * Dereferenceable address & few lanes -> scalarize speculative load/selects
353 // * Adjacent vector addresses -> masked.load
354 // * Narrow width by halfs excluding zero/undef lanes
355 // * Vector splat address w/known mask -> scalar load
356 // * Vector incrementing address -> vector masked load
357 Instruction *InstCombinerImpl::simplifyMaskedGather(IntrinsicInst &II) {
358  return nullptr;
359 }
360 
361 // TODO, Obvious Missing Transforms:
362 // * Single constant active lane -> store
363 // * Adjacent vector addresses -> masked.store
364 // * Narrow store width by halfs excluding zero/undef lanes
365 // * Vector splat address w/known mask -> scalar store
366 // * Vector incrementing address -> vector masked store
367 Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) {
368  auto *ConstMask = dyn_cast<Constant>(II.getArgOperand(3));
369  if (!ConstMask)
370  return nullptr;
371 
372  // If the mask is all zeros, a scatter does nothing.
373  if (ConstMask->isNullValue())
374  return eraseInstFromFunction(II);
375 
376  if (isa<ScalableVectorType>(ConstMask->getType()))
377  return nullptr;
378 
379  // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts
380  APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask);
381  APInt UndefElts(DemandedElts.getBitWidth(), 0);
382  if (Value *V =
383  SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, UndefElts))
384  return replaceOperand(II, 0, V);
385  if (Value *V =
386  SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts, UndefElts))
387  return replaceOperand(II, 1, V);
388 
389  return nullptr;
390 }
391 
392 /// This function transforms launder.invariant.group and strip.invariant.group
393 /// like:
394 /// launder(launder(%x)) -> launder(%x) (the result is not the argument)
395 /// launder(strip(%x)) -> launder(%x)
396 /// strip(strip(%x)) -> strip(%x) (the result is not the argument)
397 /// strip(launder(%x)) -> strip(%x)
398 /// This is legal because it preserves the most recent information about
399 /// the presence or absence of invariant.group.
401  InstCombinerImpl &IC) {
402  auto *Arg = II.getArgOperand(0);
403  auto *StrippedArg = Arg->stripPointerCasts();
404  auto *StrippedInvariantGroupsArg = StrippedArg;
405  while (auto *Intr = dyn_cast<IntrinsicInst>(StrippedInvariantGroupsArg)) {
406  if (Intr->getIntrinsicID() != Intrinsic::launder_invariant_group &&
407  Intr->getIntrinsicID() != Intrinsic::strip_invariant_group)
408  break;
409  StrippedInvariantGroupsArg = Intr->getArgOperand(0)->stripPointerCasts();
410  }
411  if (StrippedArg == StrippedInvariantGroupsArg)
412  return nullptr; // No launders/strips to remove.
413 
414  Value *Result = nullptr;
415 
416  if (II.getIntrinsicID() == Intrinsic::launder_invariant_group)
417  Result = IC.Builder.CreateLaunderInvariantGroup(StrippedInvariantGroupsArg);
418  else if (II.getIntrinsicID() == Intrinsic::strip_invariant_group)
419  Result = IC.Builder.CreateStripInvariantGroup(StrippedInvariantGroupsArg);
420  else
422  "simplifyInvariantGroupIntrinsic only handles launder and strip");
423  if (Result->getType()->getPointerAddressSpace() !=
425  Result = IC.Builder.CreateAddrSpaceCast(Result, II.getType());
426  if (Result->getType() != II.getType())
427  Result = IC.Builder.CreateBitCast(Result, II.getType());
428 
429  return cast<Instruction>(Result);
430 }
431 
433  assert((II.getIntrinsicID() == Intrinsic::cttz ||
434  II.getIntrinsicID() == Intrinsic::ctlz) &&
435  "Expected cttz or ctlz intrinsic");
436  bool IsTZ = II.getIntrinsicID() == Intrinsic::cttz;
437  Value *Op0 = II.getArgOperand(0);
438  Value *Op1 = II.getArgOperand(1);
439  Value *X;
440  // ctlz(bitreverse(x)) -> cttz(x)
441  // cttz(bitreverse(x)) -> ctlz(x)
442  if (match(Op0, m_BitReverse(m_Value(X)))) {
443  Intrinsic::ID ID = IsTZ ? Intrinsic::ctlz : Intrinsic::cttz;
445  return CallInst::Create(F, {X, II.getArgOperand(1)});
446  }
447 
448  if (II.getType()->isIntOrIntVectorTy(1)) {
449  // ctlz/cttz i1 Op0 --> not Op0
450  if (match(Op1, m_Zero()))
451  return BinaryOperator::CreateNot(Op0);
452  // If zero is undef, then the input can be assumed to be "true", so the
453  // instruction simplifies to "false".
454  assert(match(Op1, m_One()) && "Expected ctlz/cttz operand to be 0 or 1");
456  }
457 
458  // If the operand is a select with constant arm(s), try to hoist ctlz/cttz.
459  if (auto *Sel = dyn_cast<SelectInst>(Op0))
460  if (Instruction *R = IC.FoldOpIntoSelect(II, Sel))
461  return R;
462 
463  if (IsTZ) {
464  // cttz(-x) -> cttz(x)
465  if (match(Op0, m_Neg(m_Value(X))))
466  return IC.replaceOperand(II, 0, X);
467 
468  // cttz(sext(x)) -> cttz(zext(x))
469  if (match(Op0, m_OneUse(m_SExt(m_Value(X))))) {
470  auto *Zext = IC.Builder.CreateZExt(X, II.getType());
471  auto *CttzZext =
472  IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, Zext, Op1);
473  return IC.replaceInstUsesWith(II, CttzZext);
474  }
475 
476  // Zext doesn't change the number of trailing zeros, so narrow:
477  // cttz(zext(x)) -> zext(cttz(x)) if the 'ZeroIsUndef' parameter is 'true'.
478  if (match(Op0, m_OneUse(m_ZExt(m_Value(X)))) && match(Op1, m_One())) {
479  auto *Cttz = IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, X,
480  IC.Builder.getTrue());
481  auto *ZextCttz = IC.Builder.CreateZExt(Cttz, II.getType());
482  return IC.replaceInstUsesWith(II, ZextCttz);
483  }
484 
485  // cttz(abs(x)) -> cttz(x)
486  // cttz(nabs(x)) -> cttz(x)
487  Value *Y;
489  if (SPF == SPF_ABS || SPF == SPF_NABS)
490  return IC.replaceOperand(II, 0, X);
491 
492  if (match(Op0, m_Intrinsic<Intrinsic::abs>(m_Value(X))))
493  return IC.replaceOperand(II, 0, X);
494  }
495 
496  KnownBits Known = IC.computeKnownBits(Op0, 0, &II);
497 
498  // Create a mask for bits above (ctlz) or below (cttz) the first known one.
499  unsigned PossibleZeros = IsTZ ? Known.countMaxTrailingZeros()
500  : Known.countMaxLeadingZeros();
501  unsigned DefiniteZeros = IsTZ ? Known.countMinTrailingZeros()
502  : Known.countMinLeadingZeros();
503 
504  // If all bits above (ctlz) or below (cttz) the first known one are known
505  // zero, this value is constant.
506  // FIXME: This should be in InstSimplify because we're replacing an
507  // instruction with a constant.
508  if (PossibleZeros == DefiniteZeros) {
509  auto *C = ConstantInt::get(Op0->getType(), DefiniteZeros);
510  return IC.replaceInstUsesWith(II, C);
511  }
512 
513  // If the input to cttz/ctlz is known to be non-zero,
514  // then change the 'ZeroIsUndef' parameter to 'true'
515  // because we know the zero behavior can't affect the result.
516  if (!Known.One.isZero() ||
517  isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II,
518  &IC.getDominatorTree())) {
519  if (!match(II.getArgOperand(1), m_One()))
520  return IC.replaceOperand(II, 1, IC.Builder.getTrue());
521  }
522 
523  // Add range metadata since known bits can't completely reflect what we know.
524  // TODO: Handle splat vectors.
525  auto *IT = dyn_cast<IntegerType>(Op0->getType());
526  if (IT && IT->getBitWidth() != 1 && !II.getMetadata(LLVMContext::MD_range)) {
527  Metadata *LowAndHigh[] = {
529  ConstantAsMetadata::get(ConstantInt::get(IT, PossibleZeros + 1))};
530  II.setMetadata(LLVMContext::MD_range,
532  return &II;
533  }
534 
535  return nullptr;
536 }
537 
539  assert(II.getIntrinsicID() == Intrinsic::ctpop &&
540  "Expected ctpop intrinsic");
541  Type *Ty = II.getType();
542  unsigned BitWidth = Ty->getScalarSizeInBits();
543  Value *Op0 = II.getArgOperand(0);
544  Value *X, *Y;
545 
546  // ctpop(bitreverse(x)) -> ctpop(x)
547  // ctpop(bswap(x)) -> ctpop(x)
548  if (match(Op0, m_BitReverse(m_Value(X))) || match(Op0, m_BSwap(m_Value(X))))
549  return IC.replaceOperand(II, 0, X);
550 
551  // ctpop(rot(x)) -> ctpop(x)
552  if ((match(Op0, m_FShl(m_Value(X), m_Value(Y), m_Value())) ||
553  match(Op0, m_FShr(m_Value(X), m_Value(Y), m_Value()))) &&
554  X == Y)
555  return IC.replaceOperand(II, 0, X);
556 
557  // ctpop(x | -x) -> bitwidth - cttz(x, false)
558  if (Op0->hasOneUse() &&
559  match(Op0, m_c_Or(m_Value(X), m_Neg(m_Deferred(X))))) {
560  Function *F =
561  Intrinsic::getDeclaration(II.getModule(), Intrinsic::cttz, Ty);
562  auto *Cttz = IC.Builder.CreateCall(F, {X, IC.Builder.getFalse()});
563  auto *Bw = ConstantInt::get(Ty, APInt(BitWidth, BitWidth));
564  return IC.replaceInstUsesWith(II, IC.Builder.CreateSub(Bw, Cttz));
565  }
566 
567  // ctpop(~x & (x - 1)) -> cttz(x, false)
568  if (match(Op0,
570  Function *F =
571  Intrinsic::getDeclaration(II.getModule(), Intrinsic::cttz, Ty);
572  return CallInst::Create(F, {X, IC.Builder.getFalse()});
573  }
574 
575  // Zext doesn't change the number of set bits, so narrow:
576  // ctpop (zext X) --> zext (ctpop X)
577  if (match(Op0, m_OneUse(m_ZExt(m_Value(X))))) {
578  Value *NarrowPop = IC.Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, X);
579  return CastInst::Create(Instruction::ZExt, NarrowPop, Ty);
580  }
581 
582  // If the operand is a select with constant arm(s), try to hoist ctpop.
583  if (auto *Sel = dyn_cast<SelectInst>(Op0))
584  if (Instruction *R = IC.FoldOpIntoSelect(II, Sel))
585  return R;
586 
587  KnownBits Known(BitWidth);
588  IC.computeKnownBits(Op0, Known, 0, &II);
589 
590  // If all bits are zero except for exactly one fixed bit, then the result
591  // must be 0 or 1, and we can get that answer by shifting to LSB:
592  // ctpop (X & 32) --> (X & 32) >> 5
593  if ((~Known.Zero).isPowerOf2())
594  return BinaryOperator::CreateLShr(
595  Op0, ConstantInt::get(Ty, (~Known.Zero).exactLogBase2()));
596 
597  // FIXME: Try to simplify vectors of integers.
598  auto *IT = dyn_cast<IntegerType>(Ty);
599  if (!IT)
600  return nullptr;
601 
602  // Add range metadata since known bits can't completely reflect what we know.
603  unsigned MinCount = Known.countMinPopulation();
604  unsigned MaxCount = Known.countMaxPopulation();
605  if (IT->getBitWidth() != 1 && !II.getMetadata(LLVMContext::MD_range)) {
606  Metadata *LowAndHigh[] = {
609  II.setMetadata(LLVMContext::MD_range,
611  return &II;
612  }
613 
614  return nullptr;
615 }
616 
617 /// Convert a table lookup to shufflevector if the mask is constant.
618 /// This could benefit tbl1 if the mask is { 7,6,5,4,3,2,1,0 }, in
619 /// which case we could lower the shufflevector with rev64 instructions
620 /// as it's actually a byte reverse.
623  // Bail out if the mask is not a constant.
624  auto *C = dyn_cast<Constant>(II.getArgOperand(1));
625  if (!C)
626  return nullptr;
627 
628  auto *VecTy = cast<FixedVectorType>(II.getType());
629  unsigned NumElts = VecTy->getNumElements();
630 
631  // Only perform this transformation for <8 x i8> vector types.
632  if (!VecTy->getElementType()->isIntegerTy(8) || NumElts != 8)
633  return nullptr;
634 
635  int Indexes[8];
636 
637  for (unsigned I = 0; I < NumElts; ++I) {
638  Constant *COp = C->getAggregateElement(I);
639 
640  if (!COp || !isa<ConstantInt>(COp))
641  return nullptr;
642 
643  Indexes[I] = cast<ConstantInt>(COp)->getLimitedValue();
644 
645  // Make sure the mask indices are in range.
646  if ((unsigned)Indexes[I] >= NumElts)
647  return nullptr;
648  }
649 
650  auto *V1 = II.getArgOperand(0);
651  auto *V2 = Constant::getNullValue(V1->getType());
652  return Builder.CreateShuffleVector(V1, V2, makeArrayRef(Indexes));
653 }
654 
655 // Returns true iff the 2 intrinsics have the same operands, limiting the
656 // comparison to the first NumOperands.
657 static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E,
658  unsigned NumOperands) {
659  assert(I.arg_size() >= NumOperands && "Not enough operands");
660  assert(E.arg_size() >= NumOperands && "Not enough operands");
661  for (unsigned i = 0; i < NumOperands; i++)
662  if (I.getArgOperand(i) != E.getArgOperand(i))
663  return false;
664  return true;
665 }
666 
667 // Remove trivially empty start/end intrinsic ranges, i.e. a start
668 // immediately followed by an end (ignoring debuginfo or other
669 // start/end intrinsics in between). As this handles only the most trivial
670 // cases, tracking the nesting level is not needed:
671 //
672 // call @llvm.foo.start(i1 0)
673 // call @llvm.foo.start(i1 0) ; This one won't be skipped: it will be removed
674 // call @llvm.foo.end(i1 0)
675 // call @llvm.foo.end(i1 0) ; &I
676 static bool
678  std::function<bool(const IntrinsicInst &)> IsStart) {
679  // We start from the end intrinsic and scan backwards, so that InstCombine
680  // has already processed (and potentially removed) all the instructions
681  // before the end intrinsic.
682  BasicBlock::reverse_iterator BI(EndI), BE(EndI.getParent()->rend());
683  for (; BI != BE; ++BI) {
684  if (auto *I = dyn_cast<IntrinsicInst>(&*BI)) {
685  if (I->isDebugOrPseudoInst() ||
686  I->getIntrinsicID() == EndI.getIntrinsicID())
687  continue;
688  if (IsStart(*I)) {
689  if (haveSameOperands(EndI, *I, EndI.arg_size())) {
691  IC.eraseInstFromFunction(EndI);
692  return true;
693  }
694  // Skip start intrinsics that don't pair with this end intrinsic.
695  continue;
696  }
697  }
698  break;
699  }
700 
701  return false;
702 }
703 
705  removeTriviallyEmptyRange(I, *this, [](const IntrinsicInst &I) {
706  return I.getIntrinsicID() == Intrinsic::vastart ||
707  I.getIntrinsicID() == Intrinsic::vacopy;
708  });
709  return nullptr;
710 }
711 
713  assert(Call.arg_size() > 1 && "Need at least 2 args to swap");
714  Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1);
715  if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) {
716  Call.setArgOperand(0, Arg1);
717  Call.setArgOperand(1, Arg0);
718  return &Call;
719  }
720  return nullptr;
721 }
722 
723 /// Creates a result tuple for an overflow intrinsic \p II with a given
724 /// \p Result and a constant \p Overflow value.
726  Constant *Overflow) {
727  Constant *V[] = {UndefValue::get(Result->getType()), Overflow};
728  StructType *ST = cast<StructType>(II->getType());
729  Constant *Struct = ConstantStruct::get(ST, V);
730  return InsertValueInst::Create(Struct, Result, 0);
731 }
732 
733 Instruction *
734 InstCombinerImpl::foldIntrinsicWithOverflowCommon(IntrinsicInst *II) {
735  WithOverflowInst *WO = cast<WithOverflowInst>(II);
736  Value *OperationResult = nullptr;
737  Constant *OverflowResult = nullptr;
738  if (OptimizeOverflowCheck(WO->getBinaryOp(), WO->isSigned(), WO->getLHS(),
739  WO->getRHS(), *WO, OperationResult, OverflowResult))
740  return createOverflowTuple(WO, OperationResult, OverflowResult);
741  return nullptr;
742 }
743 
745  const DataLayout &DL, AssumptionCache *AC,
746  DominatorTree *DT) {
747  KnownBits Known = computeKnownBits(Op, DL, 0, AC, CxtI, DT);
748  if (Known.isNonNegative())
749  return false;
750  if (Known.isNegative())
751  return true;
752 
754  ICmpInst::ICMP_SLT, Op, Constant::getNullValue(Op->getType()), CxtI, DL);
755 }
756 
757 /// Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0. This
758 /// can trigger other combines.
761  Intrinsic::ID MinMaxID = II->getIntrinsicID();
762  assert((MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin ||
763  MinMaxID == Intrinsic::umax || MinMaxID == Intrinsic::umin) &&
764  "Expected a min or max intrinsic");
765 
766  // TODO: Match vectors with undef elements, but undef may not propagate.
767  Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
768  Value *X;
769  const APInt *C0, *C1;
770  if (!match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C0)))) ||
771  !match(Op1, m_APInt(C1)))
772  return nullptr;
773 
774  // Check for necessary no-wrap and overflow constraints.
775  bool IsSigned = MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin;
776  auto *Add = cast<BinaryOperator>(Op0);
777  if ((IsSigned && !Add->hasNoSignedWrap()) ||
778  (!IsSigned && !Add->hasNoUnsignedWrap()))
779  return nullptr;
780 
781  // If the constant difference overflows, then instsimplify should reduce the
782  // min/max to the add or C1.
783  bool Overflow;
784  APInt CDiff =
785  IsSigned ? C1->ssub_ov(*C0, Overflow) : C1->usub_ov(*C0, Overflow);
786  assert(!Overflow && "Expected simplify of min/max");
787 
788  // min/max (add X, C0), C1 --> add (min/max X, C1 - C0), C0
789  // Note: the "mismatched" no-overflow setting does not propagate.
790  Constant *NewMinMaxC = ConstantInt::get(II->getType(), CDiff);
791  Value *NewMinMax = Builder.CreateBinaryIntrinsic(MinMaxID, X, NewMinMaxC);
792  return IsSigned ? BinaryOperator::CreateNSWAdd(NewMinMax, Add->getOperand(1))
793  : BinaryOperator::CreateNUWAdd(NewMinMax, Add->getOperand(1));
794 }
795 
796 /// If we have a clamp pattern like max (min X, 42), 41 -- where the output
797 /// can only be one of two possible constant values -- turn that into a select
798 /// of constants.
801  Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
802  Value *X;
803  const APInt *C0, *C1;
804  if (!match(I1, m_APInt(C1)) || !I0->hasOneUse())
805  return nullptr;
806 
808  switch (II->getIntrinsicID()) {
809  case Intrinsic::smax:
810  if (match(I0, m_SMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)
811  Pred = ICmpInst::ICMP_SGT;
812  break;
813  case Intrinsic::smin:
814  if (match(I0, m_SMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)
815  Pred = ICmpInst::ICMP_SLT;
816  break;
817  case Intrinsic::umax:
818  if (match(I0, m_UMin(m_Value(X), m_APInt(C0))) && *C0 == *C1 + 1)
819  Pred = ICmpInst::ICMP_UGT;
820  break;
821  case Intrinsic::umin:
822  if (match(I0, m_UMax(m_Value(X), m_APInt(C0))) && *C1 == *C0 + 1)
823  Pred = ICmpInst::ICMP_ULT;
824  break;
825  default:
826  llvm_unreachable("Expected min/max intrinsic");
827  }
828  if (Pred == CmpInst::BAD_ICMP_PREDICATE)
829  return nullptr;
830 
831  // max (min X, 42), 41 --> X > 41 ? 42 : 41
832  // min (max X, 42), 43 --> X < 43 ? 42 : 43
833  Value *Cmp = Builder.CreateICmp(Pred, X, I1);
834  return SelectInst::Create(Cmp, ConstantInt::get(II->getType(), *C0), I1);
835 }
836 
837 /// Reduce a sequence of min/max intrinsics with a common operand.
839  // Match 3 of the same min/max ops. Example: umin(umin(), umin()).
840  auto *LHS = dyn_cast<IntrinsicInst>(II->getArgOperand(0));
841  auto *RHS = dyn_cast<IntrinsicInst>(II->getArgOperand(1));
842  Intrinsic::ID MinMaxID = II->getIntrinsicID();
843  if (!LHS || !RHS || LHS->getIntrinsicID() != MinMaxID ||
844  RHS->getIntrinsicID() != MinMaxID ||
845  (!LHS->hasOneUse() && !RHS->hasOneUse()))
846  return nullptr;
847 
848  Value *A = LHS->getArgOperand(0);
849  Value *B = LHS->getArgOperand(1);
850  Value *C = RHS->getArgOperand(0);
851  Value *D = RHS->getArgOperand(1);
852 
853  // Look for a common operand.
854  Value *MinMaxOp = nullptr;
855  Value *ThirdOp = nullptr;
856  if (LHS->hasOneUse()) {
857  // If the LHS is only used in this chain and the RHS is used outside of it,
858  // reuse the RHS min/max because that will eliminate the LHS.
859  if (D == A || C == A) {
860  // min(min(a, b), min(c, a)) --> min(min(c, a), b)
861  // min(min(a, b), min(a, d)) --> min(min(a, d), b)
862  MinMaxOp = RHS;
863  ThirdOp = B;
864  } else if (D == B || C == B) {
865  // min(min(a, b), min(c, b)) --> min(min(c, b), a)
866  // min(min(a, b), min(b, d)) --> min(min(b, d), a)
867  MinMaxOp = RHS;
868  ThirdOp = A;
869  }
870  } else {
871  assert(RHS->hasOneUse() && "Expected one-use operand");
872  // Reuse the LHS. This will eliminate the RHS.
873  if (D == A || D == B) {
874  // min(min(a, b), min(c, a)) --> min(min(a, b), c)
875  // min(min(a, b), min(c, b)) --> min(min(a, b), c)
876  MinMaxOp = LHS;
877  ThirdOp = C;
878  } else if (C == A || C == B) {
879  // min(min(a, b), min(b, d)) --> min(min(a, b), d)
880  // min(min(a, b), min(c, b)) --> min(min(a, b), d)
881  MinMaxOp = LHS;
882  ThirdOp = D;
883  }
884  }
885 
886  if (!MinMaxOp || !ThirdOp)
887  return nullptr;
888 
889  Module *Mod = II->getModule();
890  Function *MinMax = Intrinsic::getDeclaration(Mod, MinMaxID, II->getType());
891  return CallInst::Create(MinMax, { MinMaxOp, ThirdOp });
892 }
893 
894 /// CallInst simplification. This mostly only handles folding of intrinsic
895 /// instructions. For normal calls, it allows visitCallBase to do the heavy
896 /// lifting.
898  // Don't try to simplify calls without uses. It will not do anything useful,
899  // but will result in the following folds being skipped.
900  if (!CI.use_empty())
901  if (Value *V = SimplifyCall(&CI, SQ.getWithInstruction(&CI)))
902  return replaceInstUsesWith(CI, V);
903 
904  if (isFreeCall(&CI, &TLI))
905  return visitFree(CI);
906 
907  // If the caller function is nounwind, mark the call as nounwind, even if the
908  // callee isn't.
909  if (CI.getFunction()->doesNotThrow() && !CI.doesNotThrow()) {
910  CI.setDoesNotThrow();
911  return &CI;
912  }
913 
914  IntrinsicInst *II = dyn_cast<IntrinsicInst>(&CI);
915  if (!II) return visitCallBase(CI);
916 
917  // For atomic unordered mem intrinsics if len is not a positive or
918  // not a multiple of element size then behavior is undefined.
919  if (auto *AMI = dyn_cast<AtomicMemIntrinsic>(II))
920  if (ConstantInt *NumBytes = dyn_cast<ConstantInt>(AMI->getLength()))
921  if (NumBytes->getSExtValue() < 0 ||
922  (NumBytes->getZExtValue() % AMI->getElementSizeInBytes() != 0)) {
923  CreateNonTerminatorUnreachable(AMI);
924  assert(AMI->getType()->isVoidTy() &&
925  "non void atomic unordered mem intrinsic");
926  return eraseInstFromFunction(*AMI);
927  }
928 
929  // Intrinsics cannot occur in an invoke or a callbr, so handle them here
930  // instead of in visitCallBase.
931  if (auto *MI = dyn_cast<AnyMemIntrinsic>(II)) {
932  bool Changed = false;
933 
934  // memmove/cpy/set of zero bytes is a noop.
935  if (Constant *NumBytes = dyn_cast<Constant>(MI->getLength())) {
936  if (NumBytes->isNullValue())
937  return eraseInstFromFunction(CI);
938 
939  if (ConstantInt *CI = dyn_cast<ConstantInt>(NumBytes))
940  if (CI->getZExtValue() == 1) {
941  // Replace the instruction with just byte operations. We would
942  // transform other cases to loads/stores, but we don't know if
943  // alignment is sufficient.
944  }
945  }
946 
947  // No other transformations apply to volatile transfers.
948  if (auto *M = dyn_cast<MemIntrinsic>(MI))
949  if (M->isVolatile())
950  return nullptr;
951 
952  // If we have a memmove and the source operation is a constant global,
953  // then the source and dest pointers can't alias, so we can change this
954  // into a call to memcpy.
955  if (auto *MMI = dyn_cast<AnyMemMoveInst>(MI)) {
956  if (GlobalVariable *GVSrc = dyn_cast<GlobalVariable>(MMI->getSource()))
957  if (GVSrc->isConstant()) {
958  Module *M = CI.getModule();
959  Intrinsic::ID MemCpyID =
960  isa<AtomicMemMoveInst>(MMI)
961  ? Intrinsic::memcpy_element_unordered_atomic
963  Type *Tys[3] = { CI.getArgOperand(0)->getType(),
964  CI.getArgOperand(1)->getType(),
965  CI.getArgOperand(2)->getType() };
966  CI.setCalledFunction(Intrinsic::getDeclaration(M, MemCpyID, Tys));
967  Changed = true;
968  }
969  }
970 
971  if (AnyMemTransferInst *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
972  // memmove(x,x,size) -> noop.
973  if (MTI->getSource() == MTI->getDest())
974  return eraseInstFromFunction(CI);
975  }
976 
977  // If we can determine a pointer alignment that is bigger than currently
978  // set, update the alignment.
979  if (auto *MTI = dyn_cast<AnyMemTransferInst>(MI)) {
980  if (Instruction *I = SimplifyAnyMemTransfer(MTI))
981  return I;
982  } else if (auto *MSI = dyn_cast<AnyMemSetInst>(MI)) {
983  if (Instruction *I = SimplifyAnyMemSet(MSI))
984  return I;
985  }
986 
987  if (Changed) return II;
988  }
989 
990  // For fixed width vector result intrinsics, use the generic demanded vector
991  // support.
992  if (auto *IIFVTy = dyn_cast<FixedVectorType>(II->getType())) {
993  auto VWidth = IIFVTy->getNumElements();
994  APInt UndefElts(VWidth, 0);
995  APInt AllOnesEltMask(APInt::getAllOnes(VWidth));
996  if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) {
997  if (V != II)
998  return replaceInstUsesWith(*II, V);
999  return II;
1000  }
1001  }
1002 
1003  if (II->isCommutative()) {
1004  if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI))
1005  return NewCall;
1006  }
1007 
1008  Intrinsic::ID IID = II->getIntrinsicID();
1009  switch (IID) {
1010  case Intrinsic::objectsize:
1011  if (Value *V = lowerObjectSizeCall(II, DL, &TLI, /*MustSucceed=*/false))
1012  return replaceInstUsesWith(CI, V);
1013  return nullptr;
1014  case Intrinsic::abs: {
1015  Value *IIOperand = II->getArgOperand(0);
1016  bool IntMinIsPoison = cast<Constant>(II->getArgOperand(1))->isOneValue();
1017 
1018  // abs(-x) -> abs(x)
1019  // TODO: Copy nsw if it was present on the neg?
1020  Value *X;
1021  if (match(IIOperand, m_Neg(m_Value(X))))
1022  return replaceOperand(*II, 0, X);
1023  if (match(IIOperand, m_Select(m_Value(), m_Value(X), m_Neg(m_Deferred(X)))))
1024  return replaceOperand(*II, 0, X);
1025  if (match(IIOperand, m_Select(m_Value(), m_Neg(m_Value(X)), m_Deferred(X))))
1026  return replaceOperand(*II, 0, X);
1027 
1028  if (Optional<bool> Sign = getKnownSign(IIOperand, II, DL, &AC, &DT)) {
1029  // abs(x) -> x if x >= 0
1030  if (!*Sign)
1031  return replaceInstUsesWith(*II, IIOperand);
1032 
1033  // abs(x) -> -x if x < 0
1034  if (IntMinIsPoison)
1035  return BinaryOperator::CreateNSWNeg(IIOperand);
1036  return BinaryOperator::CreateNeg(IIOperand);
1037  }
1038 
1039  // abs (sext X) --> zext (abs X*)
1040  // Clear the IsIntMin (nsw) bit on the abs to allow narrowing.
1041  if (match(IIOperand, m_OneUse(m_SExt(m_Value(X))))) {
1042  Value *NarrowAbs =
1043  Builder.CreateBinaryIntrinsic(Intrinsic::abs, X, Builder.getFalse());
1044  return CastInst::Create(Instruction::ZExt, NarrowAbs, II->getType());
1045  }
1046 
1047  // Match a complicated way to check if a number is odd/even:
1048  // abs (srem X, 2) --> and X, 1
1049  const APInt *C;
1050  if (match(IIOperand, m_SRem(m_Value(X), m_APInt(C))) && *C == 2)
1051  return BinaryOperator::CreateAnd(X, ConstantInt::get(II->getType(), 1));
1052 
1053  break;
1054  }
1055  case Intrinsic::umin: {
1056  Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1057  // umin(x, 1) == zext(x != 0)
1058  if (match(I1, m_One())) {
1059  Value *Zero = Constant::getNullValue(I0->getType());
1060  Value *Cmp = Builder.CreateICmpNE(I0, Zero);
1061  return CastInst::Create(Instruction::ZExt, Cmp, II->getType());
1062  }
1064  }
1065  case Intrinsic::umax: {
1066  Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1067  Value *X, *Y;
1068  if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_ZExt(m_Value(Y))) &&
1069  (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
1070  Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
1071  return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
1072  }
1073  Constant *C;
1074  if (match(I0, m_ZExt(m_Value(X))) && match(I1, m_Constant(C)) &&
1075  I0->hasOneUse()) {
1076  Constant *NarrowC = ConstantExpr::getTrunc(C, X->getType());
1077  if (ConstantExpr::getZExt(NarrowC, II->getType()) == C) {
1078  Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);
1079  return CastInst::Create(Instruction::ZExt, NarrowMaxMin, II->getType());
1080  }
1081  }
1082  // If both operands of unsigned min/max are sign-extended, it is still ok
1083  // to narrow the operation.
1085  }
1086  case Intrinsic::smax:
1087  case Intrinsic::smin: {
1088  Value *I0 = II->getArgOperand(0), *I1 = II->getArgOperand(1);
1089  Value *X, *Y;
1090  if (match(I0, m_SExt(m_Value(X))) && match(I1, m_SExt(m_Value(Y))) &&
1091  (I0->hasOneUse() || I1->hasOneUse()) && X->getType() == Y->getType()) {
1092  Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, Y);
1093  return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
1094  }
1095 
1096  Constant *C;
1097  if (match(I0, m_SExt(m_Value(X))) && match(I1, m_Constant(C)) &&
1098  I0->hasOneUse()) {
1099  Constant *NarrowC = ConstantExpr::getTrunc(C, X->getType());
1100  if (ConstantExpr::getSExt(NarrowC, II->getType()) == C) {
1101  Value *NarrowMaxMin = Builder.CreateBinaryIntrinsic(IID, X, NarrowC);
1102  return CastInst::Create(Instruction::SExt, NarrowMaxMin, II->getType());
1103  }
1104  }
1105 
1106  if (IID == Intrinsic::smax || IID == Intrinsic::smin) {
1107  // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y)
1108  // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y)
1109  // TODO: Canonicalize neg after min/max if I1 is constant.
1110  if (match(I0, m_NSWNeg(m_Value(X))) && match(I1, m_NSWNeg(m_Value(Y))) &&
1111  (I0->hasOneUse() || I1->hasOneUse())) {
1113  Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, Y);
1114  return BinaryOperator::CreateNSWNeg(InvMaxMin);
1115  }
1116  }
1117 
1118  // If we can eliminate ~A and Y is free to invert:
1119  // max ~A, Y --> ~(min A, ~Y)
1120  //
1121  // Examples:
1122  // max ~A, ~Y --> ~(min A, Y)
1123  // max ~A, C --> ~(min A, ~C)
1124  // max ~A, (max ~Y, ~Z) --> ~min( A, (min Y, Z))
1125  auto moveNotAfterMinMax = [&](Value *X, Value *Y) -> Instruction * {
1126  Value *A;
1127  if (match(X, m_OneUse(m_Not(m_Value(A)))) &&
1128  !isFreeToInvert(A, A->hasOneUse()) &&
1129  isFreeToInvert(Y, Y->hasOneUse())) {
1130  Value *NotY = Builder.CreateNot(Y);
1132  Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, A, NotY);
1133  return BinaryOperator::CreateNot(InvMaxMin);
1134  }
1135  return nullptr;
1136  };
1137 
1138  if (Instruction *I = moveNotAfterMinMax(I0, I1))
1139  return I;
1140  if (Instruction *I = moveNotAfterMinMax(I1, I0))
1141  return I;
1142 
1144  return I;
1145 
1146  // smax(X, -X) --> abs(X)
1147  // smin(X, -X) --> -abs(X)
1148  // umax(X, -X) --> -abs(X)
1149  // umin(X, -X) --> abs(X)
1150  if (isKnownNegation(I0, I1)) {
1151  // We can choose either operand as the input to abs(), but if we can
1152  // eliminate the only use of a value, that's better for subsequent
1153  // transforms/analysis.
1154  if (I0->hasOneUse() && !I1->hasOneUse())
1155  std::swap(I0, I1);
1156 
1157  // This is some variant of abs(). See if we can propagate 'nsw' to the abs
1158  // operation and potentially its negation.
1159  bool IntMinIsPoison = isKnownNegation(I0, I1, /* NeedNSW */ true);
1160  Value *Abs = Builder.CreateBinaryIntrinsic(
1161  Intrinsic::abs, I0,
1162  ConstantInt::getBool(II->getContext(), IntMinIsPoison));
1163 
1164  // We don't have a "nabs" intrinsic, so negate if needed based on the
1165  // max/min operation.
1166  if (IID == Intrinsic::smin || IID == Intrinsic::umax)
1167  Abs = Builder.CreateNeg(Abs, "nabs", /* NUW */ false, IntMinIsPoison);
1168  return replaceInstUsesWith(CI, Abs);
1169  }
1170 
1171  if (Instruction *Sel = foldClampRangeOfTwo(II, Builder))
1172  return Sel;
1173 
1174  if (Instruction *SAdd = matchSAddSubSat(*II))
1175  return SAdd;
1176 
1177  if (match(I1, m_ImmConstant()))
1178  if (auto *Sel = dyn_cast<SelectInst>(I0))
1179  if (Instruction *R = FoldOpIntoSelect(*II, Sel))
1180  return R;
1181 
1182  if (Instruction *NewMinMax = factorizeMinMaxTree(II))
1183  return NewMinMax;
1184 
1185  break;
1186  }
1187  case Intrinsic::bswap: {
1188  Value *IIOperand = II->getArgOperand(0);
1189  Value *X = nullptr;
1190 
1191  // bswap(trunc(bswap(x))) -> trunc(lshr(x, c))
1192  if (match(IIOperand, m_Trunc(m_BSwap(m_Value(X))))) {
1193  unsigned C = X->getType()->getScalarSizeInBits() -
1194  IIOperand->getType()->getScalarSizeInBits();
1195  Value *CV = ConstantInt::get(X->getType(), C);
1196  Value *V = Builder.CreateLShr(X, CV);
1197  return new TruncInst(V, IIOperand->getType());
1198  }
1199  break;
1200  }
1201  case Intrinsic::masked_load:
1202  if (Value *SimplifiedMaskedOp = simplifyMaskedLoad(*II))
1203  return replaceInstUsesWith(CI, SimplifiedMaskedOp);
1204  break;
1205  case Intrinsic::masked_store:
1206  return simplifyMaskedStore(*II);
1207  case Intrinsic::masked_gather:
1208  return simplifyMaskedGather(*II);
1209  case Intrinsic::masked_scatter:
1210  return simplifyMaskedScatter(*II);
1211  case Intrinsic::launder_invariant_group:
1212  case Intrinsic::strip_invariant_group:
1213  if (auto *SkippedBarrier = simplifyInvariantGroupIntrinsic(*II, *this))
1214  return replaceInstUsesWith(*II, SkippedBarrier);
1215  break;
1216  case Intrinsic::powi:
1217  if (ConstantInt *Power = dyn_cast<ConstantInt>(II->getArgOperand(1))) {
1218  // 0 and 1 are handled in instsimplify
1219  // powi(x, -1) -> 1/x
1220  if (Power->isMinusOne())
1222  II->getArgOperand(0), II);
1223  // powi(x, 2) -> x*x
1224  if (Power->equalsInt(2))
1226  II->getArgOperand(0), II);
1227 
1228  if (!Power->getValue()[0]) {
1229  Value *X;
1230  // If power is even:
1231  // powi(-x, p) -> powi(x, p)
1232  // powi(fabs(x), p) -> powi(x, p)
1233  // powi(copysign(x, y), p) -> powi(x, p)
1234  if (match(II->getArgOperand(0), m_FNeg(m_Value(X))) ||
1235  match(II->getArgOperand(0), m_FAbs(m_Value(X))) ||
1236  match(II->getArgOperand(0),
1237  m_Intrinsic<Intrinsic::copysign>(m_Value(X), m_Value())))
1238  return replaceOperand(*II, 0, X);
1239  }
1240  }
1241  break;
1242 
1243  case Intrinsic::cttz:
1244  case Intrinsic::ctlz:
1245  if (auto *I = foldCttzCtlz(*II, *this))
1246  return I;
1247  break;
1248 
1249  case Intrinsic::ctpop:
1250  if (auto *I = foldCtpop(*II, *this))
1251  return I;
1252  break;
1253 
1254  case Intrinsic::fshl:
1255  case Intrinsic::fshr: {
1256  Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1);
1257  Type *Ty = II->getType();
1258  unsigned BitWidth = Ty->getScalarSizeInBits();
1259  Constant *ShAmtC;
1260  if (match(II->getArgOperand(2), m_ImmConstant(ShAmtC)) &&
1261  !ShAmtC->containsConstantExpression()) {
1262  // Canonicalize a shift amount constant operand to modulo the bit-width.
1263  Constant *WidthC = ConstantInt::get(Ty, BitWidth);
1264  Constant *ModuloC = ConstantExpr::getURem(ShAmtC, WidthC);
1265  if (ModuloC != ShAmtC)
1266  return replaceOperand(*II, 2, ModuloC);
1267 
1268  assert(ConstantExpr::getICmp(ICmpInst::ICMP_UGT, WidthC, ShAmtC) ==
1270  "Shift amount expected to be modulo bitwidth");
1271 
1272  // Canonicalize funnel shift right by constant to funnel shift left. This
1273  // is not entirely arbitrary. For historical reasons, the backend may
1274  // recognize rotate left patterns but miss rotate right patterns.
1275  if (IID == Intrinsic::fshr) {
1276  // fshr X, Y, C --> fshl X, Y, (BitWidth - C)
1277  Constant *LeftShiftC = ConstantExpr::getSub(WidthC, ShAmtC);
1278  Module *Mod = II->getModule();
1279  Function *Fshl = Intrinsic::getDeclaration(Mod, Intrinsic::fshl, Ty);
1280  return CallInst::Create(Fshl, { Op0, Op1, LeftShiftC });
1281  }
1282  assert(IID == Intrinsic::fshl &&
1283  "All funnel shifts by simple constants should go left");
1284 
1285  // fshl(X, 0, C) --> shl X, C
1286  // fshl(X, undef, C) --> shl X, C
1287  if (match(Op1, m_ZeroInt()) || match(Op1, m_Undef()))
1288  return BinaryOperator::CreateShl(Op0, ShAmtC);
1289 
1290  // fshl(0, X, C) --> lshr X, (BW-C)
1291  // fshl(undef, X, C) --> lshr X, (BW-C)
1292  if (match(Op0, m_ZeroInt()) || match(Op0, m_Undef()))
1293  return BinaryOperator::CreateLShr(Op1,
1294  ConstantExpr::getSub(WidthC, ShAmtC));
1295 
1296  // fshl i16 X, X, 8 --> bswap i16 X (reduce to more-specific form)
1297  if (Op0 == Op1 && BitWidth == 16 && match(ShAmtC, m_SpecificInt(8))) {
1298  Module *Mod = II->getModule();
1299  Function *Bswap = Intrinsic::getDeclaration(Mod, Intrinsic::bswap, Ty);
1300  return CallInst::Create(Bswap, { Op0 });
1301  }
1302  }
1303 
1304  // Left or right might be masked.
1305  if (SimplifyDemandedInstructionBits(*II))
1306  return &CI;
1307 
1308  // The shift amount (operand 2) of a funnel shift is modulo the bitwidth,
1309  // so only the low bits of the shift amount are demanded if the bitwidth is
1310  // a power-of-2.
1311  if (!isPowerOf2_32(BitWidth))
1312  break;
1314  KnownBits Op2Known(BitWidth);
1315  if (SimplifyDemandedBits(II, 2, Op2Demanded, Op2Known))
1316  return &CI;
1317  break;
1318  }
1319  case Intrinsic::uadd_with_overflow:
1320  case Intrinsic::sadd_with_overflow: {
1321  if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
1322  return I;
1323 
1324  // Given 2 constant operands whose sum does not overflow:
1325  // uaddo (X +nuw C0), C1 -> uaddo X, C0 + C1
1326  // saddo (X +nsw C0), C1 -> saddo X, C0 + C1
1327  Value *X;
1328  const APInt *C0, *C1;
1329  Value *Arg0 = II->getArgOperand(0);
1330  Value *Arg1 = II->getArgOperand(1);
1331  bool IsSigned = IID == Intrinsic::sadd_with_overflow;
1332  bool HasNWAdd = IsSigned ? match(Arg0, m_NSWAdd(m_Value(X), m_APInt(C0)))
1333  : match(Arg0, m_NUWAdd(m_Value(X), m_APInt(C0)));
1334  if (HasNWAdd && match(Arg1, m_APInt(C1))) {
1335  bool Overflow;
1336  APInt NewC =
1337  IsSigned ? C1->sadd_ov(*C0, Overflow) : C1->uadd_ov(*C0, Overflow);
1338  if (!Overflow)
1339  return replaceInstUsesWith(
1340  *II, Builder.CreateBinaryIntrinsic(
1341  IID, X, ConstantInt::get(Arg1->getType(), NewC)));
1342  }
1343  break;
1344  }
1345 
1346  case Intrinsic::umul_with_overflow:
1347  case Intrinsic::smul_with_overflow:
1348  case Intrinsic::usub_with_overflow:
1349  if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
1350  return I;
1351  break;
1352 
1353  case Intrinsic::ssub_with_overflow: {
1354  if (Instruction *I = foldIntrinsicWithOverflowCommon(II))
1355  return I;
1356 
1357  Constant *C;
1358  Value *Arg0 = II->getArgOperand(0);
1359  Value *Arg1 = II->getArgOperand(1);
1360  // Given a constant C that is not the minimum signed value
1361  // for an integer of a given bit width:
1362  //
1363  // ssubo X, C -> saddo X, -C
1364  if (match(Arg1, m_Constant(C)) && C->isNotMinSignedValue()) {
1365  Value *NegVal = ConstantExpr::getNeg(C);
1366  // Build a saddo call that is equivalent to the discovered
1367  // ssubo call.
1368  return replaceInstUsesWith(
1369  *II, Builder.CreateBinaryIntrinsic(Intrinsic::sadd_with_overflow,
1370  Arg0, NegVal));
1371  }
1372 
1373  break;
1374  }
1375 
1376  case Intrinsic::uadd_sat:
1377  case Intrinsic::sadd_sat:
1378  case Intrinsic::usub_sat:
1379  case Intrinsic::ssub_sat: {
1380  SaturatingInst *SI = cast<SaturatingInst>(II);
1381  Type *Ty = SI->getType();
1382  Value *Arg0 = SI->getLHS();
1383  Value *Arg1 = SI->getRHS();
1384 
1385  // Make use of known overflow information.
1386  OverflowResult OR = computeOverflow(SI->getBinaryOp(), SI->isSigned(),
1387  Arg0, Arg1, SI);
1388  switch (OR) {
1390  break;
1392  if (SI->isSigned())
1393  return BinaryOperator::CreateNSW(SI->getBinaryOp(), Arg0, Arg1);
1394  else
1395  return BinaryOperator::CreateNUW(SI->getBinaryOp(), Arg0, Arg1);
1397  unsigned BitWidth = Ty->getScalarSizeInBits();
1398  APInt Min = APSInt::getMinValue(BitWidth, !SI->isSigned());
1399  return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Min));
1400  }
1402  unsigned BitWidth = Ty->getScalarSizeInBits();
1403  APInt Max = APSInt::getMaxValue(BitWidth, !SI->isSigned());
1404  return replaceInstUsesWith(*SI, ConstantInt::get(Ty, Max));
1405  }
1406  }
1407 
1408  // ssub.sat(X, C) -> sadd.sat(X, -C) if C != MIN
1409  Constant *C;
1410  if (IID == Intrinsic::ssub_sat && match(Arg1, m_Constant(C)) &&
1411  C->isNotMinSignedValue()) {
1412  Value *NegVal = ConstantExpr::getNeg(C);
1413  return replaceInstUsesWith(
1414  *II, Builder.CreateBinaryIntrinsic(
1415  Intrinsic::sadd_sat, Arg0, NegVal));
1416  }
1417 
1418  // sat(sat(X + Val2) + Val) -> sat(X + (Val+Val2))
1419  // sat(sat(X - Val2) - Val) -> sat(X - (Val+Val2))
1420  // if Val and Val2 have the same sign
1421  if (auto *Other = dyn_cast<IntrinsicInst>(Arg0)) {
1422  Value *X;
1423  const APInt *Val, *Val2;
1424  APInt NewVal;
1425  bool IsUnsigned =
1426  IID == Intrinsic::uadd_sat || IID == Intrinsic::usub_sat;
1427  if (Other->getIntrinsicID() == IID &&
1428  match(Arg1, m_APInt(Val)) &&
1429  match(Other->getArgOperand(0), m_Value(X)) &&
1430  match(Other->getArgOperand(1), m_APInt(Val2))) {
1431  if (IsUnsigned)
1432  NewVal = Val->uadd_sat(*Val2);
1433  else if (Val->isNonNegative() == Val2->isNonNegative()) {
1434  bool Overflow;
1435  NewVal = Val->sadd_ov(*Val2, Overflow);
1436  if (Overflow) {
1437  // Both adds together may add more than SignedMaxValue
1438  // without saturating the final result.
1439  break;
1440  }
1441  } else {
1442  // Cannot fold saturated addition with different signs.
1443  break;
1444  }
1445 
1446  return replaceInstUsesWith(
1447  *II, Builder.CreateBinaryIntrinsic(
1448  IID, X, ConstantInt::get(II->getType(), NewVal)));
1449  }
1450  }
1451  break;
1452  }
1453 
1454  case Intrinsic::minnum:
1455  case Intrinsic::maxnum:
1456  case Intrinsic::minimum:
1457  case Intrinsic::maximum: {
1458  Value *Arg0 = II->getArgOperand(0);
1459  Value *Arg1 = II->getArgOperand(1);
1460  Value *X, *Y;
1461  if (match(Arg0, m_FNeg(m_Value(X))) && match(Arg1, m_FNeg(m_Value(Y))) &&
1462  (Arg0->hasOneUse() || Arg1->hasOneUse())) {
1463  // If both operands are negated, invert the call and negate the result:
1464  // min(-X, -Y) --> -(max(X, Y))
1465  // max(-X, -Y) --> -(min(X, Y))
1466  Intrinsic::ID NewIID;
1467  switch (IID) {
1468  case Intrinsic::maxnum:
1469  NewIID = Intrinsic::minnum;
1470  break;
1471  case Intrinsic::minnum:
1472  NewIID = Intrinsic::maxnum;
1473  break;
1474  case Intrinsic::maximum:
1475  NewIID = Intrinsic::minimum;
1476  break;
1477  case Intrinsic::minimum:
1478  NewIID = Intrinsic::maximum;
1479  break;
1480  default:
1481  llvm_unreachable("unexpected intrinsic ID");
1482  }
1483  Value *NewCall = Builder.CreateBinaryIntrinsic(NewIID, X, Y, II);
1484  Instruction *FNeg = UnaryOperator::CreateFNeg(NewCall);
1485  FNeg->copyIRFlags(II);
1486  return FNeg;
1487  }
1488 
1489  // m(m(X, C2), C1) -> m(X, C)
1490  const APFloat *C1, *C2;
1491  if (auto *M = dyn_cast<IntrinsicInst>(Arg0)) {
1492  if (M->getIntrinsicID() == IID && match(Arg1, m_APFloat(C1)) &&
1493  ((match(M->getArgOperand(0), m_Value(X)) &&
1494  match(M->getArgOperand(1), m_APFloat(C2))) ||
1495  (match(M->getArgOperand(1), m_Value(X)) &&
1496  match(M->getArgOperand(0), m_APFloat(C2))))) {
1497  APFloat Res(0.0);
1498  switch (IID) {
1499  case Intrinsic::maxnum:
1500  Res = maxnum(*C1, *C2);
1501  break;
1502  case Intrinsic::minnum:
1503  Res = minnum(*C1, *C2);
1504  break;
1505  case Intrinsic::maximum:
1506  Res = maximum(*C1, *C2);
1507  break;
1508  case Intrinsic::minimum:
1509  Res = minimum(*C1, *C2);
1510  break;
1511  default:
1512  llvm_unreachable("unexpected intrinsic ID");
1513  }
1514  Instruction *NewCall = Builder.CreateBinaryIntrinsic(
1515  IID, X, ConstantFP::get(Arg0->getType(), Res), II);
1516  // TODO: Conservatively intersecting FMF. If Res == C2, the transform
1517  // was a simplification (so Arg0 and its original flags could
1518  // propagate?)
1519  NewCall->andIRFlags(M);
1520  return replaceInstUsesWith(*II, NewCall);
1521  }
1522  }
1523 
1524  // m((fpext X), (fpext Y)) -> fpext (m(X, Y))
1525  if (match(Arg0, m_OneUse(m_FPExt(m_Value(X)))) &&
1526  match(Arg1, m_OneUse(m_FPExt(m_Value(Y)))) &&
1527  X->getType() == Y->getType()) {
1528  Value *NewCall =
1529  Builder.CreateBinaryIntrinsic(IID, X, Y, II, II->getName());
1530  return new FPExtInst(NewCall, II->getType());
1531  }
1532 
1533  // max X, -X --> fabs X
1534  // min X, -X --> -(fabs X)
1535  // TODO: Remove one-use limitation? That is obviously better for max.
1536  // It would be an extra instruction for min (fnabs), but that is
1537  // still likely better for analysis and codegen.
1538  if ((match(Arg0, m_OneUse(m_FNeg(m_Value(X)))) && Arg1 == X) ||
1539  (match(Arg1, m_OneUse(m_FNeg(m_Value(X)))) && Arg0 == X)) {
1540  Value *R = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, X, II);
1541  if (IID == Intrinsic::minimum || IID == Intrinsic::minnum)
1542  R = Builder.CreateFNegFMF(R, II);
1543  return replaceInstUsesWith(*II, R);
1544  }
1545 
1546  break;
1547  }
1548  case Intrinsic::fmuladd: {
1549  // Canonicalize fast fmuladd to the separate fmul + fadd.
1550  if (II->isFast()) {
1552  Builder.setFastMathFlags(II->getFastMathFlags());
1553  Value *Mul = Builder.CreateFMul(II->getArgOperand(0),
1554  II->getArgOperand(1));
1555  Value *Add = Builder.CreateFAdd(Mul, II->getArgOperand(2));
1556  Add->takeName(II);
1557  return replaceInstUsesWith(*II, Add);
1558  }
1559 
1560  // Try to simplify the underlying FMul.
1561  if (Value *V = SimplifyFMulInst(II->getArgOperand(0), II->getArgOperand(1),
1562  II->getFastMathFlags(),
1563  SQ.getWithInstruction(II))) {
1564  auto *FAdd = BinaryOperator::CreateFAdd(V, II->getArgOperand(2));
1565  FAdd->copyFastMathFlags(II);
1566  return FAdd;
1567  }
1568 
1570  }
1571  case Intrinsic::fma: {
1572  // fma fneg(x), fneg(y), z -> fma x, y, z
1573  Value *Src0 = II->getArgOperand(0);
1574  Value *Src1 = II->getArgOperand(1);
1575  Value *X, *Y;
1576  if (match(Src0, m_FNeg(m_Value(X))) && match(Src1, m_FNeg(m_Value(Y)))) {
1577  replaceOperand(*II, 0, X);
1578  replaceOperand(*II, 1, Y);
1579  return II;
1580  }
1581 
1582  // fma fabs(x), fabs(x), z -> fma x, x, z
1583  if (match(Src0, m_FAbs(m_Value(X))) &&
1584  match(Src1, m_FAbs(m_Specific(X)))) {
1585  replaceOperand(*II, 0, X);
1586  replaceOperand(*II, 1, X);
1587  return II;
1588  }
1589 
1590  // Try to simplify the underlying FMul. We can only apply simplifications
1591  // that do not require rounding.
1592  if (Value *V = SimplifyFMAFMul(II->getArgOperand(0), II->getArgOperand(1),
1593  II->getFastMathFlags(),
1594  SQ.getWithInstruction(II))) {
1595  auto *FAdd = BinaryOperator::CreateFAdd(V, II->getArgOperand(2));
1596  FAdd->copyFastMathFlags(II);
1597  return FAdd;
1598  }
1599 
1600  // fma x, y, 0 -> fmul x, y
1601  // This is always valid for -0.0, but requires nsz for +0.0 as
1602  // -0.0 + 0.0 = 0.0, which would not be the same as the fmul on its own.
1603  if (match(II->getArgOperand(2), m_NegZeroFP()) ||
1604  (match(II->getArgOperand(2), m_PosZeroFP()) &&
1606  return BinaryOperator::CreateFMulFMF(Src0, Src1, II);
1607 
1608  break;
1609  }
1610  case Intrinsic::copysign: {
1611  Value *Mag = II->getArgOperand(0), *Sign = II->getArgOperand(1);
1612  if (SignBitMustBeZero(Sign, &TLI)) {
1613  // If we know that the sign argument is positive, reduce to FABS:
1614  // copysign Mag, +Sign --> fabs Mag
1615  Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);
1616  return replaceInstUsesWith(*II, Fabs);
1617  }
1618  // TODO: There should be a ValueTracking sibling like SignBitMustBeOne.
1619  const APFloat *C;
1620  if (match(Sign, m_APFloat(C)) && C->isNegative()) {
1621  // If we know that the sign argument is negative, reduce to FNABS:
1622  // copysign Mag, -Sign --> fneg (fabs Mag)
1623  Value *Fabs = Builder.CreateUnaryIntrinsic(Intrinsic::fabs, Mag, II);
1624  return replaceInstUsesWith(*II, Builder.CreateFNegFMF(Fabs, II));
1625  }
1626 
1627  // Propagate sign argument through nested calls:
1628  // copysign Mag, (copysign ?, X) --> copysign Mag, X
1629  Value *X;
1630  if (match(Sign, m_Intrinsic<Intrinsic::copysign>(m_Value(), m_Value(X))))
1631  return replaceOperand(*II, 1, X);
1632 
1633  // Peek through changes of magnitude's sign-bit. This call rewrites those:
1634  // copysign (fabs X), Sign --> copysign X, Sign
1635  // copysign (fneg X), Sign --> copysign X, Sign
1636  if (match(Mag, m_FAbs(m_Value(X))) || match(Mag, m_FNeg(m_Value(X))))
1637  return replaceOperand(*II, 0, X);
1638 
1639  break;
1640  }
1641  case Intrinsic::fabs: {
1642  Value *Cond, *TVal, *FVal;
1643  if (match(II->getArgOperand(0),
1644  m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal)))) {
1645  // fabs (select Cond, TrueC, FalseC) --> select Cond, AbsT, AbsF
1646  if (isa<Constant>(TVal) && isa<Constant>(FVal)) {
1647  CallInst *AbsT = Builder.CreateCall(II->getCalledFunction(), {TVal});
1648  CallInst *AbsF = Builder.CreateCall(II->getCalledFunction(), {FVal});
1649  return SelectInst::Create(Cond, AbsT, AbsF);
1650  }
1651  // fabs (select Cond, -FVal, FVal) --> fabs FVal
1652  if (match(TVal, m_FNeg(m_Specific(FVal))))
1653  return replaceOperand(*II, 0, FVal);
1654  // fabs (select Cond, TVal, -TVal) --> fabs TVal
1655  if (match(FVal, m_FNeg(m_Specific(TVal))))
1656  return replaceOperand(*II, 0, TVal);
1657  }
1658 
1660  }
1661  case Intrinsic::ceil:
1662  case Intrinsic::floor:
1663  case Intrinsic::round:
1664  case Intrinsic::roundeven:
1665  case Intrinsic::nearbyint:
1666  case Intrinsic::rint:
1667  case Intrinsic::trunc: {
1668  Value *ExtSrc;
1669  if (match(II->getArgOperand(0), m_OneUse(m_FPExt(m_Value(ExtSrc))))) {
1670  // Narrow the call: intrinsic (fpext x) -> fpext (intrinsic x)
1671  Value *NarrowII = Builder.CreateUnaryIntrinsic(IID, ExtSrc, II);
1672  return new FPExtInst(NarrowII, II->getType());
1673  }
1674  break;
1675  }
1676  case Intrinsic::cos:
1677  case Intrinsic::amdgcn_cos: {
1678  Value *X;
1679  Value *Src = II->getArgOperand(0);
1680  if (match(Src, m_FNeg(m_Value(X))) || match(Src, m_FAbs(m_Value(X)))) {
1681  // cos(-x) -> cos(x)
1682  // cos(fabs(x)) -> cos(x)
1683  return replaceOperand(*II, 0, X);
1684  }
1685  break;
1686  }
1687  case Intrinsic::sin: {
1688  Value *X;
1689  if (match(II->getArgOperand(0), m_OneUse(m_FNeg(m_Value(X))))) {
1690  // sin(-x) --> -sin(x)
1691  Value *NewSin = Builder.CreateUnaryIntrinsic(Intrinsic::sin, X, II);
1692  Instruction *FNeg = UnaryOperator::CreateFNeg(NewSin);
1693  FNeg->copyFastMathFlags(II);
1694  return FNeg;
1695  }
1696  break;
1697  }
1698 
1699  case Intrinsic::arm_neon_vtbl1:
1700  case Intrinsic::aarch64_neon_tbl1:
1701  if (Value *V = simplifyNeonTbl1(*II, Builder))
1702  return replaceInstUsesWith(*II, V);
1703  break;
1704 
1705  case Intrinsic::arm_neon_vmulls:
1706  case Intrinsic::arm_neon_vmullu:
1707  case Intrinsic::aarch64_neon_smull:
1708  case Intrinsic::aarch64_neon_umull: {
1709  Value *Arg0 = II->getArgOperand(0);
1710  Value *Arg1 = II->getArgOperand(1);
1711 
1712  // Handle mul by zero first:
1713  if (isa<ConstantAggregateZero>(Arg0) || isa<ConstantAggregateZero>(Arg1)) {
1714  return replaceInstUsesWith(CI, ConstantAggregateZero::get(II->getType()));
1715  }
1716 
1717  // Check for constant LHS & RHS - in this case we just simplify.
1718  bool Zext = (IID == Intrinsic::arm_neon_vmullu ||
1719  IID == Intrinsic::aarch64_neon_umull);
1720  VectorType *NewVT = cast<VectorType>(II->getType());
1721  if (Constant *CV0 = dyn_cast<Constant>(Arg0)) {
1722  if (Constant *CV1 = dyn_cast<Constant>(Arg1)) {
1723  CV0 = ConstantExpr::getIntegerCast(CV0, NewVT, /*isSigned=*/!Zext);
1724  CV1 = ConstantExpr::getIntegerCast(CV1, NewVT, /*isSigned=*/!Zext);
1725 
1726  return replaceInstUsesWith(CI, ConstantExpr::getMul(CV0, CV1));
1727  }
1728 
1729  // Couldn't simplify - canonicalize constant to the RHS.
1730  std::swap(Arg0, Arg1);
1731  }
1732 
1733  // Handle mul by one:
1734  if (Constant *CV1 = dyn_cast<Constant>(Arg1))
1735  if (ConstantInt *Splat =
1736  dyn_cast_or_null<ConstantInt>(CV1->getSplatValue()))
1737  if (Splat->isOne())
1738  return CastInst::CreateIntegerCast(Arg0, II->getType(),
1739  /*isSigned=*/!Zext);
1740 
1741  break;
1742  }
1743  case Intrinsic::arm_neon_aesd:
1744  case Intrinsic::arm_neon_aese:
1745  case Intrinsic::aarch64_crypto_aesd:
1746  case Intrinsic::aarch64_crypto_aese: {
1747  Value *DataArg = II->getArgOperand(0);
1748  Value *KeyArg = II->getArgOperand(1);
1749 
1750  // Try to use the builtin XOR in AESE and AESD to eliminate a prior XOR
1751  Value *Data, *Key;
1752  if (match(KeyArg, m_ZeroInt()) &&
1753  match(DataArg, m_Xor(m_Value(Data), m_Value(Key)))) {
1754  replaceOperand(*II, 0, Data);
1755  replaceOperand(*II, 1, Key);
1756  return II;
1757  }
1758  break;
1759  }
1760  case Intrinsic::hexagon_V6_vandvrt:
1761  case Intrinsic::hexagon_V6_vandvrt_128B: {
1762  // Simplify Q -> V -> Q conversion.
1763  if (auto Op0 = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
1764  Intrinsic::ID ID0 = Op0->getIntrinsicID();
1765  if (ID0 != Intrinsic::hexagon_V6_vandqrt &&
1766  ID0 != Intrinsic::hexagon_V6_vandqrt_128B)
1767  break;
1768  Value *Bytes = Op0->getArgOperand(1), *Mask = II->getArgOperand(1);
1769  uint64_t Bytes1 = computeKnownBits(Bytes, 0, Op0).One.getZExtValue();
1770  uint64_t Mask1 = computeKnownBits(Mask, 0, II).One.getZExtValue();
1771  // Check if every byte has common bits in Bytes and Mask.
1772  uint64_t C = Bytes1 & Mask1;
1773  if ((C & 0xFF) && (C & 0xFF00) && (C & 0xFF0000) && (C & 0xFF000000))
1774  return replaceInstUsesWith(*II, Op0->getArgOperand(0));
1775  }
1776  break;
1777  }
1778  case Intrinsic::stackrestore: {
1779  // If the save is right next to the restore, remove the restore. This can
1780  // happen when variable allocas are DCE'd.
1781  if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) {
1782  if (SS->getIntrinsicID() == Intrinsic::stacksave) {
1783  // Skip over debug info.
1784  if (SS->getNextNonDebugInstruction() == II) {
1785  return eraseInstFromFunction(CI);
1786  }
1787  }
1788  }
1789 
1790  // Scan down this block to see if there is another stack restore in the
1791  // same block without an intervening call/alloca.
1792  BasicBlock::iterator BI(II);
1793  Instruction *TI = II->getParent()->getTerminator();
1794  bool CannotRemove = false;
1795  for (++BI; &*BI != TI; ++BI) {
1796  if (isa<AllocaInst>(BI)) {
1797  CannotRemove = true;
1798  break;
1799  }
1800  if (CallInst *BCI = dyn_cast<CallInst>(BI)) {
1801  if (auto *II2 = dyn_cast<IntrinsicInst>(BCI)) {
1802  // If there is a stackrestore below this one, remove this one.
1803  if (II2->getIntrinsicID() == Intrinsic::stackrestore)
1804  return eraseInstFromFunction(CI);
1805 
1806  // Bail if we cross over an intrinsic with side effects, such as
1807  // llvm.stacksave, or llvm.read_register.
1808  if (II2->mayHaveSideEffects()) {
1809  CannotRemove = true;
1810  break;
1811  }
1812  } else {
1813  // If we found a non-intrinsic call, we can't remove the stack
1814  // restore.
1815  CannotRemove = true;
1816  break;
1817  }
1818  }
1819  }
1820 
1821  // If the stack restore is in a return, resume, or unwind block and if there
1822  // are no allocas or calls between the restore and the return, nuke the
1823  // restore.
1824  if (!CannotRemove && (isa<ReturnInst>(TI) || isa<ResumeInst>(TI)))
1825  return eraseInstFromFunction(CI);
1826  break;
1827  }
1828  case Intrinsic::lifetime_end:
1829  // Asan needs to poison memory to detect invalid access which is possible
1830  // even for empty lifetime range.
1831  if (II->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
1832  II->getFunction()->hasFnAttribute(Attribute::SanitizeMemory) ||
1833  II->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
1834  break;
1835 
1836  if (removeTriviallyEmptyRange(*II, *this, [](const IntrinsicInst &I) {
1837  return I.getIntrinsicID() == Intrinsic::lifetime_start;
1838  }))
1839  return nullptr;
1840  break;
1841  case Intrinsic::assume: {
1842  Value *IIOperand = II->getArgOperand(0);
1844  II->getOperandBundlesAsDefs(OpBundles);
1845 
1846  /// This will remove the boolean Condition from the assume given as
1847  /// argument and remove the assume if it becomes useless.
1848  /// always returns nullptr for use as a return values.
1849  auto RemoveConditionFromAssume = [&](Instruction *Assume) -> Instruction * {
1850  assert(isa<AssumeInst>(Assume));
1851  if (isAssumeWithEmptyBundle(*cast<AssumeInst>(II)))
1852  return eraseInstFromFunction(CI);
1853  replaceUse(II->getOperandUse(0), ConstantInt::getTrue(II->getContext()));
1854  return nullptr;
1855  };
1856  // Remove an assume if it is followed by an identical assume.
1857  // TODO: Do we need this? Unless there are conflicting assumptions, the
1858  // computeKnownBits(IIOperand) below here eliminates redundant assumes.
1860  if (match(Next, m_Intrinsic<Intrinsic::assume>(m_Specific(IIOperand))))
1861  return RemoveConditionFromAssume(Next);
1862 
1863  // Canonicalize assume(a && b) -> assume(a); assume(b);
1864  // Note: New assumption intrinsics created here are registered by
1865  // the InstCombineIRInserter object.
1866  FunctionType *AssumeIntrinsicTy = II->getFunctionType();
1867  Value *AssumeIntrinsic = II->getCalledOperand();
1868  Value *A, *B;
1869  if (match(IIOperand, m_LogicalAnd(m_Value(A), m_Value(B)))) {
1870  Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, A, OpBundles,
1871  II->getName());
1872  Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic, B, II->getName());
1873  return eraseInstFromFunction(*II);
1874  }
1875  // assume(!(a || b)) -> assume(!a); assume(!b);
1876  if (match(IIOperand, m_Not(m_LogicalOr(m_Value(A), m_Value(B))))) {
1877  Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
1878  Builder.CreateNot(A), OpBundles, II->getName());
1879  Builder.CreateCall(AssumeIntrinsicTy, AssumeIntrinsic,
1880  Builder.CreateNot(B), II->getName());
1881  return eraseInstFromFunction(*II);
1882  }
1883 
1884  // assume( (load addr) != null ) -> add 'nonnull' metadata to load
1885  // (if assume is valid at the load)
1886  CmpInst::Predicate Pred;
1887  Instruction *LHS;
1888  if (match(IIOperand, m_ICmp(Pred, m_Instruction(LHS), m_Zero())) &&
1889  Pred == ICmpInst::ICMP_NE && LHS->getOpcode() == Instruction::Load &&
1890  LHS->getType()->isPointerTy() &&
1891  isValidAssumeForContext(II, LHS, &DT)) {
1892  MDNode *MD = MDNode::get(II->getContext(), None);
1893  LHS->setMetadata(LLVMContext::MD_nonnull, MD);
1894  return RemoveConditionFromAssume(II);
1895 
1896  // TODO: apply nonnull return attributes to calls and invokes
1897  // TODO: apply range metadata for range check patterns?
1898  }
1899 
1900  // Convert nonnull assume like:
1901  // %A = icmp ne i32* %PTR, null
1902  // call void @llvm.assume(i1 %A)
1903  // into
1904  // call void @llvm.assume(i1 true) [ "nonnull"(i32* %PTR) ]
1906  match(IIOperand, m_Cmp(Pred, m_Value(A), m_Zero())) &&
1907  Pred == CmpInst::ICMP_NE && A->getType()->isPointerTy()) {
1908  if (auto *Replacement = buildAssumeFromKnowledge(
1909  {RetainedKnowledge{Attribute::NonNull, 0, A}}, Next, &AC, &DT)) {
1910 
1911  Replacement->insertBefore(Next);
1912  AC.registerAssumption(Replacement);
1913  return RemoveConditionFromAssume(II);
1914  }
1915  }
1916 
1917  // Convert alignment assume like:
1918  // %B = ptrtoint i32* %A to i64
1919  // %C = and i64 %B, Constant
1920  // %D = icmp eq i64 %C, 0
1921  // call void @llvm.assume(i1 %D)
1922  // into
1923  // call void @llvm.assume(i1 true) [ "align"(i32* [[A]], i64 Constant + 1)]
1924  uint64_t AlignMask;
1926  match(IIOperand,
1927  m_Cmp(Pred, m_And(m_Value(A), m_ConstantInt(AlignMask)),
1928  m_Zero())) &&
1929  Pred == CmpInst::ICMP_EQ) {
1930  if (isPowerOf2_64(AlignMask + 1)) {
1931  uint64_t Offset = 0;
1933  if (match(A, m_PtrToInt(m_Value(A)))) {
1934  /// Note: this doesn't preserve the offset information but merges
1935  /// offset and alignment.
1936  /// TODO: we can generate a GEP instead of merging the alignment with
1937  /// the offset.
1938  RetainedKnowledge RK{Attribute::Alignment,
1939  (unsigned)MinAlign(Offset, AlignMask + 1), A};
1940  if (auto *Replacement =
1941  buildAssumeFromKnowledge(RK, Next, &AC, &DT)) {
1942 
1943  Replacement->insertAfter(II);
1944  AC.registerAssumption(Replacement);
1945  }
1946  return RemoveConditionFromAssume(II);
1947  }
1948  }
1949  }
1950 
1951  /// Canonicalize Knowledge in operand bundles.
1953  for (unsigned Idx = 0; Idx < II->getNumOperandBundles(); Idx++) {
1954  auto &BOI = II->bundle_op_info_begin()[Idx];
1955  RetainedKnowledge RK =
1956  llvm::getKnowledgeFromBundle(cast<AssumeInst>(*II), BOI);
1957  if (BOI.End - BOI.Begin > 2)
1958  continue; // Prevent reducing knowledge in an align with offset since
1959  // extracting a RetainedKnowledge form them looses offset
1960  // information
1961  RetainedKnowledge CanonRK =
1962  llvm::simplifyRetainedKnowledge(cast<AssumeInst>(II), RK,
1963  &getAssumptionCache(),
1964  &getDominatorTree());
1965  if (CanonRK == RK)
1966  continue;
1967  if (!CanonRK) {
1968  if (BOI.End - BOI.Begin > 0) {
1969  Worklist.pushValue(II->op_begin()[BOI.Begin]);
1970  Value::dropDroppableUse(II->op_begin()[BOI.Begin]);
1971  }
1972  continue;
1973  }
1974  assert(RK.AttrKind == CanonRK.AttrKind);
1975  if (BOI.End - BOI.Begin > 0)
1976  II->op_begin()[BOI.Begin].set(CanonRK.WasOn);
1977  if (BOI.End - BOI.Begin > 1)
1978  II->op_begin()[BOI.Begin + 1].set(ConstantInt::get(
1979  Type::getInt64Ty(II->getContext()), CanonRK.ArgValue));
1980  if (RK.WasOn)
1981  Worklist.pushValue(RK.WasOn);
1982  return II;
1983  }
1984  }
1985 
1986  // If there is a dominating assume with the same condition as this one,
1987  // then this one is redundant, and should be removed.
1988  KnownBits Known(1);
1989  computeKnownBits(IIOperand, Known, 0, II);
1990  if (Known.isAllOnes() && isAssumeWithEmptyBundle(cast<AssumeInst>(*II)))
1991  return eraseInstFromFunction(*II);
1992 
1993  // Update the cache of affected values for this assumption (we might be
1994  // here because we just simplified the condition).
1995  AC.updateAffectedValues(cast<AssumeInst>(II));
1996  break;
1997  }
1998  case Intrinsic::experimental_guard: {
1999  // Is this guard followed by another guard? We scan forward over a small
2000  // fixed window of instructions to handle common cases with conditions
2001  // computed between guards.
2002  Instruction *NextInst = II->getNextNonDebugInstruction();
2003  for (unsigned i = 0; i < GuardWideningWindow; i++) {
2004  // Note: Using context-free form to avoid compile time blow up
2005  if (!isSafeToSpeculativelyExecute(NextInst))
2006  break;
2007  NextInst = NextInst->getNextNonDebugInstruction();
2008  }
2009  Value *NextCond = nullptr;
2010  if (match(NextInst,
2011  m_Intrinsic<Intrinsic::experimental_guard>(m_Value(NextCond)))) {
2012  Value *CurrCond = II->getArgOperand(0);
2013 
2014  // Remove a guard that it is immediately preceded by an identical guard.
2015  // Otherwise canonicalize guard(a); guard(b) -> guard(a & b).
2016  if (CurrCond != NextCond) {
2017  Instruction *MoveI = II->getNextNonDebugInstruction();
2018  while (MoveI != NextInst) {
2019  auto *Temp = MoveI;
2020  MoveI = MoveI->getNextNonDebugInstruction();
2021  Temp->moveBefore(II);
2022  }
2023  replaceOperand(*II, 0, Builder.CreateAnd(CurrCond, NextCond));
2024  }
2025  eraseInstFromFunction(*NextInst);
2026  return II;
2027  }
2028  break;
2029  }
2030  case Intrinsic::experimental_vector_insert: {
2031  Value *Vec = II->getArgOperand(0);
2032  Value *SubVec = II->getArgOperand(1);
2033  Value *Idx = II->getArgOperand(2);
2034  auto *DstTy = dyn_cast<FixedVectorType>(II->getType());
2035  auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
2036  auto *SubVecTy = dyn_cast<FixedVectorType>(SubVec->getType());
2037 
2038  // Only canonicalize if the destination vector, Vec, and SubVec are all
2039  // fixed vectors.
2040  if (DstTy && VecTy && SubVecTy) {
2041  unsigned DstNumElts = DstTy->getNumElements();
2042  unsigned VecNumElts = VecTy->getNumElements();
2043  unsigned SubVecNumElts = SubVecTy->getNumElements();
2044  unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
2045 
2046  // An insert that entirely overwrites Vec with SubVec is a nop.
2047  if (VecNumElts == SubVecNumElts)
2048  return replaceInstUsesWith(CI, SubVec);
2049 
2050  // Widen SubVec into a vector of the same width as Vec, since
2051  // shufflevector requires the two input vectors to be the same width.
2052  // Elements beyond the bounds of SubVec within the widened vector are
2053  // undefined.
2054  SmallVector<int, 8> WidenMask;
2055  unsigned i;
2056  for (i = 0; i != SubVecNumElts; ++i)
2057  WidenMask.push_back(i);
2058  for (; i != VecNumElts; ++i)
2059  WidenMask.push_back(UndefMaskElem);
2060 
2061  Value *WidenShuffle = Builder.CreateShuffleVector(SubVec, WidenMask);
2062 
2064  for (unsigned i = 0; i != IdxN; ++i)
2065  Mask.push_back(i);
2066  for (unsigned i = DstNumElts; i != DstNumElts + SubVecNumElts; ++i)
2067  Mask.push_back(i);
2068  for (unsigned i = IdxN + SubVecNumElts; i != DstNumElts; ++i)
2069  Mask.push_back(i);
2070 
2071  Value *Shuffle = Builder.CreateShuffleVector(Vec, WidenShuffle, Mask);
2072  return replaceInstUsesWith(CI, Shuffle);
2073  }
2074  break;
2075  }
2076  case Intrinsic::experimental_vector_extract: {
2077  Value *Vec = II->getArgOperand(0);
2078  Value *Idx = II->getArgOperand(1);
2079 
2080  auto *DstTy = dyn_cast<FixedVectorType>(II->getType());
2081  auto *VecTy = dyn_cast<FixedVectorType>(Vec->getType());
2082 
2083  // Only canonicalize if the the destination vector and Vec are fixed
2084  // vectors.
2085  if (DstTy && VecTy) {
2086  unsigned DstNumElts = DstTy->getNumElements();
2087  unsigned VecNumElts = VecTy->getNumElements();
2088  unsigned IdxN = cast<ConstantInt>(Idx)->getZExtValue();
2089 
2090  // Extracting the entirety of Vec is a nop.
2091  if (VecNumElts == DstNumElts) {
2092  replaceInstUsesWith(CI, Vec);
2093  return eraseInstFromFunction(CI);
2094  }
2095 
2097  for (unsigned i = 0; i != DstNumElts; ++i)
2098  Mask.push_back(IdxN + i);
2099 
2100  Value *Shuffle = Builder.CreateShuffleVector(Vec, Mask);
2101  return replaceInstUsesWith(CI, Shuffle);
2102  }
2103  break;
2104  }
2105  case Intrinsic::experimental_vector_reverse: {
2106  Value *BO0, *BO1, *X, *Y;
2107  Value *Vec = II->getArgOperand(0);
2108  if (match(Vec, m_OneUse(m_BinOp(m_Value(BO0), m_Value(BO1))))) {
2109  auto *OldBinOp = cast<BinaryOperator>(Vec);
2110  if (match(BO0, m_Intrinsic<Intrinsic::experimental_vector_reverse>(
2111  m_Value(X)))) {
2112  // rev(binop rev(X), rev(Y)) --> binop X, Y
2113  if (match(BO1, m_Intrinsic<Intrinsic::experimental_vector_reverse>(
2114  m_Value(Y))))
2115  return replaceInstUsesWith(CI,
2117  OldBinOp->getOpcode(), X, Y, OldBinOp,
2118  OldBinOp->getName(), II));
2119  // rev(binop rev(X), BO1Splat) --> binop X, BO1Splat
2120  if (isSplatValue(BO1))
2121  return replaceInstUsesWith(CI,
2123  OldBinOp->getOpcode(), X, BO1,
2124  OldBinOp, OldBinOp->getName(), II));
2125  }
2126  // rev(binop BO0Splat, rev(Y)) --> binop BO0Splat, Y
2127  if (match(BO1, m_Intrinsic<Intrinsic::experimental_vector_reverse>(
2128  m_Value(Y))) &&
2129  isSplatValue(BO0))
2130  return replaceInstUsesWith(CI, BinaryOperator::CreateWithCopiedFlags(
2131  OldBinOp->getOpcode(), BO0, Y,
2132  OldBinOp, OldBinOp->getName(), II));
2133  }
2134  // rev(unop rev(X)) --> unop X
2135  if (match(Vec, m_OneUse(m_UnOp(
2136  m_Intrinsic<Intrinsic::experimental_vector_reverse>(
2137  m_Value(X)))))) {
2138  auto *OldUnOp = cast<UnaryOperator>(Vec);
2139  auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags(
2140  OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(), II);
2141  return replaceInstUsesWith(CI, NewUnOp);
2142  }
2143  break;
2144  }
2145  case Intrinsic::vector_reduce_or:
2146  case Intrinsic::vector_reduce_and: {
2147  // Canonicalize logical or/and reductions:
2148  // Or reduction for i1 is represented as:
2149  // %val = bitcast <ReduxWidth x i1> to iReduxWidth
2150  // %res = cmp ne iReduxWidth %val, 0
2151  // And reduction for i1 is represented as:
2152  // %val = bitcast <ReduxWidth x i1> to iReduxWidth
2153  // %res = cmp eq iReduxWidth %val, 11111
2154  Value *Arg = II->getArgOperand(0);
2155  Value *Vect;
2156  if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
2157  if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
2158  if (FTy->getElementType() == Builder.getInt1Ty()) {
2159  Value *Res = Builder.CreateBitCast(
2160  Vect, Builder.getIntNTy(FTy->getNumElements()));
2161  if (IID == Intrinsic::vector_reduce_and) {
2162  Res = Builder.CreateICmpEQ(
2163  Res, ConstantInt::getAllOnesValue(Res->getType()));
2164  } else {
2165  assert(IID == Intrinsic::vector_reduce_or &&
2166  "Expected or reduction.");
2167  Res = Builder.CreateIsNotNull(Res);
2168  }
2169  if (Arg != Vect)
2170  Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
2171  II->getType());
2172  return replaceInstUsesWith(CI, Res);
2173  }
2174  }
2176  }
2177  case Intrinsic::vector_reduce_add: {
2178  if (IID == Intrinsic::vector_reduce_add) {
2179  // Convert vector_reduce_add(ZExt(<n x i1>)) to
2180  // ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
2181  // Convert vector_reduce_add(SExt(<n x i1>)) to
2182  // -ZExtOrTrunc(ctpop(bitcast <n x i1> to in)).
2183  // Convert vector_reduce_add(<n x i1>) to
2184  // Trunc(ctpop(bitcast <n x i1> to in)).
2185  Value *Arg = II->getArgOperand(0);
2186  Value *Vect;
2187  if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
2188  if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
2189  if (FTy->getElementType() == Builder.getInt1Ty()) {
2190  Value *V = Builder.CreateBitCast(
2191  Vect, Builder.getIntNTy(FTy->getNumElements()));
2192  Value *Res = Builder.CreateUnaryIntrinsic(Intrinsic::ctpop, V);
2193  if (Res->getType() != II->getType())
2194  Res = Builder.CreateZExtOrTrunc(Res, II->getType());
2195  if (Arg != Vect &&
2196  cast<Instruction>(Arg)->getOpcode() == Instruction::SExt)
2197  Res = Builder.CreateNeg(Res);
2198  return replaceInstUsesWith(CI, Res);
2199  }
2200  }
2201  }
2203  }
2204  case Intrinsic::vector_reduce_xor: {
2205  if (IID == Intrinsic::vector_reduce_xor) {
2206  // Exclusive disjunction reduction over the vector with
2207  // (potentially-extended) i1 element type is actually a
2208  // (potentially-extended) arithmetic `add` reduction over the original
2209  // non-extended value:
2210  // vector_reduce_xor(?ext(<n x i1>))
2211  // -->
2212  // ?ext(vector_reduce_add(<n x i1>))
2213  Value *Arg = II->getArgOperand(0);
2214  Value *Vect;
2215  if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
2216  if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
2217  if (FTy->getElementType() == Builder.getInt1Ty()) {
2218  Value *Res = Builder.CreateAddReduce(Vect);
2219  if (Arg != Vect)
2220  Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
2221  II->getType());
2222  return replaceInstUsesWith(CI, Res);
2223  }
2224  }
2225  }
2227  }
2228  case Intrinsic::vector_reduce_mul: {
2229  if (IID == Intrinsic::vector_reduce_mul) {
2230  // Multiplicative reduction over the vector with (potentially-extended)
2231  // i1 element type is actually a (potentially zero-extended)
2232  // logical `and` reduction over the original non-extended value:
2233  // vector_reduce_mul(?ext(<n x i1>))
2234  // -->
2235  // zext(vector_reduce_and(<n x i1>))
2236  Value *Arg = II->getArgOperand(0);
2237  Value *Vect;
2238  if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
2239  if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
2240  if (FTy->getElementType() == Builder.getInt1Ty()) {
2241  Value *Res = Builder.CreateAndReduce(Vect);
2242  if (Res->getType() != II->getType())
2243  Res = Builder.CreateZExt(Res, II->getType());
2244  return replaceInstUsesWith(CI, Res);
2245  }
2246  }
2247  }
2249  }
2250  case Intrinsic::vector_reduce_umin:
2251  case Intrinsic::vector_reduce_umax: {
2252  if (IID == Intrinsic::vector_reduce_umin ||
2253  IID == Intrinsic::vector_reduce_umax) {
2254  // UMin/UMax reduction over the vector with (potentially-extended)
2255  // i1 element type is actually a (potentially-extended)
2256  // logical `and`/`or` reduction over the original non-extended value:
2257  // vector_reduce_u{min,max}(?ext(<n x i1>))
2258  // -->
2259  // ?ext(vector_reduce_{and,or}(<n x i1>))
2260  Value *Arg = II->getArgOperand(0);
2261  Value *Vect;
2262  if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
2263  if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
2264  if (FTy->getElementType() == Builder.getInt1Ty()) {
2265  Value *Res = IID == Intrinsic::vector_reduce_umin
2266  ? Builder.CreateAndReduce(Vect)
2267  : Builder.CreateOrReduce(Vect);
2268  if (Arg != Vect)
2269  Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res,
2270  II->getType());
2271  return replaceInstUsesWith(CI, Res);
2272  }
2273  }
2274  }
2276  }
2277  case Intrinsic::vector_reduce_smin:
2278  case Intrinsic::vector_reduce_smax: {
2279  if (IID == Intrinsic::vector_reduce_smin ||
2280  IID == Intrinsic::vector_reduce_smax) {
2281  // SMin/SMax reduction over the vector with (potentially-extended)
2282  // i1 element type is actually a (potentially-extended)
2283  // logical `and`/`or` reduction over the original non-extended value:
2284  // vector_reduce_s{min,max}(<n x i1>)
2285  // -->
2286  // vector_reduce_{or,and}(<n x i1>)
2287  // and
2288  // vector_reduce_s{min,max}(sext(<n x i1>))
2289  // -->
2290  // sext(vector_reduce_{or,and}(<n x i1>))
2291  // and
2292  // vector_reduce_s{min,max}(zext(<n x i1>))
2293  // -->
2294  // zext(vector_reduce_{and,or}(<n x i1>))
2295  Value *Arg = II->getArgOperand(0);
2296  Value *Vect;
2297  if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) {
2298  if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType()))
2299  if (FTy->getElementType() == Builder.getInt1Ty()) {
2300  Instruction::CastOps ExtOpc = Instruction::CastOps::CastOpsEnd;
2301  if (Arg != Vect)
2302  ExtOpc = cast<CastInst>(Arg)->getOpcode();
2303  Value *Res = ((IID == Intrinsic::vector_reduce_smin) ==
2304  (ExtOpc == Instruction::CastOps::ZExt))
2305  ? Builder.CreateAndReduce(Vect)
2306  : Builder.CreateOrReduce(Vect);
2307  if (Arg != Vect)
2308  Res = Builder.CreateCast(ExtOpc, Res, II->getType());
2309  return replaceInstUsesWith(CI, Res);
2310  }
2311  }
2312  }
2314  }
2315  case Intrinsic::vector_reduce_fmax:
2316  case Intrinsic::vector_reduce_fmin:
2317  case Intrinsic::vector_reduce_fadd:
2318  case Intrinsic::vector_reduce_fmul: {
2319  bool CanBeReassociated = (IID != Intrinsic::vector_reduce_fadd &&
2320  IID != Intrinsic::vector_reduce_fmul) ||
2321  II->hasAllowReassoc();
2322  const unsigned ArgIdx = (IID == Intrinsic::vector_reduce_fadd ||
2323  IID == Intrinsic::vector_reduce_fmul)
2324  ? 1
2325  : 0;
2326  Value *Arg = II->getArgOperand(ArgIdx);
2327  Value *V;
2329  if (!isa<FixedVectorType>(Arg->getType()) || !CanBeReassociated ||
2330  !match(Arg, m_Shuffle(m_Value(V), m_Undef(), m_Mask(Mask))) ||
2331  !cast<ShuffleVectorInst>(Arg)->isSingleSource())
2332  break;
2333  int Sz = Mask.size();
2334  SmallBitVector UsedIndices(Sz);
2335  for (int Idx : Mask) {
2336  if (Idx == UndefMaskElem || UsedIndices.test(Idx))
2337  break;
2338  UsedIndices.set(Idx);
2339  }
2340  // Can remove shuffle iff just shuffled elements, no repeats, undefs, or
2341  // other changes.
2342  if (UsedIndices.all()) {
2343  replaceUse(II->getOperandUse(ArgIdx), V);
2344  return nullptr;
2345  }
2346  break;
2347  }
2348  default: {
2349  // Handle target specific intrinsics
2350  Optional<Instruction *> V = targetInstCombineIntrinsic(*II);
2351  if (V.hasValue())
2352  return V.getValue();
2353  break;
2354  }
2355  }
2356  // Some intrinsics (like experimental_gc_statepoint) can be used in invoke
2357  // context, so it is handled in visitCallBase and we should trigger it.
2358  return visitCallBase(*II);
2359 }
2360 
2361 // Fence instruction simplification
2363  // Remove identical consecutive fences.
2365  if (auto *NFI = dyn_cast<FenceInst>(Next))
2366  if (FI.isIdenticalTo(NFI))
2367  return eraseInstFromFunction(FI);
2368  return nullptr;
2369 }
2370 
2371 // InvokeInst simplification
2373  return visitCallBase(II);
2374 }
2375 
2376 // CallBrInst simplification
2378  return visitCallBase(CBI);
2379 }
2380 
2381 /// If this cast does not affect the value passed through the varargs area, we
2382 /// can eliminate the use of the cast.
2383 static bool isSafeToEliminateVarargsCast(const CallBase &Call,
2384  const DataLayout &DL,
2385  const CastInst *const CI,
2386  const int ix) {
2387  if (!CI->isLosslessCast())
2388  return false;
2389 
2390  // If this is a GC intrinsic, avoid munging types. We need types for
2391  // statepoint reconstruction in SelectionDAG.
2392  // TODO: This is probably something which should be expanded to all
2393  // intrinsics since the entire point of intrinsics is that
2394  // they are understandable by the optimizer.
2395  if (isa<GCStatepointInst>(Call) || isa<GCRelocateInst>(Call) ||
2396  isa<GCResultInst>(Call))
2397  return false;
2398 
2399  // Opaque pointers are compatible with any byval types.
2400  PointerType *SrcTy = cast<PointerType>(CI->getOperand(0)->getType());
2401  if (SrcTy->isOpaque())
2402  return true;
2403 
2404  // The size of ByVal or InAlloca arguments is derived from the type, so we
2405  // can't change to a type with a different size. If the size were
2406  // passed explicitly we could avoid this check.
2407  if (!Call.isPassPointeeByValueArgument(ix))
2408  return true;
2409 
2410  // The transform currently only handles type replacement for byval, not other
2411  // type-carrying attributes.
2412  if (!Call.isByValArgument(ix))
2413  return false;
2414 
2415  Type *SrcElemTy = SrcTy->getElementType();
2416  Type *DstElemTy = Call.getParamByValType(ix);
2417  if (!SrcElemTy->isSized() || !DstElemTy->isSized())
2418  return false;
2419  if (DL.getTypeAllocSize(SrcElemTy) != DL.getTypeAllocSize(DstElemTy))
2420  return false;
2421  return true;
2422 }
2423 
2424 Instruction *InstCombinerImpl::tryOptimizeCall(CallInst *CI) {
2425  if (!CI->getCalledFunction()) return nullptr;
2426 
2427  auto InstCombineRAUW = [this](Instruction *From, Value *With) {
2428  replaceInstUsesWith(*From, With);
2429  };
2430  auto InstCombineErase = [this](Instruction *I) {
2431  eraseInstFromFunction(*I);
2432  };
2433  LibCallSimplifier Simplifier(DL, &TLI, ORE, BFI, PSI, InstCombineRAUW,
2434  InstCombineErase);
2435  if (Value *With = Simplifier.optimizeCall(CI, Builder)) {
2436  ++NumSimplified;
2437  return CI->use_empty() ? CI : replaceInstUsesWith(*CI, With);
2438  }
2439 
2440  return nullptr;
2441 }
2442 
2444  // Strip off at most one level of pointer casts, looking for an alloca. This
2445  // is good enough in practice and simpler than handling any number of casts.
2446  Value *Underlying = TrampMem->stripPointerCasts();
2447  if (Underlying != TrampMem &&
2448  (!Underlying->hasOneUse() || Underlying->user_back() != TrampMem))
2449  return nullptr;
2450  if (!isa<AllocaInst>(Underlying))
2451  return nullptr;
2452 
2453  IntrinsicInst *InitTrampoline = nullptr;
2454  for (User *U : TrampMem->users()) {
2455  IntrinsicInst *II = dyn_cast<IntrinsicInst>(U);
2456  if (!II)
2457  return nullptr;
2458  if (II->getIntrinsicID() == Intrinsic::init_trampoline) {
2459  if (InitTrampoline)
2460  // More than one init_trampoline writes to this value. Give up.
2461  return nullptr;
2462  InitTrampoline = II;
2463  continue;
2464  }
2465  if (II->getIntrinsicID() == Intrinsic::adjust_trampoline)
2466  // Allow any number of calls to adjust.trampoline.
2467  continue;
2468  return nullptr;
2469  }
2470 
2471  // No call to init.trampoline found.
2472  if (!InitTrampoline)
2473  return nullptr;
2474 
2475  // Check that the alloca is being used in the expected way.
2476  if (InitTrampoline->getOperand(0) != TrampMem)
2477  return nullptr;
2478 
2479  return InitTrampoline;
2480 }
2481 
2483  Value *TrampMem) {
2484  // Visit all the previous instructions in the basic block, and try to find a
2485  // init.trampoline which has a direct path to the adjust.trampoline.
2486  for (BasicBlock::iterator I = AdjustTramp->getIterator(),
2487  E = AdjustTramp->getParent()->begin();
2488  I != E;) {
2489  Instruction *Inst = &*--I;
2490  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
2491  if (II->getIntrinsicID() == Intrinsic::init_trampoline &&
2492  II->getOperand(0) == TrampMem)
2493  return II;
2494  if (Inst->mayWriteToMemory())
2495  return nullptr;
2496  }
2497  return nullptr;
2498 }
2499 
2500 // Given a call to llvm.adjust.trampoline, find and return the corresponding
2501 // call to llvm.init.trampoline if the call to the trampoline can be optimized
2502 // to a direct call to a function. Otherwise return NULL.
2504  Callee = Callee->stripPointerCasts();
2505  IntrinsicInst *AdjustTramp = dyn_cast<IntrinsicInst>(Callee);
2506  if (!AdjustTramp ||
2507  AdjustTramp->getIntrinsicID() != Intrinsic::adjust_trampoline)
2508  return nullptr;
2509 
2510  Value *TrampMem = AdjustTramp->getOperand(0);
2511 
2513  return IT;
2514  if (IntrinsicInst *IT = findInitTrampolineFromBB(AdjustTramp, TrampMem))
2515  return IT;
2516  return nullptr;
2517 }
2518 
2519 void InstCombinerImpl::annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) {
2520  unsigned NumArgs = Call.arg_size();
2521  ConstantInt *Op0C = dyn_cast<ConstantInt>(Call.getOperand(0));
2522  ConstantInt *Op1C =
2523  (NumArgs == 1) ? nullptr : dyn_cast<ConstantInt>(Call.getOperand(1));
2524  // Bail out if the allocation size is zero (or an invalid alignment of zero
2525  // with aligned_alloc).
2526  if ((Op0C && Op0C->isNullValue()) || (Op1C && Op1C->isNullValue()))
2527  return;
2528 
2529  if (isMallocLikeFn(&Call, TLI) && Op0C) {
2530  if (isOpNewLikeFn(&Call, TLI))
2532  Call.getContext(), Op0C->getZExtValue()));
2533  else
2535  Call.getContext(), Op0C->getZExtValue()));
2536  } else if (isAlignedAllocLikeFn(&Call, TLI)) {
2537  if (Op1C)
2539  Call.getContext(), Op1C->getZExtValue()));
2540  // Add alignment attribute if alignment is a power of two constant.
2541  if (Op0C && Op0C->getValue().ult(llvm::Value::MaximumAlignment) &&
2542  isKnownNonZero(Call.getOperand(1), DL, 0, &AC, &Call, &DT)) {
2543  uint64_t AlignmentVal = Op0C->getZExtValue();
2544  if (llvm::isPowerOf2_64(AlignmentVal)) {
2545  Call.removeRetAttr(Attribute::Alignment);
2546  Call.addRetAttr(Attribute::getWithAlignment(Call.getContext(),
2547  Align(AlignmentVal)));
2548  }
2549  }
2550  } else if (isReallocLikeFn(&Call, TLI) && Op1C) {
2552  Call.getContext(), Op1C->getZExtValue()));
2553  } else if (isCallocLikeFn(&Call, TLI) && Op0C && Op1C) {
2554  bool Overflow;
2555  const APInt &N = Op0C->getValue();
2556  APInt Size = N.umul_ov(Op1C->getValue(), Overflow);
2557  if (!Overflow)
2559  Call.getContext(), Size.getZExtValue()));
2560  } else if (isStrdupLikeFn(&Call, TLI)) {
2561  uint64_t Len = GetStringLength(Call.getOperand(0));
2562  if (Len) {
2563  // strdup
2564  if (NumArgs == 1)
2566  Call.getContext(), Len));
2567  // strndup
2568  else if (NumArgs == 2 && Op1C)
2570  Call.getContext(), std::min(Len, Op1C->getZExtValue() + 1)));
2571  }
2572  }
2573 }
2574 
2575 /// Improvements for call, callbr and invoke instructions.
2576 Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) {
2577  if (isAllocationFn(&Call, &TLI))
2578  annotateAnyAllocSite(Call, &TLI);
2579 
2580  bool Changed = false;
2581 
2582  // Mark any parameters that are known to be non-null with the nonnull
2583  // attribute. This is helpful for inlining calls to functions with null
2584  // checks on their arguments.
2585  SmallVector<unsigned, 4> ArgNos;
2586  unsigned ArgNo = 0;
2587 
2588  for (Value *V : Call.args()) {
2589  if (V->getType()->isPointerTy() &&
2590  !Call.paramHasAttr(ArgNo, Attribute::NonNull) &&
2591  isKnownNonZero(V, DL, 0, &AC, &Call, &DT))
2592  ArgNos.push_back(ArgNo);
2593  ArgNo++;
2594  }
2595 
2596  assert(ArgNo == Call.arg_size() && "sanity check");
2597 
2598  if (!ArgNos.empty()) {
2599  AttributeList AS = Call.getAttributes();
2600  LLVMContext &Ctx = Call.getContext();
2601  AS = AS.addParamAttribute(Ctx, ArgNos,
2602  Attribute::get(Ctx, Attribute::NonNull));
2603  Call.setAttributes(AS);
2604  Changed = true;
2605  }
2606 
2607  // If the callee is a pointer to a function, attempt to move any casts to the
2608  // arguments of the call/callbr/invoke.
2609  Value *Callee = Call.getCalledOperand();
2610  if (!isa<Function>(Callee) && transformConstExprCastCall(Call))
2611  return nullptr;
2612 
2613  if (Function *CalleeF = dyn_cast<Function>(Callee)) {
2614  // Remove the convergent attr on calls when the callee is not convergent.
2615  if (Call.isConvergent() && !CalleeF->isConvergent() &&
2616  !CalleeF->isIntrinsic()) {
2617  LLVM_DEBUG(dbgs() << "Removing convergent attr from instr " << Call
2618  << "\n");
2619  Call.setNotConvergent();
2620  return &Call;
2621  }
2622 
2623  // If the call and callee calling conventions don't match, and neither one
2624  // of the calling conventions is compatible with C calling convention
2625  // this call must be unreachable, as the call is undefined.
2626  if ((CalleeF->getCallingConv() != Call.getCallingConv() &&
2627  !(CalleeF->getCallingConv() == llvm::CallingConv::C &&
2629  !(Call.getCallingConv() == llvm::CallingConv::C &&
2631  // Only do this for calls to a function with a body. A prototype may
2632  // not actually end up matching the implementation's calling conv for a
2633  // variety of reasons (e.g. it may be written in assembly).
2634  !CalleeF->isDeclaration()) {
2635  Instruction *OldCall = &Call;
2636  CreateNonTerminatorUnreachable(OldCall);
2637  // If OldCall does not return void then replaceInstUsesWith poison.
2638  // This allows ValueHandlers and custom metadata to adjust itself.
2639  if (!OldCall->getType()->isVoidTy())
2640  replaceInstUsesWith(*OldCall, PoisonValue::get(OldCall->getType()));
2641  if (isa<CallInst>(OldCall))
2642  return eraseInstFromFunction(*OldCall);
2643 
2644  // We cannot remove an invoke or a callbr, because it would change thexi
2645  // CFG, just change the callee to a null pointer.
2646  cast<CallBase>(OldCall)->setCalledFunction(
2647  CalleeF->getFunctionType(),
2648  Constant::getNullValue(CalleeF->getType()));
2649  return nullptr;
2650  }
2651  }
2652 
2653  // Calling a null function pointer is undefined if a null address isn't
2654  // dereferenceable.
2655  if ((isa<ConstantPointerNull>(Callee) &&
2656  !NullPointerIsDefined(Call.getFunction())) ||
2657  isa<UndefValue>(Callee)) {
2658  // If Call does not return void then replaceInstUsesWith poison.
2659  // This allows ValueHandlers and custom metadata to adjust itself.
2660  if (!Call.getType()->isVoidTy())
2661  replaceInstUsesWith(Call, PoisonValue::get(Call.getType()));
2662 
2663  if (Call.isTerminator()) {
2664  // Can't remove an invoke or callbr because we cannot change the CFG.
2665  return nullptr;
2666  }
2667 
2668  // This instruction is not reachable, just remove it.
2669  CreateNonTerminatorUnreachable(&Call);
2670  return eraseInstFromFunction(Call);
2671  }
2672 
2673  if (IntrinsicInst *II = findInitTrampoline(Callee))
2674  return transformCallThroughTrampoline(Call, *II);
2675 
2676  // TODO: Drop this transform once opaque pointer transition is done.
2677  FunctionType *FTy = Call.getFunctionType();
2678  if (FTy->isVarArg()) {
2679  int ix = FTy->getNumParams();
2680  // See if we can optimize any arguments passed through the varargs area of
2681  // the call.
2682  for (auto I = Call.arg_begin() + FTy->getNumParams(), E = Call.arg_end();
2683  I != E; ++I, ++ix) {
2684  CastInst *CI = dyn_cast<CastInst>(*I);
2685  if (CI && isSafeToEliminateVarargsCast(Call, DL, CI, ix)) {
2686  replaceUse(*I, CI->getOperand(0));
2687 
2688  // Update the byval type to match the pointer type.
2689  // Not necessary for opaque pointers.
2690  PointerType *NewTy = cast<PointerType>(CI->getOperand(0)->getType());
2691  if (!NewTy->isOpaque() && Call.isByValArgument(ix)) {
2692  Call.removeParamAttr(ix, Attribute::ByVal);
2693  Call.addParamAttr(
2695  Call.getContext(), NewTy->getElementType()));
2696  }
2697  Changed = true;
2698  }
2699  }
2700  }
2701 
2702  if (isa<InlineAsm>(Callee) && !Call.doesNotThrow()) {
2703  InlineAsm *IA = cast<InlineAsm>(Callee);
2704  if (!IA->canThrow()) {
2705  // Normal inline asm calls cannot throw - mark them
2706  // 'nounwind'.
2707  Call.setDoesNotThrow();
2708  Changed = true;
2709  }
2710  }
2711 
2712  // Try to optimize the call if possible, we require DataLayout for most of
2713  // this. None of these calls are seen as possibly dead so go ahead and
2714  // delete the instruction now.
2715  if (CallInst *CI = dyn_cast<CallInst>(&Call)) {
2716  Instruction *I = tryOptimizeCall(CI);
2717  // If we changed something return the result, etc. Otherwise let
2718  // the fallthrough check.
2719  if (I) return eraseInstFromFunction(*I);
2720  }
2721 
2722  if (!Call.use_empty() && !Call.isMustTailCall())
2723  if (Value *ReturnedArg = Call.getReturnedArgOperand()) {
2724  Type *CallTy = Call.getType();
2725  Type *RetArgTy = ReturnedArg->getType();
2726  if (RetArgTy->canLosslesslyBitCastTo(CallTy))
2727  return replaceInstUsesWith(
2728  Call, Builder.CreateBitOrPointerCast(ReturnedArg, CallTy));
2729  }
2730 
2731  if (isAllocLikeFn(&Call, &TLI))
2732  return visitAllocSite(Call);
2733 
2734  // Handle intrinsics which can be used in both call and invoke context.
2735  switch (Call.getIntrinsicID()) {
2736  case Intrinsic::experimental_gc_statepoint: {
2737  GCStatepointInst &GCSP = *cast<GCStatepointInst>(&Call);
2738  SmallPtrSet<Value *, 32> LiveGcValues;
2739  for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {
2740  GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);
2741 
2742  // Remove the relocation if unused.
2743  if (GCR.use_empty()) {
2744  eraseInstFromFunction(GCR);
2745  continue;
2746  }
2747 
2748  Value *DerivedPtr = GCR.getDerivedPtr();
2749  Value *BasePtr = GCR.getBasePtr();
2750 
2751  // Undef is undef, even after relocation.
2752  if (isa<UndefValue>(DerivedPtr) || isa<UndefValue>(BasePtr)) {
2753  replaceInstUsesWith(GCR, UndefValue::get(GCR.getType()));
2754  eraseInstFromFunction(GCR);
2755  continue;
2756  }
2757 
2758  if (auto *PT = dyn_cast<PointerType>(GCR.getType())) {
2759  // The relocation of null will be null for most any collector.
2760  // TODO: provide a hook for this in GCStrategy. There might be some
2761  // weird collector this property does not hold for.
2762  if (isa<ConstantPointerNull>(DerivedPtr)) {
2763  // Use null-pointer of gc_relocate's type to replace it.
2764  replaceInstUsesWith(GCR, ConstantPointerNull::get(PT));
2765  eraseInstFromFunction(GCR);
2766  continue;
2767  }
2768 
2769  // isKnownNonNull -> nonnull attribute
2770  if (!GCR.hasRetAttr(Attribute::NonNull) &&
2771  isKnownNonZero(DerivedPtr, DL, 0, &AC, &Call, &DT)) {
2772  GCR.addRetAttr(Attribute::NonNull);
2773  // We discovered new fact, re-check users.
2774  Worklist.pushUsersToWorkList(GCR);
2775  }
2776  }
2777 
2778  // If we have two copies of the same pointer in the statepoint argument
2779  // list, canonicalize to one. This may let us common gc.relocates.
2780  if (GCR.getBasePtr() == GCR.getDerivedPtr() &&
2781  GCR.getBasePtrIndex() != GCR.getDerivedPtrIndex()) {
2782  auto *OpIntTy = GCR.getOperand(2)->getType();
2783  GCR.setOperand(2, ConstantInt::get(OpIntTy, GCR.getBasePtrIndex()));
2784  }
2785 
2786  // TODO: bitcast(relocate(p)) -> relocate(bitcast(p))
2787  // Canonicalize on the type from the uses to the defs
2788 
2789  // TODO: relocate((gep p, C, C2, ...)) -> gep(relocate(p), C, C2, ...)
2790  LiveGcValues.insert(BasePtr);
2791  LiveGcValues.insert(DerivedPtr);
2792  }
2795  unsigned NumOfGCLives = LiveGcValues.size();
2796  if (!Bundle.hasValue() || NumOfGCLives == Bundle->Inputs.size())
2797  break;
2798  // We can reduce the size of gc live bundle.
2800  std::vector<Value *> NewLiveGc;
2801  for (unsigned I = 0, E = Bundle->Inputs.size(); I < E; ++I) {
2802  Value *V = Bundle->Inputs[I];
2803  if (Val2Idx.count(V))
2804  continue;
2805  if (LiveGcValues.count(V)) {
2806  Val2Idx[V] = NewLiveGc.size();
2807  NewLiveGc.push_back(V);
2808  } else
2809  Val2Idx[V] = NumOfGCLives;
2810  }
2811  // Update all gc.relocates
2812  for (const GCRelocateInst *Reloc : GCSP.getGCRelocates()) {
2813  GCRelocateInst &GCR = *const_cast<GCRelocateInst *>(Reloc);
2814  Value *BasePtr = GCR.getBasePtr();
2815  assert(Val2Idx.count(BasePtr) && Val2Idx[BasePtr] != NumOfGCLives &&
2816  "Missed live gc for base pointer");
2817  auto *OpIntTy1 = GCR.getOperand(1)->getType();
2818  GCR.setOperand(1, ConstantInt::get(OpIntTy1, Val2Idx[BasePtr]));
2819  Value *DerivedPtr = GCR.getDerivedPtr();
2820  assert(Val2Idx.count(DerivedPtr) && Val2Idx[DerivedPtr] != NumOfGCLives &&
2821  "Missed live gc for derived pointer");
2822  auto *OpIntTy2 = GCR.getOperand(2)->getType();
2823  GCR.setOperand(2, ConstantInt::get(OpIntTy2, Val2Idx[DerivedPtr]));
2824  }
2825  // Create new statepoint instruction.
2826  OperandBundleDef NewBundle("gc-live", NewLiveGc);
2827  return CallBase::Create(&Call, NewBundle);
2828  }
2829  default: { break; }
2830  }
2831 
2832  return Changed ? &Call : nullptr;
2833 }
2834 
2835 /// If the callee is a constexpr cast of a function, attempt to move the cast to
2836 /// the arguments of the call/callbr/invoke.
2837 bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) {
2838  auto *Callee =
2839  dyn_cast<Function>(Call.getCalledOperand()->stripPointerCasts());
2840  if (!Callee)
2841  return false;
2842 
2843  // If this is a call to a thunk function, don't remove the cast. Thunks are
2844  // used to transparently forward all incoming parameters and outgoing return
2845  // values, so it's important to leave the cast in place.
2846  if (Callee->hasFnAttribute("thunk"))
2847  return false;
2848 
2849  // If this is a musttail call, the callee's prototype must match the caller's
2850  // prototype with the exception of pointee types. The code below doesn't
2851  // implement that, so we can't do this transform.
2852  // TODO: Do the transform if it only requires adding pointer casts.
2853  if (Call.isMustTailCall())
2854  return false;
2855 
2856  Instruction *Caller = &Call;
2857  const AttributeList &CallerPAL = Call.getAttributes();
2858 
2859  // Okay, this is a cast from a function to a different type. Unless doing so
2860  // would cause a type conversion of one of our arguments, change this call to
2861  // be a direct call with arguments casted to the appropriate types.
2863  Type *OldRetTy = Caller->getType();
2864  Type *NewRetTy = FT->getReturnType();
2865 
2866  // Check to see if we are changing the return type...
2867  if (OldRetTy != NewRetTy) {
2868 
2869  if (NewRetTy->isStructTy())
2870  return false; // TODO: Handle multiple return values.
2871 
2872  if (!CastInst::isBitOrNoopPointerCastable(NewRetTy, OldRetTy, DL)) {
2873  if (Callee->isDeclaration())
2874  return false; // Cannot transform this return value.
2875 
2876  if (!Caller->use_empty() &&
2877  // void -> non-void is handled specially
2878  !NewRetTy->isVoidTy())
2879  return false; // Cannot transform this return value.
2880  }
2881 
2882  if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
2883  AttrBuilder RAttrs(CallerPAL, AttributeList::ReturnIndex);
2884  if (RAttrs.overlaps(AttributeFuncs::typeIncompatible(NewRetTy)))
2885  return false; // Attribute not compatible with transformed value.
2886  }
2887 
2888  // If the callbase is an invoke/callbr instruction, and the return value is
2889  // used by a PHI node in a successor, we cannot change the return type of
2890  // the call because there is no place to put the cast instruction (without
2891  // breaking the critical edge). Bail out in this case.
2892  if (!Caller->use_empty()) {
2893  if (InvokeInst *II = dyn_cast<InvokeInst>(Caller))
2894  for (User *U : II->users())
2895  if (PHINode *PN = dyn_cast<PHINode>(U))
2896  if (PN->getParent() == II->getNormalDest() ||
2897  PN->getParent() == II->getUnwindDest())
2898  return false;
2899  // FIXME: Be conservative for callbr to avoid a quadratic search.
2900  if (isa<CallBrInst>(Caller))
2901  return false;
2902  }
2903  }
2904 
2905  unsigned NumActualArgs = Call.arg_size();
2906  unsigned NumCommonArgs = std::min(FT->getNumParams(), NumActualArgs);
2907 
2908  // Prevent us turning:
2909  // declare void @takes_i32_inalloca(i32* inalloca)
2910  // call void bitcast (void (i32*)* @takes_i32_inalloca to void (i32)*)(i32 0)
2911  //
2912  // into:
2913  // call void @takes_i32_inalloca(i32* null)
2914  //
2915  // Similarly, avoid folding away bitcasts of byval calls.
2916  if (Callee->getAttributes().hasAttrSomewhere(Attribute::InAlloca) ||
2917  Callee->getAttributes().hasAttrSomewhere(Attribute::Preallocated) ||
2918  Callee->getAttributes().hasAttrSomewhere(Attribute::ByVal))
2919  return false;
2920 
2921  auto AI = Call.arg_begin();
2922  for (unsigned i = 0, e = NumCommonArgs; i != e; ++i, ++AI) {
2923  Type *ParamTy = FT->getParamType(i);
2924  Type *ActTy = (*AI)->getType();
2925 
2926  if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL))
2927  return false; // Cannot transform this parameter value.
2928 
2929  if (AttrBuilder(CallerPAL.getParamAttrs(i))
2931  return false; // Attribute not compatible with transformed value.
2932 
2933  if (Call.isInAllocaArgument(i))
2934  return false; // Cannot transform to and from inalloca.
2935 
2936  if (CallerPAL.hasParamAttr(i, Attribute::SwiftError))
2937  return false;
2938 
2939  // If the parameter is passed as a byval argument, then we have to have a
2940  // sized type and the sized type has to have the same size as the old type.
2941  if (ParamTy != ActTy && CallerPAL.hasParamAttr(i, Attribute::ByVal)) {
2942  PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
2943  if (!ParamPTy || !ParamPTy->getElementType()->isSized())
2944  return false;
2945 
2946  Type *CurElTy = Call.getParamByValType(i);
2947  if (DL.getTypeAllocSize(CurElTy) !=
2948  DL.getTypeAllocSize(ParamPTy->getElementType()))
2949  return false;
2950  }
2951  }
2952 
2953  if (Callee->isDeclaration()) {
2954  // Do not delete arguments unless we have a function body.
2955  if (FT->getNumParams() < NumActualArgs && !FT->isVarArg())
2956  return false;
2957 
2958  // If the callee is just a declaration, don't change the varargsness of the
2959  // call. We don't want to introduce a varargs call where one doesn't
2960  // already exist.
2961  PointerType *APTy = cast<PointerType>(Call.getCalledOperand()->getType());
2962  if (FT->isVarArg()!=cast<FunctionType>(APTy->getElementType())->isVarArg())
2963  return false;
2964 
2965  // If both the callee and the cast type are varargs, we still have to make
2966  // sure the number of fixed parameters are the same or we have the same
2967  // ABI issues as if we introduce a varargs call.
2968  if (FT->isVarArg() &&
2969  cast<FunctionType>(APTy->getElementType())->isVarArg() &&
2970  FT->getNumParams() !=
2971  cast<FunctionType>(APTy->getElementType())->getNumParams())
2972  return false;
2973  }
2974 
2975  if (FT->getNumParams() < NumActualArgs && FT->isVarArg() &&
2976  !CallerPAL.isEmpty()) {
2977  // In this case we have more arguments than the new function type, but we
2978  // won't be dropping them. Check that these extra arguments have attributes
2979  // that are compatible with being a vararg call argument.
2980  unsigned SRetIdx;
2981  if (CallerPAL.hasAttrSomewhere(Attribute::StructRet, &SRetIdx) &&
2982  SRetIdx - AttributeList::FirstArgIndex >= FT->getNumParams())
2983  return false;
2984  }
2985 
2986  // Okay, we decided that this is a safe thing to do: go ahead and start
2987  // inserting cast instructions as necessary.
2990  Args.reserve(NumActualArgs);
2991  ArgAttrs.reserve(NumActualArgs);
2992 
2993  // Get any return attributes.
2994  AttrBuilder RAttrs(CallerPAL, AttributeList::ReturnIndex);
2995 
2996  // If the return value is not being used, the type may not be compatible
2997  // with the existing attributes. Wipe out any problematic attributes.
2998  RAttrs.remove(AttributeFuncs::typeIncompatible(NewRetTy));
2999 
3000  LLVMContext &Ctx = Call.getContext();
3001  AI = Call.arg_begin();
3002  for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
3003  Type *ParamTy = FT->getParamType(i);
3004 
3005  Value *NewArg = *AI;
3006  if ((*AI)->getType() != ParamTy)
3007  NewArg = Builder.CreateBitOrPointerCast(*AI, ParamTy);
3008  Args.push_back(NewArg);
3009 
3010  // Add any parameter attributes.
3011  if (CallerPAL.hasParamAttr(i, Attribute::ByVal)) {
3012  AttrBuilder AB(CallerPAL.getParamAttrs(i));
3013  AB.addByValAttr(NewArg->getType()->getPointerElementType());
3014  ArgAttrs.push_back(AttributeSet::get(Ctx, AB));
3015  } else
3016  ArgAttrs.push_back(CallerPAL.getParamAttrs(i));
3017  }
3018 
3019  // If the function takes more arguments than the call was taking, add them
3020  // now.
3021  for (unsigned i = NumCommonArgs; i != FT->getNumParams(); ++i) {
3022  Args.push_back(Constant::getNullValue(FT->getParamType(i)));
3023  ArgAttrs.push_back(AttributeSet());
3024  }
3025 
3026  // If we are removing arguments to the function, emit an obnoxious warning.
3027  if (FT->getNumParams() < NumActualArgs) {
3028  // TODO: if (!FT->isVarArg()) this call may be unreachable. PR14722
3029  if (FT->isVarArg()) {
3030  // Add all of the arguments in their promoted form to the arg list.
3031  for (unsigned i = FT->getNumParams(); i != NumActualArgs; ++i, ++AI) {
3032  Type *PTy = getPromotedType((*AI)->getType());
3033  Value *NewArg = *AI;
3034  if (PTy != (*AI)->getType()) {
3035  // Must promote to pass through va_arg area!
3036  Instruction::CastOps opcode =
3037  CastInst::getCastOpcode(*AI, false, PTy, false);
3038  NewArg = Builder.CreateCast(opcode, *AI, PTy);
3039  }
3040  Args.push_back(NewArg);
3041 
3042  // Add any parameter attributes.
3043  ArgAttrs.push_back(CallerPAL.getParamAttrs(i));
3044  }
3045  }
3046  }
3047 
3048  AttributeSet FnAttrs = CallerPAL.getFnAttrs();
3049 
3050  if (NewRetTy->isVoidTy())
3051  Caller->setName(""); // Void type should not have a name.
3052 
3053  assert((ArgAttrs.size() == FT->getNumParams() || FT->isVarArg()) &&
3054  "missing argument attributes");
3055  AttributeList NewCallerPAL = AttributeList::get(
3056  Ctx, FnAttrs, AttributeSet::get(Ctx, RAttrs), ArgAttrs);
3057 
3059  Call.getOperandBundlesAsDefs(OpBundles);
3060 
3061  CallBase *NewCall;
3062  if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
3063  NewCall = Builder.CreateInvoke(Callee, II->getNormalDest(),
3064  II->getUnwindDest(), Args, OpBundles);
3065  } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
3066  NewCall = Builder.CreateCallBr(Callee, CBI->getDefaultDest(),
3067  CBI->getIndirectDests(), Args, OpBundles);
3068  } else {
3069  NewCall = Builder.CreateCall(Callee, Args, OpBundles);
3070  cast<CallInst>(NewCall)->setTailCallKind(
3071  cast<CallInst>(Caller)->getTailCallKind());
3072  }
3073  NewCall->takeName(Caller);
3074  NewCall->setCallingConv(Call.getCallingConv());
3075  NewCall->setAttributes(NewCallerPAL);
3076 
3077  // Preserve prof metadata if any.
3078  NewCall->copyMetadata(*Caller, {LLVMContext::MD_prof});
3079 
3080  // Insert a cast of the return type as necessary.
3081  Instruction *NC = NewCall;
3082  Value *NV = NC;
3083  if (OldRetTy != NV->getType() && !Caller->use_empty()) {
3084  if (!NV->getType()->isVoidTy()) {
3085  NV = NC = CastInst::CreateBitOrPointerCast(NC, OldRetTy);
3086  NC->setDebugLoc(Caller->getDebugLoc());
3087 
3088  // If this is an invoke/callbr instruction, we should insert it after the
3089  // first non-phi instruction in the normal successor block.
3090  if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
3091  BasicBlock::iterator I = II->getNormalDest()->getFirstInsertionPt();
3092  InsertNewInstBefore(NC, *I);
3093  } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(Caller)) {
3094  BasicBlock::iterator I = CBI->getDefaultDest()->getFirstInsertionPt();
3095  InsertNewInstBefore(NC, *I);
3096  } else {
3097  // Otherwise, it's a call, just insert cast right after the call.
3098  InsertNewInstBefore(NC, *Caller);
3099  }
3100  Worklist.pushUsersToWorkList(*Caller);
3101  } else {
3102  NV = UndefValue::get(Caller->getType());
3103  }
3104  }
3105 
3106  if (!Caller->use_empty())
3107  replaceInstUsesWith(*Caller, NV);
3108  else if (Caller->hasValueHandle()) {
3109  if (OldRetTy == NV->getType())
3111  else
3112  // We cannot call ValueIsRAUWd with a different type, and the
3113  // actual tracked value will disappear.
3115  }
3116 
3117  eraseInstFromFunction(*Caller);
3118  return true;
3119 }
3120 
3121 /// Turn a call to a function created by init_trampoline / adjust_trampoline
3122 /// intrinsic pair into a direct call to the underlying function.
3123 Instruction *
3124 InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call,
3125  IntrinsicInst &Tramp) {
3126  Value *Callee = Call.getCalledOperand();
3127  Type *CalleeTy = Callee->getType();
3128  FunctionType *FTy = Call.getFunctionType();
3129  AttributeList Attrs = Call.getAttributes();
3130 
3131  // If the call already has the 'nest' attribute somewhere then give up -
3132  // otherwise 'nest' would occur twice after splicing in the chain.
3133  if (Attrs.hasAttrSomewhere(Attribute::Nest))
3134  return nullptr;
3135 
3136  Function *NestF = cast<Function>(Tramp.getArgOperand(1)->stripPointerCasts());
3137  FunctionType *NestFTy = NestF->getFunctionType();
3138 
3139  AttributeList NestAttrs = NestF->getAttributes();
3140  if (!NestAttrs.isEmpty()) {
3141  unsigned NestArgNo = 0;
3142  Type *NestTy = nullptr;
3143  AttributeSet NestAttr;
3144 
3145  // Look for a parameter marked with the 'nest' attribute.
3146  for (FunctionType::param_iterator I = NestFTy->param_begin(),
3147  E = NestFTy->param_end();
3148  I != E; ++NestArgNo, ++I) {
3149  AttributeSet AS = NestAttrs.getParamAttrs(NestArgNo);
3150  if (AS.hasAttribute(Attribute::Nest)) {
3151  // Record the parameter type and any other attributes.
3152  NestTy = *I;
3153  NestAttr = AS;
3154  break;
3155  }
3156  }
3157 
3158  if (NestTy) {
3159  std::vector<Value*> NewArgs;
3160  std::vector<AttributeSet> NewArgAttrs;
3161  NewArgs.reserve(Call.arg_size() + 1);
3162  NewArgAttrs.reserve(Call.arg_size());
3163 
3164  // Insert the nest argument into the call argument list, which may
3165  // mean appending it. Likewise for attributes.
3166 
3167  {
3168  unsigned ArgNo = 0;
3169  auto I = Call.arg_begin(), E = Call.arg_end();
3170  do {
3171  if (ArgNo == NestArgNo) {
3172  // Add the chain argument and attributes.
3173  Value *NestVal = Tramp.getArgOperand(2);
3174  if (NestVal->getType() != NestTy)
3175  NestVal = Builder.CreateBitCast(NestVal, NestTy, "nest");
3176  NewArgs.push_back(NestVal);
3177  NewArgAttrs.push_back(NestAttr);
3178  }
3179 
3180  if (I == E)
3181  break;
3182 
3183  // Add the original argument and attributes.
3184  NewArgs.push_back(*I);
3185  NewArgAttrs.push_back(Attrs.getParamAttrs(ArgNo));
3186 
3187  ++ArgNo;
3188  ++I;
3189  } while (true);
3190  }
3191 
3192  // The trampoline may have been bitcast to a bogus type (FTy).
3193  // Handle this by synthesizing a new function type, equal to FTy
3194  // with the chain parameter inserted.
3195 
3196  std::vector<Type*> NewTypes;
3197  NewTypes.reserve(FTy->getNumParams()+1);
3198 
3199  // Insert the chain's type into the list of parameter types, which may
3200  // mean appending it.
3201  {
3202  unsigned ArgNo = 0;
3204  E = FTy->param_end();
3205 
3206  do {
3207  if (ArgNo == NestArgNo)
3208  // Add the chain's type.
3209  NewTypes.push_back(NestTy);
3210 
3211  if (I == E)
3212  break;
3213 
3214  // Add the original type.
3215  NewTypes.push_back(*I);
3216 
3217  ++ArgNo;
3218  ++I;
3219  } while (true);
3220  }
3221 
3222  // Replace the trampoline call with a direct call. Let the generic
3223  // code sort out any function type mismatches.
3224  FunctionType *NewFTy = FunctionType::get(FTy->getReturnType(), NewTypes,
3225  FTy->isVarArg());
3226  Constant *NewCallee =
3227  NestF->getType() == PointerType::getUnqual(NewFTy) ?
3228  NestF : ConstantExpr::getBitCast(NestF,
3229  PointerType::getUnqual(NewFTy));
3230  AttributeList NewPAL =
3231  AttributeList::get(FTy->getContext(), Attrs.getFnAttrs(),
3232  Attrs.getRetAttrs(), NewArgAttrs);
3233 
3235  Call.getOperandBundlesAsDefs(OpBundles);
3236 
3237  Instruction *NewCaller;
3238  if (InvokeInst *II = dyn_cast<InvokeInst>(&Call)) {
3239  NewCaller = InvokeInst::Create(NewFTy, NewCallee,
3240  II->getNormalDest(), II->getUnwindDest(),
3241  NewArgs, OpBundles);
3242  cast<InvokeInst>(NewCaller)->setCallingConv(II->getCallingConv());
3243  cast<InvokeInst>(NewCaller)->setAttributes(NewPAL);
3244  } else if (CallBrInst *CBI = dyn_cast<CallBrInst>(&Call)) {
3245  NewCaller =
3246  CallBrInst::Create(NewFTy, NewCallee, CBI->getDefaultDest(),
3247  CBI->getIndirectDests(), NewArgs, OpBundles);
3248  cast<CallBrInst>(NewCaller)->setCallingConv(CBI->getCallingConv());
3249  cast<CallBrInst>(NewCaller)->setAttributes(NewPAL);
3250  } else {
3251  NewCaller = CallInst::Create(NewFTy, NewCallee, NewArgs, OpBundles);
3252  cast<CallInst>(NewCaller)->setTailCallKind(
3253  cast<CallInst>(Call).getTailCallKind());
3254  cast<CallInst>(NewCaller)->setCallingConv(
3255  cast<CallInst>(Call).getCallingConv());
3256  cast<CallInst>(NewCaller)->setAttributes(NewPAL);
3257  }
3258  NewCaller->setDebugLoc(Call.getDebugLoc());
3259 
3260  return NewCaller;
3261  }
3262  }
3263 
3264  // Replace the trampoline call with a direct call. Since there is no 'nest'
3265  // parameter, there is no need to adjust the argument list. Let the generic
3266  // code sort out any function type mismatches.
3267  Constant *NewCallee = ConstantExpr::getBitCast(NestF, CalleeTy);
3268  Call.setCalledFunction(FTy, NewCallee);
3269  return &Call;
3270 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::RetainedKnowledge::WasOn
Value * WasOn
Definition: AssumeBundleQueries.h:103
i
i
Definition: README.txt:29
llvm::CallBase::getNumOperandBundles
unsigned getNumOperandBundles() const
Return the number of operand bundles associated with this User.
Definition: InstrTypes.h:1890
FloatingPointMode.h
llvm::SmallBitVector::set
SmallBitVector & set()
Definition: SmallBitVector.h:365
AssumptionCache.h
llvm::BinaryOpIntrinsic::getBinaryOp
Instruction::BinaryOps getBinaryOp() const
Returns the binary operation underlying the intrinsic.
Definition: IntrinsicInst.cpp:558
llvm::PatternMatch::m_FShr
m_Intrinsic_Ty< Opnd0, Opnd1, Opnd2 >::Ty m_FShr(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2)
Definition: PatternMatch.h:2203
Attrs
Function Attrs
Definition: README_ALTIVEC.txt:215
llvm::Type::isSized
bool isSized(SmallPtrSetImpl< Type * > *Visited=nullptr) const
Return true if it makes sense to take the size of this type.
Definition: Type.h:263
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:105
MathExtras.h
llvm::APInt::sadd_ov
APInt sadd_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1920
llvm
This file implements support for optimizing divisions by a constant.
Definition: AllocatorList.h:23
llvm::isCallocLikeFn
bool isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates zero-filled memory (such as...
Definition: MemoryBuiltins.cpp:290
llvm::maskIsAllOneOrUndef
bool maskIsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
Definition: VectorUtils.cpp:929
llvm::CallBase::getOperandBundlesAsDefs
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: Instructions.cpp:375
llvm::InvokeInst::Create
static InvokeInst * Create(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef< Value * > Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3793
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::Value::dropDroppableUse
static void dropDroppableUse(Use &U)
Remove the droppable use U.
Definition: Value.cpp:217
llvm::CmpInst::ICMP_EQ
@ ICMP_EQ
equal
Definition: InstrTypes.h:741
llvm::OverflowResult::NeverOverflows
@ NeverOverflows
Never overflows.
llvm::CallBase::getOperandBundle
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
Definition: InstrTypes.h:1977
Optional.h
llvm::PatternMatch::m_Mask
Definition: PatternMatch.h:1524
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:633
llvm::Value::hasOneUse
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
llvm::AttrBuilder::overlaps
bool overlaps(const AttrBuilder &B) const
Return true if the builder has any attribute that's in the specified builder.
Definition: Attributes.cpp:1754
InstCombiner.h
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1379
Metadata.h
llvm::AttributeList::hasAttrSomewhere
bool hasAttrSomewhere(Attribute::AttrKind Kind, unsigned *Index=nullptr) const
Return true if the specified attribute is set for at least one parameter or for the return value.
Definition: Attributes.cpp:1387
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:720
llvm::Value::MaximumAlignment
static constexpr uint64_t MaximumAlignment
Definition: Value.h:793
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::isSafeToSpeculativelyExecute
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
Definition: ValueTracking.cpp:4622
llvm::ConstantInt::getType
IntegerType * getType() const
getType - Specialize the getType() method to always return an IntegerType, which reduces the amount o...
Definition: Constants.h:173
llvm::InstCombiner::getDominatorTree
DominatorTree & getDominatorTree() const
Definition: InstCombiner.h:370
IntrinsicInst.h
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:217
ceil
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g ceil
Definition: README-FPStack.txt:54
AtomicOrdering.h
llvm::ConstantExpr::getZExt
static Constant * getZExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2123
llvm::Instruction::getNextNonDebugInstruction
const Instruction * getNextNonDebugInstruction(bool SkipPseudoOp=false) const
Return a pointer to the next non-debug instruction in the same basic block as 'this',...
Definition: Instruction.cpp:728
Loads.h
GuardWideningWindow
static cl::opt< unsigned > GuardWideningWindow("instcombine-guard-widening-window", cl::init(3), cl::desc("How wide an instruction window to bypass looking for " "another guard"))
llvm::PointerType::getElementType
Type * getElementType() const
Definition: DerivedTypes.h:673
llvm::Function
Definition: Function.h:62
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1352
llvm::BinaryOperator::CreateWithCopiedFlags
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
Definition: InstrTypes.h:250
llvm::BinaryOperator::CreateNot
static BinaryOperator * CreateNot(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
Definition: Instructions.cpp:2730
llvm::IntrinsicInst::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:52
llvm::SelectPatternResult::Flavor
SelectPatternFlavor Flavor
Definition: ValueTracking.h:700
llvm::CallBase::setCalledFunction
void setCalledFunction(Function *Fn)
Sets the function called, including updating the function type.
Definition: InstrTypes.h:1419
llvm::possiblyDemandedEltsInMask
APInt possiblyDemandedEltsInMask(Value *Mask)
Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) for each lane which may be ...
Definition: VectorUtils.cpp:957
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:729
minimum
Should compile r2 movcc movcs str strb mov lr r1 movcs movcc mov lr r1 str mov mov cmp r1 movlo r2 str bx lr r0 mov mov cmp r0 movhs r2 mov r1 bx lr Some of the NEON intrinsics may be appropriate for more general either as target independent intrinsics or perhaps elsewhere in the ARM backend Some of them may also be lowered to target independent and perhaps some new SDNodes could be added For minimum
Definition: README.txt:489
llvm::KnownBits::Zero
APInt Zero
Definition: KnownBits.h:24
C1
instcombine should handle this C2 when C1
Definition: README.txt:263
llvm::Attribute::get
static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val=0)
Return a uniquified Attribute object.
Definition: Attributes.cpp:92
llvm::ConstantInt::getValue
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:133
llvm::BinaryOpIntrinsic::getRHS
Value * getRHS() const
Definition: IntrinsicInst.h:576
llvm::ConstantExpr::getSExt
static Constant * getSExt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2109
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::InstCombiner::Builder
BuilderTy & Builder
Definition: InstCombiner.h:58
Statistic.h
InlineAsm.h
llvm::AttributeList::addParamAttribute
LLVM_NODISCARD AttributeList addParamAttribute(LLVMContext &C, unsigned ArgNo, Attribute::AttrKind Kind) const
Add an argument attribute to the list.
Definition: Attributes.h:533
llvm::ConstantExpr::getICmp
static Constant * getICmp(unsigned short pred, Constant *LHS, Constant *RHS, bool OnlyIfReduced=false)
get* - Return some common constants without having to specify the full Instruction::OPCODE identifier...
Definition: Constants.cpp:2502
llvm::BinaryOperator::CreateFDivFMF
static BinaryOperator * CreateFDivFMF(Value *V1, Value *V2, Instruction *FMFSource, const Twine &Name="")
Definition: InstrTypes.h:273
llvm::PatternMatch::m_Add
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1008
llvm::BinaryOpIntrinsic::getLHS
Value * getLHS() const
Definition: IntrinsicInst.h:575
ErrorHandling.h
llvm::CmpInst::makeCmpResultType
static Type * makeCmpResultType(Type *opnd_type)
Create a result type for fcmp/icmp.
Definition: InstrTypes.h:1031
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:734
llvm::IRBuilder< TargetFolder, IRBuilderCallbackInserter >
llvm::InsertValueInst::Create
static InsertValueInst * Create(Value *Agg, Value *Val, ArrayRef< unsigned > Idxs, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:2530
llvm::X86::SecondMacroFusionInstKind::AB
@ AB
llvm::GlobalVariable
Definition: GlobalVariable.h:40
llvm::CastInst::Create
static CastInst * Create(Instruction::CastOps, Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Provides a way to construct any of the CastInst subclasses using an opcode instead of the subclass's ...
Definition: Instructions.cpp:3076
llvm::CmpInst::ICMP_NE
@ ICMP_NE
not equal
Definition: InstrTypes.h:742
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2233
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:363
llvm::SignBitMustBeZero
bool SignBitMustBeZero(const Value *V, const TargetLibraryInfo *TLI)
Return true if we can prove that the specified FP value's sign bit is 0.
Definition: ValueTracking.cpp:3682
ValueTracking.h
llvm::isAssumeWithEmptyBundle
bool isAssumeWithEmptyBundle(AssumeInst &Assume)
Return true iff the operand bundles of the provided llvm.assume doesn't contain any valuable informat...
Definition: AssumeBundleQueries.cpp:125
Local.h
llvm::AttributeList::get
static AttributeList get(LLVMContext &C, ArrayRef< std::pair< unsigned, Attribute >> Attrs)
Create an AttributeList with the specified parameters in it.
Definition: Attributes.cpp:1011
llvm::isAllocLikeFn
bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates memory (either malloc,...
Definition: MemoryBuiltins.cpp:305
llvm::PatternMatch::m_APFloat
apfloat_match m_APFloat(const APFloat *&Res)
Match a ConstantFP or splatted ConstantVector, binding the specified pointer to the contained APFloat...
Definition: PatternMatch.h:287
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::minimum
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 minimum semantics.
Definition: APFloat.h:1318
llvm::IRBuilderBase::CreateUnaryIntrinsic
CallInst * CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 1 operand which is mangled on its type.
Definition: IRBuilder.cpp:823
llvm::PatternMatch::m_BitReverse
m_Intrinsic_Ty< Opnd0 >::Ty m_BitReverse(const Opnd0 &Op0)
Definition: PatternMatch.h:2164
llvm::AttributeFuncs::typeIncompatible
AttrBuilder typeIncompatible(Type *Ty)
Which attributes cannot be applied to a type.
Definition: Attributes.cpp:1811
llvm::isKnownNonZero
bool isKnownNonZero(const Value *V, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to be non-zero when defined.
Definition: ValueTracking.cpp:316
llvm::ConstantInt::getLimitedValue
uint64_t getLimitedValue(uint64_t Limit=~0ULL) const
getLimitedValue - If the value is smaller than the specified limit, return it, otherwise return the l...
Definition: Constants.h:244
APInt.h
llvm::IRBuilderBase::CreateStripInvariantGroup
Value * CreateStripInvariantGroup(Value *Ptr)
Create a strip.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1033
llvm::PatternMatch::m_NegZeroFP
cstfp_pred_ty< is_neg_zero_fp > m_NegZeroFP()
Match a floating-point negative zero.
Definition: PatternMatch.h:714
llvm::InstCombinerImpl::visitInvokeInst
Instruction * visitInvokeInst(InvokeInst &II)
Definition: InstCombineCalls.cpp:2372
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:747
llvm::BitmaskEnumDetail::Underlying
std::underlying_type_t< E > Underlying(E Val)
Check that Val is in range for E, and return Val cast to E's underlying type.
Definition: BitmaskEnum.h:90
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::OverflowResult::AlwaysOverflowsLow
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
llvm::APInt::getBitWidth
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1410
llvm::InstCombinerImpl::visitCallInst
Instruction * visitCallInst(CallInst &CI)
CallInst simplification.
Definition: InstCombineCalls.cpp:897
llvm::InstCombinerImpl::FoldOpIntoSelect
Instruction * FoldOpIntoSelect(Instruction &Op, SelectInst *SI)
Given an instruction with a select as one operand and a constant as the other operand,...
Definition: InstructionCombining.cpp:1016
MemoryBuiltins.h
llvm::AttributeList
Definition: Attributes.h:399
llvm::OperandBundleDefT
A container for an operand bundle being viewed as a set of values rather than a set of uses.
Definition: InstrTypes.h:1114
llvm::CallBase::getFunctionType
FunctionType * getFunctionType() const
Definition: InstrTypes.h:1241
llvm::simplifyRetainedKnowledge
RetainedKnowledge simplifyRetainedKnowledge(AssumeInst *Assume, RetainedKnowledge RK, AssumptionCache *AC, DominatorTree *DT)
canonicalize the RetainedKnowledge RK.
Definition: AssumeBundleBuilder.cpp:315
llvm::FastMathFlags::noSignedZeros
bool noSignedZeros() const
Definition: Operator.h:209
llvm::AttributeList::getFnAttrs
AttributeSet getFnAttrs() const
The function attributes are returned.
Definition: Attributes.cpp:1362
llvm::GCRelocateInst::getBasePtrIndex
unsigned getBasePtrIndex() const
The index into the associate statepoint's argument list which contains the base pointer of the pointe...
Definition: IntrinsicInst.h:1291
llvm::PatternMatch::m_ZExtOrSExtOrSelf
match_combine_or< match_combine_or< CastClass_match< OpTy, Instruction::ZExt >, CastClass_match< OpTy, Instruction::SExt > >, OpTy > m_ZExtOrSExtOrSelf(const OpTy &Op)
Definition: PatternMatch.h:1667
llvm::EnableKnowledgeRetention
cl::opt< bool > EnableKnowledgeRetention
enable preservation of attributes in assume like: call void @llvm.assume(i1 true) [ "nonnull"(i32* PT...
Definition: InstCombineCalls.cpp:98
llvm::InstCombinerImpl::replaceInstUsesWith
Instruction * replaceInstUsesWith(Instruction &I, Value *V)
A combiner-aware RAUW-like routine.
Definition: InstCombineInternal.h:399
llvm::IRBuilderBase::CreateBinaryIntrinsic
CallInst * CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with 2 operands which is mangled on the first type.
Definition: IRBuilder.cpp:831
llvm::Optional< bool >
foldCttzCtlz
static Instruction * foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC)
Definition: InstCombineCalls.cpp:432
llvm::DenseMapBase::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
llvm::CastInst::getCastOpcode
static Instruction::CastOps getCastOpcode(const Value *Val, bool SrcIsSigned, Type *Ty, bool DstIsSigned)
Returns the opcode necessary to cast Val into Ty using usual casting rules.
Definition: Instructions.cpp:3358
llvm::FenceInst
An instruction for ordering other memory operations.
Definition: Instructions.h:444
llvm::PatternMatch::m_BinOp
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
Definition: PatternMatch.h:84
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:419
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::Instruction::copyMetadata
void copyMetadata(const Instruction &SrcInst, ArrayRef< unsigned > WL=ArrayRef< unsigned >())
Copy metadata from SrcInst to this instruction.
Definition: Instruction.cpp:829
llvm::FunctionType::getNumParams
unsigned getNumParams() const
Return the number of fixed parameters this function type requires.
Definition: DerivedTypes.h:139
llvm::CastInst::CreateIntegerCast
static CastInst * CreateIntegerCast(Value *S, Type *Ty, bool isSigned, const Twine &Name="", Instruction *InsertBefore=nullptr)
Create a ZExt, BitCast, or Trunc for int -> int casts.
Definition: Instructions.cpp:3242
STLExtras.h
llvm::SmallBitVector::test
bool test(unsigned Idx) const
Definition: SmallBitVector.h:465
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:491
llvm::matchSelectPattern
SelectPatternResult matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, Instruction::CastOps *CastOp=nullptr, unsigned Depth=0)
Pattern match integer [SU]MIN, [SU]MAX and ABS idioms, returning the kind and providing the out param...
Definition: ValueTracking.cpp:6242
llvm::PatternMatch::m_c_And
BinaryOp_match< LHS, RHS, Instruction::And, true > m_c_And(const LHS &L, const RHS &R)
Matches an And with LHS and RHS in either order.
Definition: PatternMatch.h:2242
llvm::BinaryOperator::CreateNeg
NUW NUW NUW NUW Exact static Exact BinaryOperator * CreateNeg(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
Helper functions to construct and inspect unary operations (NEG and NOT) via binary operators SUB and...
Definition: Instructions.cpp:2690
llvm::CallBase::setDoesNotThrow
void setDoesNotThrow()
Definition: InstrTypes.h:1850
findInitTrampolineFromBB
static IntrinsicInst * findInitTrampolineFromBB(IntrinsicInst *AdjustTramp, Value *TrampMem)
Definition: InstCombineCalls.cpp:2482
llvm::PatternMatch::m_Not
BinaryOp_match< ValTy, cst_pred_ty< is_all_ones >, Instruction::Xor, true > m_Not(const ValTy &V)
Matches a 'Not' as 'xor V, -1' or 'xor -1, V'.
Definition: PatternMatch.h:2280
llvm::SelectPatternFlavor
SelectPatternFlavor
Specific patterns of select instructions we can match.
Definition: ValueTracking.h:676
llvm::CallBrInst::Create
static CallBrInst * Create(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, ArrayRef< BasicBlock * > IndirectDests, ArrayRef< Value * > Args, const Twine &NameStr, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:4001
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::InstCombinerImpl::SimplifyAnyMemTransfer
Instruction * SimplifyAnyMemTransfer(AnyMemTransferInst *MI)
Definition: InstCombineCalls.cpp:111
llvm::BasicBlock::rend
reverse_iterator rend()
Definition: BasicBlock.h:303
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:241
llvm::PatternMatch::m_Deferred
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:820
llvm::isAllocationFn
bool isAllocationFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates or reallocates memory (eith...
Definition: MemoryBuiltins.cpp:241
llvm::APIntOps::umin
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition: APInt.h:2128
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1233
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::FunctionType::isVarArg
bool isVarArg() const
Definition: DerivedTypes.h:123
llvm::Instruction::setMetadata
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1336
KnownBits.h
llvm::KnownBits::countMinTrailingZeros
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:226
floor
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g floor
Definition: README-FPStack.txt:54
llvm::GCRelocateInst::getDerivedPtrIndex
unsigned getDerivedPtrIndex() const
The index into the associate statepoint's argument list which contains the pointer whose relocation t...
Definition: IntrinsicInst.h:1297
llvm::UndefMaskElem
constexpr int UndefMaskElem
Definition: Instructions.h:1997
AliasAnalysis.h
llvm::PatternMatch::m_OneUse
OneUse_match< T > m_OneUse(const T &SubPattern)
Definition: PatternMatch.h:67
llvm::Optional::hasValue
constexpr bool hasValue() const
Definition: Optional.h:288
factorizeMinMaxTree
static Instruction * factorizeMinMaxTree(IntrinsicInst *II)
Reduce a sequence of min/max intrinsics with a common operand.
Definition: InstCombineCalls.cpp:838
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ConstantExpr::getSub
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2701
llvm::InstCombinerImpl::eraseInstFromFunction
Instruction * eraseInstFromFunction(Instruction &I) override
Combiner aware instruction erasure.
Definition: InstCombineInternal.h:447
llvm::GCStatepointInst
Represents a gc.statepoint intrinsic call.
Definition: Statepoint.h:63
llvm::SmallBitVector
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
Definition: SmallBitVector.h:34
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::User::getOperandUse
const Use & getOperandUse(unsigned i) const
Definition: User.h:182
Instruction.h
llvm::PatternMatch::m_APInt
apint_match m_APInt(const APInt *&Res)
Match a ConstantInt or splatted ConstantVector, binding the specified pointer to the contained APInt.
Definition: PatternMatch.h:270
llvm::SPF_NABS
@ SPF_NABS
Absolute value.
Definition: ValueTracking.h:685
CommandLine.h
llvm::PatternMatch::m_UMax
MaxMin_match< ICmpInst, LHS, RHS, umax_pred_ty > m_UMax(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1875
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::KnownBits::isNonNegative
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:99
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:160
llvm::MinAlign
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
Definition: MathExtras.h:672
llvm::APInt::isNonNegative
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:317
llvm::isValidAssumeForContext
bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, const DominatorTree *DT=nullptr)
Return true if it is valid to use the assumptions provided by an assume intrinsic,...
Definition: ValueTracking.cpp:529
llvm::isSplatValue
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
Definition: VectorUtils.cpp:381
llvm::SelectInst::Create
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr="", Instruction *InsertBefore=nullptr, Instruction *MDFrom=nullptr)
Definition: Instructions.h:1769
InstCombineInternal.h
llvm::PatternMatch::m_Select
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
Definition: PatternMatch.h:1472
llvm::SimplifyCall
Value * SimplifyCall(CallBase *Call, const SimplifyQuery &Q)
Given a callsite, fold the result or return null.
Definition: InstructionSimplify.cpp:6056
llvm::Constant::containsConstantExpression
bool containsConstantExpression() const
Return true if this is a fixed width vector constant that includes any constant expressions.
Definition: Constants.cpp:338
Constants.h
llvm::maximum
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2018 maximum semantics.
Definition: APFloat.h:1331
llvm::PatternMatch::match
bool match(Val *V, const Pattern &P)
Definition: PatternMatch.h:49
llvm::Constant::isNullValue
bool isNullValue() const
Return true if this is the value that would be returned by getNullValue.
Definition: Constants.cpp:74
llvm::PatternMatch::m_FShl
m_Intrinsic_Ty< Opnd0, Opnd1, Opnd2 >::Ty m_FShl(const Opnd0 &Op0, const Opnd1 &Op1, const Opnd2 &Op2)
Definition: PatternMatch.h:2197
llvm::APInt::isZero
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:359
llvm::OverflowResult::MayOverflow
@ MayOverflow
May or may not overflow.
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::User
Definition: User.h:44
Intr
unsigned Intr
Definition: AMDGPUBaseInfo.cpp:1991
llvm::PatternMatch::m_UnOp
class_match< UnaryOperator > m_UnOp()
Match an arbitrary unary operation and ignore it.
Definition: PatternMatch.h:79
llvm::getKnownAlignment
Align getKnownAlignment(Value *V, const DataLayout &DL, const Instruction *CxtI=nullptr, AssumptionCache *AC=nullptr, const DominatorTree *DT=nullptr)
Try to infer an alignment for the specified pointer.
Definition: Local.h:224
Intrinsics.h
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::FunctionType::param_end
param_iterator param_end() const
Definition: DerivedTypes.h:129
llvm::AttributeSet::hasAttribute
bool hasAttribute(Attribute::AttrKind Kind) const
Return true if the attribute exists in this set.
Definition: Attributes.cpp:659
Twine.h
round
static uint64_t round(uint64_t Acc, uint64_t Input)
Definition: xxhash.cpp:57
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1383
llvm::CallBase::setAttributes
void setAttributes(AttributeList A)
Set the parameter attributes for this call.
Definition: InstrTypes.h:1462
llvm::BinaryOperator::CreateNSW
static BinaryOperator * CreateNSW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition: InstrTypes.h:284
llvm::BasicBlock::begin
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:296
llvm::KnownBits::One
APInt One
Definition: KnownBits.h:25
Y
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
llvm::PatternMatch::m_PosZeroFP
cstfp_pred_ty< is_pos_zero_fp > m_PosZeroFP()
Match a floating-point positive zero.
Definition: PatternMatch.h:705
InstructionWorklist.h
llvm::LoadInst::setVolatile
void setVolatile(bool V)
Specify whether this is a volatile load or not.
Definition: Instructions.h:215
llvm::CallInst::Create
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Definition: Instructions.h:1518
AssumeBundleQueries.h
getOpcode
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:199
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:214
llvm::AMDGPU::PALMD::Key
Key
PAL metadata keys.
Definition: AMDGPUMetadata.h:481
llvm::PatternMatch::m_Instruction
bind_ty< Instruction > m_Instruction(Instruction *&I)
Match an instruction, capturing it if we match.
Definition: PatternMatch.h:748
AssumeBundleBuilder.h
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:109
llvm::PatternMatch::m_ZExt
CastClass_match< OpTy, Instruction::ZExt > m_ZExt(const OpTy &Op)
Matches ZExt.
Definition: PatternMatch.h:1639
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::SmallBitVector::all
bool all() const
Returns true if all bits are set.
Definition: SmallBitVector.h:215
llvm::IntegerType
Class to represent integer types.
Definition: DerivedTypes.h:40
llvm::Instruction::CastOps
CastOps
Definition: Instruction.h:799
llvm::PatternMatch::m_FNeg
FNeg_match< OpTy > m_FNeg(const OpTy &X)
Match 'fneg X' as 'fsub -0.0, X'.
Definition: PatternMatch.h:1063
llvm::SaturatingInst
Represents a saturating add/sub intrinsic.
Definition: IntrinsicInst.h:610
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:405
llvm::pdb::PDB_SymType::Caller
@ Caller
llvm::PatternMatch::m_ConstantInt
class_match< ConstantInt > m_ConstantInt()
Match an arbitrary ConstantInt and ignore it.
Definition: PatternMatch.h:145
llvm::FunctionType::param_begin
param_iterator param_begin() const
Definition: DerivedTypes.h:128
llvm::Instruction
Definition: Instruction.h:45
llvm::Type::getScalarSizeInBits
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition: Type.cpp:191
llvm::isDereferenceablePointer
bool isDereferenceablePointer(const Value *V, Type *Ty, const DataLayout &DL, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if this is always a dereferenceable pointer.
Definition: Loads.cpp:234
llvm::PatternMatch::m_UMin
MaxMin_match< ICmpInst, LHS, RHS, umin_pred_ty > m_UMin(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1881
llvm::PatternMatch::m_c_Or
BinaryOp_match< LHS, RHS, Instruction::Or, true > m_c_Or(const LHS &L, const RHS &R)
Matches an Or with LHS and RHS in either order.
Definition: PatternMatch.h:2249
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
APFloat.h
This file declares a class to represent arbitrary precision floating point values and provide a varie...
llvm::UnaryOperator::CreateWithCopiedFlags
static UnaryOperator * CreateWithCopiedFlags(UnaryOps Opc, Value *V, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
Definition: InstrTypes.h:157
llvm::codeview::EncodedFramePtrReg::BasePtr
@ BasePtr
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1796
llvm::CallBase::bundle_op_info_begin
bundle_op_iterator bundle_op_info_begin()
Return the start of the list of BundleOpInfo instances associated with this OperandBundleUser.
Definition: InstrTypes.h:2184
llvm::ConstantExpr::getURem
static Constant * getURem(Constant *C1, Constant *C2)
Definition: Constants.cpp:2737
llvm::InstCombinerImpl
Definition: InstCombineInternal.h:61
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:925
llvm::isReallocLikeFn
bool isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that reallocates memory (e....
Definition: MemoryBuiltins.cpp:312
llvm::FunctionCallee::getFunctionType
FunctionType * getFunctionType()
Definition: DerivedTypes.h:182
llvm::LLVMContext::OB_gc_live
@ OB_gc_live
Definition: LLVMContext.h:95
llvm::APSInt::getMinValue
static APSInt getMinValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the minimum integer value with the given bit width and signedness.
Definition: APSInt.h:289
llvm::getKnowledgeFromBundle
RetainedKnowledge getKnowledgeFromBundle(AssumeInst &Assume, const CallBase::BundleOpInfo &BOI)
This extracts the Knowledge from an element of an operand bundle.
Definition: AssumeBundleQueries.cpp:99
llvm::KnownBits::isNegative
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:96
canonicalizeConstantArg0ToArg1
static CallInst * canonicalizeConstantArg0ToArg1(CallInst &Call)
Definition: InstCombineCalls.cpp:712
PatternMatch.h
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:153
llvm::Metadata
Root of the metadata hierarchy.
Definition: Metadata.h:62
llvm::Instruction::mayWriteToMemory
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
Definition: Instruction.cpp:584
llvm::PointerType::isOpaque
bool isOpaque() const
Definition: DerivedTypes.h:678
llvm::CallBase::getCallingConv
CallingConv::ID getCallingConv() const
Definition: InstrTypes.h:1439
llvm::None
const NoneType None
Definition: None.h:23
Statepoint.h
llvm::Value::use_empty
bool use_empty() const
Definition: Value.h:344
Type.h
llvm::IRBuilderBase::CreateAddrSpaceCast
Value * CreateAddrSpaceCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2130
X
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
llvm::AnyMemTransferInst
Definition: IntrinsicInst.h:1029
llvm::Log2_32_Ceil
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:609
llvm::CastInst::isLosslessCast
bool isLosslessCast() const
A lossless cast is one that does not alter the basic value.
Definition: Instructions.cpp:2786
llvm::PatternMatch::m_One
cst_pred_ty< is_one > m_One()
Match an integer 1 or a vector with all elements equal to 1.
Definition: PatternMatch.h:513
llvm::Instruction::getMetadata
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:282
llvm::SimplifyFMulInst
Value * SimplifyFMulInst(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior=fp::ebIgnore, RoundingMode Rounding=RoundingMode::NearestTiesToEven)
Given operands for an FMul, fold the result or return null.
Definition: InstructionSimplify.cpp:5138
llvm::OverflowResult
OverflowResult
Definition: ValueTracking.h:487
llvm::maxnum
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1307
findInitTrampolineFromAlloca
static IntrinsicInst * findInitTrampolineFromAlloca(Value *TrampMem)
Definition: InstCombineCalls.cpp:2443
llvm::ValueHandleBase::ValueIsRAUWd
static void ValueIsRAUWd(Value *Old, Value *New)
Definition: Value.cpp:1193
llvm::PatternMatch::m_NSWAdd
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoSignedWrap > m_NSWAdd(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1186
llvm::IRBuilderBase::CreateLaunderInvariantGroup
Value * CreateLaunderInvariantGroup(Value *Ptr)
Create a launder.invariant.group intrinsic call.
Definition: IRBuilder.cpp:1009
llvm::Function::getAttributes
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:327
llvm::InvokeInst
Invoke instruction.
Definition: Instructions.h:3749
llvm::IRBuilderBase::CreateZExt
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2033
llvm::PatternMatch::m_Xor
BinaryOp_match< LHS, RHS, Instruction::Xor > m_Xor(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1135
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:190
removeTriviallyEmptyRange
static bool removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC, std::function< bool(const IntrinsicInst &)> IsStart)
Definition: InstCombineCalls.cpp:677
llvm::InlineAsm
Definition: InlineAsm.h:31
llvm::VectorType
Base class of all SIMD vector types.
Definition: DerivedTypes.h:389
llvm::Instruction::andIRFlags
void andIRFlags(const Value *V)
Logical 'and' of any supported wrapping, exact, and fast-math flags of V and this instruction.
Definition: Instruction.cpp:313
llvm::IRBuilderBase::FastMathFlagGuard
Definition: IRBuilder.h:389
VectorUtils.h
llvm::KnownBits::isAllOnes
bool isAllOnes() const
Returns true if value is all one bits.
Definition: KnownBits.h:78
llvm::Use::set
void set(Value *Val)
Definition: Value.h:868
llvm::ConstantExpr::getTrunc
static Constant * getTrunc(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2095
llvm::Function::hasFnAttribute
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:626
llvm::GCStatepointInst::getGCRelocates
std::vector< const GCRelocateInst * > getGCRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
Definition: Statepoint.h:214
BasicBlock.h
llvm::cl::opt
Definition: CommandLine.h:1432
llvm::APFloat
Definition: APFloat.h:701
llvm::AttributeList::ReturnIndex
@ ReturnIndex
Definition: Attributes.h:402
llvm::IRBuilderBase::CreateBitCast
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2125
llvm::RetainedKnowledge::AttrKind
Attribute::AttrKind AttrKind
Definition: AssumeBundleQueries.h:101
llvm::PatternMatch::m_Zero
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:535
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:304
llvm::PatternMatch::m_NUWAdd
OverflowingBinaryOp_match< LHS, RHS, Instruction::Add, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWAdd(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1219
llvm::FunctionType::param_iterator
Type::subtype_iterator param_iterator
Definition: DerivedTypes.h:126
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::isMallocLikeFn
bool isMallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates uninitialized memory (such ...
Definition: MemoryBuiltins.cpp:263
llvm::GCRelocateInst
Represents calls to the gc.relocate intrinsic.
Definition: IntrinsicInst.h:1278
findInitTrampoline
static IntrinsicInst * findInitTrampoline(Value *Callee)
Definition: InstCombineCalls.cpp:2503
llvm::PointerType::getUnqual
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:651
llvm::PatternMatch::m_ImmConstant
match_combine_and< class_match< Constant >, match_unless< class_match< ConstantExpr > > > m_ImmConstant()
Match an arbitrary immediate Constant and ignore it.
Definition: PatternMatch.h:781
llvm::InlineAsm::canThrow
bool canThrow() const
Definition: InlineAsm.h:71
llvm::LibCallSimplifier
LibCallSimplifier - This class implements a collection of optimizations that replace well formed call...
Definition: SimplifyLibCalls.h:100
llvm::InstCombiner::getAssumptionCache
AssumptionCache & getAssumptionCache() const
Definition: InstCombiner.h:368
llvm::BinaryOperator::CreateFMulFMF
static BinaryOperator * CreateFMulFMF(Value *V1, Value *V2, Instruction *FMFSource, const Twine &Name="")
Definition: InstrTypes.h:268
llvm::BinaryOpIntrinsic::isSigned
bool isSigned() const
Whether the intrinsic is signed or unsigned.
Definition: IntrinsicInst.cpp:578
uint64_t
llvm::InstCombiner::getDataLayout
const DataLayout & getDataLayout() const
Definition: InstCombiner.h:371
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
llvm::PatternMatch::m_LogicalOr
LogicalOp_match< LHS, RHS, Instruction::Or > m_LogicalOr(const LHS &L, const RHS &R)
Matches L || R either in the form of L | R or L ? true : R.
Definition: PatternMatch.h:2522
llvm::RetainedKnowledge::ArgValue
uint64_t ArgValue
Definition: AssumeBundleQueries.h:102
llvm::LoadInst::setOrdering
void setOrdering(AtomicOrdering Ordering)
Sets the ordering constraint of this load instruction.
Definition: Instructions.h:237
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::TruncInst
This class represents a truncation of integer types.
Definition: Instructions.h:4753
llvm::GCRelocateInst::getDerivedPtr
Value * getDerivedPtr() const
Definition: IntrinsicInst.cpp:623
llvm::AttributeSet::get
static AttributeSet get(LLVMContext &C, const AttrBuilder &B)
Definition: Attributes.cpp:590
llvm::isStrdupLikeFn
bool isStrdupLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates memory (strdup,...
Definition: MemoryBuiltins.cpp:332
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1782
llvm::CallBrInst
CallBr instruction, tracking function calls that may not return control but instead transfer it to a ...
Definition: Instructions.h:3958
llvm::PatternMatch::m_AllOnes
cst_pred_ty< is_all_ones > m_AllOnes()
Match an integer or vector with all bits set.
Definition: PatternMatch.h:445
simplifyNeonTbl1
static Value * simplifyNeonTbl1(const IntrinsicInst &II, InstCombiner::BuilderTy &Builder)
Convert a table lookup to shufflevector if the mask is constant.
Definition: InstCombineCalls.cpp:621
llvm::Instruction::isIdenticalTo
bool isIdenticalTo(const Instruction *I) const
Return true if the specified instruction is exactly identical to the current one.
Definition: Instruction.cpp:490
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::assumeAligned
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
Definition: Alignment.h:103
llvm::DenseMap< Value *, unsigned >
llvm::InstCombinerImpl::visitFenceInst
Instruction * visitFenceInst(FenceInst &FI)
Definition: InstCombineCalls.cpp:2362
llvm::AtomicOrdering::Unordered
@ Unordered
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::AttrBuilder
Definition: Attributes.h:931
llvm::FunctionType::getParamType
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
llvm::LoadInst::setAlignment
void setAlignment(Align Align)
Definition: Instructions.h:227
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
llvm::lowerObjectSizeCall
Value * lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, const TargetLibraryInfo *TLI, bool MustSucceed)
Try to turn a call to @llvm.objectsize into an integer value of the given Type.
Definition: MemoryBuiltins.cpp:525
llvm::PatternMatch::m_FAbs
m_Intrinsic_Ty< Opnd0 >::Ty m_FAbs(const Opnd0 &Op0)
Definition: PatternMatch.h:2174
llvm::Attribute::getWithDereferenceableBytes
static Attribute getWithDereferenceableBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:176
llvm::PatternMatch::m_And
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1123
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
ArrayRef.h
llvm::CallBase::addRetAttr
void addRetAttr(Attribute::AttrKind Kind)
Adds the attribute to the return value.
Definition: InstrTypes.h:1500
llvm::SimplifyFMAFMul
Value * SimplifyFMAFMul(Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q, fp::ExceptionBehavior ExBehavior=fp::ebIgnore, RoundingMode Rounding=RoundingMode::NearestTiesToEven)
Given operands for the multiplication of a FMA, fold the result or return null.
Definition: InstructionSimplify.cpp:5146
llvm::Instruction::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:367
llvm::PatternMatch::m_SRem
BinaryOp_match< LHS, RHS, Instruction::SRem > m_SRem(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1111
llvm::computeKnownBits
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, OptimizationRemarkEmitter *ORE=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
Definition: ValueTracking.cpp:224
maximum
Should compile r2 movcc movcs str strb mov lr r1 movcs movcc mov lr r1 str mov mov cmp r1 movlo r2 str bx lr r0 mov mov cmp r0 movhs r2 mov r1 bx lr Some of the NEON intrinsics may be appropriate for more general either as target independent intrinsics or perhaps elsewhere in the ARM backend Some of them may also be lowered to target independent and perhaps some new SDNodes could be added For maximum
Definition: README.txt:489
llvm::KnownBits::countMaxLeadingZeros
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition: KnownBits.h:266
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
foldClampRangeOfTwo
static Instruction * foldClampRangeOfTwo(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
If we have a clamp pattern like max (min X, 42), 41 – where the output can only be one of two possibl...
Definition: InstCombineCalls.cpp:799
llvm::KnownBits::countMaxTrailingZeros
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:256
llvm::SPF_ABS
@ SPF_ABS
Floating point maxnum.
Definition: ValueTracking.h:684
llvm::CallBase::hasOperandBundles
bool hasOperandBundles() const
Return true if this User has any operand bundles.
Definition: InstrTypes.h:1895
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
memcpy
<%struct.s * > cast struct s *S to sbyte *< sbyte * > sbyte uint cast struct s *agg result to sbyte *< sbyte * > sbyte uint cast struct s *memtmp to sbyte *< sbyte * > sbyte uint ret void llc ends up issuing two memcpy or custom lower memcpy(of small size) to be ldmia/stmia. I think option 2 is better but the current register allocator cannot allocate a chunk of registers at a time. A feasible temporary solution is to use specific physical registers at the lowering time for small(<
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::isKnownNegation
bool isKnownNegation(const Value *X, const Value *Y, bool NeedNSW=false)
Return true if the two given values are negation.
Definition: ValueTracking.cpp:5931
llvm::isAlignedAllocLikeFn
bool isAlignedAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates uninitialized memory with a...
Definition: MemoryBuiltins.cpp:276
llvm::Instruction::isFast
bool isFast() const
Determine whether all fast-math-flags are set.
Definition: Instruction.cpp:240
llvm::KnownBits::countMinLeadingZeros
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
Definition: KnownBits.h:236
llvm::KnownBits::countMaxPopulation
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:281
llvm::CmpInst::BAD_ICMP_PREDICATE
@ BAD_ICMP_PREDICATE
Definition: InstrTypes.h:753
APSInt.h
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:138
llvm::PatternMatch::m_SMin
MaxMin_match< ICmpInst, LHS, RHS, smin_pred_ty > m_SMin(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1869
simplifyInvariantGroupIntrinsic
static Instruction * simplifyInvariantGroupIntrinsic(IntrinsicInst &II, InstCombinerImpl &IC)
This function transforms launder.invariant.group and strip.invariant.group like: launder(launder(x)) ...
Definition: InstCombineCalls.cpp:400
foldCtpop
static Instruction * foldCtpop(IntrinsicInst &II, InstCombinerImpl &IC)
Definition: InstCombineCalls.cpp:538
llvm::IRBuilderBase::getTrue
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:458
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
llvm::MDNode
Metadata node.
Definition: Metadata.h:906
llvm::CallBase::Create
static CallBase * Create(CallBase *CB, ArrayRef< OperandBundleDef > Bundles, Instruction *InsertPt=nullptr)
Create a clone of CB with a different set of operand bundles and insert it before InsertPt.
Definition: Instructions.cpp:255
llvm::PatternMatch::m_Constant
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
Definition: PatternMatch.h:142
haveSameOperands
static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E, unsigned NumOperands)
Definition: InstCombineCalls.cpp:657
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
llvm::PatternMatch::m_Value
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:76
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::User::setOperand
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:749
llvm::APSInt::getMaxValue
static APSInt getMaxValue(uint32_t numBits, bool Unsigned)
Return the APSInt representing the maximum integer value with the given bit width and signedness.
Definition: APSInt.h:282
llvm::APIntOps::smin
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition: APInt.h:2118
llvm::PatternMatch::m_SExt
CastClass_match< OpTy, Instruction::SExt > m_SExt(const OpTy &Op)
Matches SExt.
Definition: PatternMatch.h:1633
llvm::PatternMatch::m_SpecificInt
specific_intval< false > m_SpecificInt(APInt V)
Match a specific integer value or vector with all elements equal to the value.
Definition: PatternMatch.h:885
llvm::ValueHandleBase::ValueIsDeleted
static void ValueIsDeleted(Value *V)
Definition: Value.cpp:1140
llvm::ArrayRef< int >
IT
static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::ZeroOrMore, cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate IT block based on arch"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow deprecated IT based on ARMv8"), clEnumValN(NoRestrictedIT, "arm-no-restrict-it", "Allow IT blocks based on ARMv7")))
llvm::VAEndInst
This represents the llvm.va_end intrinsic.
Definition: IntrinsicInst.h:1101
llvm::Instruction::getFastMathFlags
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition: Instruction.cpp:280
None.h
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
DataLayout.h
llvm::StructType
Class to represent struct types.
Definition: DerivedTypes.h:213
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:179
llvm::PatternMatch::m_Undef
auto m_Undef()
Match an arbitrary undef constant.
Definition: PatternMatch.h:136
llvm::AssumptionCache
A cache of @llvm.assume calls within a function.
Definition: AssumptionCache.h:41
SimplifyLibCalls.h
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:745
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:134
llvm::Constant::getAggregateElement
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:420
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
isSafeToEliminateVarargsCast
static bool isSafeToEliminateVarargsCast(const CallBase &Call, const DataLayout &DL, const CastInst *const CI, const int ix)
If this cast does not affect the value passed through the varargs area, we can eliminate the use of t...
Definition: InstCombineCalls.cpp:2383
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
llvm::PatternMatch::m_Shuffle
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
Definition: PatternMatch.h:1561
moveAddAfterMinMax
static Instruction * moveAddAfterMinMax(IntrinsicInst *II, InstCombiner::BuilderTy &Builder)
Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0.
Definition: InstCombineCalls.cpp:759
uint32_t
Compiler.h
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:993
llvm::InstCombinerImpl::visitVAEndInst
Instruction * visitVAEndInst(VAEndInst &I)
Definition: InstCombineCalls.cpp:704
llvm::MinMax
Definition: AssumeBundleQueries.h:71
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::CastInst::isBitOrNoopPointerCastable
static bool isBitOrNoopPointerCastable(Type *SrcTy, Type *DestTy, const DataLayout &DL)
Check whether a bitcast, inttoptr, or ptrtoint cast between these types is valid and a no-op.
Definition: Instructions.cpp:3336
llvm::ConstantExpr::getIntegerCast
static Constant * getIntegerCast(Constant *C, Type *Ty, bool IsSigned)
Create a ZExt, Bitcast or Trunc for integer -> integer casts.
Definition: Constants.cpp:2071
llvm::RecurKind::Mul
@ Mul
Product of integers.
llvm::FPExtInst
This class represents an extension of floating point types.
Definition: Instructions.h:4909
trunc
We have fiadd patterns now but the followings have the same cost and complexity We need a way to specify the later is more profitable def def The FP stackifier should handle simple permutates to reduce number of shuffle e g trunc
Definition: README-FPStack.txt:63
llvm::RetainedKnowledge
Represent one information held inside an operand bundle of an llvm.assume.
Definition: AssumeBundleQueries.h:100
llvm::Instruction::copyFastMathFlags
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
Definition: Instruction.cpp:235
llvm::CastInst
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:430
llvm::WithOverflowInst
Represents an op.with.overflow intrinsic.
Definition: IntrinsicInst.h:589
llvm::AMDGPUISD::BFI
@ BFI
Definition: AMDGPUISelLowering.h:421
llvm::APInt::ult
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1044
llvm::Attribute::getWithDereferenceableOrNullBytes
static Attribute getWithDereferenceableOrNullBytes(LLVMContext &Context, uint64_t Bytes)
Definition: Attributes.cpp:182
llvm::CallBase::hasRetAttr
bool hasRetAttr(Attribute::AttrKind Kind) const
Determine whether the return value has the given attribute.
Definition: InstrTypes.h:1579
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:286
llvm::PatternMatch::m_SMax
MaxMin_match< ICmpInst, LHS, RHS, smax_pred_ty > m_SMax(const LHS &L, const RHS &R)
Definition: PatternMatch.h:1863
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
llvm::Type::getContext
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:127
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:175
llvm::isOpNewLikeFn
bool isOpNewLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast=false)
Tests if a value is a call or invoke to a library function that allocates memory and throws if an all...
Definition: MemoryBuiltins.cpp:325
ValueHandle.h
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:152
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::Value::stripPointerCasts
const Value * stripPointerCasts() const
Strip off pointer casts, all-zero GEPs and address space casts.
Definition: Value.cpp:687
llvm::SmallPtrSetImplBase::size
size_type size() const
Definition: SmallPtrSet.h:92
llvm::CallBase::doesNotThrow
bool doesNotThrow() const
Determine if the call cannot unwind.
Definition: InstrTypes.h:1849
llvm::OperandBundleUse::Inputs
ArrayRef< Use > Inputs
Definition: InstrTypes.h:1057
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:206
getPromotedType
static Type * getPromotedType(Type *Ty)
Return the specified type promoted as it would be to pass though a va_arg area.
Definition: InstCombineCalls.cpp:103
llvm::Attribute::getWithAlignment
static Attribute getWithAlignment(LLVMContext &Context, Align Alignment)
Return a uniquified Attribute object that has the specific alignment set.
Definition: Attributes.cpp:166
llvm::GetStringLength
uint64_t GetStringLength(const Value *V, unsigned CharSize=8)
If we can compute the length of the string pointed to by the specified pointer, return 'len+1'.
Definition: ValueTracking.cpp:4322
llvm::Function::doesNotThrow
bool doesNotThrow() const
Determine if the function cannot unwind.
Definition: Function.h:567
Attributes.h
llvm::PatternMatch::m_FPExt
CastClass_match< OpTy, Instruction::FPExt > m_FPExt(const OpTy &Op)
Definition: PatternMatch.h:1697
llvm::APIntOps::umax
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition: APInt.h:2133
Constant.h
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::GCRelocateInst::getBasePtr
Value * getBasePtr() const
Definition: IntrinsicInst.cpp:617
llvm::minnum
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1296
llvm::IRBuilderBase::getFalse
ConstantInt * getFalse()
Get the constant value for i1 false.
Definition: IRBuilder.h:463
llvm::APInt::uadd_sat
APInt uadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2013
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:242
llvm::ConstantInt::getTrue
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:873
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:348
llvm::KnownBits
Definition: KnownBits.h:23
llvm::ConstantExpr::getNeg
static Constant * getNeg(Constant *C, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2671
llvm::AnyMemSetInst
This class represents any memset intrinsic.
Definition: IntrinsicInst.h:1010
llvm::CastInst::CreateBitOrPointerCast
static CastInst * CreateBitOrPointerCast(Value *S, Type *Ty, const Twine &Name="", Instruction *InsertBefore=nullptr)
Create a BitCast, a PtrToInt, or an IntToPTr cast instruction.
Definition: Instructions.cpp:3231
llvm::KnownBits::countMinPopulation
unsigned countMinPopulation() const
Returns the number of bits known to be one.
Definition: KnownBits.h:276
NC
#define NC
Definition: regutils.h:42
llvm::BinaryOperator::CreateNUW
static BinaryOperator * CreateNUW(BinaryOps Opc, Value *V1, Value *V2, const Twine &Name="")
Definition: InstrTypes.h:303
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:324
llvm::ConstantInt::getBool
static ConstantInt * getBool(LLVMContext &Context, bool V)
Definition: Constants.cpp:887
llvm::CallBase::arg_size
unsigned arg_size() const
Definition: InstrTypes.h:1326
llvm::Type::isIntOrIntVectorTy
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:196
GlobalVariable.h
llvm::ConstantFP::get
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:972
Casting.h
llvm::InstCombinerImpl::visitCallBrInst
Instruction * visitCallBrInst(CallBrInst &CBI)
Definition: InstCombineCalls.cpp:2377
LowAndHigh
Metadata * LowAndHigh[]
Definition: NVVMIntrRange.cpp:68
Function.h
llvm::BitWidth
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:147
llvm::BinaryOperator::CreateNSWNeg
static BinaryOperator * CreateNSWNeg(Value *Op, const Twine &Name="", Instruction *InsertBefore=nullptr)
Definition: Instructions.cpp:2706
llvm::DenseMapBase::size
unsigned size() const
Definition: DenseMap.h:100
llvm::PatternMatch::m_BSwap
m_Intrinsic_Ty< Opnd0 >::Ty m_BSwap(const Opnd0 &Op0)
Definition: PatternMatch.h:2169
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:221
powi
This is blocked on not handling X *X *X powi(X, 3)(see note above). The issue is that we end up getting t
llvm::OverflowResult::AlwaysOverflowsHigh
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
llvm::Function::getFunctionType
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:177
llvm::PatternMatch::m_ZeroInt
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
Definition: PatternMatch.h:522
llvm::getInverseMinMaxIntrinsic
Intrinsic::ID getInverseMinMaxIntrinsic(Intrinsic::ID MinMaxID)
Definition: ValueTracking.cpp:6320
llvm::PatternMatch::m_PtrToInt
CastClass_match< OpTy, Instruction::PtrToInt > m_PtrToInt(const OpTy &Op)
Matches PtrToInt.
Definition: PatternMatch.h:1609
llvm::Instruction::hasAllowReassoc
bool hasAllowReassoc() const
Determine whether the allow-reassociation flag is set.
Definition: Instruction.cpp:245
llvm::CallBase::getCalledOperand
Value * getCalledOperand() const
Definition: InstrTypes.h:1376
createOverflowTuple
static Instruction * createOverflowTuple(IntrinsicInst *II, Value *Result, Constant *Overflow)
Creates a result tuple for an overflow intrinsic II with a given Result and a constant Overflow value...
Definition: InstCombineCalls.cpp:725
llvm::IntrinsicInst
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:45
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:476
llvm::InstCombinerImpl::replaceOperand
Instruction * replaceOperand(Instruction &I, unsigned OpNum, Value *V)
Replace operand of instruction and add old operand to the worklist.
Definition: InstCombineInternal.h:420
llvm::BasicBlock::reverse_iterator
InstListType::reverse_iterator reverse_iterator
Definition: BasicBlock.h:92
llvm::X86AS::SS
@ SS
Definition: X86.h:189
llvm::RecurKind::FAdd
@ FAdd
Sum of floats.
Instructions.h
llvm::isImpliedByDomCondition
Optional< bool > isImpliedByDomCondition(const Value *Cond, const Instruction *ContextI, const DataLayout &DL)
Return the boolean condition value in the context of the given instruction if it is known based on do...
Definition: ValueTracking.cpp:6745
llvm::Type::isStructTy
bool isStructTy() const
True if this is an instance of StructType.
Definition: Type.h:211
llvm::AttributeSet
Definition: Attributes.h:267
SmallVector.h
getKnownSign
static Optional< bool > getKnownSign(Value *Op, Instruction *CxtI, const DataLayout &DL, AssumptionCache *AC, DominatorTree *DT)
Definition: InstCombineCalls.cpp:744
llvm::PatternMatch::m_Specific
specificval_ty m_Specific(const Value *V)
Match if we have a specific specified value.
Definition: PatternMatch.h:802
User.h
llvm::PatternMatch::m_ICmp
CmpClass_match< LHS, RHS, ICmpInst, ICmpInst::Predicate > m_ICmp(ICmpInst::Predicate &Pred, const LHS &L, const RHS &R)
Definition: PatternMatch.h:1404
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:743
llvm::User::op_begin
op_iterator op_begin()
Definition: User.h:234
llvm::Type::getPointerElementType
Type * getPointerElementType() const
Definition: Type.h:369
llvm::CallBase::getArgOperand
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1328
SmallBitVector.h
N
#define N
llvm::IntrinsicInst::isCommutative
bool isCommutative() const
Return true if swapping the first two arguments to the intrinsic produces the same result.
Definition: IntrinsicInst.h:58
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
InstructionSimplify.h
llvm::ConstantExpr::getMul
static Constant * getMul(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2712
TargetTransformInfo.h
llvm::PatternMatch::m_NSWNeg
OverflowingBinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub, OverflowingBinaryOperator::NoSignedWrap > m_NSWNeg(const ValTy &V)
Matches a 'Neg' as 'sub nsw 0, V'.
Definition: PatternMatch.h:2273
llvm::PHINode
Definition: Instructions.h:2633
llvm::Instruction::copyIRFlags
void copyIRFlags(const Value *V, bool IncludeWrapFlags=true)
Convenience method to copy supported exact, fast-math, and (optionally) wrapping flags from V to this...
Definition: Instruction.cpp:289
llvm::buildAssumeFromKnowledge
AssumeInst * buildAssumeFromKnowledge(ArrayRef< RetainedKnowledge > Knowledge, Instruction *CtxI, AssumptionCache *AC=nullptr, DominatorTree *DT=nullptr)
Build and return a new assume created from the provided knowledge if the knowledge in the assume is f...
Definition: AssumeBundleBuilder.cpp:306
llvm::CallBase
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
Definition: InstrTypes.h:1161
llvm::GlobalValue::getType
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:271
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:401
DerivedTypes.h
llvm::PatternMatch::m_Neg
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
Definition: PatternMatch.h:2264
llvm::APInt::getLowBitsSet
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:289
llvm::IntegerType::get
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:313
llvm::TargetLibraryInfoImpl::isCallingConvCCompatible
static bool isCallingConvCCompatible(CallBase *CI)
Returns true if call site / callee has cdecl-compatible calling conventions.
Definition: TargetLibraryInfo.cpp:101
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1475
llvm::CallingConv::C
@ C
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
LLVMContext.h
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::IRBuilderBase::CreateSub
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1229
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:382
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:389
llvm::InstCombinerImpl::SimplifyAnyMemSet
Instruction * SimplifyAnyMemSet(AnyMemSetInst *MI)
Definition: InstCombineCalls.cpp:226
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::cl::desc
Definition: CommandLine.h:412
Mod
Module * Mod
Definition: PassBuilderBindings.cpp:54
raw_ostream.h
llvm::PatternMatch::m_Trunc
CastClass_match< OpTy, Instruction::Trunc > m_Trunc(const OpTy &Op)
Matches Trunc.
Definition: PatternMatch.h:1621
llvm::NullPointerIsDefined
bool NullPointerIsDefined(const Function *F, unsigned AS=0)
Check whether null pointer dereferencing is considered undefined behavior for a given function or an ...
Definition: Function.cpp:1965
llvm::AttributeList::hasParamAttr
bool hasParamAttr(unsigned ArgNo, Attribute::AttrKind Kind) const
Return true if the attribute exists for the given argument.
Definition: Attributes.h:722
llvm::Attribute::getWithByValType
static Attribute getWithByValType(LLVMContext &Context, Type *Ty)
Definition: Attributes.cpp:188
llvm::AttributeList::getParamAttrs
AttributeSet getParamAttrs(unsigned ArgNo) const
The attributes for the argument or parameter at the given index are returned.
Definition: Attributes.cpp:1354
llvm::ConstantAggregateZero::get
static ConstantAggregateZero * get(Type *Ty)
Definition: Constants.cpp:1675
llvm::Type::canLosslesslyBitCastTo
bool canLosslesslyBitCastTo(Type *Ty) const
Return true if this type could be converted with a lossless BitCast to type 'Ty'.
Definition: Type.cpp:107
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:624
llvm::isPowerOf2_64
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:496
Value.h
llvm::abs
APFloat abs(APFloat X)
Returns the absolute value of the argument.
Definition: APFloat.h:1282
llvm::PatternMatch::m_Cmp
class_match< CmpInst > m_Cmp()
Matches any compare instruction and ignore it.
Definition: PatternMatch.h:89
llvm::FunctionType::getReturnType
Type * getReturnType() const
Definition: DerivedTypes.h:124
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
llvm::AttributeList::FirstArgIndex
@ FirstArgIndex
Definition: Attributes.h:404
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
llvm::InstCombinerImpl::computeKnownBits
void computeKnownBits(const Value *V, KnownBits &Known, unsigned Depth, const Instruction *CxtI) const
Definition: InstCombineInternal.h:464
llvm::APIntOps::smax
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition: APInt.h:2123
llvm::AttributeList::isEmpty
bool isEmpty() const
Return true if there are no attributes.
Definition: Attributes.h:892
llvm::PatternMatch::m_LogicalAnd
LogicalOp_match< LHS, RHS, Instruction::And > m_LogicalAnd(const LHS &L, const RHS &R)
Matches L && R either in the form of L & R or L ? R : false.
Definition: PatternMatch.h:2504
llvm::IRBuilderBase::CreateCall
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2403
llvm::Optional::getValue
constexpr const T & getValue() const LLVM_LVALUE_FUNCTION
Definition: Optional.h:282
llvm::isFreeCall
const CallInst * isFreeCall(const Value *I, const TargetLibraryInfo *TLI)
isFreeCall - Returns non-null if the value is a call to the builtin free()
Definition: MemoryBuiltins.cpp:486
llvm::Instruction::moveBefore
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Definition: Instruction.cpp:97
Other
Optional< std::vector< StOtherPiece > > Other
Definition: ELFYAML.cpp:1184
llvm::CallBase::setCallingConv
void setCallingConv(CallingConv::ID CC)
Definition: InstrTypes.h:1443
llvm::FunctionType
Class to represent function types.
Definition: DerivedTypes.h:103
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1815