LLVM  14.0.0git
ExpandVectorPredication.cpp
Go to the documentation of this file.
1 //===----- CodeGen/ExpandVectorPredication.cpp - Expand VP intrinsics -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements IR expansion for vector predication intrinsics, allowing
10 // targets to enable vector predication until just before codegen.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "llvm/ADT/Statistic.h"
18 #include "llvm/CodeGen/Passes.h"
19 #include "llvm/IR/Constants.h"
20 #include "llvm/IR/Function.h"
21 #include "llvm/IR/IRBuilder.h"
22 #include "llvm/IR/InstIterator.h"
23 #include "llvm/IR/Instructions.h"
24 #include "llvm/IR/IntrinsicInst.h"
25 #include "llvm/IR/Intrinsics.h"
26 #include "llvm/IR/Module.h"
27 #include "llvm/InitializePasses.h"
28 #include "llvm/Pass.h"
30 #include "llvm/Support/Compiler.h"
31 #include "llvm/Support/Debug.h"
33 
34 using namespace llvm;
35 
38 
39 // Keep this in sync with TargetTransformInfo::VPLegalization.
40 #define VPINTERNAL_VPLEGAL_CASES \
41  VPINTERNAL_CASE(Legal) \
42  VPINTERNAL_CASE(Discard) \
43  VPINTERNAL_CASE(Convert)
44 
45 #define VPINTERNAL_CASE(X) "|" #X
46 
47 // Override options.
49  "expandvp-override-evl-transform", cl::init(""), cl::Hidden,
50  cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
51  ". If non-empty, ignore "
52  "TargetTransformInfo and "
53  "always use this transformation for the %evl parameter (Used in "
54  "testing)."));
55 
57  "expandvp-override-mask-transform", cl::init(""), cl::Hidden,
58  cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES
59  ". If non-empty, Ignore "
60  "TargetTransformInfo and "
61  "always use this transformation for the %mask parameter (Used in "
62  "testing)."));
63 
64 #undef VPINTERNAL_CASE
65 #define VPINTERNAL_CASE(X) .Case(#X, VPLegalization::X)
66 
67 static VPTransform parseOverrideOption(const std::string &TextOpt) {
69 }
70 
71 #undef VPINTERNAL_VPLEGAL_CASES
72 
73 // Whether any override options are set.
74 static bool anyExpandVPOverridesSet() {
75  return !EVLTransformOverride.empty() || !MaskTransformOverride.empty();
76 }
77 
78 #define DEBUG_TYPE "expandvp"
79 
80 STATISTIC(NumFoldedVL, "Number of folded vector length params");
81 STATISTIC(NumLoweredVPOps, "Number of folded vector predication operations");
82 
83 ///// Helpers {
84 
85 /// \returns Whether the vector mask \p MaskVal has all lane bits set.
86 static bool isAllTrueMask(Value *MaskVal) {
87  auto *ConstVec = dyn_cast<ConstantVector>(MaskVal);
88  return ConstVec && ConstVec->isAllOnesValue();
89 }
90 
91 /// \returns A non-excepting divisor constant for this type.
92 static Constant *getSafeDivisor(Type *DivTy) {
93  assert(DivTy->isIntOrIntVectorTy() && "Unsupported divisor type");
94  return ConstantInt::get(DivTy, 1u, false);
95 }
96 
97 /// Transfer operation properties from \p OldVPI to \p NewVal.
98 static void transferDecorations(Value &NewVal, VPIntrinsic &VPI) {
99  auto *NewInst = dyn_cast<Instruction>(&NewVal);
100  if (!NewInst || !isa<FPMathOperator>(NewVal))
101  return;
102 
103  auto *OldFMOp = dyn_cast<FPMathOperator>(&VPI);
104  if (!OldFMOp)
105  return;
106 
107  NewInst->setFastMathFlags(OldFMOp->getFastMathFlags());
108 }
109 
110 /// Transfer all properties from \p OldOp to \p NewOp and replace all uses.
111 /// OldVP gets erased.
112 static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp) {
113  transferDecorations(NewOp, OldOp);
114  OldOp.replaceAllUsesWith(&NewOp);
115  OldOp.eraseFromParent();
116 }
117 
118 //// } Helpers
119 
120 namespace {
121 
122 // Expansion pass state at function scope.
123 struct CachingVPExpander {
124  Function &F;
125  const TargetTransformInfo &TTI;
126 
127  /// \returns A (fixed length) vector with ascending integer indices
128  /// (<0, 1, ..., NumElems-1>).
129  /// \p Builder
130  /// Used for instruction creation.
131  /// \p LaneTy
132  /// Integer element type of the result vector.
133  /// \p NumElems
134  /// Number of vector elements.
135  Value *createStepVector(IRBuilder<> &Builder, Type *LaneTy,
136  unsigned NumElems);
137 
138  /// \returns A bitmask that is true where the lane position is less-than \p
139  /// EVLParam
140  ///
141  /// \p Builder
142  /// Used for instruction creation.
143  /// \p VLParam
144  /// The explicit vector length parameter to test against the lane
145  /// positions.
146  /// \p ElemCount
147  /// Static (potentially scalable) number of vector elements.
148  Value *convertEVLToMask(IRBuilder<> &Builder, Value *EVLParam,
149  ElementCount ElemCount);
150 
151  Value *foldEVLIntoMask(VPIntrinsic &VPI);
152 
153  /// "Remove" the %evl parameter of \p PI by setting it to the static vector
154  /// length of the operation.
155  void discardEVLParameter(VPIntrinsic &PI);
156 
157  /// \brief Lower this VP binary operator to a unpredicated binary operator.
158  Value *expandPredicationInBinaryOperator(IRBuilder<> &Builder,
159  VPIntrinsic &PI);
160 
161  /// \brief Lower this VP reduction to a call to an unpredicated reduction
162  /// intrinsic.
163  Value *expandPredicationInReduction(IRBuilder<> &Builder,
165 
166  /// \brief Query TTI and expand the vector predication in \p P accordingly.
167  Value *expandPredication(VPIntrinsic &PI);
168 
169  /// \brief Determine how and whether the VPIntrinsic \p VPI shall be
170  /// expanded. This overrides TTI with the cl::opts listed at the top of this
171  /// file.
172  VPLegalization getVPLegalizationStrategy(const VPIntrinsic &VPI) const;
173  bool UsingTTIOverrides;
174 
175 public:
176  CachingVPExpander(Function &F, const TargetTransformInfo &TTI)
177  : F(F), TTI(TTI), UsingTTIOverrides(anyExpandVPOverridesSet()) {}
178 
179  bool expandVectorPredication();
180 };
181 
182 //// CachingVPExpander {
183 
184 Value *CachingVPExpander::createStepVector(IRBuilder<> &Builder, Type *LaneTy,
185  unsigned NumElems) {
186  // TODO add caching
187  SmallVector<Constant *, 16> ConstElems;
188 
189  for (unsigned Idx = 0; Idx < NumElems; ++Idx)
190  ConstElems.push_back(ConstantInt::get(LaneTy, Idx, false));
191 
192  return ConstantVector::get(ConstElems);
193 }
194 
195 Value *CachingVPExpander::convertEVLToMask(IRBuilder<> &Builder,
196  Value *EVLParam,
197  ElementCount ElemCount) {
198  // TODO add caching
199  // Scalable vector %evl conversion.
200  if (ElemCount.isScalable()) {
201  auto *M = Builder.GetInsertBlock()->getModule();
202  Type *BoolVecTy = VectorType::get(Builder.getInt1Ty(), ElemCount);
203  Function *ActiveMaskFunc = Intrinsic::getDeclaration(
204  M, Intrinsic::get_active_lane_mask, {BoolVecTy, EVLParam->getType()});
205  // `get_active_lane_mask` performs an implicit less-than comparison.
206  Value *ConstZero = Builder.getInt32(0);
207  return Builder.CreateCall(ActiveMaskFunc, {ConstZero, EVLParam});
208  }
209 
210  // Fixed vector %evl conversion.
211  Type *LaneTy = EVLParam->getType();
212  unsigned NumElems = ElemCount.getFixedValue();
213  Value *VLSplat = Builder.CreateVectorSplat(NumElems, EVLParam);
214  Value *IdxVec = createStepVector(Builder, LaneTy, NumElems);
215  return Builder.CreateICmp(CmpInst::ICMP_ULT, IdxVec, VLSplat);
216 }
217 
218 Value *
219 CachingVPExpander::expandPredicationInBinaryOperator(IRBuilder<> &Builder,
220  VPIntrinsic &VPI) {
223  "Implicitly dropping %evl in non-speculatable operator!");
224 
225  auto OC = static_cast<Instruction::BinaryOps>(*VPI.getFunctionalOpcode());
227 
228  Value *Op0 = VPI.getOperand(0);
229  Value *Op1 = VPI.getOperand(1);
230  Value *Mask = VPI.getMaskParam();
231 
232  // Blend in safe operands.
233  if (Mask && !isAllTrueMask(Mask)) {
234  switch (OC) {
235  default:
236  // Can safely ignore the predicate.
237  break;
238 
239  // Division operators need a safe divisor on masked-off lanes (1).
240  case Instruction::UDiv:
241  case Instruction::SDiv:
242  case Instruction::URem:
243  case Instruction::SRem:
244  // 2nd operand must not be zero.
245  Value *SafeDivisor = getSafeDivisor(VPI.getType());
246  Op1 = Builder.CreateSelect(Mask, Op1, SafeDivisor);
247  }
248  }
249 
250  Value *NewBinOp = Builder.CreateBinOp(OC, Op0, Op1, VPI.getName());
251 
252  replaceOperation(*NewBinOp, VPI);
253  return NewBinOp;
254 }
255 
256 static Value *getNeutralReductionElement(const VPReductionIntrinsic &VPI,
257  Type *EltTy) {
258  bool Negative = false;
259  unsigned EltBits = EltTy->getScalarSizeInBits();
260  switch (VPI.getIntrinsicID()) {
261  default:
262  llvm_unreachable("Expecting a VP reduction intrinsic");
263  case Intrinsic::vp_reduce_add:
264  case Intrinsic::vp_reduce_or:
265  case Intrinsic::vp_reduce_xor:
266  case Intrinsic::vp_reduce_umax:
267  return Constant::getNullValue(EltTy);
268  case Intrinsic::vp_reduce_mul:
269  return ConstantInt::get(EltTy, 1, /*IsSigned*/ false);
270  case Intrinsic::vp_reduce_and:
271  case Intrinsic::vp_reduce_umin:
272  return ConstantInt::getAllOnesValue(EltTy);
273  case Intrinsic::vp_reduce_smin:
274  return ConstantInt::get(EltTy->getContext(),
275  APInt::getSignedMaxValue(EltBits));
276  case Intrinsic::vp_reduce_smax:
277  return ConstantInt::get(EltTy->getContext(),
278  APInt::getSignedMinValue(EltBits));
279  case Intrinsic::vp_reduce_fmax:
280  Negative = true;
282  case Intrinsic::vp_reduce_fmin: {
283  FastMathFlags Flags = VPI.getFastMathFlags();
284  const fltSemantics &Semantics = EltTy->getFltSemantics();
285  return !Flags.noNaNs() ? ConstantFP::getQNaN(EltTy, Negative)
286  : !Flags.noInfs()
287  ? ConstantFP::getInfinity(EltTy, Negative)
288  : ConstantFP::get(EltTy,
289  APFloat::getLargest(Semantics, Negative));
290  }
291  case Intrinsic::vp_reduce_fadd:
292  return ConstantFP::getNegativeZero(EltTy);
293  case Intrinsic::vp_reduce_fmul:
294  return ConstantFP::get(EltTy, 1.0);
295  }
296 }
297 
298 Value *
299 CachingVPExpander::expandPredicationInReduction(IRBuilder<> &Builder,
300  VPReductionIntrinsic &VPI) {
303  "Implicitly dropping %evl in non-speculatable operator!");
304 
305  Value *Mask = VPI.getMaskParam();
306  Value *RedOp = VPI.getOperand(VPI.getVectorParamPos());
307 
308  // Insert neutral element in masked-out positions
309  if (Mask && !isAllTrueMask(Mask)) {
310  auto *NeutralElt = getNeutralReductionElement(VPI, VPI.getType());
311  auto *NeutralVector = Builder.CreateVectorSplat(
312  cast<VectorType>(RedOp->getType())->getElementCount(), NeutralElt);
313  RedOp = Builder.CreateSelect(Mask, RedOp, NeutralVector);
314  }
315 
316  Value *Reduction;
317  Value *Start = VPI.getOperand(VPI.getStartParamPos());
318 
319  switch (VPI.getIntrinsicID()) {
320  default:
321  llvm_unreachable("Impossible reduction kind");
322  case Intrinsic::vp_reduce_add:
323  Reduction = Builder.CreateAddReduce(RedOp);
324  Reduction = Builder.CreateAdd(Reduction, Start);
325  break;
326  case Intrinsic::vp_reduce_mul:
327  Reduction = Builder.CreateMulReduce(RedOp);
328  Reduction = Builder.CreateMul(Reduction, Start);
329  break;
330  case Intrinsic::vp_reduce_and:
331  Reduction = Builder.CreateAndReduce(RedOp);
332  Reduction = Builder.CreateAnd(Reduction, Start);
333  break;
334  case Intrinsic::vp_reduce_or:
335  Reduction = Builder.CreateOrReduce(RedOp);
336  Reduction = Builder.CreateOr(Reduction, Start);
337  break;
338  case Intrinsic::vp_reduce_xor:
339  Reduction = Builder.CreateXorReduce(RedOp);
340  Reduction = Builder.CreateXor(Reduction, Start);
341  break;
342  case Intrinsic::vp_reduce_smax:
343  Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ true);
344  Reduction =
345  Builder.CreateBinaryIntrinsic(Intrinsic::smax, Reduction, Start);
346  break;
347  case Intrinsic::vp_reduce_smin:
348  Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ true);
349  Reduction =
350  Builder.CreateBinaryIntrinsic(Intrinsic::smin, Reduction, Start);
351  break;
352  case Intrinsic::vp_reduce_umax:
353  Reduction = Builder.CreateIntMaxReduce(RedOp, /*IsSigned*/ false);
354  Reduction =
355  Builder.CreateBinaryIntrinsic(Intrinsic::umax, Reduction, Start);
356  break;
357  case Intrinsic::vp_reduce_umin:
358  Reduction = Builder.CreateIntMinReduce(RedOp, /*IsSigned*/ false);
359  Reduction =
360  Builder.CreateBinaryIntrinsic(Intrinsic::umin, Reduction, Start);
361  break;
362  case Intrinsic::vp_reduce_fmax:
363  Reduction = Builder.CreateFPMaxReduce(RedOp);
365  Reduction =
366  Builder.CreateBinaryIntrinsic(Intrinsic::maxnum, Reduction, Start);
367  break;
368  case Intrinsic::vp_reduce_fmin:
369  Reduction = Builder.CreateFPMinReduce(RedOp);
371  Reduction =
372  Builder.CreateBinaryIntrinsic(Intrinsic::minnum, Reduction, Start);
373  break;
374  case Intrinsic::vp_reduce_fadd:
375  Reduction = Builder.CreateFAddReduce(Start, RedOp);
376  break;
377  case Intrinsic::vp_reduce_fmul:
378  Reduction = Builder.CreateFMulReduce(Start, RedOp);
379  break;
380  }
381 
383  return Reduction;
384 }
385 
386 void CachingVPExpander::discardEVLParameter(VPIntrinsic &VPI) {
387  LLVM_DEBUG(dbgs() << "Discard EVL parameter in " << VPI << "\n");
388 
389  if (VPI.canIgnoreVectorLengthParam())
390  return;
391 
392  Value *EVLParam = VPI.getVectorLengthParam();
393  if (!EVLParam)
394  return;
395 
396  ElementCount StaticElemCount = VPI.getStaticVectorLength();
397  Value *MaxEVL = nullptr;
399  if (StaticElemCount.isScalable()) {
400  // TODO add caching
401  auto *M = VPI.getModule();
402  Function *VScaleFunc =
403  Intrinsic::getDeclaration(M, Intrinsic::vscale, Int32Ty);
405  Value *FactorConst = Builder.getInt32(StaticElemCount.getKnownMinValue());
406  Value *VScale = Builder.CreateCall(VScaleFunc, {}, "vscale");
407  MaxEVL = Builder.CreateMul(VScale, FactorConst, "scalable_size",
408  /*NUW*/ true, /*NSW*/ false);
409  } else {
410  MaxEVL = ConstantInt::get(Int32Ty, StaticElemCount.getFixedValue(), false);
411  }
412  VPI.setVectorLengthParam(MaxEVL);
413 }
414 
415 Value *CachingVPExpander::foldEVLIntoMask(VPIntrinsic &VPI) {
416  LLVM_DEBUG(dbgs() << "Folding vlen for " << VPI << '\n');
417 
418  IRBuilder<> Builder(&VPI);
419 
420  // Ineffective %evl parameter and so nothing to do here.
421  if (VPI.canIgnoreVectorLengthParam())
422  return &VPI;
423 
424  // Only VP intrinsics can have an %evl parameter.
425  Value *OldMaskParam = VPI.getMaskParam();
426  Value *OldEVLParam = VPI.getVectorLengthParam();
427  assert(OldMaskParam && "no mask param to fold the vl param into");
428  assert(OldEVLParam && "no EVL param to fold away");
429 
430  LLVM_DEBUG(dbgs() << "OLD evl: " << *OldEVLParam << '\n');
431  LLVM_DEBUG(dbgs() << "OLD mask: " << *OldMaskParam << '\n');
432 
433  // Convert the %evl predication into vector mask predication.
434  ElementCount ElemCount = VPI.getStaticVectorLength();
435  Value *VLMask = convertEVLToMask(Builder, OldEVLParam, ElemCount);
436  Value *NewMaskParam = Builder.CreateAnd(VLMask, OldMaskParam);
437  VPI.setMaskParam(NewMaskParam);
438 
439  // Drop the %evl parameter.
440  discardEVLParameter(VPI);
442  "transformation did not render the evl param ineffective!");
443 
444  // Reassess the modified instruction.
445  return &VPI;
446 }
447 
448 Value *CachingVPExpander::expandPredication(VPIntrinsic &VPI) {
449  LLVM_DEBUG(dbgs() << "Lowering to unpredicated op: " << VPI << '\n');
450 
451  IRBuilder<> Builder(&VPI);
452 
453  // Try lowering to a LLVM instruction first.
454  auto OC = VPI.getFunctionalOpcode();
455 
456  if (OC && Instruction::isBinaryOp(*OC))
457  return expandPredicationInBinaryOperator(Builder, VPI);
458 
459  if (auto *VPRI = dyn_cast<VPReductionIntrinsic>(&VPI))
460  return expandPredicationInReduction(Builder, *VPRI);
461 
462  return &VPI;
463 }
464 
465 //// } CachingVPExpander
466 
467 struct TransformJob {
468  VPIntrinsic *PI;
470  TransformJob(VPIntrinsic *PI, TargetTransformInfo::VPLegalization InitStrat)
471  : PI(PI), Strategy(InitStrat) {}
472 
473  bool isDone() const { return Strategy.shouldDoNothing(); }
474 };
475 
476 void sanitizeStrategy(Instruction &I, VPLegalization &LegalizeStrat) {
477  // Speculatable instructions do not strictly need predication.
479  // Converting a speculatable VP intrinsic means dropping %mask and %evl.
480  // No need to expand %evl into the %mask only to ignore that code.
481  if (LegalizeStrat.OpStrategy == VPLegalization::Convert)
483  return;
484  }
485 
486  // We have to preserve the predicating effect of %evl for this
487  // non-speculatable VP intrinsic.
488  // 1) Never discard %evl.
489  // 2) If this VP intrinsic will be expanded to non-VP code, make sure that
490  // %evl gets folded into %mask.
491  if ((LegalizeStrat.EVLParamStrategy == VPLegalization::Discard) ||
492  (LegalizeStrat.OpStrategy == VPLegalization::Convert)) {
494  }
495 }
496 
498 CachingVPExpander::getVPLegalizationStrategy(const VPIntrinsic &VPI) const {
499  auto VPStrat = TTI.getVPLegalizationStrategy(VPI);
500  if (LLVM_LIKELY(!UsingTTIOverrides)) {
501  // No overrides - we are in production.
502  return VPStrat;
503  }
504 
505  // Overrides set - we are in testing, the following does not need to be
506  // efficient.
508  VPStrat.OpStrategy = parseOverrideOption(MaskTransformOverride);
509  return VPStrat;
510 }
511 
512 /// \brief Expand llvm.vp.* intrinsics as requested by \p TTI.
513 bool CachingVPExpander::expandVectorPredication() {
515 
516  // Collect all VPIntrinsics that need expansion and determine their expansion
517  // strategy.
518  for (auto &I : instructions(F)) {
519  auto *VPI = dyn_cast<VPIntrinsic>(&I);
520  if (!VPI)
521  continue;
522  auto VPStrat = getVPLegalizationStrategy(*VPI);
523  sanitizeStrategy(I, VPStrat);
524  if (!VPStrat.shouldDoNothing())
525  Worklist.emplace_back(VPI, VPStrat);
526  }
527  if (Worklist.empty())
528  return false;
529 
530  // Transform all VPIntrinsics on the worklist.
531  LLVM_DEBUG(dbgs() << "\n:::: Transforming " << Worklist.size()
532  << " instructions ::::\n");
533  for (TransformJob Job : Worklist) {
534  // Transform the EVL parameter.
535  switch (Job.Strategy.EVLParamStrategy) {
537  break;
539  discardEVLParameter(*Job.PI);
540  break;
542  if (foldEVLIntoMask(*Job.PI))
543  ++NumFoldedVL;
544  break;
545  }
546  Job.Strategy.EVLParamStrategy = VPLegalization::Legal;
547 
548  // Replace with a non-predicated operation.
549  switch (Job.Strategy.OpStrategy) {
551  break;
553  llvm_unreachable("Invalid strategy for operators.");
555  expandPredication(*Job.PI);
556  ++NumLoweredVPOps;
557  break;
558  }
559  Job.Strategy.OpStrategy = VPLegalization::Legal;
560 
561  assert(Job.isDone() && "incomplete transformation");
562  }
563 
564  return true;
565 }
566 class ExpandVectorPredication : public FunctionPass {
567 public:
568  static char ID;
569  ExpandVectorPredication() : FunctionPass(ID) {
571  }
572 
573  bool runOnFunction(Function &F) override {
574  const auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
575  CachingVPExpander VPExpander(F, *TTI);
576  return VPExpander.expandVectorPredication();
577  }
578 
579  void getAnalysisUsage(AnalysisUsage &AU) const override {
581  AU.setPreservesCFG();
582  }
583 };
584 } // namespace
585 
587 INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp",
588  "Expand vector predication intrinsics", false, false)
591 INITIALIZE_PASS_END(ExpandVectorPredication, "expandvp",
592  "Expand vector predication intrinsics", false, false)
593 
595  return new ExpandVectorPredication();
596 }
597 
600  const auto &TTI = AM.getResult<TargetIRAnalysis>(F);
601  CachingVPExpander VPExpander(F, TTI);
602  if (!VPExpander.expandVectorPredication())
603  return PreservedAnalyses::all();
605  PA.preserveSet<CFGAnalyses>();
606  return PA;
607 }
transferDecorations
static void transferDecorations(Value &NewVal, VPIntrinsic &VPI)
Transfer operation properties from OldVPI to NewVal.
Definition: ExpandVectorPredication.cpp:98
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::TargetIRAnalysis
Analysis pass providing the TargetTransformInfo.
Definition: TargetTransformInfo.h:2331
llvm::initializeExpandVectorPredicationPass
void initializeExpandVectorPredicationPass(PassRegistry &)
MathExtras.h
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
expandvp
expandvp
Definition: ExpandVectorPredication.cpp:591
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1379
llvm::isSafeToSpeculativelyExecute
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
Definition: ValueTracking.cpp:4589
IntrinsicInst.h
llvm::ElementCount
Definition: TypeSize.h:386
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:779
InstIterator.h
llvm::Function
Definition: Function.h:61
Pass.h
llvm::IntrinsicInst::getIntrinsicID
Intrinsic::ID getIntrinsicID() const
Return the intrinsic ID of this intrinsic.
Definition: IntrinsicInst.h:52
llvm::TargetTransformInfo::getVPLegalizationStrategy
VPLegalization getVPLegalizationStrategy(const VPIntrinsic &PI) const
Definition: TargetTransformInfo.cpp:1041
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
Statistic.h
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:168
llvm::VPIntrinsic::setVectorLengthParam
void setVectorLengthParam(Value *)
Definition: IntrinsicInst.cpp:313
llvm::IRBuilder<>
ValueTracking.h
llvm::VPIntrinsic::canIgnoreVectorLengthParam
bool canIgnoreVectorLengthParam() const
Definition: IntrinsicInst.cpp:429
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::TargetTransformInfo::VPLegalization
Definition: TargetTransformInfo.h:1381
llvm::APInt::getSignedMaxValue
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:191
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::TargetTransformInfo::VPLegalization::Convert
@ Convert
Definition: TargetTransformInfo.h:1388
llvm::ConstantFP::getNegativeZero
static Constant * getNegativeZero(Type *Ty)
Definition: Constants.cpp:1021
parseOverrideOption
static VPTransform parseOverrideOption(const std::string &TextOpt)
Definition: ExpandVectorPredication.cpp:67
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: Operator.h:161
llvm::FastMathFlags::noNaNs
bool noNaNs() const
Definition: Operator.h:207
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:299
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:203
llvm::APIntOps::umin
const APInt & umin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be unsigned.
Definition: APInt.h:2121
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::VPIntrinsic::setMaskParam
void setMaskParam(Value *)
Definition: IntrinsicInst.cpp:302
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
CommandLine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
intrinsics
Expand vector predication intrinsics
Definition: ExpandVectorPredication.cpp:592
Constants.h
llvm::VPReductionIntrinsic
This represents vector predication reduction intrinsics.
Definition: IntrinsicInst.h:452
Intrinsics.h
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
false
Definition: StackSlotColoring.cpp:142
llvm::Type::getFltSemantics
const fltSemantics & getFltSemantics() const
Definition: Type.h:169
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:405
llvm::Instruction
Definition: Instruction.h:45
llvm::Type::getScalarSizeInBits
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
Definition: Type.cpp:153
llvm::DominatorTreeWrapperPass
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:287
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::ConstantFP
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:257
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:900
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::maxnum
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE maxNum semantics.
Definition: APFloat.h:1309
llvm::VPIntrinsic::getVectorLengthParam
Value * getVectorLengthParam() const
Definition: IntrinsicInst.cpp:307
llvm::TargetTransformInfo::VPLegalization::EVLParamStrategy
VPTransform EVLParamStrategy
Definition: TargetTransformInfo.h:1395
Passes.h
llvm::VPIntrinsic::getFunctionalOpcode
Optional< unsigned > getFunctionalOpcode() const
Definition: IntrinsicInst.h:443
ExpandVectorPredication.h
llvm::TargetTransformInfo::VPLegalization::VPTransform
VPTransform
Definition: TargetTransformInfo.h:1382
llvm::cl::opt
Definition: CommandLine.h:1434
llvm::APFloat
Definition: APFloat.h:701
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::ConstantFP::getQNaN
static Constant * getQNaN(Type *Ty, bool Negative=false, APInt *Payload=nullptr)
Definition: Constants.cpp:999
llvm::VPIntrinsic::getStaticVectorLength
ElementCount getStaticVectorLength() const
Definition: IntrinsicInst.cpp:284
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:78
EVLTransformOverride
static cl::opt< std::string > EVLTransformOverride("expandvp-override-evl-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, ignore " "TargetTransformInfo and " "always use this transformation for the %evl parameter (Used in " "testing)."))
llvm::TargetTransformInfo::VPLegalization::OpStrategy
VPTransform OpStrategy
Definition: TargetTransformInfo.h:1401
llvm::TargetTransformInfoWrapperPass
Wrapper pass for TargetTransformInfo.
Definition: TargetTransformInfo.h:2387
replaceOperation
static void replaceOperation(Value &NewOp, VPIntrinsic &OldOp)
Transfer all properties from OldOp to NewOp and replace all uses.
Definition: ExpandVectorPredication.cpp:112
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::TargetTransformInfo::VPLegalization::shouldDoNothing
bool shouldDoNothing() const
Definition: TargetTransformInfo.h:1403
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
anyExpandVPOverridesSet
static bool anyExpandVPOverridesSet()
Definition: ExpandVectorPredication.cpp:74
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetTransformInfo::VPLegalization::Discard
@ Discard
Definition: TargetTransformInfo.h:1386
llvm::SystemZISD::OC
@ OC
Definition: SystemZISelLowering.h:128
llvm::createExpandVectorPredicationPass
FunctionPass * createExpandVectorPredicationPass()
This pass expands the vector predication intrinsics into unpredicated instructions with selects or ju...
Definition: ExpandVectorPredication.cpp:594
llvm::TTI
TargetTransformInfo TTI
Definition: TargetTransformInfo.h:163
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition: Instruction.h:165
predication
loop predication
Definition: LoopPredication.cpp:359
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:297
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
llvm::APIntOps::smin
const APInt & smin(const APInt &A, const APInt &B)
Determine the smaller of two APInts considered to be signed.
Definition: APInt.h:2111
llvm::Instruction::getFastMathFlags
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition: Instruction.cpp:280
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:745
llvm::LinearPolySize::getFixedValue
ScalarTy getFixedValue() const
Definition: TypeSize.h:313
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:116
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:520
Compiler.h
llvm::Value::getContext
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:978
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
llvm::ConstantVector::get
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1369
llvm::VPReductionIntrinsic::getStartParamPos
unsigned getStartParamPos() const
Definition: IntrinsicInst.cpp:532
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:273
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:297
llvm::Type::getContext
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:127
MaskTransformOverride
static cl::opt< std::string > MaskTransformOverride("expandvp-override-mask-transform", cl::init(""), cl::Hidden, cl::desc("Options: <empty>" VPINTERNAL_VPLEGAL_CASES ". If non-empty, Ignore " "TargetTransformInfo and " "always use this transformation for the %mask parameter (Used in " "testing)."))
llvm::ExpandVectorPredicationPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: ExpandVectorPredication.cpp:599
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(ExpandVectorPredication, "expandvp", "Expand vector predication intrinsics", false, false) INITIALIZE_PASS_END(ExpandVectorPredication
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::APIntOps::umax
const APInt & umax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be unsigned.
Definition: APInt.h:2126
llvm::minnum
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE minNum semantics.
Definition: APFloat.h:1298
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:348
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::VPIntrinsic
This is the common base class for vector predication intrinsics.
Definition: IntrinsicInst.h:390
llvm::Type::isIntOrIntVectorTy
bool isIntOrIntVectorTy() const
Return true if this is an integer type or a vector of integer types.
Definition: Type.h:207
VPLegalization
TargetTransformInfo::VPLegalization VPLegalization
Definition: ExpandVectorPredication.cpp:36
llvm::VPReductionIntrinsic::getVectorParamPos
unsigned getVectorParamPos() const
Definition: IntrinsicInst.cpp:528
llvm::ConstantFP::get
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:947
llvm::fltSemantics
Definition: APFloat.cpp:54
Function.h
VPINTERNAL_VPLEGAL_CASES
#define VPINTERNAL_VPLEGAL_CASES
Definition: ExpandVectorPredication.cpp:40
llvm::APInt::getSignedMinValue
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:201
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:785
Instructions.h
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:191
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::TargetTransformInfo::VPLegalization::Legal
@ Legal
Definition: TargetTransformInfo.h:1384
getSafeDivisor
static Constant * getSafeDivisor(Type *DivTy)
Definition: ExpandVectorPredication.cpp:92
TargetTransformInfo.h
LLVM_LIKELY
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:219
llvm::StringSwitch
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::VPIntrinsic::getMaskParam
Value * getMaskParam() const
Definition: IntrinsicInst.cpp:296
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::cl::desc
Definition: CommandLine.h:414
isAllTrueMask
static bool isAllTrueMask(Value *MaskVal)
Definition: ExpandVectorPredication.cpp:86
Reduction
loop Loop Strength Reduction
Definition: LoopStrengthReduce.cpp:6410
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
Debug.h
llvm::VectorType::get
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:632
llvm::APIntOps::smax
const APInt & smax(const APInt &A, const APInt &B)
Determine the larger of two APInts considered to be signed.
Definition: APInt.h:2116
llvm::SmallVectorImpl::emplace_back
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:908
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37