LLVM  13.0.0git
SpeculativeExecution.cpp
Go to the documentation of this file.
1 //===- SpeculativeExecution.cpp ---------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass hoists instructions to enable speculative execution on
10 // targets where branches are expensive. This is aimed at GPUs. It
11 // currently works on simple if-then and if-then-else
12 // patterns.
13 //
14 // Removing branches is not the only motivation for this
15 // pass. E.g. consider this code and assume that there is no
16 // addressing mode for multiplying by sizeof(*a):
17 //
18 // if (b > 0)
19 // c = a[i + 1]
20 // if (d > 0)
21 // e = a[i + 2]
22 //
23 // turns into
24 //
25 // p = &a[i + 1];
26 // if (b > 0)
27 // c = *p;
28 // q = &a[i + 2];
29 // if (d > 0)
30 // e = *q;
31 //
32 // which could later be optimized to
33 //
34 // r = &a[i];
35 // if (b > 0)
36 // c = r[1];
37 // if (d > 0)
38 // e = r[2];
39 //
40 // Later passes sink back much of the speculated code that did not enable
41 // further optimization.
42 //
43 // This pass is more aggressive than the function SpeculativeyExecuteBB in
44 // SimplifyCFG. SimplifyCFG will not speculate if no selects are introduced and
45 // it will speculate at most one instruction. It also will not speculate if
46 // there is a value defined in the if-block that is only used in the then-block.
47 // These restrictions make sense since the speculation in SimplifyCFG seems
48 // aimed at introducing cheap selects, while this pass is intended to do more
49 // aggressive speculation while counting on later passes to either capitalize on
50 // that or clean it up.
51 //
52 // If the pass was created by calling
53 // createSpeculativeExecutionIfHasBranchDivergencePass or the
54 // -spec-exec-only-if-divergent-target option is present, this pass only has an
55 // effect on targets where TargetTransformInfo::hasBranchDivergence() is true;
56 // on other targets, it is a nop.
57 //
58 // This lets you include this pass unconditionally in the IR pass pipeline, but
59 // only enable it for relevant targets.
60 //
61 //===----------------------------------------------------------------------===//
62 
64 #include "llvm/ADT/SmallPtrSet.h"
67 #include "llvm/IR/Instructions.h"
68 #include "llvm/IR/IntrinsicInst.h"
69 #include "llvm/IR/Module.h"
70 #include "llvm/IR/Operator.h"
71 #include "llvm/InitializePasses.h"
73 #include "llvm/Support/Debug.h"
74 
75 using namespace llvm;
76 
77 #define DEBUG_TYPE "speculative-execution"
78 
79 // The risk that speculation will not pay off increases with the
80 // number of instructions speculated, so we put a limit on that.
82  "spec-exec-max-speculation-cost", cl::init(7), cl::Hidden,
83  cl::desc("Speculative execution is not applied to basic blocks where "
84  "the cost of the instructions to speculatively execute "
85  "exceeds this limit."));
86 
87 // Speculating just a few instructions from a larger block tends not
88 // to be profitable and this limit prevents that. A reason for that is
89 // that small basic blocks are more likely to be candidates for
90 // further optimization.
92  "spec-exec-max-not-hoisted", cl::init(5), cl::Hidden,
93  cl::desc("Speculative execution is not applied to basic blocks where the "
94  "number of instructions that would not be speculatively executed "
95  "exceeds this limit."));
96 
98  "spec-exec-only-if-divergent-target", cl::init(false), cl::Hidden,
99  cl::desc("Speculative execution is applied only to targets with divergent "
100  "branches, even if the pass was configured to apply only to all "
101  "targets."));
102 
103 namespace {
104 
105 class SpeculativeExecutionLegacyPass : public FunctionPass {
106 public:
107  static char ID;
108  explicit SpeculativeExecutionLegacyPass(bool OnlyIfDivergentTarget = false)
109  : FunctionPass(ID), OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
111  Impl(OnlyIfDivergentTarget) {}
112 
113  void getAnalysisUsage(AnalysisUsage &AU) const override;
114  bool runOnFunction(Function &F) override;
115 
116  StringRef getPassName() const override {
117  if (OnlyIfDivergentTarget)
118  return "Speculatively execute instructions if target has divergent "
119  "branches";
120  return "Speculatively execute instructions";
121  }
122 
123 private:
124  // Variable preserved purely for correct name printing.
125  const bool OnlyIfDivergentTarget;
126 
128 };
129 } // namespace
130 
132 INITIALIZE_PASS_BEGIN(SpeculativeExecutionLegacyPass, "speculative-execution",
133  "Speculatively execute instructions", false, false)
135 INITIALIZE_PASS_END(SpeculativeExecutionLegacyPass, "speculative-execution",
136  "Speculatively execute instructions", false, false)
137 
138 void SpeculativeExecutionLegacyPass::getAnalysisUsage(AnalysisUsage &AU) const {
139  AU.addRequired<TargetTransformInfoWrapperPass>();
140  AU.addPreserved<GlobalsAAWrapperPass>();
141  AU.setPreservesCFG();
142 }
143 
145  if (skipFunction(F))
146  return false;
147 
148  auto *TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
149  return Impl.runImpl(F, TTI);
150 }
151 
152 namespace llvm {
153 
155  if (OnlyIfDivergentTarget && !TTI->hasBranchDivergence()) {
156  LLVM_DEBUG(dbgs() << "Not running SpeculativeExecution because "
157  "TTI->hasBranchDivergence() is false.\n");
158  return false;
159  }
160 
161  this->TTI = TTI;
162  bool Changed = false;
163  for (auto& B : F) {
164  Changed |= runOnBasicBlock(B);
165  }
166  return Changed;
167 }
168 
169 bool SpeculativeExecutionPass::runOnBasicBlock(BasicBlock &B) {
170  BranchInst *BI = dyn_cast<BranchInst>(B.getTerminator());
171  if (BI == nullptr)
172  return false;
173 
174  if (BI->getNumSuccessors() != 2)
175  return false;
176  BasicBlock &Succ0 = *BI->getSuccessor(0);
177  BasicBlock &Succ1 = *BI->getSuccessor(1);
178 
179  if (&B == &Succ0 || &B == &Succ1 || &Succ0 == &Succ1) {
180  return false;
181  }
182 
183  // Hoist from if-then (triangle).
184  if (Succ0.getSinglePredecessor() != nullptr &&
185  Succ0.getSingleSuccessor() == &Succ1) {
186  return considerHoistingFromTo(Succ0, B);
187  }
188 
189  // Hoist from if-else (triangle).
190  if (Succ1.getSinglePredecessor() != nullptr &&
191  Succ1.getSingleSuccessor() == &Succ0) {
192  return considerHoistingFromTo(Succ1, B);
193  }
194 
195  // Hoist from if-then-else (diamond), but only if it is equivalent to
196  // an if-else or if-then due to one of the branches doing nothing.
197  if (Succ0.getSinglePredecessor() != nullptr &&
198  Succ1.getSinglePredecessor() != nullptr &&
199  Succ1.getSingleSuccessor() != nullptr &&
200  Succ1.getSingleSuccessor() != &B &&
201  Succ1.getSingleSuccessor() == Succ0.getSingleSuccessor()) {
202  // If a block has only one instruction, then that is a terminator
203  // instruction so that the block does nothing. This does happen.
204  if (Succ1.size() == 1) // equivalent to if-then
205  return considerHoistingFromTo(Succ0, B);
206  if (Succ0.size() == 1) // equivalent to if-else
207  return considerHoistingFromTo(Succ1, B);
208  }
209 
210  return false;
211 }
212 
214  const TargetTransformInfo &TTI) {
215  switch (Operator::getOpcode(I)) {
216  case Instruction::GetElementPtr:
217  case Instruction::Add:
218  case Instruction::Mul:
219  case Instruction::And:
220  case Instruction::Or:
221  case Instruction::Select:
222  case Instruction::Shl:
223  case Instruction::Sub:
224  case Instruction::LShr:
225  case Instruction::AShr:
226  case Instruction::Xor:
227  case Instruction::ZExt:
228  case Instruction::SExt:
229  case Instruction::Call:
230  case Instruction::BitCast:
231  case Instruction::PtrToInt:
232  case Instruction::IntToPtr:
233  case Instruction::AddrSpaceCast:
234  case Instruction::FPToUI:
235  case Instruction::FPToSI:
236  case Instruction::UIToFP:
237  case Instruction::SIToFP:
238  case Instruction::FPExt:
239  case Instruction::FPTrunc:
240  case Instruction::FAdd:
241  case Instruction::FSub:
242  case Instruction::FMul:
243  case Instruction::FDiv:
244  case Instruction::FRem:
245  case Instruction::FNeg:
246  case Instruction::ICmp:
247  case Instruction::FCmp:
248  case Instruction::Trunc:
249  case Instruction::Freeze:
250  case Instruction::ExtractElement:
251  case Instruction::InsertElement:
252  case Instruction::ShuffleVector:
253  case Instruction::ExtractValue:
254  case Instruction::InsertValue:
256 
257  default:
258  return InstructionCost::getInvalid(); // Disallow anything not explicitly
259  // listed.
260  }
261 }
262 
263 bool SpeculativeExecutionPass::considerHoistingFromTo(
264  BasicBlock &FromBlock, BasicBlock &ToBlock) {
266  const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](const User *U) {
267  // Debug variable has special operand to check it's not hoisted.
268  if (const auto *DVI = dyn_cast<DbgVariableIntrinsic>(U)) {
269  return all_of(DVI->location_ops(), [&NotHoisted](Value *V) {
270  if (const auto *I = dyn_cast_or_null<Instruction>(V)) {
271  if (NotHoisted.count(I) == 0)
272  return true;
273  }
274  return false;
275  });
276  }
277 
278  // Usially debug label instrinsic corresponds to label in LLVM IR. In these
279  // cases we should not move it here.
280  // TODO: Possible special processing needed to detect it is related to a
281  // hoisted instruction.
282  if (isa<DbgLabelInst>(U))
283  return false;
284 
285  for (const Value *V : U->operand_values()) {
286  if (const Instruction *I = dyn_cast<Instruction>(V)) {
287  if (NotHoisted.contains(I))
288  return false;
289  }
290  }
291  return true;
292  };
293 
294  InstructionCost TotalSpeculationCost = 0;
295  unsigned NotHoistedInstCount = 0;
296  for (const auto &I : FromBlock) {
297  const InstructionCost Cost = ComputeSpeculationCost(&I, *TTI);
298  if (Cost.isValid() && isSafeToSpeculativelyExecute(&I) &&
299  AllPrecedingUsesFromBlockHoisted(&I)) {
300  TotalSpeculationCost += Cost;
301  if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
302  return false; // too much to hoist
303  } else {
304  // Debug info instrinsics should not be counted for threshold.
305  if (!isa<DbgInfoIntrinsic>(I))
306  NotHoistedInstCount++;
307  if (NotHoistedInstCount > SpecExecMaxNotHoisted)
308  return false; // too much left behind
309  NotHoisted.insert(&I);
310  }
311  }
312 
313  for (auto I = FromBlock.begin(); I != FromBlock.end();) {
314  // We have to increment I before moving Current as moving Current
315  // changes the list that I is iterating through.
316  auto Current = I;
317  ++I;
318  if (!NotHoisted.count(&*Current)) {
319  Current->moveBefore(ToBlock.getTerminator());
320  }
321  }
322  return true;
323 }
324 
326  return new SpeculativeExecutionLegacyPass();
327 }
328 
330  return new SpeculativeExecutionLegacyPass(/* OnlyIfDivergentTarget = */ true);
331 }
332 
333 SpeculativeExecutionPass::SpeculativeExecutionPass(bool OnlyIfDivergentTarget)
334  : OnlyIfDivergentTarget(OnlyIfDivergentTarget ||
336 
339  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
340 
341  bool Changed = runImpl(F, TTI);
342 
343  if (!Changed)
344  return PreservedAnalyses::all();
346  PA.preserve<GlobalsAA>();
347  PA.preserveSet<CFGAnalyses>();
348  return PA;
349 }
350 } // namespace llvm
llvm::GlobalsAA
Analysis pass providing a never-invalidated alias analysis result.
Definition: GlobalsModRef.h:132
llvm::InstructionCost
Definition: InstructionCost.h:26
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::TargetIRAnalysis
Analysis pass providing the TargetTransformInfo.
Definition: TargetTransformInfo.h:2319
llvm
Definition: AllocatorList.h:23
llvm::isSafeToSpeculativelyExecute
bool isSafeToSpeculativelyExecute(const Value *V, const Instruction *CtxI=nullptr, const DominatorTree *DT=nullptr, const TargetLibraryInfo *TLI=nullptr)
Return true if the instruction does not have any effects besides calculating the result and does not ...
Definition: ValueTracking.cpp:4541
IntrinsicInst.h
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:785
llvm::Function
Definition: Function.h:61
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:167
ValueTracking.h
GlobalsModRef.h
SpecExecMaxNotHoisted
static cl::opt< unsigned > SpecExecMaxNotHoisted("spec-exec-max-not-hoisted", cl::init(5), cl::Hidden, cl::desc("Speculative execution is not applied to basic blocks where the " "number of instructions that would not be speculatively executed " "exceeds this limit."))
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
Module.h
llvm::BasicBlock::getSingleSuccessor
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:294
llvm::BranchInst::getNumSuccessors
unsigned getNumSuccessors() const
Definition: Instructions.h:3098
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
Operator.h
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:264
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
CommandLine.h
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1482
llvm::User
Definition: User.h:44
llvm::TargetTransformInfo::hasBranchDivergence
bool hasBranchDivergence() const
Return true if branch divergence exists.
Definition: TargetTransformInfo.cpp:236
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
false
Definition: StackSlotColoring.cpp:142
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::Instruction
Definition: Instruction.h:45
llvm::Operator::getOpcode
unsigned getOpcode() const
Return the opcode for this Instruction or ConstantExpr.
Definition: Operator.h:40
SpecExecMaxSpeculationCost
static cl::opt< unsigned > SpecExecMaxSpeculationCost("spec-exec-max-speculation-cost", cl::init(7), cl::Hidden, cl::desc("Speculative execution is not applied to basic blocks where " "the cost of the instructions to speculatively execute " "exceeds this limit."))
SmallPtrSet.h
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:153
llvm::createSpeculativeExecutionPass
FunctionPass * createSpeculativeExecutionPass()
Definition: SpeculativeExecution.cpp:325
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::cl::opt
Definition: CommandLine.h:1422
SpeculativeExecution.h
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(SpeculativeExecutionLegacyPass, "speculative-execution", "Speculatively execute instructions", false, false) INITIALIZE_PASS_END(SpeculativeExecutionLegacyPass
llvm::TargetTransformInfoWrapperPass
Wrapper pass for TargetTransformInfo.
Definition: TargetTransformInfo.h:2375
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:176
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::SpeculativeExecutionPass
Definition: SpeculativeExecution.h:69
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:382
llvm::SpeculativeExecutionPass::runImpl
bool runImpl(Function &F, TargetTransformInfo *TTI)
Definition: SpeculativeExecution.cpp:154
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::TargetTransformInfo::TCK_SizeAndLatency
@ TCK_SizeAndLatency
The weighted sum of size and latency.
Definition: TargetTransformInfo.h:214
llvm::CFGAnalyses
Represents analyses that only rely on functions' control flow.
Definition: PassManager.h:116
llvm::TargetTransformInfo::getUserCost
InstructionCost getUserCost(const User *U, ArrayRef< const Value * > Operands, TargetCostKind CostKind) const
Estimate the cost of a given IR user when lowered.
Definition: TargetTransformInfo.cpp:223
llvm::InstructionCost::isValid
bool isValid() const
Definition: InstructionCost.h:60
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:148
llvm::ComputeSpeculationCost
static InstructionCost ComputeSpeculationCost(const Instruction *I, const TargetTransformInfo &TTI)
Definition: SpeculativeExecution.cpp:213
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:162
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
SpecExecOnlyIfDivergentTarget
static cl::opt< bool > SpecExecOnlyIfDivergentTarget("spec-exec-only-if-divergent-target", cl::init(false), cl::Hidden, cl::desc("Speculative execution is applied only to targets with divergent " "branches, even if the pass was configured to apply only to all " "targets."))
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:183
llvm::InstructionCost::getInvalid
static InstructionCost getInvalid(CostType Val=0)
Definition: InstructionCost.h:54
Instructions.h
llvm::PreservedAnalyses::preserveSet
void preserveSet()
Mark an analysis set as preserved.
Definition: PassManager.h:191
llvm::BasicBlock::size
size_t size() const
Definition: BasicBlock.h:306
llvm::GlobalsAAWrapperPass
Legacy wrapper pass to provide the GlobalsAAResult object.
Definition: GlobalsModRef.h:143
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::createSpeculativeExecutionIfHasBranchDivergencePass
FunctionPass * createSpeculativeExecutionIfHasBranchDivergencePass()
Definition: SpeculativeExecution.cpp:329
llvm::cl::desc
Definition: CommandLine.h:414
execution
speculative execution
Definition: SpeculativeExecution.cpp:135
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3007
llvm::SmallPtrSetImpl::contains
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:388
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
Debug.h
llvm::SpeculativeExecutionPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: SpeculativeExecution.cpp:337
llvm::BranchInst::getSuccessor
BasicBlock * getSuccessor(unsigned i) const
Definition: Instructions.h:3100
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38