LLVM  16.0.0git
AMDGPUSetWavePriority.cpp
Go to the documentation of this file.
1 //===- AMDGPUSetWavePriority.cpp - Set wave priority ----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// Pass to temporarily raise the wave priority beginning the start of
11 /// the shader function until its last VMEM instructions to allow younger
12 /// waves to issue their VMEM instructions as well.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "AMDGPU.h"
17 #include "GCNSubtarget.h"
19 #include "SIInstrInfo.h"
22 #include "llvm/InitializePasses.h"
23 #include "llvm/Support/Allocator.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "amdgpu-set-wave-priority"
28 
29 namespace {
30 
31 struct MBBInfo {
32  MBBInfo() = default;
33  bool MayReachVMEMLoad = false;
34 };
35 
37 
38 class AMDGPUSetWavePriority : public MachineFunctionPass {
39 public:
40  static char ID;
41 
42  AMDGPUSetWavePriority() : MachineFunctionPass(ID) {}
43 
44  StringRef getPassName() const override { return "Set wave priority"; }
45 
46  bool runOnMachineFunction(MachineFunction &MF) override;
47 
48 private:
49  MachineInstr *BuildSetprioMI(MachineFunction &MF, unsigned priority) const;
50 
51  const SIInstrInfo *TII;
52 };
53 
54 } // End anonymous namespace.
55 
56 INITIALIZE_PASS(AMDGPUSetWavePriority, DEBUG_TYPE, "Set wave priority", false,
57  false)
58 
59 char AMDGPUSetWavePriority::ID = 0;
60 
62  return new AMDGPUSetWavePriority();
63 }
64 
65 MachineInstr *AMDGPUSetWavePriority::BuildSetprioMI(MachineFunction &MF,
66  unsigned priority) const {
67  return BuildMI(MF, DebugLoc(), TII->get(AMDGPU::S_SETPRIO)).addImm(priority);
68 }
69 
70 // Checks that for every predecessor Pred that can reach a VMEM load,
71 // none of Pred's successors can reach a VMEM load.
73  MBBInfoSet &MBBInfos) {
74  for (const MachineBasicBlock *Pred : MBB.predecessors()) {
75  if (!MBBInfos[Pred].MayReachVMEMLoad)
76  continue;
77  for (const MachineBasicBlock *Succ : Pred->successors()) {
78  if (MBBInfos[Succ].MayReachVMEMLoad)
79  return false;
80  }
81  }
82  return true;
83 }
84 
85 static bool isVMEMLoad(const MachineInstr &MI) {
86  return SIInstrInfo::isVMEM(MI) && MI.mayLoad();
87 }
88 
89 bool AMDGPUSetWavePriority::runOnMachineFunction(MachineFunction &MF) {
90  const unsigned HighPriority = 3;
91  const unsigned LowPriority = 0;
92 
93  Function &F = MF.getFunction();
94  if (skipFunction(F) || !AMDGPU::isEntryFunctionCC(F.getCallingConv()))
95  return false;
96 
98  TII = ST.getInstrInfo();
99 
100  MBBInfoSet MBBInfos;
102  for (MachineBasicBlock &MBB : MF) {
103  if (any_of(MBB, isVMEMLoad))
104  Worklist.push_back(&MBB);
105  }
106 
107  // Mark blocks from which control may reach VMEM loads.
108  while (!Worklist.empty()) {
109  const MachineBasicBlock *MBB = Worklist.pop_back_val();
110  MBBInfo &Info = MBBInfos[MBB];
111  if (!Info.MayReachVMEMLoad) {
112  Info.MayReachVMEMLoad = true;
113  Worklist.append(MBB->pred_begin(), MBB->pred_end());
114  }
115  }
116 
117  MachineBasicBlock &Entry = MF.front();
118  if (!MBBInfos[&Entry].MayReachVMEMLoad)
119  return false;
120 
121  // Raise the priority at the beginning of the shader.
122  MachineBasicBlock::iterator I = Entry.begin(), E = Entry.end();
123  while (I != E && !SIInstrInfo::isVALU(*I) && !I->isTerminator())
124  ++I;
125  Entry.insert(I, BuildSetprioMI(MF, HighPriority));
126 
127  // Lower the priority on edges where control leaves blocks from which
128  // VMEM loads are reachable.
129  SmallSet<MachineBasicBlock *, 16> PriorityLoweringBlocks;
130  for (MachineBasicBlock &MBB : MF) {
131  if (MBBInfos[&MBB].MayReachVMEMLoad) {
132  if (MBB.succ_empty())
133  PriorityLoweringBlocks.insert(&MBB);
134  continue;
135  }
136 
138  for (MachineBasicBlock *Pred : MBB.predecessors()) {
139  if (MBBInfos[Pred].MayReachVMEMLoad)
140  PriorityLoweringBlocks.insert(Pred);
141  }
142  continue;
143  }
144 
145  // Where lowering the priority in predecessors is not possible, the
146  // block receiving control either was not part of a loop in the first
147  // place or the loop simplification/canonicalization pass should have
148  // already tried to split the edge and insert a preheader, and if for
149  // whatever reason it failed to do so, then this leaves us with the
150  // only option of lowering the priority within the loop.
151  PriorityLoweringBlocks.insert(&MBB);
152  }
153 
154  for (MachineBasicBlock *MBB : PriorityLoweringBlocks) {
156  while (I != B) {
157  if (isVMEMLoad(*--I)) {
158  ++I;
159  break;
160  }
161  }
162  MBB->insert(I, BuildSetprioMI(MF, LowPriority));
163  }
164 
165  return true;
166 }
llvm::MachineBasicBlock::pred_begin
pred_iterator pred_begin()
Definition: MachineBasicBlock.h:326
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:105
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::createAMDGPUSetWavePriorityPass
FunctionPass * createAMDGPUSetWavePriorityPass()
llvm::Function
Definition: Function.h:60
isVMEMLoad
static bool isVMEMLoad(const MachineInstr &MI)
Definition: AMDGPUSetWavePriority.cpp:85
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1183
Allocator.h
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
CanLowerPriorityDirectlyInPredecessors
static bool CanLowerPriorityDirectlyInPredecessors(const MachineBasicBlock &MBB, MBBInfoSet &MBBInfos)
Definition: AMDGPUSetWavePriority.cpp:72
llvm::SmallSet
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:136
llvm::GCNSubtarget
Definition: GCNSubtarget.h:31
F
#define F(x, y, z)
Definition: MD5.cpp:55
GCNSubtarget.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
INITIALIZE_PASS
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:37
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Info
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
llvm::SmallVectorImpl::append
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:671
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::AMDGPU::isEntryFunctionCC
bool isEntryFunctionCC(CallingConv::ID CC)
Definition: AMDGPUBaseInfo.cpp:1687
llvm::MachineFunction::getSubtarget
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Definition: MachineFunction.h:656
AMDGPUMCTargetDesc.h
llvm::MachineBasicBlock::pred_end
pred_iterator pred_end()
Definition: MachineBasicBlock.h:328
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::DenseMap
Definition: DenseMap.h:714
I
#define I(x, y, z)
Definition: MD5.cpp:58
MachineFunctionPass.h
llvm::MachineBasicBlock::predecessors
iterator_range< pred_iterator > predecessors()
Definition: MachineBasicBlock.h:359
SIInstrInfo.h
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::MachineBasicBlock::succ_empty
bool succ_empty() const
Definition: MachineBasicBlock.h:357
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1625
llvm::MachineBasicBlock::successors
iterator_range< succ_iterator > successors()
Definition: MachineBasicBlock.h:365
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
AMDGPU.h
MBB
MachineBasicBlock & MBB
Definition: AArch64SLSHardening.cpp:74
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:622
llvm::MachineBasicBlock::insert
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Definition: MachineBasicBlock.cpp:1312
DEBUG_TYPE
#define DEBUG_TYPE
Definition: AMDGPUSetWavePriority.cpp:27
llvm::SIInstrInfo::isVMEM
static bool isVMEM(const MachineInstr &MI)
Definition: SIInstrInfo.h:367
llvm::SmallSet::insert
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:178
llvm::SIInstrInfo
Definition: SIInstrInfo.h:44
PostOrderIterator.h
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:278
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
llvm::SIInstrInfo::isVALU
static bool isVALU(const MachineInstr &MI)
Definition: SIInstrInfo.h:359
llvm::SmallVectorImpl::pop_back_val
T pop_back_val()
Definition: SmallVector.h:661
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::MachineInstrBundleIterator< MachineInstr >
InitializePasses.h
llvm::MachineBasicBlock::end
iterator end()
Definition: MachineBasicBlock.h:280
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38