LLVM  16.0.0git
GCNHazardRecognizer.h
Go to the documentation of this file.
1 //===-- GCNHazardRecognizers.h - GCN Hazard Recognizers ---------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines hazard recognizers for scheduling on GCN processors.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #ifndef LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
14 #define LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
15 
16 #include "llvm/ADT/BitVector.h"
17 #include "llvm/ADT/STLExtras.h"
20 #include <list>
21 
22 namespace llvm {
23 
24 class MachineFunction;
25 class MachineInstr;
26 class MachineOperand;
27 class MachineRegisterInfo;
28 class SIInstrInfo;
29 class SIRegisterInfo;
30 class GCNSubtarget;
31 
33 public:
34  typedef function_ref<bool(const MachineInstr &)> IsHazardFn;
35 
36 private:
37  // Distinguish if we are called from scheduler or hazard recognizer
38  bool IsHazardRecognizerMode;
39 
40  // This variable stores the instruction that has been emitted this cycle. It
41  // will be added to EmittedInstrs, when AdvanceCycle() or RecedeCycle() is
42  // called.
43  MachineInstr *CurrCycleInstr;
44  std::list<MachineInstr*> EmittedInstrs;
45  const MachineFunction &MF;
46  const GCNSubtarget &ST;
47  const SIInstrInfo &TII;
48  const SIRegisterInfo &TRI;
49  TargetSchedModel TSchedModel;
50  bool RunLdsBranchVmemWARHazardFixup;
51 
52  /// RegUnits of uses in the current soft memory clause.
53  BitVector ClauseUses;
54 
55  /// RegUnits of defs in the current soft memory clause.
56  BitVector ClauseDefs;
57 
58  void resetClause() {
59  ClauseUses.reset();
60  ClauseDefs.reset();
61  }
62 
63  void addClauseInst(const MachineInstr &MI);
64 
65  /// \returns the number of wait states before another MFMA instruction can be
66  /// issued after \p MI.
67  unsigned getMFMAPipelineWaitStates(const MachineInstr &MI) const;
68 
69  // Advance over a MachineInstr bundle. Look for hazards in the bundled
70  // instructions.
71  void processBundle();
72 
73  // Run on an individual instruction in hazard recognizer mode. This can be
74  // used on a newly inserted instruction before returning from PreEmitNoops.
75  void runOnInstruction(MachineInstr *MI);
76 
77  int getWaitStatesSince(IsHazardFn IsHazard, int Limit);
78  int getWaitStatesSinceDef(unsigned Reg, IsHazardFn IsHazardDef, int Limit);
79  int getWaitStatesSinceSetReg(IsHazardFn IsHazard, int Limit);
80 
81  int checkSoftClauseHazards(MachineInstr *SMEM);
82  int checkSMRDHazards(MachineInstr *SMRD);
83  int checkVMEMHazards(MachineInstr* VMEM);
84  int checkDPPHazards(MachineInstr *DPP);
85  int checkDivFMasHazards(MachineInstr *DivFMas);
86  int checkGetRegHazards(MachineInstr *GetRegInstr);
87  int checkSetRegHazards(MachineInstr *SetRegInstr);
88  int createsVALUHazard(const MachineInstr &MI);
89  int checkVALUHazards(MachineInstr *VALU);
90  int checkVALUHazardsHelper(const MachineOperand &Def, const MachineRegisterInfo &MRI);
91  int checkRWLaneHazards(MachineInstr *RWLane);
92  int checkRFEHazards(MachineInstr *RFE);
93  int checkInlineAsmHazards(MachineInstr *IA);
94  int checkReadM0Hazards(MachineInstr *SMovRel);
95  int checkNSAtoVMEMHazard(MachineInstr *MI);
96  int checkFPAtomicToDenormModeHazard(MachineInstr *MI);
97  void fixHazards(MachineInstr *MI);
98  bool fixVcmpxPermlaneHazards(MachineInstr *MI);
99  bool fixVMEMtoScalarWriteHazards(MachineInstr *MI);
100  bool fixSMEMtoVectorWriteHazards(MachineInstr *MI);
101  bool fixVcmpxExecWARHazard(MachineInstr *MI);
102  bool fixLdsBranchVmemWARHazard(MachineInstr *MI);
103  bool fixLdsDirectVALUHazard(MachineInstr *MI);
104  bool fixLdsDirectVMEMHazard(MachineInstr *MI);
105  bool fixVALUPartialForwardingHazard(MachineInstr *MI);
106  bool fixVALUTransUseHazard(MachineInstr *MI);
107  bool fixWMMAHazards(MachineInstr *MI);
108  bool fixShift64HighRegBug(MachineInstr *MI);
109  bool fixVALUMaskWriteHazard(MachineInstr *MI);
110 
111  int checkMAIHazards(MachineInstr *MI);
112  int checkMAIHazards908(MachineInstr *MI);
113  int checkMAIHazards90A(MachineInstr *MI);
114  /// Pad the latency between neighboring MFMA instructions with s_nops. The
115  /// percentage of wait states to fill with s_nops is specified by the command
116  /// line option '-amdgpu-mfma-padding-ratio'.
117  ///
118  /// For example, with '-amdgpu-mfma-padding-ratio=100':
119  ///
120  /// 2 pass MFMA instructions have a latency of 2 wait states. Therefore, a
121  /// 'S_NOP 1' will be added between sequential MFMA instructions.
122  ///
123  /// V_MFMA_F32_4X4X1F32
124  /// V_MFMA_F32_4X4X1F32
125  ///-->
126  /// V_MFMA_F32_4X4X1F32
127  /// S_NOP 1
128  /// V_MFMA_F32_4X4X1F32
129  int checkMFMAPadding(MachineInstr *MI);
130  int checkMAIVALUHazards(MachineInstr *MI);
131  int checkMAILdStHazards(MachineInstr *MI);
132 
133 public:
135  // We can only issue one instruction per cycle.
136  bool atIssueLimit() const override { return true; }
137  void EmitInstruction(SUnit *SU) override;
138  void EmitInstruction(MachineInstr *MI) override;
139  HazardType getHazardType(SUnit *SU, int Stalls) override;
140  void EmitNoop() override;
141  unsigned PreEmitNoops(MachineInstr *) override;
142  unsigned PreEmitNoopsCommon(MachineInstr *);
143  void AdvanceCycle() override;
144  void RecedeCycle() override;
145  bool ShouldPreferAnother(SUnit *SU) override;
146  void Reset() override;
147 };
148 
149 } // end namespace llvm
150 
151 #endif //LLVM_LIB_TARGET_AMDGPUHAZARDRECOGNIZERS_H
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:109
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::GCNHazardRecognizer::getHazardType
HazardType getHazardType(SUnit *SU, int Stalls) override
getHazardType - Return the hazard type of emitting this node.
Definition: GCNHazardRecognizer.cpp:181
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::GCNHazardRecognizer::GCNHazardRecognizer
GCNHazardRecognizer(const MachineFunction &MF)
Definition: GCNHazardRecognizer.cpp:53
llvm::GCNHazardRecognizer::atIssueLimit
bool atIssueLimit() const override
atIssueLimit - Return true if no more instructions may be issued in this cycle.
Definition: GCNHazardRecognizer.h:136
llvm::SIInstrFlags::DPP
@ DPP
Definition: SIDefines.h:50
llvm::GCNSubtarget
Definition: GCNSubtarget.h:31
STLExtras.h
ScheduleHazardRecognizer.h
llvm::GCNHazardRecognizer
Definition: GCNHazardRecognizer.h:32
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:48
BitVector.h
llvm::AArch64PACKey::IA
@ IA
Definition: AArch64BaseInfo.h:819
llvm::SIRegisterInfo
Definition: SIRegisterInfo.h:30
llvm::BitVector
Definition: BitVector.h:75
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::ScheduleHazardRecognizer::HazardType
HazardType
Definition: ScheduleHazardRecognizer.h:37
TargetSchedule.h
llvm::GCNHazardRecognizer::PreEmitNoopsCommon
unsigned PreEmitNoopsCommon(MachineInstr *)
Definition: GCNHazardRecognizer.cpp:326
llvm::TargetSchedModel
Provide an instruction scheduling machine model to CodeGen passes.
Definition: TargetSchedule.h:30
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::GCNHazardRecognizer::PreEmitNoops
unsigned PreEmitNoops(MachineInstr *) override
This overload will be used when the hazard recognizer is being used by a non-scheduling pass,...
Definition: GCNHazardRecognizer.cpp:317
llvm::GCNHazardRecognizer::EmitInstruction
void EmitInstruction(SUnit *SU) override
EmitInstruction - This callback is invoked when an instruction is emitted, to advance the hazard stat...
Definition: GCNHazardRecognizer.cpp:71
llvm::GCNHazardRecognizer::EmitNoop
void EmitNoop() override
EmitNoop - This callback is invoked when a noop was added to the instruction stream.
Definition: GCNHazardRecognizer.cpp:395
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::GCNHazardRecognizer::Reset
void Reset() override
Reset - This callback is invoked when a new block of instructions is about to be schedule.
Definition: GCNHazardRecognizer.cpp:67
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::SIInstrFlags::SMRD
@ SMRD
Definition: SIDefines.h:56
llvm::GCNHazardRecognizer::RecedeCycle
void RecedeCycle() override
RecedeCycle - This callback is invoked whenever the next bottom-up instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:437
llvm::BitVector::reset
BitVector & reset()
Definition: BitVector.h:385
llvm::SIInstrInfo
Definition: SIInstrInfo.h:44
llvm::GCNHazardRecognizer::AdvanceCycle
void AdvanceCycle() override
AdvanceCycle - This callback is invoked whenever the next top-down instruction to be scheduled cannot...
Definition: GCNHazardRecognizer.cpp:399
llvm::GCNHazardRecognizer::ShouldPreferAnother
bool ShouldPreferAnother(SUnit *SU) override
ShouldPreferAnother - This callback may be invoked if getHazardType returns NoHazard.
Definition: GCNHazardRecognizer.cpp:2692
llvm::GCNHazardRecognizer::IsHazardFn
function_ref< bool(const MachineInstr &)> IsHazardFn
Definition: GCNHazardRecognizer.h:34
llvm::SUnit
Scheduling unit. This is a node in the scheduling DAG.
Definition: ScheduleDAG.h:242
llvm::ScheduleHazardRecognizer
HazardRecognizer - This determines whether or not an instruction can be issued this cycle,...
Definition: ScheduleHazardRecognizer.h:25