LLVM 24.0.0git
AMDGPUISelDAGToDAG.h
Go to the documentation of this file.
1//===-- AMDGPUISelDAGToDAG.h - A dag to dag inst selector for AMDGPU ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//==-----------------------------------------------------------------------===//
8//
9/// \file
10/// Defines an instruction selector for the AMDGPU target.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
15#define LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
16
18#include "GCNSubtarget.h"
25
26namespace llvm {
27
28static inline bool getConstantValue(SDValue N, uint32_t &Out) {
29 // This is only used for packed vectors, where using 0 for undef should
30 // always be good.
31 if (N.isUndef()) {
32 Out = 0;
33 return true;
34 }
35
37 Out = C->getAPIntValue().getSExtValue();
38 return true;
39 }
40
42 Out = C->getValueAPF().bitcastToAPInt().getSExtValue();
43 return true;
44 }
45
46 return false;
47}
48
49/// AMDGPU specific code to select AMDGPU machine instructions for
50/// SelectionDAG operations.
52 // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can
53 // make the right decision when generating code for different targets.
54 const GCNSubtarget *Subtarget;
55
56 // Default FP mode for the current function.
58
59 // Instructions that will be lowered with a final instruction that zeros the
60 // high result bits.
61 bool fp16SrcZerosHighBits(unsigned Opc) const;
62
63public:
65
67
70 bool widenRegionLoad16(SDNode *N) const;
71 void PreprocessISelDAG() override;
72 void Select(SDNode *N) override;
73 void PostprocessISelDAG() override;
74
75protected:
76 void SelectBuildVector(SDNode *N, unsigned RegClassID);
78 bool isSDWAOperand(const SDNode *N) const;
79
80private:
81 std::pair<SDValue, SDValue> foldFrameIndex(SDValue N) const;
82
83 bool isInlineImmediate(const SDNode *N) const;
84
85 bool isInlineImmediate(const APInt &Imm) const {
86 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
87 }
88
89 bool isInlineImmediate(const APFloat &Imm) const {
90 return Subtarget->getInstrInfo()->isInlineConstant(Imm);
91 }
92
93 bool isVGPRImm(const SDNode *N) const;
94 bool isUniformBr(const SDNode *N) const;
95
96 MachineSDNode *buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
97 const SDLoc &DL) const;
98 MachineSDNode *buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
99 const SDLoc &DL) const;
100 MachineSDNode *buildRegSequence(SmallVectorImpl<SDValue> &Elts,
101 const SDLoc &DL, unsigned ElementSize) const;
102
103 void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods,
104 SmallVectorImpl<SDValue> &Elts, SDValue &Src,
105 const SDLoc &DL, unsigned ElementSize) const;
106
107 // Returns true if ISD::AND SDNode `N`'s masking of the shift amount operand's
108 // `ShAmtBits` bits is unneeded.
109 bool isUnneededShiftMask(const SDNode *N, unsigned ShAmtBits) const;
110
111 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &LHS,
112 SDValue &RHS) const;
113
114 MachineSDNode *buildSMovImm64(SDLoc &DL, uint64_t Val, EVT VT) const;
115
116 SDNode *packConstantV2I16(const SDNode *N, SelectionDAG &DAG) const;
117
118 SDNode *glueCopyToOp(SDNode *N, SDValue NewChain, SDValue Glue) const;
119 SDNode *glueCopyToM0(SDNode *N, SDValue Val) const;
120 SDNode *glueCopyToM0LDSInit(SDNode *N) const;
121
122 const TargetRegisterClass *getOperandRegClass(SDNode *N, unsigned OpNo) const;
123 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &Base, SDValue &Offset);
124 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &Base, SDValue &Offset);
125 bool isDSOffsetLegal(SDValue Base, unsigned Offset) const;
126 bool isDSOffset2Legal(SDValue Base, unsigned Offset0, unsigned Offset1,
127 unsigned Size) const;
128
129 bool isFlatScratchBaseLegal(SDValue Addr) const;
130 bool isFlatScratchBaseLegalSV(SDValue Addr) const;
131 bool isFlatScratchBaseLegalSVImm(SDValue Addr) const;
132 bool isSOffsetLegalWithImmOffset(SDValue *SOffset, bool Imm32Only,
133 bool IsBuffer, int64_t ImmOffset = 0) const;
134
135 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &Base, SDValue &Offset) const;
136 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
137 SDValue &Offset1) const;
138 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &Base, SDValue &Offset0,
139 SDValue &Offset1) const;
140 bool SelectDSReadWrite2(SDValue Ptr, SDValue &Base, SDValue &Offset0,
141 SDValue &Offset1, unsigned Size) const;
142 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
143 SDValue &SOffset, SDValue &Offset, SDValue &Offen,
144 SDValue &Idxen, SDValue &Addr64) const;
145 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
146 SDValue &SOffset, SDValue &Offset) const;
147 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
148 SDValue &VAddr, SDValue &SOffset,
149 SDValue &ImmOffset) const;
150 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
151 SDValue &Soffset, SDValue &Offset) const;
152
153 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
154 SDValue &Offset) const;
155 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset) const;
156
157 bool SelectFlatOffsetImpl(SDNode *N, SDValue Addr, SDValue &VAddr,
158 SDValue &Offset,
159 AMDGPU::FlatAddrSpace FlatVariant) const;
160 bool SelectFlatOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
161 SDValue &Offset) const;
162 bool SelectGlobalOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
163 SDValue &Offset) const;
164 bool SelectScratchOffset(SDNode *N, SDValue Addr, SDValue &VAddr,
165 SDValue &Offset) const;
166 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
167 SDValue &VOffset, SDValue &Offset, bool &ScaleOffset,
168 bool NeedIOffset = true) const;
169 bool SelectGlobalSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
170 SDValue &VOffset, SDValue &Offset,
171 SDValue &CPol) const;
172 bool SelectGlobalSAddrCPol(SDNode *N, SDValue Addr, SDValue &SAddr,
173 SDValue &VOffset, SDValue &Offset,
174 SDValue &CPol) const;
175 bool SelectGlobalSAddrCPolM0(SDNode *N, SDValue Addr, SDValue &SAddr,
176 SDValue &VOffset, SDValue &Offset,
177 SDValue &CPol) const;
178 bool SelectGlobalSAddrGLC(SDNode *N, SDValue Addr, SDValue &SAddr,
179 SDValue &VOffset, SDValue &Offset,
180 SDValue &CPol) const;
181 bool SelectGlobalSAddrNoIOffset(SDNode *N, SDValue Addr, SDValue &SAddr,
182 SDValue &VOffset, SDValue &CPol) const;
183 bool SelectGlobalSAddrNoIOffsetM0(SDNode *N, SDValue Addr, SDValue &SAddr,
184 SDValue &VOffset, SDValue &CPol) const;
185 bool SelectScratchSAddr(SDNode *N, SDValue Addr, SDValue &SAddr,
186 SDValue &Offset) const;
187 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
188 uint64_t ImmOffset) const;
189 bool SelectScratchSVAddr(SDNode *N, SDValue Addr, SDValue &VAddr,
190 SDValue &SAddr, SDValue &Offset,
191 SDValue &CPol) const;
192
193 bool SelectSMRDOffset(SDNode *N, SDValue ByteOffsetNode, SDValue *SOffset,
194 SDValue *Offset, bool Imm32Only = false,
195 bool IsBuffer = false, bool HasSOffset = false,
196 int64_t ImmOffset = 0,
197 bool *ScaleOffset = nullptr) const;
198 SDValue Expand32BitAddress(SDValue Addr) const;
199 bool SelectSMRDBaseOffset(SDNode *N, SDValue Addr, SDValue &SBase,
200 SDValue *SOffset, SDValue *Offset,
201 bool Imm32Only = false, bool IsBuffer = false,
202 bool HasSOffset = false, int64_t ImmOffset = 0,
203 bool *ScaleOffset = nullptr) const;
204 bool SelectSMRD(SDNode *N, SDValue Addr, SDValue &SBase, SDValue *SOffset,
205 SDValue *Offset, bool Imm32Only = false,
206 bool *ScaleOffset = nullptr) const;
207 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
208 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &Offset) const;
209 bool SelectScaleOffset(SDNode *N, SDValue &Offset, bool IsSigned,
210 bool IsSMEM = false) const;
211 bool SelectSMRDSgpr(SDNode *N, SDValue Addr, SDValue &SBase, SDValue &SOffset,
212 SDValue &CPol) const;
213 bool SelectSMRDSgprImm(SDNode *N, SDValue Addr, SDValue &SBase,
214 SDValue &SOffset, SDValue &Offset,
215 SDValue &CPol) const;
216 bool SelectSMRDBufferImm(SDValue N, SDValue &Offset) const;
217 bool SelectSMRDBufferImm32(SDValue N, SDValue &Offset) const;
218 bool SelectSMRDBufferSgprImm(SDValue N, SDValue &SOffset,
219 SDValue &Offset) const;
220 bool SelectMOVRELOffset(SDValue Index, SDValue &Base, SDValue &Offset) const;
221
222 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src, unsigned &SrcMods,
223 bool IsCanonicalizing = true,
224 bool AllowAbs = true) const;
225 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
226 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
227 SDValue &SrcMods) const;
228 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
229 bool SelectVOP3NoMods(SDValue In, SDValue &Src) const;
230 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
231 SDValue &Clamp, SDValue &Omod) const;
232 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
233 SDValue &Clamp, SDValue &Omod) const;
234
235 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
236 bool OpSel) const;
237 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
238 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods) const;
239
240 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
241 SDValue &Omod) const;
242
243 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
244 bool IsDOT = false, bool NoOpSel0 = false) const;
245 bool SelectVOP3PModsPkFmaOpSel0(SDValue In, SDValue &Src,
246 SDValue &SrcMods) const;
247 bool SelectVOP3PModsPkAddMulOpSel0(SDValue In, SDValue &Src,
248 SDValue &SrcMods) const;
249 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods) const;
250 bool SelectVOP3PNoModsDOT(SDValue In, SDValue &Src) const;
251 bool SelectVOP3PModsF32(SDValue In, SDValue &Src, SDValue &SrcMods) const;
252 bool SelectVOP3PNoModsF32(SDValue In, SDValue &Src) const;
253
254 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src) const;
255
256 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
257 SDValue &SrcMods) const;
258 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods) const;
259 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
260 SDValue &SrcMods) const;
261 bool SelectWMMAVISrc(SDValue In, SDValue &Src) const;
262
263 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey) const;
264 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey) const;
265 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey) const;
266
267 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods) const;
268
269 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
270 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src, unsigned &Mods,
271 MVT VT) const;
272 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
273 SDValue &SrcMods) const;
274 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods) const;
275 bool SelectVOP3PMadMixModsExtNeg(SDValue In, SDValue &Src,
276 SDValue &SrcMods) const;
277 bool SelectVOP3PMadMixModsNeg(SDValue In, SDValue &Src,
278 SDValue &SrcMods) const;
279 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
280 SDValue &SrcMods) const;
281 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
282 SDValue &SrcMods) const;
283 bool SelectVOP3PMadMixBF16ModsExtNeg(SDValue In, SDValue &Src,
284 SDValue &SrcMods) const;
285 bool SelectVOP3PMadMixBF16ModsNeg(SDValue In, SDValue &Src,
286 SDValue &SrcMods) const;
287
288 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,
289 SDValue &Tbl) const;
290
291 SDValue getHi16Elt(SDValue In) const;
292
293 SDValue getMaterializedScalarImm32(int64_t Val, const SDLoc &DL) const;
294
295 void SelectAddcSubb(SDNode *N);
296 void SelectAddcSubbI64(SDNode *N);
297 void SelectUADDO_USUBO(SDNode *N);
298 void SelectDIV_SCALE(SDNode *N);
299 void SelectMAD_64_32(SDNode *N);
300 void SelectMUL_LOHI(SDNode *N);
301 void SelectFMA_W_CHAIN(SDNode *N);
302 void SelectFMUL_W_CHAIN(SDNode *N);
303 SDNode *getBFE32(bool IsSigned, const SDLoc &DL, SDValue Val, uint32_t Offset,
304 uint32_t Width);
305 void SelectS_BFEFromShifts(SDNode *N);
306 void SelectS_BFE(SDNode *N);
307 bool isCBranchSCC(const SDNode *N) const;
308 void SelectBRCOND(SDNode *N);
309 void SelectFP_EXTEND(SDNode *N);
310 void SelectDSAppendConsume(SDNode *N, unsigned IntrID);
311 void SelectDSBvhStackIntrinsic(SDNode *N, unsigned IntrID);
312 void SelectTensorLoadStore(SDNode *N, unsigned IntrID);
313 void SelectDS_GWS(SDNode *N, unsigned IntrID);
314 void SelectInterpP1F16(SDNode *N);
315 void SelectINTRINSIC_W_CHAIN(SDNode *N);
316 void SelectINTRINSIC_WO_CHAIN(SDNode *N);
317 void SelectINTRINSIC_VOID(SDNode *N);
318 void SelectWAVE_ADDRESS(SDNode *N);
319 void SelectSTACKRESTORE(SDNode *N);
320 void SelectWRITE_REGISTER(SDNode *N);
321
322protected:
323 // Include the pieces autogenerated from the target description.
324#include "AMDGPUGenDAGISel.inc"
325};
326
334
336public:
337 static char ID;
338
340
341 bool runOnMachineFunction(MachineFunction &MF) override;
342 void getAnalysisUsage(AnalysisUsage &AU) const override;
343 StringRef getPassName() const override;
344};
345
346} // namespace llvm
347
348#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUISELDAGTODAG_H
AMDGPU address space definition.
uint64_t uint64_t
uint64_t Imm
static Register buildRegSequence(SmallVectorImpl< Register > &Elts, MachineInstr *InsertPt, MachineRegisterInfo &MRI)
static void selectWMMAModsNegAbs(unsigned ModOpcode, unsigned &Mods, SmallVectorImpl< Register > &Elts, Register &Src, MachineInstr *InsertPt, MachineRegisterInfo &MRI)
AMDGPU Register Bank Select
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
AMD GCN specific subclass of TargetSubtarget.
Value * RHS
Value * LHS
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - This function should be overriden by passes that need analysis information to do t...
AMDGPUDAGToDAGISelLegacy(TargetMachine &TM, CodeGenOptLevel OptLevel)
bool runOnMachineFunction(MachineFunction &MF) override
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
StringRef getPassName() const override
getPassName - Return a nice clean name for a pass.
bool isSDWAOperand(const SDNode *N) const
void SelectBuildVector(SDNode *N, unsigned RegClassID)
bool widenRegionLoad16(SDNode *N) const
bool runOnMachineFunction(MachineFunction &MF) override
void PreprocessISelDAG() override
PreprocessISelDAG - This hook allows targets to hack on the graph before instruction selection starts...
void PostprocessISelDAG() override
PostprocessISelDAG() - This hook allows the target to hack on the graph right after selection.
bool matchLoadD16FromBuildVector(SDNode *N) const
PreservedAnalyses run(MachineFunction &MF, MachineFunctionAnalysisManager &MFAM)
Class for arbitrary precision integers.
Definition APInt.h:78
Represent the analysis usage information of a pass.
const SIInstrInfo * getInstrInfo() const override
A set of analyses that are preserved following a run of a transformation pass.
Definition Analysis.h:112
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isInlineConstant(const APInt &Imm) const
SelectionDAGISelLegacy(char &ID, std::unique_ptr< SelectionDAGISel > S)
SelectionDAGISelPass(std::unique_ptr< SelectionDAGISel > Selector)
SelectionDAGISel(TargetMachine &tm, CodeGenOptLevel OL=CodeGenOptLevel::Default)
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
Primary interface to the complete machine description for the target machine.
This is an optimization pass for GlobalISel generic memory operations.
@ Offset
Definition DWP.cpp:577
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
AnalysisManager< MachineFunction > MachineFunctionAnalysisManager
static bool getConstantValue(SDValue N, uint32_t &Out)
CodeGenOptLevel
Code generation optimization level.
Definition CodeGen.h:280
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
#define N