61 bool fp16SrcZerosHighBits(
unsigned Opc)
const;
81 std::pair<SDValue, SDValue> foldFrameIndex(
SDValue N)
const;
83 bool isInlineImmediate(
const SDNode *
N)
const;
85 bool isInlineImmediate(
const APInt &
Imm)
const {
86 return Subtarget->getInstrInfo()->isInlineConstant(
Imm);
89 bool isInlineImmediate(
const APFloat &
Imm)
const {
93 bool isVGPRImm(
const SDNode *
N)
const;
94 bool isUniformBr(
const SDNode *
N)
const;
96 MachineSDNode *buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
97 const SDLoc &
DL)
const;
98 MachineSDNode *buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
99 const SDLoc &
DL)
const;
101 const SDLoc &
DL,
unsigned ElementSize)
const;
104 SmallVectorImpl<SDValue> &Elts, SDValue &Src,
105 const SDLoc &
DL,
unsigned ElementSize)
const;
109 bool isUnneededShiftMask(
const SDNode *
N,
unsigned ShAmtBits)
const;
111 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &
LHS,
114 MachineSDNode *buildSMovImm64(SDLoc &
DL,
uint64_t Val, EVT VT)
const;
116 SDNode *packConstantV2I16(
const SDNode *
N, SelectionDAG &DAG)
const;
118 SDNode *glueCopyToOp(SDNode *
N, SDValue NewChain, SDValue Glue)
const;
119 SDNode *glueCopyToM0(SDNode *
N, SDValue Val)
const;
120 SDNode *glueCopyToM0LDSInit(SDNode *
N)
const;
123 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &
Base, SDValue &
Offset);
124 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &
Base, SDValue &
Offset);
125 bool isDSOffsetLegal(SDValue
Base,
unsigned Offset)
const;
126 bool isDSOffset2Legal(SDValue
Base,
unsigned Offset0,
unsigned Offset1,
127 unsigned Size)
const;
129 bool isFlatScratchBaseLegal(SDValue Addr)
const;
130 bool isFlatScratchBaseLegalSV(SDValue Addr)
const;
131 bool isFlatScratchBaseLegalSVImm(SDValue Addr)
const;
132 bool isSOffsetLegalWithImmOffset(SDValue *SOffset,
bool Imm32Only,
133 bool IsBuffer, int64_t ImmOffset = 0)
const;
135 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &
Base, SDValue &
Offset)
const;
136 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &
Base, SDValue &Offset0,
137 SDValue &Offset1)
const;
138 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &
Base, SDValue &Offset0,
139 SDValue &Offset1)
const;
140 bool SelectDSReadWrite2(SDValue Ptr, SDValue &
Base, SDValue &Offset0,
141 SDValue &Offset1,
unsigned Size)
const;
142 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
143 SDValue &SOffset, SDValue &
Offset, SDValue &Offen,
144 SDValue &Idxen, SDValue &Addr64)
const;
145 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
146 SDValue &SOffset, SDValue &
Offset)
const;
147 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
148 SDValue &VAddr, SDValue &SOffset,
149 SDValue &ImmOffset)
const;
150 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
151 SDValue &Soffset, SDValue &
Offset)
const;
153 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
155 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset)
const;
157 bool SelectFlatOffsetImpl(SDNode *
N, SDValue Addr, SDValue &VAddr,
160 bool SelectFlatOffset(SDNode *
N, SDValue Addr, SDValue &VAddr,
162 bool SelectGlobalOffset(SDNode *
N, SDValue Addr, SDValue &VAddr,
164 bool SelectScratchOffset(SDNode *
N, SDValue Addr, SDValue &VAddr,
166 bool SelectGlobalSAddr(SDNode *
N, SDValue Addr, SDValue &SAddr,
167 SDValue &VOffset, SDValue &
Offset,
bool &ScaleOffset,
168 bool NeedIOffset =
true)
const;
169 bool SelectGlobalSAddr(SDNode *
N, SDValue Addr, SDValue &SAddr,
170 SDValue &VOffset, SDValue &
Offset,
171 SDValue &CPol)
const;
172 bool SelectGlobalSAddrCPol(SDNode *
N, SDValue Addr, SDValue &SAddr,
173 SDValue &VOffset, SDValue &
Offset,
174 SDValue &CPol)
const;
175 bool SelectGlobalSAddrCPolM0(SDNode *
N, SDValue Addr, SDValue &SAddr,
176 SDValue &VOffset, SDValue &
Offset,
177 SDValue &CPol)
const;
178 bool SelectGlobalSAddrGLC(SDNode *
N, SDValue Addr, SDValue &SAddr,
179 SDValue &VOffset, SDValue &
Offset,
180 SDValue &CPol)
const;
181 bool SelectGlobalSAddrNoIOffset(SDNode *
N, SDValue Addr, SDValue &SAddr,
182 SDValue &VOffset, SDValue &CPol)
const;
183 bool SelectGlobalSAddrNoIOffsetM0(SDNode *
N, SDValue Addr, SDValue &SAddr,
184 SDValue &VOffset, SDValue &CPol)
const;
185 bool SelectScratchSAddr(SDNode *
N, SDValue Addr, SDValue &SAddr,
187 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
189 bool SelectScratchSVAddr(SDNode *
N, SDValue Addr, SDValue &VAddr,
190 SDValue &SAddr, SDValue &
Offset,
191 SDValue &CPol)
const;
193 bool SelectSMRDOffset(SDNode *
N, SDValue ByteOffsetNode, SDValue *SOffset,
194 SDValue *
Offset,
bool Imm32Only =
false,
195 bool IsBuffer =
false,
bool HasSOffset =
false,
196 int64_t ImmOffset = 0,
197 bool *ScaleOffset =
nullptr)
const;
198 SDValue Expand32BitAddress(SDValue Addr)
const;
199 bool SelectSMRDBaseOffset(SDNode *
N, SDValue Addr, SDValue &SBase,
200 SDValue *SOffset, SDValue *
Offset,
201 bool Imm32Only =
false,
bool IsBuffer =
false,
202 bool HasSOffset =
false, int64_t ImmOffset = 0,
203 bool *ScaleOffset =
nullptr)
const;
204 bool SelectSMRD(SDNode *
N, SDValue Addr, SDValue &SBase, SDValue *SOffset,
205 SDValue *
Offset,
bool Imm32Only =
false,
206 bool *ScaleOffset =
nullptr)
const;
207 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &
Offset)
const;
208 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &
Offset)
const;
209 bool SelectScaleOffset(SDNode *
N, SDValue &
Offset,
bool IsSigned,
210 bool IsSMEM =
false)
const;
211 bool SelectSMRDSgpr(SDNode *
N, SDValue Addr, SDValue &SBase, SDValue &SOffset,
212 SDValue &CPol)
const;
213 bool SelectSMRDSgprImm(SDNode *
N, SDValue Addr, SDValue &SBase,
214 SDValue &SOffset, SDValue &
Offset,
215 SDValue &CPol)
const;
216 bool SelectSMRDBufferImm(SDValue
N, SDValue &
Offset)
const;
217 bool SelectSMRDBufferImm32(SDValue
N, SDValue &
Offset)
const;
218 bool SelectSMRDBufferSgprImm(SDValue
N, SDValue &SOffset,
220 bool SelectMOVRELOffset(SDValue Index, SDValue &
Base, SDValue &
Offset)
const;
222 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src,
unsigned &SrcMods,
223 bool IsCanonicalizing =
true,
224 bool AllowAbs =
true)
const;
225 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
226 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
227 SDValue &SrcMods)
const;
228 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
229 bool SelectVOP3NoMods(SDValue In, SDValue &Src)
const;
230 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
231 SDValue &Clamp, SDValue &Omod)
const;
232 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
233 SDValue &Clamp, SDValue &Omod)
const;
235 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
237 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
238 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
240 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
241 SDValue &Omod)
const;
243 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
244 bool IsDOT =
false,
bool NoOpSel0 =
false)
const;
245 bool SelectVOP3PModsPkFmaOpSel0(SDValue In, SDValue &Src,
246 SDValue &SrcMods)
const;
247 bool SelectVOP3PModsPkAddMulOpSel0(SDValue In, SDValue &Src,
248 SDValue &SrcMods)
const;
249 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
250 bool SelectVOP3PNoModsDOT(SDValue In, SDValue &Src)
const;
251 bool SelectVOP3PModsF32(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
252 bool SelectVOP3PNoModsF32(SDValue In, SDValue &Src)
const;
254 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src)
const;
256 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
257 SDValue &SrcMods)
const;
258 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
259 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
260 SDValue &SrcMods)
const;
261 bool SelectWMMAVISrc(SDValue In, SDValue &Src)
const;
263 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
264 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
265 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
267 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
269 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
270 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
unsigned &Mods,
272 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
273 SDValue &SrcMods)
const;
274 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
275 bool SelectVOP3PMadMixModsExtNeg(SDValue In, SDValue &Src,
276 SDValue &SrcMods)
const;
277 bool SelectVOP3PMadMixModsNeg(SDValue In, SDValue &Src,
278 SDValue &SrcMods)
const;
279 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
280 SDValue &SrcMods)
const;
281 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
282 SDValue &SrcMods)
const;
283 bool SelectVOP3PMadMixBF16ModsExtNeg(SDValue In, SDValue &Src,
284 SDValue &SrcMods)
const;
285 bool SelectVOP3PMadMixBF16ModsNeg(SDValue In, SDValue &Src,
286 SDValue &SrcMods)
const;
288 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,
291 SDValue getHi16Elt(SDValue In)
const;
293 SDValue getMaterializedScalarImm32(int64_t Val,
const SDLoc &
DL)
const;
295 void SelectAddcSubb(SDNode *
N);
296 void SelectAddcSubbI64(SDNode *
N);
297 void SelectUADDO_USUBO(SDNode *
N);
298 void SelectDIV_SCALE(SDNode *
N);
299 void SelectMAD_64_32(SDNode *
N);
300 void SelectMUL_LOHI(SDNode *
N);
301 void SelectFMA_W_CHAIN(SDNode *
N);
302 void SelectFMUL_W_CHAIN(SDNode *
N);
303 SDNode *getBFE32(
bool IsSigned,
const SDLoc &
DL, SDValue Val,
uint32_t Offset,
305 void SelectS_BFEFromShifts(SDNode *
N);
306 void SelectS_BFE(SDNode *
N);
307 bool isCBranchSCC(
const SDNode *
N)
const;
308 void SelectBRCOND(SDNode *
N);
309 void SelectFP_EXTEND(SDNode *
N);
310 void SelectDSAppendConsume(SDNode *
N,
unsigned IntrID);
311 void SelectDSBvhStackIntrinsic(SDNode *
N,
unsigned IntrID);
312 void SelectTensorLoadStore(SDNode *
N,
unsigned IntrID);
313 void SelectDS_GWS(SDNode *
N,
unsigned IntrID);
314 void SelectInterpP1F16(SDNode *
N);
315 void SelectINTRINSIC_W_CHAIN(SDNode *
N);
316 void SelectINTRINSIC_WO_CHAIN(SDNode *
N);
317 void SelectINTRINSIC_VOID(SDNode *
N);
318 void SelectWAVE_ADDRESS(SDNode *
N);
319 void SelectSTACKRESTORE(SDNode *
N);
320 void SelectWRITE_REGISTER(SDNode *
N);
324#include "AMDGPUGenDAGISel.inc"