61 bool fp16SrcZerosHighBits(
unsigned Opc)
const;
80 std::pair<SDValue, SDValue> foldFrameIndex(
SDValue N)
const;
82 bool isInlineImmediate(
const SDNode *
N)
const;
84 bool isInlineImmediate(
const APInt &
Imm)
const {
85 return Subtarget->getInstrInfo()->isInlineConstant(
Imm);
88 bool isInlineImmediate(
const APFloat &
Imm)
const {
92 bool isVGPRImm(
const SDNode *
N)
const;
93 bool isUniformBr(
const SDNode *
N)
const;
95 MachineSDNode *buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
96 const SDLoc &
DL)
const;
97 MachineSDNode *buildRegSequence32(SmallVectorImpl<SDValue> &Elts,
98 const SDLoc &
DL)
const;
100 const SDLoc &
DL,
unsigned ElementSize)
const;
103 SmallVectorImpl<SDValue> &Elts, SDValue &Src,
104 const SDLoc &
DL,
unsigned ElementSize)
const;
108 bool isUnneededShiftMask(
const SDNode *
N,
unsigned ShAmtBits)
const;
110 bool isBaseWithConstantOffset64(SDValue Addr, SDValue &
LHS,
113 MachineSDNode *buildSMovImm64(SDLoc &
DL,
uint64_t Val, EVT VT)
const;
115 SDNode *packConstantV2I16(
const SDNode *
N, SelectionDAG &DAG)
const;
117 SDNode *glueCopyToOp(SDNode *
N, SDValue NewChain, SDValue Glue)
const;
118 SDNode *glueCopyToM0(SDNode *
N, SDValue Val)
const;
119 SDNode *glueCopyToM0LDSInit(SDNode *
N)
const;
122 virtual bool SelectADDRVTX_READ(SDValue Addr, SDValue &
Base, SDValue &
Offset);
123 virtual bool SelectADDRIndirect(SDValue Addr, SDValue &
Base, SDValue &
Offset);
124 bool isDSOffsetLegal(SDValue
Base,
unsigned Offset)
const;
125 bool isDSOffset2Legal(SDValue
Base,
unsigned Offset0,
unsigned Offset1,
126 unsigned Size)
const;
128 bool isFlatScratchBaseLegal(SDValue Addr)
const;
129 bool isFlatScratchBaseLegalSV(SDValue Addr)
const;
130 bool isFlatScratchBaseLegalSVImm(SDValue Addr)
const;
131 bool isSOffsetLegalWithImmOffset(SDValue *SOffset,
bool Imm32Only,
132 bool IsBuffer, int64_t ImmOffset = 0)
const;
134 bool SelectDS1Addr1Offset(SDValue Ptr, SDValue &
Base, SDValue &
Offset)
const;
135 bool SelectDS64Bit4ByteAligned(SDValue Ptr, SDValue &
Base, SDValue &Offset0,
136 SDValue &Offset1)
const;
137 bool SelectDS128Bit8ByteAligned(SDValue Ptr, SDValue &
Base, SDValue &Offset0,
138 SDValue &Offset1)
const;
139 bool SelectDSReadWrite2(SDValue Ptr, SDValue &
Base, SDValue &Offset0,
140 SDValue &Offset1,
unsigned Size)
const;
141 bool SelectMUBUF(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
142 SDValue &SOffset, SDValue &
Offset, SDValue &Offen,
143 SDValue &Idxen, SDValue &Addr64)
const;
144 bool SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, SDValue &VAddr,
145 SDValue &SOffset, SDValue &
Offset)
const;
146 bool SelectMUBUFScratchOffen(SDNode *Parent, SDValue Addr, SDValue &RSrc,
147 SDValue &VAddr, SDValue &SOffset,
148 SDValue &ImmOffset)
const;
149 bool SelectMUBUFScratchOffset(SDNode *Parent, SDValue Addr, SDValue &SRsrc,
150 SDValue &Soffset, SDValue &
Offset)
const;
152 bool SelectMUBUFOffset(SDValue Addr, SDValue &SRsrc, SDValue &Soffset,
154 bool SelectBUFSOffset(SDValue Addr, SDValue &SOffset)
const;
156 bool SelectFlatOffsetImpl(SDNode *
N, SDValue Addr, SDValue &VAddr,
159 bool SelectFlatOffset(SDNode *
N, SDValue Addr, SDValue &VAddr,
161 bool SelectGlobalOffset(SDNode *
N, SDValue Addr, SDValue &VAddr,
163 bool SelectScratchOffset(SDNode *
N, SDValue Addr, SDValue &VAddr,
165 bool SelectGlobalSAddr(SDNode *
N, SDValue Addr, SDValue &SAddr,
166 SDValue &VOffset, SDValue &
Offset,
bool &ScaleOffset,
167 bool NeedIOffset =
true)
const;
168 bool SelectGlobalSAddr(SDNode *
N, SDValue Addr, SDValue &SAddr,
169 SDValue &VOffset, SDValue &
Offset,
170 SDValue &CPol)
const;
171 bool SelectGlobalSAddrCPol(SDNode *
N, SDValue Addr, SDValue &SAddr,
172 SDValue &VOffset, SDValue &
Offset,
173 SDValue &CPol)
const;
174 bool SelectGlobalSAddrCPolM0(SDNode *
N, SDValue Addr, SDValue &SAddr,
175 SDValue &VOffset, SDValue &
Offset,
176 SDValue &CPol)
const;
177 bool SelectGlobalSAddrGLC(SDNode *
N, SDValue Addr, SDValue &SAddr,
178 SDValue &VOffset, SDValue &
Offset,
179 SDValue &CPol)
const;
180 bool SelectGlobalSAddrNoIOffset(SDNode *
N, SDValue Addr, SDValue &SAddr,
181 SDValue &VOffset, SDValue &CPol)
const;
182 bool SelectGlobalSAddrNoIOffsetM0(SDNode *
N, SDValue Addr, SDValue &SAddr,
183 SDValue &VOffset, SDValue &CPol)
const;
184 bool SelectScratchSAddr(SDNode *
N, SDValue Addr, SDValue &SAddr,
186 bool checkFlatScratchSVSSwizzleBug(SDValue VAddr, SDValue SAddr,
188 bool SelectScratchSVAddr(SDNode *
N, SDValue Addr, SDValue &VAddr,
189 SDValue &SAddr, SDValue &
Offset,
190 SDValue &CPol)
const;
192 bool SelectSMRDOffset(SDNode *
N, SDValue ByteOffsetNode, SDValue *SOffset,
193 SDValue *
Offset,
bool Imm32Only =
false,
194 bool IsBuffer =
false,
bool HasSOffset =
false,
195 int64_t ImmOffset = 0,
196 bool *ScaleOffset =
nullptr)
const;
197 SDValue Expand32BitAddress(SDValue Addr)
const;
198 bool SelectSMRDBaseOffset(SDNode *
N, SDValue Addr, SDValue &SBase,
199 SDValue *SOffset, SDValue *
Offset,
200 bool Imm32Only =
false,
bool IsBuffer =
false,
201 bool HasSOffset =
false, int64_t ImmOffset = 0,
202 bool *ScaleOffset =
nullptr)
const;
203 bool SelectSMRD(SDNode *
N, SDValue Addr, SDValue &SBase, SDValue *SOffset,
204 SDValue *
Offset,
bool Imm32Only =
false,
205 bool *ScaleOffset =
nullptr)
const;
206 bool SelectSMRDImm(SDValue Addr, SDValue &SBase, SDValue &
Offset)
const;
207 bool SelectSMRDImm32(SDValue Addr, SDValue &SBase, SDValue &
Offset)
const;
208 bool SelectScaleOffset(SDNode *
N, SDValue &
Offset,
bool IsSigned)
const;
209 bool SelectSMRDSgpr(SDNode *
N, SDValue Addr, SDValue &SBase, SDValue &SOffset,
210 SDValue &CPol)
const;
211 bool SelectSMRDSgprImm(SDNode *
N, SDValue Addr, SDValue &SBase,
212 SDValue &SOffset, SDValue &
Offset,
213 SDValue &CPol)
const;
214 bool SelectSMRDBufferImm(SDValue
N, SDValue &
Offset)
const;
215 bool SelectSMRDBufferImm32(SDValue
N, SDValue &
Offset)
const;
216 bool SelectSMRDBufferSgprImm(SDValue
N, SDValue &SOffset,
218 bool SelectMOVRELOffset(SDValue Index, SDValue &
Base, SDValue &
Offset)
const;
220 bool SelectVOP3ModsImpl(SDValue In, SDValue &Src,
unsigned &SrcMods,
221 bool IsCanonicalizing =
true,
222 bool AllowAbs =
true)
const;
223 bool SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
224 bool SelectVOP3ModsNonCanonicalizing(SDValue In, SDValue &Src,
225 SDValue &SrcMods)
const;
226 bool SelectVOP3BMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
227 bool SelectVOP3NoMods(SDValue In, SDValue &Src)
const;
228 bool SelectVOP3Mods0(SDValue In, SDValue &Src, SDValue &SrcMods,
229 SDValue &Clamp, SDValue &Omod)
const;
230 bool SelectVOP3BMods0(SDValue In, SDValue &Src, SDValue &SrcMods,
231 SDValue &Clamp, SDValue &Omod)
const;
233 bool SelectVINTERPModsImpl(SDValue In, SDValue &Src, SDValue &SrcMods,
235 bool SelectVINTERPMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
236 bool SelectVINTERPModsHi(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
238 bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp,
239 SDValue &Omod)
const;
241 bool SelectVOP3PMods(SDValue In, SDValue &Src, SDValue &SrcMods,
242 bool IsDOT =
false)
const;
243 bool SelectVOP3PModsDOT(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
244 bool SelectVOP3PNoModsDOT(SDValue In, SDValue &Src)
const;
245 bool SelectVOP3PModsF32(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
246 bool SelectVOP3PNoModsF32(SDValue In, SDValue &Src)
const;
248 bool SelectWMMAOpSelVOP3PMods(SDValue In, SDValue &Src)
const;
250 bool SelectWMMAModsF32NegAbs(SDValue In, SDValue &Src,
251 SDValue &SrcMods)
const;
252 bool SelectWMMAModsF16Neg(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
253 bool SelectWMMAModsF16NegAbs(SDValue In, SDValue &Src,
254 SDValue &SrcMods)
const;
255 bool SelectWMMAVISrc(SDValue In, SDValue &Src)
const;
257 bool SelectSWMMACIndex8(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
258 bool SelectSWMMACIndex16(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
259 bool SelectSWMMACIndex32(SDValue In, SDValue &Src, SDValue &IndexKey)
const;
261 bool SelectVOP3OpSel(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
263 bool SelectVOP3OpSelMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
264 bool SelectVOP3PMadMixModsImpl(SDValue In, SDValue &Src,
unsigned &Mods,
266 bool SelectVOP3PMadMixModsExt(SDValue In, SDValue &Src,
267 SDValue &SrcMods)
const;
268 bool SelectVOP3PMadMixMods(SDValue In, SDValue &Src, SDValue &SrcMods)
const;
269 bool SelectVOP3PMadMixModsExtNeg(SDValue In, SDValue &Src,
270 SDValue &SrcMods)
const;
271 bool SelectVOP3PMadMixModsNeg(SDValue In, SDValue &Src,
272 SDValue &SrcMods)
const;
273 bool SelectVOP3PMadMixBF16ModsExt(SDValue In, SDValue &Src,
274 SDValue &SrcMods)
const;
275 bool SelectVOP3PMadMixBF16Mods(SDValue In, SDValue &Src,
276 SDValue &SrcMods)
const;
277 bool SelectVOP3PMadMixBF16ModsExtNeg(SDValue In, SDValue &Src,
278 SDValue &SrcMods)
const;
279 bool SelectVOP3PMadMixBF16ModsNeg(SDValue In, SDValue &Src,
280 SDValue &SrcMods)
const;
282 bool SelectBITOP3(SDValue In, SDValue &Src0, SDValue &Src1, SDValue &Src2,
285 SDValue getHi16Elt(SDValue In)
const;
287 SDValue getMaterializedScalarImm32(int64_t Val,
const SDLoc &
DL)
const;
289 void SelectAddcSubb(SDNode *
N);
290 void SelectAddcSubbI64(SDNode *
N);
291 void SelectUADDO_USUBO(SDNode *
N);
292 void SelectDIV_SCALE(SDNode *
N);
293 void SelectMAD_64_32(SDNode *
N);
294 void SelectMUL_LOHI(SDNode *
N);
295 void SelectFMA_W_CHAIN(SDNode *
N);
296 void SelectFMUL_W_CHAIN(SDNode *
N);
297 SDNode *getBFE32(
bool IsSigned,
const SDLoc &
DL, SDValue Val,
uint32_t Offset,
299 void SelectS_BFEFromShifts(SDNode *
N);
300 void SelectS_BFE(SDNode *
N);
301 bool isCBranchSCC(
const SDNode *
N)
const;
302 void SelectBRCOND(SDNode *
N);
303 void SelectFP_EXTEND(SDNode *
N);
304 void SelectDSAppendConsume(SDNode *
N,
unsigned IntrID);
305 void SelectDSBvhStackIntrinsic(SDNode *
N,
unsigned IntrID);
306 void SelectTensorLoadStore(SDNode *
N,
unsigned IntrID);
307 void SelectDS_GWS(SDNode *
N,
unsigned IntrID);
308 void SelectInterpP1F16(SDNode *
N);
309 void SelectINTRINSIC_W_CHAIN(SDNode *
N);
310 void SelectINTRINSIC_WO_CHAIN(SDNode *
N);
311 void SelectINTRINSIC_VOID(SDNode *
N);
312 void SelectWAVE_ADDRESS(SDNode *
N);
313 void SelectSTACKRESTORE(SDNode *
N);
317#include "AMDGPUGenDAGISel.inc"