LLVM 19.0.0git
LegalizationArtifactCombiner.h
Go to the documentation of this file.
1//===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h -----*- C++ -*-//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8// This file contains some helper functions which try to cleanup artifacts
9// such as G_TRUNCs/G_[ZSA]EXTENDS that were created during legalization to make
10// the types match. This file also contains some combines of merges that happens
11// at the end of the legalization.
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H
15#define LLVM_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H
16
28#include "llvm/IR/Constants.h"
29#include "llvm/Support/Debug.h"
30
31#define DEBUG_TYPE "legalizer"
32
33namespace llvm {
35 MachineIRBuilder &Builder;
37 const LegalizerInfo &LI;
39
40 static bool isArtifactCast(unsigned Opc) {
41 switch (Opc) {
42 case TargetOpcode::G_TRUNC:
43 case TargetOpcode::G_SEXT:
44 case TargetOpcode::G_ZEXT:
45 case TargetOpcode::G_ANYEXT:
46 return true;
47 default:
48 return false;
49 }
50 }
51
52public:
54 const LegalizerInfo &LI,
55 GISelKnownBits *KB = nullptr)
56 : Builder(B), MRI(MRI), LI(LI), KB(KB) {}
57
60 SmallVectorImpl<Register> &UpdatedDefs,
61 GISelObserverWrapper &Observer) {
62 using namespace llvm::MIPatternMatch;
63 assert(MI.getOpcode() == TargetOpcode::G_ANYEXT);
64
65 Builder.setInstrAndDebugLoc(MI);
66 Register DstReg = MI.getOperand(0).getReg();
67 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
68
69 // aext(trunc x) - > aext/copy/trunc x
70 Register TruncSrc;
71 if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
72 LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
73 if (MRI.getType(DstReg) == MRI.getType(TruncSrc))
74 replaceRegOrBuildCopy(DstReg, TruncSrc, MRI, Builder, UpdatedDefs,
75 Observer);
76 else
77 Builder.buildAnyExtOrTrunc(DstReg, TruncSrc);
78 UpdatedDefs.push_back(DstReg);
79 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
80 return true;
81 }
82
83 // aext([asz]ext x) -> [asz]ext x
84 Register ExtSrc;
85 MachineInstr *ExtMI;
86 if (mi_match(SrcReg, MRI,
87 m_all_of(m_MInstr(ExtMI), m_any_of(m_GAnyExt(m_Reg(ExtSrc)),
88 m_GSExt(m_Reg(ExtSrc)),
89 m_GZExt(m_Reg(ExtSrc)))))) {
90 Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc});
91 UpdatedDefs.push_back(DstReg);
92 markInstAndDefDead(MI, *ExtMI, DeadInsts);
93 return true;
94 }
95
96 // Try to fold aext(g_constant) when the larger constant type is legal.
97 auto *SrcMI = MRI.getVRegDef(SrcReg);
98 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {
99 const LLT DstTy = MRI.getType(DstReg);
100 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
101 auto &CstVal = SrcMI->getOperand(1);
102 Builder.buildConstant(
103 DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits()));
104 UpdatedDefs.push_back(DstReg);
105 markInstAndDefDead(MI, *SrcMI, DeadInsts);
106 return true;
107 }
108 }
109 return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs);
110 }
111
114 SmallVectorImpl<Register> &UpdatedDefs,
115 GISelObserverWrapper &Observer) {
116 using namespace llvm::MIPatternMatch;
117 assert(MI.getOpcode() == TargetOpcode::G_ZEXT);
118
119 Builder.setInstrAndDebugLoc(MI);
120 Register DstReg = MI.getOperand(0).getReg();
121 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
122
123 // zext(trunc x) - > and (aext/copy/trunc x), mask
124 // zext(sext x) -> and (sext x), mask
125 Register TruncSrc;
126 Register SextSrc;
127 if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc))) ||
128 mi_match(SrcReg, MRI, m_GSExt(m_Reg(SextSrc)))) {
129 LLT DstTy = MRI.getType(DstReg);
130 if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) ||
131 isConstantUnsupported(DstTy))
132 return false;
133 LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
134 LLT SrcTy = MRI.getType(SrcReg);
135 APInt MaskVal = APInt::getAllOnes(SrcTy.getScalarSizeInBits());
136 if (SextSrc && (DstTy != MRI.getType(SextSrc)))
137 SextSrc = Builder.buildSExtOrTrunc(DstTy, SextSrc).getReg(0);
138 if (TruncSrc && (DstTy != MRI.getType(TruncSrc)))
139 TruncSrc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc).getReg(0);
140 APInt ExtMaskVal = MaskVal.zext(DstTy.getScalarSizeInBits());
141 Register AndSrc = SextSrc ? SextSrc : TruncSrc;
142 // Elide G_AND and mask constant if possible.
143 // The G_AND would also be removed by the post-legalize redundant_and
144 // combine, but in this very common case, eliding early and regardless of
145 // OptLevel results in significant compile-time and O0 code-size
146 // improvements. Inserting unnecessary instructions between boolean defs
147 // and uses hinders a lot of folding during ISel.
148 if (KB && (KB->getKnownZeroes(AndSrc) | ExtMaskVal).isAllOnes()) {
149 replaceRegOrBuildCopy(DstReg, AndSrc, MRI, Builder, UpdatedDefs,
150 Observer);
151 } else {
152 auto Mask = Builder.buildConstant(DstTy, ExtMaskVal);
153 Builder.buildAnd(DstReg, AndSrc, Mask);
154 }
155 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
156 return true;
157 }
158
159 // zext(zext x) -> (zext x)
160 Register ZextSrc;
161 if (mi_match(SrcReg, MRI, m_GZExt(m_Reg(ZextSrc)))) {
162 LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI);
163 Observer.changingInstr(MI);
164 MI.getOperand(1).setReg(ZextSrc);
165 Observer.changedInstr(MI);
166 UpdatedDefs.push_back(DstReg);
167 markDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
168 return true;
169 }
170
171 // Try to fold zext(g_constant) when the larger constant type is legal.
172 auto *SrcMI = MRI.getVRegDef(SrcReg);
173 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {
174 const LLT DstTy = MRI.getType(DstReg);
175 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
176 auto &CstVal = SrcMI->getOperand(1);
177 Builder.buildConstant(
178 DstReg, CstVal.getCImm()->getValue().zext(DstTy.getSizeInBits()));
179 UpdatedDefs.push_back(DstReg);
180 markInstAndDefDead(MI, *SrcMI, DeadInsts);
181 return true;
182 }
183 }
184 return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs);
185 }
186
189 SmallVectorImpl<Register> &UpdatedDefs) {
190 using namespace llvm::MIPatternMatch;
191 assert(MI.getOpcode() == TargetOpcode::G_SEXT);
192
193 Builder.setInstrAndDebugLoc(MI);
194 Register DstReg = MI.getOperand(0).getReg();
195 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
196
197 // sext(trunc x) - > (sext_inreg (aext/copy/trunc x), c)
198 Register TruncSrc;
199 if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
200 LLT DstTy = MRI.getType(DstReg);
201 if (isInstUnsupported({TargetOpcode::G_SEXT_INREG, {DstTy}}))
202 return false;
203 LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;);
204 LLT SrcTy = MRI.getType(SrcReg);
205 uint64_t SizeInBits = SrcTy.getScalarSizeInBits();
206 if (DstTy != MRI.getType(TruncSrc))
207 TruncSrc = Builder.buildAnyExtOrTrunc(DstTy, TruncSrc).getReg(0);
208 Builder.buildSExtInReg(DstReg, TruncSrc, SizeInBits);
209 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
210 return true;
211 }
212
213 // sext(zext x) -> (zext x)
214 // sext(sext x) -> (sext x)
215 Register ExtSrc;
216 MachineInstr *ExtMI;
217 if (mi_match(SrcReg, MRI,
218 m_all_of(m_MInstr(ExtMI), m_any_of(m_GZExt(m_Reg(ExtSrc)),
219 m_GSExt(m_Reg(ExtSrc)))))) {
220 LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI);
221 Builder.buildInstr(ExtMI->getOpcode(), {DstReg}, {ExtSrc});
222 UpdatedDefs.push_back(DstReg);
223 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
224 return true;
225 }
226
227 // Try to fold sext(g_constant) when the larger constant type is legal.
228 auto *SrcMI = MRI.getVRegDef(SrcReg);
229 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {
230 const LLT DstTy = MRI.getType(DstReg);
231 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
232 auto &CstVal = SrcMI->getOperand(1);
233 Builder.buildConstant(
234 DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits()));
235 UpdatedDefs.push_back(DstReg);
236 markInstAndDefDead(MI, *SrcMI, DeadInsts);
237 return true;
238 }
239 }
240
241 return tryFoldImplicitDef(MI, DeadInsts, UpdatedDefs);
242 }
243
246 SmallVectorImpl<Register> &UpdatedDefs,
247 GISelObserverWrapper &Observer) {
248 using namespace llvm::MIPatternMatch;
249 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
250
251 Builder.setInstr(MI);
252 Register DstReg = MI.getOperand(0).getReg();
253 const LLT DstTy = MRI.getType(DstReg);
254 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
255
256 // Try to fold trunc(g_constant) when the smaller constant type is legal.
257 auto *SrcMI = MRI.getVRegDef(SrcReg);
258 if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) {
259 if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) {
260 auto &CstVal = SrcMI->getOperand(1);
261 Builder.buildConstant(
262 DstReg, CstVal.getCImm()->getValue().trunc(DstTy.getSizeInBits()));
263 UpdatedDefs.push_back(DstReg);
264 markInstAndDefDead(MI, *SrcMI, DeadInsts);
265 return true;
266 }
267 }
268
269 // Try to fold trunc(merge) to directly use the source of the merge.
270 // This gets rid of large, difficult to legalize, merges
271 if (auto *SrcMerge = dyn_cast<GMerge>(SrcMI)) {
272 const Register MergeSrcReg = SrcMerge->getSourceReg(0);
273 const LLT MergeSrcTy = MRI.getType(MergeSrcReg);
274
275 // We can only fold if the types are scalar
276 const unsigned DstSize = DstTy.getSizeInBits();
277 const unsigned MergeSrcSize = MergeSrcTy.getSizeInBits();
278 if (!DstTy.isScalar() || !MergeSrcTy.isScalar())
279 return false;
280
281 if (DstSize < MergeSrcSize) {
282 // When the merge source is larger than the destination, we can just
283 // truncate the merge source directly
284 if (isInstUnsupported({TargetOpcode::G_TRUNC, {DstTy, MergeSrcTy}}))
285 return false;
286
287 LLVM_DEBUG(dbgs() << "Combining G_TRUNC(G_MERGE_VALUES) to G_TRUNC: "
288 << MI);
289
290 Builder.buildTrunc(DstReg, MergeSrcReg);
291 UpdatedDefs.push_back(DstReg);
292 } else if (DstSize == MergeSrcSize) {
293 // If the sizes match we can simply try to replace the register
295 dbgs() << "Replacing G_TRUNC(G_MERGE_VALUES) with merge input: "
296 << MI);
297 replaceRegOrBuildCopy(DstReg, MergeSrcReg, MRI, Builder, UpdatedDefs,
298 Observer);
299 } else if (DstSize % MergeSrcSize == 0) {
300 // If the trunc size is a multiple of the merge source size we can use
301 // a smaller merge instead
302 if (isInstUnsupported(
303 {TargetOpcode::G_MERGE_VALUES, {DstTy, MergeSrcTy}}))
304 return false;
305
307 dbgs() << "Combining G_TRUNC(G_MERGE_VALUES) to G_MERGE_VALUES: "
308 << MI);
309
310 const unsigned NumSrcs = DstSize / MergeSrcSize;
311 assert(NumSrcs < SrcMI->getNumOperands() - 1 &&
312 "trunc(merge) should require less inputs than merge");
313 SmallVector<Register, 8> SrcRegs(NumSrcs);
314 for (unsigned i = 0; i < NumSrcs; ++i)
315 SrcRegs[i] = SrcMerge->getSourceReg(i);
316
317 Builder.buildMergeValues(DstReg, SrcRegs);
318 UpdatedDefs.push_back(DstReg);
319 } else {
320 // Unable to combine
321 return false;
322 }
323
324 markInstAndDefDead(MI, *SrcMerge, DeadInsts);
325 return true;
326 }
327
328 // trunc(trunc) -> trunc
329 Register TruncSrc;
330 if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) {
331 // Always combine trunc(trunc) since the eventual resulting trunc must be
332 // legal anyway as it must be legal for all outputs of the consumer type
333 // set.
334 LLVM_DEBUG(dbgs() << ".. Combine G_TRUNC(G_TRUNC): " << MI);
335
336 Builder.buildTrunc(DstReg, TruncSrc);
337 UpdatedDefs.push_back(DstReg);
338 markInstAndDefDead(MI, *MRI.getVRegDef(TruncSrc), DeadInsts);
339 return true;
340 }
341
342 // trunc(ext x) -> x
343 ArtifactValueFinder Finder(MRI, Builder, LI);
344 if (Register FoundReg =
345 Finder.findValueFromDef(DstReg, 0, DstTy.getSizeInBits())) {
346 LLT FoundRegTy = MRI.getType(FoundReg);
347 if (DstTy == FoundRegTy) {
348 LLVM_DEBUG(dbgs() << ".. Combine G_TRUNC(G_[S,Z,ANY]EXT/G_TRUNC...): "
349 << MI;);
350
351 replaceRegOrBuildCopy(DstReg, FoundReg, MRI, Builder, UpdatedDefs,
352 Observer);
353 UpdatedDefs.push_back(DstReg);
354 markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts);
355 return true;
356 }
357 }
358
359 return false;
360 }
361
362 /// Try to fold G_[ASZ]EXT (G_IMPLICIT_DEF).
365 SmallVectorImpl<Register> &UpdatedDefs) {
366 unsigned Opcode = MI.getOpcode();
367 assert(Opcode == TargetOpcode::G_ANYEXT || Opcode == TargetOpcode::G_ZEXT ||
368 Opcode == TargetOpcode::G_SEXT);
369
370 if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
371 MI.getOperand(1).getReg(), MRI)) {
372 Builder.setInstr(MI);
373 Register DstReg = MI.getOperand(0).getReg();
374 LLT DstTy = MRI.getType(DstReg);
375
376 if (Opcode == TargetOpcode::G_ANYEXT) {
377 // G_ANYEXT (G_IMPLICIT_DEF) -> G_IMPLICIT_DEF
378 if (!isInstLegal({TargetOpcode::G_IMPLICIT_DEF, {DstTy}}))
379 return false;
380 LLVM_DEBUG(dbgs() << ".. Combine G_ANYEXT(G_IMPLICIT_DEF): " << MI;);
381 Builder.buildInstr(TargetOpcode::G_IMPLICIT_DEF, {DstReg}, {});
382 UpdatedDefs.push_back(DstReg);
383 } else {
384 // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top
385 // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT.
386 if (isConstantUnsupported(DstTy))
387 return false;
388 LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;);
389 Builder.buildConstant(DstReg, 0);
390 UpdatedDefs.push_back(DstReg);
391 }
392
393 markInstAndDefDead(MI, *DefMI, DeadInsts);
394 return true;
395 }
396 return false;
397 }
398
401 SmallVectorImpl<Register> &UpdatedDefs) {
402
403 assert(MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
404
405 const unsigned CastOpc = CastMI.getOpcode();
406
407 if (!isArtifactCast(CastOpc))
408 return false;
409
410 const unsigned NumDefs = MI.getNumOperands() - 1;
411
412 const Register CastSrcReg = CastMI.getOperand(1).getReg();
413 const LLT CastSrcTy = MRI.getType(CastSrcReg);
414 const LLT DestTy = MRI.getType(MI.getOperand(0).getReg());
415 const LLT SrcTy = MRI.getType(MI.getOperand(NumDefs).getReg());
416
417 const unsigned CastSrcSize = CastSrcTy.getSizeInBits();
418 const unsigned DestSize = DestTy.getSizeInBits();
419
420 if (CastOpc == TargetOpcode::G_TRUNC) {
421 if (SrcTy.isVector() && SrcTy.getScalarType() == DestTy.getScalarType()) {
422 // %1:_(<4 x s8>) = G_TRUNC %0(<4 x s32>)
423 // %2:_(s8), %3:_(s8), %4:_(s8), %5:_(s8) = G_UNMERGE_VALUES %1
424 // =>
425 // %6:_(s32), %7:_(s32), %8:_(s32), %9:_(s32) = G_UNMERGE_VALUES %0
426 // %2:_(s8) = G_TRUNC %6
427 // %3:_(s8) = G_TRUNC %7
428 // %4:_(s8) = G_TRUNC %8
429 // %5:_(s8) = G_TRUNC %9
430
431 unsigned UnmergeNumElts =
432 DestTy.isVector() ? CastSrcTy.getNumElements() / NumDefs : 1;
433 LLT UnmergeTy = CastSrcTy.changeElementCount(
434 ElementCount::getFixed(UnmergeNumElts));
435 LLT SrcWideTy =
436 SrcTy.changeElementCount(ElementCount::getFixed(UnmergeNumElts));
437
438 if (isInstUnsupported(
439 {TargetOpcode::G_UNMERGE_VALUES, {UnmergeTy, CastSrcTy}}) ||
440 LI.getAction({TargetOpcode::G_TRUNC, {SrcWideTy, UnmergeTy}})
442 return false;
443
444 Builder.setInstr(MI);
445 auto NewUnmerge = Builder.buildUnmerge(UnmergeTy, CastSrcReg);
446
447 for (unsigned I = 0; I != NumDefs; ++I) {
448 Register DefReg = MI.getOperand(I).getReg();
449 UpdatedDefs.push_back(DefReg);
450 Builder.buildTrunc(DefReg, NewUnmerge.getReg(I));
451 }
452
453 markInstAndDefDead(MI, CastMI, DeadInsts);
454 return true;
455 }
456
457 if (CastSrcTy.isScalar() && SrcTy.isScalar() && !DestTy.isVector()) {
458 // %1:_(s16) = G_TRUNC %0(s32)
459 // %2:_(s8), %3:_(s8) = G_UNMERGE_VALUES %1
460 // =>
461 // %2:_(s8), %3:_(s8), %4:_(s8), %5:_(s8) = G_UNMERGE_VALUES %0
462
463 // Unmerge(trunc) can be combined if the trunc source size is a multiple
464 // of the unmerge destination size
465 if (CastSrcSize % DestSize != 0)
466 return false;
467
468 // Check if the new unmerge is supported
469 if (isInstUnsupported(
470 {TargetOpcode::G_UNMERGE_VALUES, {DestTy, CastSrcTy}}))
471 return false;
472
473 // Gather the original destination registers and create new ones for the
474 // unused bits
475 const unsigned NewNumDefs = CastSrcSize / DestSize;
476 SmallVector<Register, 8> DstRegs(NewNumDefs);
477 for (unsigned Idx = 0; Idx < NewNumDefs; ++Idx) {
478 if (Idx < NumDefs)
479 DstRegs[Idx] = MI.getOperand(Idx).getReg();
480 else
481 DstRegs[Idx] = MRI.createGenericVirtualRegister(DestTy);
482 }
483
484 // Build new unmerge
485 Builder.setInstr(MI);
486 Builder.buildUnmerge(DstRegs, CastSrcReg);
487 UpdatedDefs.append(DstRegs.begin(), DstRegs.begin() + NewNumDefs);
488 markInstAndDefDead(MI, CastMI, DeadInsts);
489 return true;
490 }
491 }
492
493 // TODO: support combines with other casts as well
494 return false;
495 }
496
497 static bool canFoldMergeOpcode(unsigned MergeOp, unsigned ConvertOp,
498 LLT OpTy, LLT DestTy) {
499 // Check if we found a definition that is like G_MERGE_VALUES.
500 switch (MergeOp) {
501 default:
502 return false;
503 case TargetOpcode::G_BUILD_VECTOR:
504 case TargetOpcode::G_MERGE_VALUES:
505 // The convert operation that we will need to insert is
506 // going to convert the input of that type of instruction (scalar)
507 // to the destination type (DestTy).
508 // The conversion needs to stay in the same domain (scalar to scalar
509 // and vector to vector), so if we were to allow to fold the merge
510 // we would need to insert some bitcasts.
511 // E.g.,
512 // <2 x s16> = build_vector s16, s16
513 // <2 x s32> = zext <2 x s16>
514 // <2 x s16>, <2 x s16> = unmerge <2 x s32>
515 //
516 // As is the folding would produce:
517 // <2 x s16> = zext s16 <-- scalar to vector
518 // <2 x s16> = zext s16 <-- scalar to vector
519 // Which is invalid.
520 // Instead we would want to generate:
521 // s32 = zext s16
522 // <2 x s16> = bitcast s32
523 // s32 = zext s16
524 // <2 x s16> = bitcast s32
525 //
526 // That is not done yet.
527 if (ConvertOp == 0)
528 return true;
529 return !DestTy.isVector() && OpTy.isVector() &&
530 DestTy == OpTy.getElementType();
531 case TargetOpcode::G_CONCAT_VECTORS: {
532 if (ConvertOp == 0)
533 return true;
534 if (!DestTy.isVector())
535 return false;
536
537 const unsigned OpEltSize = OpTy.getElementType().getSizeInBits();
538
539 // Don't handle scalarization with a cast that isn't in the same
540 // direction as the vector cast. This could be handled, but it would
541 // require more intermediate unmerges.
542 if (ConvertOp == TargetOpcode::G_TRUNC)
543 return DestTy.getSizeInBits() <= OpEltSize;
544 return DestTy.getSizeInBits() >= OpEltSize;
545 }
546 }
547 }
548
549 /// Try to replace DstReg with SrcReg or build a COPY instruction
550 /// depending on the register constraints.
551 static void replaceRegOrBuildCopy(Register DstReg, Register SrcReg,
553 MachineIRBuilder &Builder,
554 SmallVectorImpl<Register> &UpdatedDefs,
555 GISelChangeObserver &Observer) {
556 if (!llvm::canReplaceReg(DstReg, SrcReg, MRI)) {
557 Builder.buildCopy(DstReg, SrcReg);
558 UpdatedDefs.push_back(DstReg);
559 return;
560 }
562 // Get the users and notify the observer before replacing.
563 for (auto &UseMI : MRI.use_instructions(DstReg)) {
564 UseMIs.push_back(&UseMI);
565 Observer.changingInstr(UseMI);
566 }
567 // Replace the registers.
568 MRI.replaceRegWith(DstReg, SrcReg);
569 UpdatedDefs.push_back(SrcReg);
570 // Notify the observer that we changed the instructions.
571 for (auto *UseMI : UseMIs)
572 Observer.changedInstr(*UseMI);
573 }
574
575 /// Return the operand index in \p MI that defines \p Def
576 static unsigned getDefIndex(const MachineInstr &MI, Register SearchDef) {
577 unsigned DefIdx = 0;
578 for (const MachineOperand &Def : MI.defs()) {
579 if (Def.getReg() == SearchDef)
580 break;
581 ++DefIdx;
582 }
583
584 return DefIdx;
585 }
586
587 /// This class provides utilities for finding source registers of specific
588 /// bit ranges in an artifact. The routines can look through the source
589 /// registers if they're other artifacts to try to find a non-artifact source
590 /// of a value.
593 MachineIRBuilder &MIB;
594 const LegalizerInfo &LI;
595
596 // Stores the best register found in the current query so far.
597 Register CurrentBest = Register();
598
599 /// Given an concat_vector op \p Concat and a start bit and size, try to
600 /// find the origin of the value defined by that start position and size.
601 ///
602 /// \returns a register with the requested size, or the current best
603 /// register found during the current query.
604 Register findValueFromConcat(GConcatVectors &Concat, unsigned StartBit,
605 unsigned Size) {
606 assert(Size > 0);
607
608 // Find the source operand that provides the bits requested.
609 Register Src1Reg = Concat.getSourceReg(0);
610 unsigned SrcSize = MRI.getType(Src1Reg).getSizeInBits();
611
612 // Operand index of the source that provides the start of the bit range.
613 unsigned StartSrcIdx = (StartBit / SrcSize) + 1;
614 // Offset into the source at which the bit range starts.
615 unsigned InRegOffset = StartBit % SrcSize;
616 // Check that the bits don't span multiple sources.
617 // FIXME: we might be able return multiple sources? Or create an
618 // appropriate concat to make it fit.
619 if (InRegOffset + Size > SrcSize)
620 return CurrentBest;
621
622 Register SrcReg = Concat.getReg(StartSrcIdx);
623 if (InRegOffset == 0 && Size == SrcSize) {
624 CurrentBest = SrcReg;
625 return findValueFromDefImpl(SrcReg, 0, Size);
626 }
627
628 return findValueFromDefImpl(SrcReg, InRegOffset, Size);
629 }
630
631 /// Given an build_vector op \p BV and a start bit and size, try to find
632 /// the origin of the value defined by that start position and size.
633 ///
634 /// \returns a register with the requested size, or the current best
635 /// register found during the current query.
636 Register findValueFromBuildVector(GBuildVector &BV, unsigned StartBit,
637 unsigned Size) {
638 assert(Size > 0);
639
640 // Find the source operand that provides the bits requested.
641 Register Src1Reg = BV.getSourceReg(0);
642 unsigned SrcSize = MRI.getType(Src1Reg).getSizeInBits();
643
644 // Operand index of the source that provides the start of the bit range.
645 unsigned StartSrcIdx = (StartBit / SrcSize) + 1;
646 // Offset into the source at which the bit range starts.
647 unsigned InRegOffset = StartBit % SrcSize;
648
649 if (InRegOffset != 0)
650 return CurrentBest; // Give up, bits don't start at a scalar source.
651 if (Size < SrcSize)
652 return CurrentBest; // Scalar source is too large for requested bits.
653
654 // If the bits cover multiple sources evenly, then create a new
655 // build_vector to synthesize the required size, if that's been requested.
656 if (Size > SrcSize) {
657 if (Size % SrcSize > 0)
658 return CurrentBest; // Isn't covered exactly by sources.
659
660 unsigned NumSrcsUsed = Size / SrcSize;
661 // If we're requesting all of the sources, just return this def.
662 if (NumSrcsUsed == BV.getNumSources())
663 return BV.getReg(0);
664
665 LLT SrcTy = MRI.getType(Src1Reg);
666 LLT NewBVTy = LLT::fixed_vector(NumSrcsUsed, SrcTy);
667
668 // Check if the resulting build vector would be legal.
669 LegalizeActionStep ActionStep =
670 LI.getAction({TargetOpcode::G_BUILD_VECTOR, {NewBVTy, SrcTy}});
671 if (ActionStep.Action != LegalizeActions::Legal)
672 return CurrentBest;
673
674 SmallVector<Register> NewSrcs;
675 for (unsigned SrcIdx = StartSrcIdx; SrcIdx < StartSrcIdx + NumSrcsUsed;
676 ++SrcIdx)
677 NewSrcs.push_back(BV.getReg(SrcIdx));
678 MIB.setInstrAndDebugLoc(BV);
679 return MIB.buildBuildVector(NewBVTy, NewSrcs).getReg(0);
680 }
681 // A single source is requested, just return it.
682 return BV.getReg(StartSrcIdx);
683 }
684
685 /// Given an G_INSERT op \p MI and a start bit and size, try to find
686 /// the origin of the value defined by that start position and size.
687 ///
688 /// \returns a register with the requested size, or the current best
689 /// register found during the current query.
690 Register findValueFromInsert(MachineInstr &MI, unsigned StartBit,
691 unsigned Size) {
692 assert(MI.getOpcode() == TargetOpcode::G_INSERT);
693 assert(Size > 0);
694
695 Register ContainerSrcReg = MI.getOperand(1).getReg();
696 Register InsertedReg = MI.getOperand(2).getReg();
697 LLT InsertedRegTy = MRI.getType(InsertedReg);
698 unsigned InsertOffset = MI.getOperand(3).getImm();
699
700 // There are 4 possible container/insertreg + requested bit-range layouts
701 // that the instruction and query could be representing.
702 // For: %_ = G_INSERT %CONTAINER, %INS, InsOff (abbrev. to 'IO')
703 // and a start bit 'SB', with size S, giving an end bit 'EB', we could
704 // have...
705 // Scenario A:
706 // --------------------------
707 // | INS | CONTAINER |
708 // --------------------------
709 // | |
710 // SB EB
711 //
712 // Scenario B:
713 // --------------------------
714 // | INS | CONTAINER |
715 // --------------------------
716 // | |
717 // SB EB
718 //
719 // Scenario C:
720 // --------------------------
721 // | CONTAINER | INS |
722 // --------------------------
723 // | |
724 // SB EB
725 //
726 // Scenario D:
727 // --------------------------
728 // | CONTAINER | INS |
729 // --------------------------
730 // | |
731 // SB EB
732 //
733 // So therefore, A and D are requesting data from the INS operand, while
734 // B and C are requesting from the container operand.
735
736 unsigned InsertedEndBit = InsertOffset + InsertedRegTy.getSizeInBits();
737 unsigned EndBit = StartBit + Size;
738 unsigned NewStartBit;
739 Register SrcRegToUse;
740 if (EndBit <= InsertOffset || InsertedEndBit <= StartBit) {
741 SrcRegToUse = ContainerSrcReg;
742 NewStartBit = StartBit;
743 return findValueFromDefImpl(SrcRegToUse, NewStartBit, Size);
744 }
745 if (InsertOffset <= StartBit && EndBit <= InsertedEndBit) {
746 SrcRegToUse = InsertedReg;
747 NewStartBit = StartBit - InsertOffset;
748 if (NewStartBit == 0 &&
749 Size == MRI.getType(SrcRegToUse).getSizeInBits())
750 CurrentBest = SrcRegToUse;
751 return findValueFromDefImpl(SrcRegToUse, NewStartBit, Size);
752 }
753 // The bit range spans both the inserted and container regions.
754 return Register();
755 }
756
757 /// Given an G_SEXT, G_ZEXT, G_ANYEXT op \p MI and a start bit and
758 /// size, try to find the origin of the value defined by that start
759 /// position and size.
760 ///
761 /// \returns a register with the requested size, or the current best
762 /// register found during the current query.
763 Register findValueFromExt(MachineInstr &MI, unsigned StartBit,
764 unsigned Size) {
765 assert(MI.getOpcode() == TargetOpcode::G_SEXT ||
766 MI.getOpcode() == TargetOpcode::G_ZEXT ||
767 MI.getOpcode() == TargetOpcode::G_ANYEXT);
768 assert(Size > 0);
769
770 Register SrcReg = MI.getOperand(1).getReg();
771 LLT SrcType = MRI.getType(SrcReg);
772 unsigned SrcSize = SrcType.getSizeInBits();
773
774 // Currently we don't go into vectors.
775 if (!SrcType.isScalar())
776 return CurrentBest;
777
778 if (StartBit + Size > SrcSize)
779 return CurrentBest;
780
781 if (StartBit == 0 && SrcType.getSizeInBits() == Size)
782 CurrentBest = SrcReg;
783 return findValueFromDefImpl(SrcReg, StartBit, Size);
784 }
785
786 /// Given an G_TRUNC op \p MI and a start bit and size, try to find
787 /// the origin of the value defined by that start position and size.
788 ///
789 /// \returns a register with the requested size, or the current best
790 /// register found during the current query.
791 Register findValueFromTrunc(MachineInstr &MI, unsigned StartBit,
792 unsigned Size) {
793 assert(MI.getOpcode() == TargetOpcode::G_TRUNC);
794 assert(Size > 0);
795
796 Register SrcReg = MI.getOperand(1).getReg();
797 LLT SrcType = MRI.getType(SrcReg);
798
799 // Currently we don't go into vectors.
800 if (!SrcType.isScalar())
801 return CurrentBest;
802
803 return findValueFromDefImpl(SrcReg, StartBit, Size);
804 }
805
806 /// Internal implementation for findValueFromDef(). findValueFromDef()
807 /// initializes some data like the CurrentBest register, which this method
808 /// and its callees rely upon.
809 Register findValueFromDefImpl(Register DefReg, unsigned StartBit,
810 unsigned Size) {
811 std::optional<DefinitionAndSourceRegister> DefSrcReg =
813 MachineInstr *Def = DefSrcReg->MI;
814 DefReg = DefSrcReg->Reg;
815 // If the instruction has a single def, then simply delegate the search.
816 // For unmerge however with multiple defs, we need to compute the offset
817 // into the source of the unmerge.
818 switch (Def->getOpcode()) {
819 case TargetOpcode::G_CONCAT_VECTORS:
820 return findValueFromConcat(cast<GConcatVectors>(*Def), StartBit, Size);
821 case TargetOpcode::G_UNMERGE_VALUES: {
822 unsigned DefStartBit = 0;
823 unsigned DefSize = MRI.getType(DefReg).getSizeInBits();
824 for (const auto &MO : Def->defs()) {
825 if (MO.getReg() == DefReg)
826 break;
827 DefStartBit += DefSize;
828 }
829 Register SrcReg = Def->getOperand(Def->getNumOperands() - 1).getReg();
830 Register SrcOriginReg =
831 findValueFromDefImpl(SrcReg, StartBit + DefStartBit, Size);
832 if (SrcOriginReg)
833 return SrcOriginReg;
834 // Failed to find a further value. If the StartBit and Size perfectly
835 // covered the requested DefReg, return that since it's better than
836 // nothing.
837 if (StartBit == 0 && Size == DefSize)
838 return DefReg;
839 return CurrentBest;
840 }
841 case TargetOpcode::G_BUILD_VECTOR:
842 return findValueFromBuildVector(cast<GBuildVector>(*Def), StartBit,
843 Size);
844 case TargetOpcode::G_INSERT:
845 return findValueFromInsert(*Def, StartBit, Size);
846 case TargetOpcode::G_TRUNC:
847 return findValueFromTrunc(*Def, StartBit, Size);
848 case TargetOpcode::G_SEXT:
849 case TargetOpcode::G_ZEXT:
850 case TargetOpcode::G_ANYEXT:
851 return findValueFromExt(*Def, StartBit, Size);
852 default:
853 return CurrentBest;
854 }
855 }
856
857 public:
859 const LegalizerInfo &Info)
860 : MRI(Mri), MIB(Builder), LI(Info) {}
861
862 /// Try to find a source of the value defined in the def \p DefReg, starting
863 /// at position \p StartBit with size \p Size.
864 /// \returns a register with the requested size, or an empty Register if no
865 /// better value could be found.
866 Register findValueFromDef(Register DefReg, unsigned StartBit,
867 unsigned Size) {
868 CurrentBest = Register();
869 Register FoundReg = findValueFromDefImpl(DefReg, StartBit, Size);
870 return FoundReg != DefReg ? FoundReg : Register();
871 }
872
873 /// Try to combine the defs of an unmerge \p MI by attempting to find
874 /// values that provides the bits for each def reg.
875 /// \returns true if all the defs of the unmerge have been made dead.
877 SmallVectorImpl<Register> &UpdatedDefs) {
878 unsigned NumDefs = MI.getNumDefs();
879 LLT DestTy = MRI.getType(MI.getReg(0));
880
881 SmallBitVector DeadDefs(NumDefs);
882 for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) {
883 Register DefReg = MI.getReg(DefIdx);
884 if (MRI.use_nodbg_empty(DefReg)) {
885 DeadDefs[DefIdx] = true;
886 continue;
887 }
888 Register FoundVal = findValueFromDef(DefReg, 0, DestTy.getSizeInBits());
889 if (!FoundVal)
890 continue;
891 if (MRI.getType(FoundVal) != DestTy)
892 continue;
893
894 replaceRegOrBuildCopy(DefReg, FoundVal, MRI, MIB, UpdatedDefs,
895 Observer);
896 // We only want to replace the uses, not the def of the old reg.
897 Observer.changingInstr(MI);
898 MI.getOperand(DefIdx).setReg(DefReg);
899 Observer.changedInstr(MI);
900 DeadDefs[DefIdx] = true;
901 }
902 return DeadDefs.all();
903 }
904
906 unsigned &DefOperandIdx) {
907 if (Register Def = findValueFromDefImpl(Reg, 0, Size)) {
908 if (auto *Unmerge = dyn_cast<GUnmerge>(MRI.getVRegDef(Def))) {
909 DefOperandIdx = Unmerge->findRegisterDefOperandIdx(Def);
910 return Unmerge;
911 }
912 }
913 return nullptr;
914 }
915
916 // Check if sequence of elements from merge-like instruction is defined by
917 // another sequence of elements defined by unmerge. Most often this is the
918 // same sequence. Search for elements using findValueFromDefImpl.
919 bool isSequenceFromUnmerge(GMergeLikeInstr &MI, unsigned MergeStartIdx,
920 GUnmerge *Unmerge, unsigned UnmergeIdxStart,
921 unsigned NumElts, unsigned EltSize,
922 bool AllowUndef) {
923 assert(MergeStartIdx + NumElts <= MI.getNumSources());
924 for (unsigned i = MergeStartIdx; i < MergeStartIdx + NumElts; ++i) {
925 unsigned EltUnmergeIdx;
926 GUnmerge *EltUnmerge = findUnmergeThatDefinesReg(
927 MI.getSourceReg(i), EltSize, EltUnmergeIdx);
928 // Check if source i comes from the same Unmerge.
929 if (EltUnmerge == Unmerge) {
930 // Check that source i's def has same index in sequence in Unmerge.
931 if (i - MergeStartIdx != EltUnmergeIdx - UnmergeIdxStart)
932 return false;
933 } else if (!AllowUndef ||
934 MRI.getVRegDef(MI.getSourceReg(i))->getOpcode() !=
935 TargetOpcode::G_IMPLICIT_DEF)
936 return false;
937 }
938 return true;
939 }
940
943 SmallVectorImpl<Register> &UpdatedDefs,
944 GISelChangeObserver &Observer) {
945 Register Elt0 = MI.getSourceReg(0);
946 LLT EltTy = MRI.getType(Elt0);
947 unsigned EltSize = EltTy.getSizeInBits();
948
949 unsigned Elt0UnmergeIdx;
950 // Search for unmerge that will be candidate for combine.
951 auto *Unmerge = findUnmergeThatDefinesReg(Elt0, EltSize, Elt0UnmergeIdx);
952 if (!Unmerge)
953 return false;
954
955 unsigned NumMIElts = MI.getNumSources();
956 Register Dst = MI.getReg(0);
957 LLT DstTy = MRI.getType(Dst);
958 Register UnmergeSrc = Unmerge->getSourceReg();
959 LLT UnmergeSrcTy = MRI.getType(UnmergeSrc);
960
961 // Recognize copy of UnmergeSrc to Dst.
962 // Unmerge UnmergeSrc and reassemble it using merge-like opcode into Dst.
963 //
964 // %0:_(EltTy), %1, ... = G_UNMERGE_VALUES %UnmergeSrc:_(Ty)
965 // %Dst:_(Ty) = G_merge_like_opcode %0:_(EltTy), %1, ...
966 //
967 // %Dst:_(Ty) = COPY %UnmergeSrc:_(Ty)
968 if ((DstTy == UnmergeSrcTy) && (Elt0UnmergeIdx == 0)) {
969 if (!isSequenceFromUnmerge(MI, 0, Unmerge, 0, NumMIElts, EltSize,
970 /*AllowUndef=*/DstTy.isVector()))
971 return false;
972
973 replaceRegOrBuildCopy(Dst, UnmergeSrc, MRI, MIB, UpdatedDefs, Observer);
974 DeadInsts.push_back(&MI);
975 return true;
976 }
977
978 // Recognize UnmergeSrc that can be unmerged to DstTy directly.
979 // Types have to be either both vector or both non-vector types.
980 // Merge-like opcodes are combined one at the time. First one creates new
981 // unmerge, following should use the same unmerge (builder performs CSE).
982 //
983 // %0:_(EltTy), %1, %2, %3 = G_UNMERGE_VALUES %UnmergeSrc:_(UnmergeSrcTy)
984 // %Dst:_(DstTy) = G_merge_like_opcode %0:_(EltTy), %1
985 // %AnotherDst:_(DstTy) = G_merge_like_opcode %2:_(EltTy), %3
986 //
987 // %Dst:_(DstTy), %AnotherDst = G_UNMERGE_VALUES %UnmergeSrc
988 if ((DstTy.isVector() == UnmergeSrcTy.isVector()) &&
989 (Elt0UnmergeIdx % NumMIElts == 0) &&
990 getCoverTy(UnmergeSrcTy, DstTy) == UnmergeSrcTy) {
991 if (!isSequenceFromUnmerge(MI, 0, Unmerge, Elt0UnmergeIdx, NumMIElts,
992 EltSize, false))
993 return false;
995 auto NewUnmerge = MIB.buildUnmerge(DstTy, Unmerge->getSourceReg());
996 unsigned DstIdx = (Elt0UnmergeIdx * EltSize) / DstTy.getSizeInBits();
997 replaceRegOrBuildCopy(Dst, NewUnmerge.getReg(DstIdx), MRI, MIB,
998 UpdatedDefs, Observer);
999 DeadInsts.push_back(&MI);
1000 return true;
1001 }
1002
1003 // Recognize when multiple unmerged sources with UnmergeSrcTy type
1004 // can be merged into Dst with DstTy type directly.
1005 // Types have to be either both vector or both non-vector types.
1006
1007 // %0:_(EltTy), %1 = G_UNMERGE_VALUES %UnmergeSrc:_(UnmergeSrcTy)
1008 // %2:_(EltTy), %3 = G_UNMERGE_VALUES %AnotherUnmergeSrc:_(UnmergeSrcTy)
1009 // %Dst:_(DstTy) = G_merge_like_opcode %0:_(EltTy), %1, %2, %3
1010 //
1011 // %Dst:_(DstTy) = G_merge_like_opcode %UnmergeSrc, %AnotherUnmergeSrc
1012
1013 if ((DstTy.isVector() == UnmergeSrcTy.isVector()) &&
1014 getCoverTy(DstTy, UnmergeSrcTy) == DstTy) {
1015 SmallVector<Register, 4> ConcatSources;
1016 unsigned NumElts = Unmerge->getNumDefs();
1017 for (unsigned i = 0; i < MI.getNumSources(); i += NumElts) {
1018 unsigned EltUnmergeIdx;
1019 auto *UnmergeI = findUnmergeThatDefinesReg(MI.getSourceReg(i),
1020 EltSize, EltUnmergeIdx);
1021 // All unmerges have to be the same size.
1022 if ((!UnmergeI) || (UnmergeI->getNumDefs() != NumElts) ||
1023 (EltUnmergeIdx != 0))
1024 return false;
1025 if (!isSequenceFromUnmerge(MI, i, UnmergeI, 0, NumElts, EltSize,
1026 false))
1027 return false;
1028 ConcatSources.push_back(UnmergeI->getSourceReg());
1029 }
1030
1032 MIB.buildMergeLikeInstr(Dst, ConcatSources);
1033 DeadInsts.push_back(&MI);
1034 return true;
1035 }
1036
1037 return false;
1038 }
1039 };
1040
1043 SmallVectorImpl<Register> &UpdatedDefs,
1044 GISelChangeObserver &Observer) {
1045 unsigned NumDefs = MI.getNumDefs();
1046 Register SrcReg = MI.getSourceReg();
1047 MachineInstr *SrcDef = getDefIgnoringCopies(SrcReg, MRI);
1048 if (!SrcDef)
1049 return false;
1050
1051 LLT OpTy = MRI.getType(SrcReg);
1052 LLT DestTy = MRI.getType(MI.getReg(0));
1053 unsigned SrcDefIdx = getDefIndex(*SrcDef, SrcReg);
1054
1055 Builder.setInstrAndDebugLoc(MI);
1056
1057 ArtifactValueFinder Finder(MRI, Builder, LI);
1058 if (Finder.tryCombineUnmergeDefs(MI, Observer, UpdatedDefs)) {
1059 markInstAndDefDead(MI, *SrcDef, DeadInsts, SrcDefIdx);
1060 return true;
1061 }
1062
1063 if (auto *SrcUnmerge = dyn_cast<GUnmerge>(SrcDef)) {
1064 // %0:_(<4 x s16>) = G_FOO
1065 // %1:_(<2 x s16>), %2:_(<2 x s16>) = G_UNMERGE_VALUES %0
1066 // %3:_(s16), %4:_(s16) = G_UNMERGE_VALUES %1
1067 //
1068 // %3:_(s16), %4:_(s16), %5:_(s16), %6:_(s16) = G_UNMERGE_VALUES %0
1069 Register SrcUnmergeSrc = SrcUnmerge->getSourceReg();
1070 LLT SrcUnmergeSrcTy = MRI.getType(SrcUnmergeSrc);
1071
1072 // If we need to decrease the number of vector elements in the result type
1073 // of an unmerge, this would involve the creation of an equivalent unmerge
1074 // to copy back to the original result registers.
1075 LegalizeActionStep ActionStep = LI.getAction(
1076 {TargetOpcode::G_UNMERGE_VALUES, {OpTy, SrcUnmergeSrcTy}});
1077 switch (ActionStep.Action) {
1080 break;
1083 if (ActionStep.TypeIdx == 1)
1084 return false;
1085 break;
1086 default:
1087 return false;
1088 }
1089
1090 auto NewUnmerge = Builder.buildUnmerge(DestTy, SrcUnmergeSrc);
1091
1092 // TODO: Should we try to process out the other defs now? If the other
1093 // defs of the source unmerge are also unmerged, we end up with a separate
1094 // unmerge for each one.
1095 for (unsigned I = 0; I != NumDefs; ++I) {
1096 Register Def = MI.getReg(I);
1097 replaceRegOrBuildCopy(Def, NewUnmerge.getReg(SrcDefIdx * NumDefs + I),
1098 MRI, Builder, UpdatedDefs, Observer);
1099 }
1100
1101 markInstAndDefDead(MI, *SrcUnmerge, DeadInsts, SrcDefIdx);
1102 return true;
1103 }
1104
1105 MachineInstr *MergeI = SrcDef;
1106 unsigned ConvertOp = 0;
1107
1108 // Handle intermediate conversions
1109 unsigned SrcOp = SrcDef->getOpcode();
1110 if (isArtifactCast(SrcOp)) {
1111 ConvertOp = SrcOp;
1112 MergeI = getDefIgnoringCopies(SrcDef->getOperand(1).getReg(), MRI);
1113 }
1114
1115 if (!MergeI || !canFoldMergeOpcode(MergeI->getOpcode(),
1116 ConvertOp, OpTy, DestTy)) {
1117 // We might have a chance to combine later by trying to combine
1118 // unmerge(cast) first
1119 return tryFoldUnmergeCast(MI, *SrcDef, DeadInsts, UpdatedDefs);
1120 }
1121
1122 const unsigned NumMergeRegs = MergeI->getNumOperands() - 1;
1123
1124 if (NumMergeRegs < NumDefs) {
1125 if (NumDefs % NumMergeRegs != 0)
1126 return false;
1127
1128 Builder.setInstr(MI);
1129 // Transform to UNMERGEs, for example
1130 // %1 = G_MERGE_VALUES %4, %5
1131 // %9, %10, %11, %12 = G_UNMERGE_VALUES %1
1132 // to
1133 // %9, %10 = G_UNMERGE_VALUES %4
1134 // %11, %12 = G_UNMERGE_VALUES %5
1135
1136 const unsigned NewNumDefs = NumDefs / NumMergeRegs;
1137 for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) {
1139 for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs;
1140 ++j, ++DefIdx)
1141 DstRegs.push_back(MI.getReg(DefIdx));
1142
1143 if (ConvertOp) {
1144 LLT MergeDstTy = MRI.getType(SrcDef->getOperand(0).getReg());
1145
1146 // This is a vector that is being split and casted. Extract to the
1147 // element type, and do the conversion on the scalars (or smaller
1148 // vectors).
1149 LLT MergeEltTy = MergeDstTy.divide(NumMergeRegs);
1150
1151 // Handle split to smaller vectors, with conversions.
1152 // %2(<8 x s8>) = G_CONCAT_VECTORS %0(<4 x s8>), %1(<4 x s8>)
1153 // %3(<8 x s16>) = G_SEXT %2
1154 // %4(<2 x s16>), %5(<2 x s16>), %6(<2 x s16>), %7(<2 x s16>) =
1155 // G_UNMERGE_VALUES %3
1156 //
1157 // =>
1158 //
1159 // %8(<4 x s16>) = G_SEXT %0
1160 // %9(<4 x s16>) = G_SEXT %1
1161 // %4(<2 x s16>), %5(<2 x s16>) = G_UNMERGE_VALUES %8
1162 // %7(<2 x s16>), %7(<2 x s16>) = G_UNMERGE_VALUES %9
1163
1164 Register TmpReg = MRI.createGenericVirtualRegister(MergeEltTy);
1165 Builder.buildInstr(ConvertOp, {TmpReg},
1166 {MergeI->getOperand(Idx + 1).getReg()});
1167 Builder.buildUnmerge(DstRegs, TmpReg);
1168 } else {
1169 Builder.buildUnmerge(DstRegs, MergeI->getOperand(Idx + 1).getReg());
1170 }
1171 UpdatedDefs.append(DstRegs.begin(), DstRegs.end());
1172 }
1173
1174 } else if (NumMergeRegs > NumDefs) {
1175 if (ConvertOp != 0 || NumMergeRegs % NumDefs != 0)
1176 return false;
1177
1178 Builder.setInstr(MI);
1179 // Transform to MERGEs
1180 // %6 = G_MERGE_VALUES %17, %18, %19, %20
1181 // %7, %8 = G_UNMERGE_VALUES %6
1182 // to
1183 // %7 = G_MERGE_VALUES %17, %18
1184 // %8 = G_MERGE_VALUES %19, %20
1185
1186 const unsigned NumRegs = NumMergeRegs / NumDefs;
1187 for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) {
1189 for (unsigned j = 0, Idx = NumRegs * DefIdx + 1; j < NumRegs;
1190 ++j, ++Idx)
1191 Regs.push_back(MergeI->getOperand(Idx).getReg());
1192
1193 Register DefReg = MI.getReg(DefIdx);
1194 Builder.buildMergeLikeInstr(DefReg, Regs);
1195 UpdatedDefs.push_back(DefReg);
1196 }
1197
1198 } else {
1199 LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg());
1200
1201 if (!ConvertOp && DestTy != MergeSrcTy)
1202 ConvertOp = TargetOpcode::G_BITCAST;
1203
1204 if (ConvertOp) {
1205 Builder.setInstr(MI);
1206
1207 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1208 Register DefReg = MI.getOperand(Idx).getReg();
1209 Register MergeSrc = MergeI->getOperand(Idx + 1).getReg();
1210
1211 if (!MRI.use_empty(DefReg)) {
1212 Builder.buildInstr(ConvertOp, {DefReg}, {MergeSrc});
1213 UpdatedDefs.push_back(DefReg);
1214 }
1215 }
1216
1217 markInstAndDefDead(MI, *MergeI, DeadInsts);
1218 return true;
1219 }
1220
1221 assert(DestTy == MergeSrcTy &&
1222 "Bitcast and the other kinds of conversions should "
1223 "have happened earlier");
1224
1225 Builder.setInstr(MI);
1226 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1227 Register DstReg = MI.getOperand(Idx).getReg();
1228 Register SrcReg = MergeI->getOperand(Idx + 1).getReg();
1229 replaceRegOrBuildCopy(DstReg, SrcReg, MRI, Builder, UpdatedDefs,
1230 Observer);
1231 }
1232 }
1233
1234 markInstAndDefDead(MI, *MergeI, DeadInsts);
1235 return true;
1236 }
1237
1240 SmallVectorImpl<Register> &UpdatedDefs) {
1241 assert(MI.getOpcode() == TargetOpcode::G_EXTRACT);
1242
1243 // Try to use the source registers from a G_MERGE_VALUES
1244 //
1245 // %2 = G_MERGE_VALUES %0, %1
1246 // %3 = G_EXTRACT %2, N
1247 // =>
1248 //
1249 // for N < %2.getSizeInBits() / 2
1250 // %3 = G_EXTRACT %0, N
1251 //
1252 // for N >= %2.getSizeInBits() / 2
1253 // %3 = G_EXTRACT %1, (N - %0.getSizeInBits()
1254
1255 Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg());
1256 MachineInstr *MergeI = MRI.getVRegDef(SrcReg);
1257 if (!MergeI || !isa<GMergeLikeInstr>(MergeI))
1258 return false;
1259
1260 Register DstReg = MI.getOperand(0).getReg();
1261 LLT DstTy = MRI.getType(DstReg);
1262 LLT SrcTy = MRI.getType(SrcReg);
1263
1264 // TODO: Do we need to check if the resulting extract is supported?
1265 unsigned ExtractDstSize = DstTy.getSizeInBits();
1266 unsigned Offset = MI.getOperand(2).getImm();
1267 unsigned NumMergeSrcs = MergeI->getNumOperands() - 1;
1268 unsigned MergeSrcSize = SrcTy.getSizeInBits() / NumMergeSrcs;
1269 unsigned MergeSrcIdx = Offset / MergeSrcSize;
1270
1271 // Compute the offset of the last bit the extract needs.
1272 unsigned EndMergeSrcIdx = (Offset + ExtractDstSize - 1) / MergeSrcSize;
1273
1274 // Can't handle the case where the extract spans multiple inputs.
1275 if (MergeSrcIdx != EndMergeSrcIdx)
1276 return false;
1277
1278 // TODO: We could modify MI in place in most cases.
1279 Builder.setInstr(MI);
1280 Builder.buildExtract(DstReg, MergeI->getOperand(MergeSrcIdx + 1).getReg(),
1281 Offset - MergeSrcIdx * MergeSrcSize);
1282 UpdatedDefs.push_back(DstReg);
1283 markInstAndDefDead(MI, *MergeI, DeadInsts);
1284 return true;
1285 }
1286
1287 /// Try to combine away MI.
1288 /// Returns true if it combined away the MI.
1289 /// Adds instructions that are dead as a result of the combine
1290 /// into DeadInsts, which can include MI.
1293 GISelObserverWrapper &WrapperObserver) {
1294 ArtifactValueFinder Finder(MRI, Builder, LI);
1295
1296 // This might be a recursive call, and we might have DeadInsts already
1297 // populated. To avoid bad things happening later with multiple vreg defs
1298 // etc, process the dead instructions now if any.
1299 if (!DeadInsts.empty())
1300 deleteMarkedDeadInsts(DeadInsts, WrapperObserver);
1301
1302 // Put here every vreg that was redefined in such a way that it's at least
1303 // possible that one (or more) of its users (immediate or COPY-separated)
1304 // could become artifact combinable with the new definition (or the
1305 // instruction reachable from it through a chain of copies if any).
1306 SmallVector<Register, 4> UpdatedDefs;
1307 bool Changed = false;
1308 switch (MI.getOpcode()) {
1309 default:
1310 return false;
1311 case TargetOpcode::G_ANYEXT:
1312 Changed = tryCombineAnyExt(MI, DeadInsts, UpdatedDefs, WrapperObserver);
1313 break;
1314 case TargetOpcode::G_ZEXT:
1315 Changed = tryCombineZExt(MI, DeadInsts, UpdatedDefs, WrapperObserver);
1316 break;
1317 case TargetOpcode::G_SEXT:
1318 Changed = tryCombineSExt(MI, DeadInsts, UpdatedDefs);
1319 break;
1320 case TargetOpcode::G_UNMERGE_VALUES:
1321 Changed = tryCombineUnmergeValues(cast<GUnmerge>(MI), DeadInsts,
1322 UpdatedDefs, WrapperObserver);
1323 break;
1324 case TargetOpcode::G_MERGE_VALUES:
1325 case TargetOpcode::G_BUILD_VECTOR:
1326 case TargetOpcode::G_CONCAT_VECTORS:
1327 // If any of the users of this merge are an unmerge, then add them to the
1328 // artifact worklist in case there's folding that can be done looking up.
1329 for (MachineInstr &U : MRI.use_instructions(MI.getOperand(0).getReg())) {
1330 if (U.getOpcode() == TargetOpcode::G_UNMERGE_VALUES ||
1331 U.getOpcode() == TargetOpcode::G_TRUNC) {
1332 UpdatedDefs.push_back(MI.getOperand(0).getReg());
1333 break;
1334 }
1335 }
1336 Changed = Finder.tryCombineMergeLike(cast<GMergeLikeInstr>(MI), DeadInsts,
1337 UpdatedDefs, WrapperObserver);
1338 break;
1339 case TargetOpcode::G_EXTRACT:
1340 Changed = tryCombineExtract(MI, DeadInsts, UpdatedDefs);
1341 break;
1342 case TargetOpcode::G_TRUNC:
1343 Changed = tryCombineTrunc(MI, DeadInsts, UpdatedDefs, WrapperObserver);
1344 if (!Changed) {
1345 // Try to combine truncates away even if they are legal. As all artifact
1346 // combines at the moment look only "up" the def-use chains, we achieve
1347 // that by throwing truncates' users (with look through copies) into the
1348 // ArtifactList again.
1349 UpdatedDefs.push_back(MI.getOperand(0).getReg());
1350 }
1351 break;
1352 }
1353 // If the main loop through the ArtifactList found at least one combinable
1354 // pair of artifacts, not only combine it away (as done above), but also
1355 // follow the def-use chain from there to combine everything that can be
1356 // combined within this def-use chain of artifacts.
1357 while (!UpdatedDefs.empty()) {
1358 Register NewDef = UpdatedDefs.pop_back_val();
1359 assert(NewDef.isVirtual() && "Unexpected redefinition of a physreg");
1360 for (MachineInstr &Use : MRI.use_instructions(NewDef)) {
1361 switch (Use.getOpcode()) {
1362 // Keep this list in sync with the list of all artifact combines.
1363 case TargetOpcode::G_ANYEXT:
1364 case TargetOpcode::G_ZEXT:
1365 case TargetOpcode::G_SEXT:
1366 case TargetOpcode::G_UNMERGE_VALUES:
1367 case TargetOpcode::G_EXTRACT:
1368 case TargetOpcode::G_TRUNC:
1369 case TargetOpcode::G_BUILD_VECTOR:
1370 // Adding Use to ArtifactList.
1371 WrapperObserver.changedInstr(Use);
1372 break;
1373 case TargetOpcode::G_ASSERT_SEXT:
1374 case TargetOpcode::G_ASSERT_ZEXT:
1375 case TargetOpcode::G_ASSERT_ALIGN:
1376 case TargetOpcode::COPY: {
1377 Register Copy = Use.getOperand(0).getReg();
1378 if (Copy.isVirtual())
1379 UpdatedDefs.push_back(Copy);
1380 break;
1381 }
1382 default:
1383 // If we do not have an artifact combine for the opcode, there is no
1384 // point in adding it to the ArtifactList as nothing interesting will
1385 // be done to it anyway.
1386 break;
1387 }
1388 }
1389 }
1390 return Changed;
1391 }
1392
1393private:
1394 static Register getArtifactSrcReg(const MachineInstr &MI) {
1395 switch (MI.getOpcode()) {
1396 case TargetOpcode::COPY:
1397 case TargetOpcode::G_TRUNC:
1398 case TargetOpcode::G_ZEXT:
1399 case TargetOpcode::G_ANYEXT:
1400 case TargetOpcode::G_SEXT:
1401 case TargetOpcode::G_EXTRACT:
1402 case TargetOpcode::G_ASSERT_SEXT:
1403 case TargetOpcode::G_ASSERT_ZEXT:
1404 case TargetOpcode::G_ASSERT_ALIGN:
1405 return MI.getOperand(1).getReg();
1406 case TargetOpcode::G_UNMERGE_VALUES:
1407 return MI.getOperand(MI.getNumOperands() - 1).getReg();
1408 default:
1409 llvm_unreachable("Not a legalization artifact happen");
1410 }
1411 }
1412
1413 /// Mark a def of one of MI's original operands, DefMI, as dead if changing MI
1414 /// (either by killing it or changing operands) results in DefMI being dead
1415 /// too. In-between COPYs or artifact-casts are also collected if they are
1416 /// dead.
1417 /// MI is not marked dead.
1418 void markDefDead(MachineInstr &MI, MachineInstr &DefMI,
1420 unsigned DefIdx = 0) {
1421 // Collect all the copy instructions that are made dead, due to deleting
1422 // this instruction. Collect all of them until the Trunc(DefMI).
1423 // Eg,
1424 // %1(s1) = G_TRUNC %0(s32)
1425 // %2(s1) = COPY %1(s1)
1426 // %3(s1) = COPY %2(s1)
1427 // %4(s32) = G_ANYEXT %3(s1)
1428 // In this case, we would have replaced %4 with a copy of %0,
1429 // and as a result, %3, %2, %1 are dead.
1430 MachineInstr *PrevMI = &MI;
1431 while (PrevMI != &DefMI) {
1432 Register PrevRegSrc = getArtifactSrcReg(*PrevMI);
1433
1434 MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc);
1435 if (MRI.hasOneUse(PrevRegSrc)) {
1436 if (TmpDef != &DefMI) {
1437 assert((TmpDef->getOpcode() == TargetOpcode::COPY ||
1438 isArtifactCast(TmpDef->getOpcode()) ||
1440 "Expecting copy or artifact cast here");
1441
1442 DeadInsts.push_back(TmpDef);
1443 }
1444 } else
1445 break;
1446 PrevMI = TmpDef;
1447 }
1448
1449 if (PrevMI == &DefMI) {
1450 unsigned I = 0;
1451 bool IsDead = true;
1452 for (MachineOperand &Def : DefMI.defs()) {
1453 if (I != DefIdx) {
1454 if (!MRI.use_empty(Def.getReg())) {
1455 IsDead = false;
1456 break;
1457 }
1458 } else {
1459 if (!MRI.hasOneUse(DefMI.getOperand(DefIdx).getReg()))
1460 break;
1461 }
1462
1463 ++I;
1464 }
1465
1466 if (IsDead)
1467 DeadInsts.push_back(&DefMI);
1468 }
1469 }
1470
1471 /// Mark MI as dead. If a def of one of MI's operands, DefMI, would also be
1472 /// dead due to MI being killed, then mark DefMI as dead too.
1473 /// Some of the combines (extends(trunc)), try to walk through redundant
1474 /// copies in between the extends and the truncs, and this attempts to collect
1475 /// the in between copies if they're dead.
1476 void markInstAndDefDead(MachineInstr &MI, MachineInstr &DefMI,
1478 unsigned DefIdx = 0) {
1479 DeadInsts.push_back(&MI);
1480 markDefDead(MI, DefMI, DeadInsts, DefIdx);
1481 }
1482
1483 /// Erase the dead instructions in the list and call the observer hooks.
1484 /// Normally the Legalizer will deal with erasing instructions that have been
1485 /// marked dead. However, for the trunc(ext(x)) cases we can end up trying to
1486 /// process instructions which have been marked dead, but otherwise break the
1487 /// MIR by introducing multiple vreg defs. For those cases, allow the combines
1488 /// to explicitly delete the instructions before we run into trouble.
1489 void deleteMarkedDeadInsts(SmallVectorImpl<MachineInstr *> &DeadInsts,
1490 GISelObserverWrapper &WrapperObserver) {
1491 for (auto *DeadMI : DeadInsts) {
1492 LLVM_DEBUG(dbgs() << *DeadMI << "Is dead, eagerly deleting\n");
1493 WrapperObserver.erasingInstr(*DeadMI);
1494 DeadMI->eraseFromParent();
1495 }
1496 DeadInsts.clear();
1497 }
1498
1499 /// Checks if the target legalizer info has specified anything about the
1500 /// instruction, or if unsupported.
1501 bool isInstUnsupported(const LegalityQuery &Query) const {
1502 using namespace LegalizeActions;
1503 auto Step = LI.getAction(Query);
1504 return Step.Action == Unsupported || Step.Action == NotFound;
1505 }
1506
1507 bool isInstLegal(const LegalityQuery &Query) const {
1508 return LI.getAction(Query).Action == LegalizeActions::Legal;
1509 }
1510
1511 bool isConstantUnsupported(LLT Ty) const {
1512 if (!Ty.isVector())
1513 return isInstUnsupported({TargetOpcode::G_CONSTANT, {Ty}});
1514
1515 LLT EltTy = Ty.getElementType();
1516 return isInstUnsupported({TargetOpcode::G_CONSTANT, {EltTy}}) ||
1517 isInstUnsupported({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}});
1518 }
1519
1520 /// Looks through copy instructions and returns the actual
1521 /// source register.
1522 Register lookThroughCopyInstrs(Register Reg) {
1524 return TmpReg.isValid() ? TmpReg : Reg;
1525 }
1526};
1527
1528} // namespace llvm
1529
1530#endif // LLVM_CODEGEN_GLOBALISEL_LEGALIZATIONARTIFACTCOMBINER_H
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
This contains common code to allow clients to notify changes to machine instr.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
IRTranslator LLVM IR MI
Interface for Targets to specify which operations they can successfully select and how the others sho...
#define I(x, y, z)
Definition: MD5.cpp:58
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
unsigned Reg
Promote Memory to Register
Definition: Mem2Reg.cpp:110
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool IsDead
This file implements the SmallBitVector class.
static constexpr int Concat[]
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:296
Represents a G_BUILD_VECTOR.
Represents a G_CONCAT_VECTORS.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
APInt getKnownZeroes(Register R)
Simple wrapper observer that takes several observers, and calls each one for each event.
void changedInstr(MachineInstr &MI) override
This instruction was mutated in some way.
void changingInstr(MachineInstr &MI) override
This instruction is about to be mutated in some way.
void erasingInstr(MachineInstr &MI) override
An instruction is about to be erased.
Represents G_BUILD_VECTOR, G_CONCAT_VECTORS or G_MERGE_VALUES.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a G_UNMERGE_VALUES.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
constexpr unsigned getScalarSizeInBits() const
Definition: LowLevelType.h:267
constexpr bool isScalar() const
Definition: LowLevelType.h:146
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelType.h:159
constexpr bool isVector() const
Definition: LowLevelType.h:148
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelType.h:290
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
Definition: LowLevelType.h:230
constexpr LLT getScalarType() const
Definition: LowLevelType.h:208
constexpr LLT divide(int Factor) const
Return a type that is Factor times smaller.
Definition: LowLevelType.h:237
This class provides utilities for finding source registers of specific bit ranges in an artifact.
bool tryCombineUnmergeDefs(GUnmerge &MI, GISelChangeObserver &Observer, SmallVectorImpl< Register > &UpdatedDefs)
Try to combine the defs of an unmerge MI by attempting to find values that provides the bits for each...
bool isSequenceFromUnmerge(GMergeLikeInstr &MI, unsigned MergeStartIdx, GUnmerge *Unmerge, unsigned UnmergeIdxStart, unsigned NumElts, unsigned EltSize, bool AllowUndef)
Register findValueFromDef(Register DefReg, unsigned StartBit, unsigned Size)
Try to find a source of the value defined in the def DefReg, starting at position StartBit with size ...
GUnmerge * findUnmergeThatDefinesReg(Register Reg, unsigned Size, unsigned &DefOperandIdx)
bool tryCombineMergeLike(GMergeLikeInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs, GISelChangeObserver &Observer)
ArtifactValueFinder(MachineRegisterInfo &Mri, MachineIRBuilder &Builder, const LegalizerInfo &Info)
LegalizationArtifactCombiner(MachineIRBuilder &B, MachineRegisterInfo &MRI, const LegalizerInfo &LI, GISelKnownBits *KB=nullptr)
bool tryFoldUnmergeCast(MachineInstr &MI, MachineInstr &CastMI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs)
bool tryCombineZExt(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs, GISelObserverWrapper &Observer)
bool tryCombineInstruction(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, GISelObserverWrapper &WrapperObserver)
Try to combine away MI.
bool tryCombineTrunc(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs, GISelObserverWrapper &Observer)
static bool canFoldMergeOpcode(unsigned MergeOp, unsigned ConvertOp, LLT OpTy, LLT DestTy)
bool tryCombineSExt(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs)
static unsigned getDefIndex(const MachineInstr &MI, Register SearchDef)
Return the operand index in MI that defines Def.
static void replaceRegOrBuildCopy(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI, MachineIRBuilder &Builder, SmallVectorImpl< Register > &UpdatedDefs, GISelChangeObserver &Observer)
Try to replace DstReg with SrcReg or build a COPY instruction depending on the register constraints.
bool tryCombineUnmergeValues(GUnmerge &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs, GISelChangeObserver &Observer)
bool tryCombineExtract(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs)
bool tryFoldImplicitDef(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs)
Try to fold G_[ASZ]EXT (G_IMPLICIT_DEF).
bool tryCombineAnyExt(MachineInstr &MI, SmallVectorImpl< MachineInstr * > &DeadInsts, SmallVectorImpl< Register > &UpdatedDefs, GISelObserverWrapper &Observer)
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
Helper class to build MachineInstr.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Res = COPY Op depending on the differing sizes of Res and Op.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildMergeValues(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ...
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
Representation of each machine instruction.
Definition: MachineInstr.h:69
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:546
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:549
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:556
MachineOperand class - Representation of each machine instruction operand.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isValid() const
Definition: Register.h:116
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition: Register.h:91
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
bool empty() const
Definition: SmallVector.h:94
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Unsupported
This operation is completely unsupported on the target.
@ NotFound
Sentinel value for when no action was found in the specified table.
@ FewerElements
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
Definition: LegalizerInfo.h:65
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
Definition: LegalizerInfo.h:47
@ Unsupported
This operation is completely unsupported on the target.
Definition: LegalizerInfo.h:91
@ Lower
The operation itself must be expressed in terms of simpler actions on this target.
Definition: LegalizerInfo.h:78
@ NarrowScalar
The operation should be synthesized from multiple instructions acting on a narrower scalar base-type.
Definition: LegalizerInfo.h:52
@ MoreElements
The (vector) operation should be implemented by widening the input vector and ignoring the lanes adde...
Definition: LegalizerInfo.h:71
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition: Utils.cpp:625
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition: Utils.cpp:465
bool isPreISelGenericOptimizationHint(unsigned Opcode)
Definition: TargetOpcodes.h:42
bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
Definition: Utils.cpp:199
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
LLVM_READNONE LLT getCoverTy(LLT OrigTy, LLT TargetTy)
Return smallest type that covers both OrigTy and TargetTy and is multiple of TargetTy.
Definition: Utils.cpp:1143
std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Definition: Utils.cpp:446
Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
Definition: Utils.cpp:472
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
The result of a query.
LegalizeAction Action
The action to take or the final answer.
unsigned TypeIdx
If describing an action, the type index to change. Otherwise zero.