LLVM  9.0.0svn
LegalizerHelper.cpp
Go to the documentation of this file.
1 //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file This file implements the LegalizerHelper class to legalize
10 /// individual instructions and the LegalizeMachineIR wrapper pass for the
11 /// primary legalization.
12 //
13 //===----------------------------------------------------------------------===//
14 
23 #include "llvm/Support/Debug.h"
26 
27 #define DEBUG_TYPE "legalizer"
28 
29 using namespace llvm;
30 using namespace LegalizeActions;
31 
32 /// Try to break down \p OrigTy into \p NarrowTy sized pieces.
33 ///
34 /// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy,
35 /// with any leftover piece as type \p LeftoverTy
36 ///
37 /// Returns -1 in the first element of the pair if the breakdown is not
38 /// satisfiable.
39 static std::pair<int, int>
40 getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) {
41  assert(!LeftoverTy.isValid() && "this is an out argument");
42 
43  unsigned Size = OrigTy.getSizeInBits();
44  unsigned NarrowSize = NarrowTy.getSizeInBits();
45  unsigned NumParts = Size / NarrowSize;
46  unsigned LeftoverSize = Size - NumParts * NarrowSize;
47  assert(Size > NarrowSize);
48 
49  if (LeftoverSize == 0)
50  return {NumParts, 0};
51 
52  if (NarrowTy.isVector()) {
53  unsigned EltSize = OrigTy.getScalarSizeInBits();
54  if (LeftoverSize % EltSize != 0)
55  return {-1, -1};
56  LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
57  } else {
58  LeftoverTy = LLT::scalar(LeftoverSize);
59  }
60 
61  int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits();
62  return std::make_pair(NumParts, NumLeftover);
63 }
64 
66  GISelChangeObserver &Observer,
67  MachineIRBuilder &Builder)
68  : MIRBuilder(Builder), MRI(MF.getRegInfo()),
69  LI(*MF.getSubtarget().getLegalizerInfo()), Observer(Observer) {
70  MIRBuilder.setMF(MF);
71  MIRBuilder.setChangeObserver(Observer);
72 }
73 
75  GISelChangeObserver &Observer,
77  : MIRBuilder(B), MRI(MF.getRegInfo()), LI(LI), Observer(Observer) {
78  MIRBuilder.setMF(MF);
79  MIRBuilder.setChangeObserver(Observer);
80 }
83  LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs()));
84 
85  auto Step = LI.getAction(MI, MRI);
86  switch (Step.Action) {
87  case Legal:
88  LLVM_DEBUG(dbgs() << ".. Already legal\n");
89  return AlreadyLegal;
90  case Libcall:
91  LLVM_DEBUG(dbgs() << ".. Convert to libcall\n");
92  return libcall(MI);
93  case NarrowScalar:
94  LLVM_DEBUG(dbgs() << ".. Narrow scalar\n");
95  return narrowScalar(MI, Step.TypeIdx, Step.NewType);
96  case WidenScalar:
97  LLVM_DEBUG(dbgs() << ".. Widen scalar\n");
98  return widenScalar(MI, Step.TypeIdx, Step.NewType);
99  case Lower:
100  LLVM_DEBUG(dbgs() << ".. Lower\n");
101  return lower(MI, Step.TypeIdx, Step.NewType);
102  case FewerElements:
103  LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n");
104  return fewerElementsVector(MI, Step.TypeIdx, Step.NewType);
105  case MoreElements:
106  LLVM_DEBUG(dbgs() << ".. Increase number of elements\n");
107  return moreElementsVector(MI, Step.TypeIdx, Step.NewType);
108  case Custom:
109  LLVM_DEBUG(dbgs() << ".. Custom legalization\n");
110  return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized
112  default:
113  LLVM_DEBUG(dbgs() << ".. Unable to legalize\n");
114  return UnableToLegalize;
115  }
116 }
117 
118 void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts,
119  SmallVectorImpl<unsigned> &VRegs) {
120  for (int i = 0; i < NumParts; ++i)
122  MIRBuilder.buildUnmerge(VRegs, Reg);
123 }
124 
125 bool LegalizerHelper::extractParts(unsigned Reg, LLT RegTy,
126  LLT MainTy, LLT &LeftoverTy,
128  SmallVectorImpl<unsigned> &LeftoverRegs) {
129  assert(!LeftoverTy.isValid() && "this is an out argument");
130 
131  unsigned RegSize = RegTy.getSizeInBits();
132  unsigned MainSize = MainTy.getSizeInBits();
133  unsigned NumParts = RegSize / MainSize;
134  unsigned LeftoverSize = RegSize - NumParts * MainSize;
135 
136  // Use an unmerge when possible.
137  if (LeftoverSize == 0) {
138  for (unsigned I = 0; I < NumParts; ++I)
139  VRegs.push_back(MRI.createGenericVirtualRegister(MainTy));
140  MIRBuilder.buildUnmerge(VRegs, Reg);
141  return true;
142  }
143 
144  if (MainTy.isVector()) {
145  unsigned EltSize = MainTy.getScalarSizeInBits();
146  if (LeftoverSize % EltSize != 0)
147  return false;
148  LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize);
149  } else {
150  LeftoverTy = LLT::scalar(LeftoverSize);
151  }
152 
153  // For irregular sizes, extract the individual parts.
154  for (unsigned I = 0; I != NumParts; ++I) {
155  unsigned NewReg = MRI.createGenericVirtualRegister(MainTy);
156  VRegs.push_back(NewReg);
157  MIRBuilder.buildExtract(NewReg, Reg, MainSize * I);
158  }
159 
160  for (unsigned Offset = MainSize * NumParts; Offset < RegSize;
161  Offset += LeftoverSize) {
162  unsigned NewReg = MRI.createGenericVirtualRegister(LeftoverTy);
163  LeftoverRegs.push_back(NewReg);
164  MIRBuilder.buildExtract(NewReg, Reg, Offset);
165  }
166 
167  return true;
168 }
169 
170 void LegalizerHelper::insertParts(unsigned DstReg,
171  LLT ResultTy, LLT PartTy,
172  ArrayRef<unsigned> PartRegs,
173  LLT LeftoverTy,
174  ArrayRef<unsigned> LeftoverRegs) {
175  if (!LeftoverTy.isValid()) {
176  assert(LeftoverRegs.empty());
177 
178  if (!ResultTy.isVector()) {
179  MIRBuilder.buildMerge(DstReg, PartRegs);
180  return;
181  }
182 
183  if (PartTy.isVector())
184  MIRBuilder.buildConcatVectors(DstReg, PartRegs);
185  else
186  MIRBuilder.buildBuildVector(DstReg, PartRegs);
187  return;
188  }
189 
190  unsigned PartSize = PartTy.getSizeInBits();
191  unsigned LeftoverPartSize = LeftoverTy.getSizeInBits();
192 
193  unsigned CurResultReg = MRI.createGenericVirtualRegister(ResultTy);
194  MIRBuilder.buildUndef(CurResultReg);
195 
196  unsigned Offset = 0;
197  for (unsigned PartReg : PartRegs) {
198  unsigned NewResultReg = MRI.createGenericVirtualRegister(ResultTy);
199  MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset);
200  CurResultReg = NewResultReg;
201  Offset += PartSize;
202  }
203 
204  for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) {
205  // Use the original output register for the final insert to avoid a copy.
206  unsigned NewResultReg = (I + 1 == E) ?
207  DstReg : MRI.createGenericVirtualRegister(ResultTy);
208 
209  MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset);
210  CurResultReg = NewResultReg;
211  Offset += LeftoverPartSize;
212  }
213 }
214 
215 static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) {
216  switch (Opcode) {
217  case TargetOpcode::G_SDIV:
218  assert((Size == 32 || Size == 64) && "Unsupported size");
219  return Size == 64 ? RTLIB::SDIV_I64 : RTLIB::SDIV_I32;
220  case TargetOpcode::G_UDIV:
221  assert((Size == 32 || Size == 64) && "Unsupported size");
222  return Size == 64 ? RTLIB::UDIV_I64 : RTLIB::UDIV_I32;
223  case TargetOpcode::G_SREM:
224  assert((Size == 32 || Size == 64) && "Unsupported size");
225  return Size == 64 ? RTLIB::SREM_I64 : RTLIB::SREM_I32;
226  case TargetOpcode::G_UREM:
227  assert((Size == 32 || Size == 64) && "Unsupported size");
228  return Size == 64 ? RTLIB::UREM_I64 : RTLIB::UREM_I32;
229  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
230  assert(Size == 32 && "Unsupported size");
231  return RTLIB::CTLZ_I32;
232  case TargetOpcode::G_FADD:
233  assert((Size == 32 || Size == 64) && "Unsupported size");
234  return Size == 64 ? RTLIB::ADD_F64 : RTLIB::ADD_F32;
235  case TargetOpcode::G_FSUB:
236  assert((Size == 32 || Size == 64) && "Unsupported size");
237  return Size == 64 ? RTLIB::SUB_F64 : RTLIB::SUB_F32;
238  case TargetOpcode::G_FMUL:
239  assert((Size == 32 || Size == 64) && "Unsupported size");
240  return Size == 64 ? RTLIB::MUL_F64 : RTLIB::MUL_F32;
241  case TargetOpcode::G_FDIV:
242  assert((Size == 32 || Size == 64) && "Unsupported size");
243  return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32;
244  case TargetOpcode::G_FEXP:
245  assert((Size == 32 || Size == 64) && "Unsupported size");
246  return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32;
247  case TargetOpcode::G_FREM:
248  return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32;
249  case TargetOpcode::G_FPOW:
250  return Size == 64 ? RTLIB::POW_F64 : RTLIB::POW_F32;
251  case TargetOpcode::G_FMA:
252  assert((Size == 32 || Size == 64) && "Unsupported size");
253  return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32;
254  case TargetOpcode::G_FSIN:
255  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
256  return Size == 128 ? RTLIB::SIN_F128
257  : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32;
258  case TargetOpcode::G_FCOS:
259  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
260  return Size == 128 ? RTLIB::COS_F128
261  : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32;
262  case TargetOpcode::G_FLOG10:
263  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
264  return Size == 128 ? RTLIB::LOG10_F128
265  : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32;
266  case TargetOpcode::G_FLOG:
267  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
268  return Size == 128 ? RTLIB::LOG_F128
269  : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32;
270  case TargetOpcode::G_FLOG2:
271  assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size");
272  return Size == 128 ? RTLIB::LOG2_F128
273  : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32;
274  }
275  llvm_unreachable("Unknown libcall function");
276 }
277 
280  const CallLowering::ArgInfo &Result,
282  auto &CLI = *MIRBuilder.getMF().getSubtarget().getCallLowering();
283  auto &TLI = *MIRBuilder.getMF().getSubtarget().getTargetLowering();
284  const char *Name = TLI.getLibcallName(Libcall);
285 
286  MIRBuilder.getMF().getFrameInfo().setHasCalls(true);
287  if (!CLI.lowerCall(MIRBuilder, TLI.getLibcallCallingConv(Libcall),
288  MachineOperand::CreateES(Name), Result, Args))
290 
292 }
293 
294 // Useful for libcalls where all operands have the same type.
297  Type *OpType) {
298  auto Libcall = getRTLibDesc(MI.getOpcode(), Size);
299 
301  for (unsigned i = 1; i < MI.getNumOperands(); i++)
302  Args.push_back({MI.getOperand(i).getReg(), OpType});
303  return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), OpType},
304  Args);
305 }
306 
307 static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType,
308  Type *FromType) {
309  auto ToMVT = MVT::getVT(ToType);
310  auto FromMVT = MVT::getVT(FromType);
311 
312  switch (Opcode) {
313  case TargetOpcode::G_FPEXT:
314  return RTLIB::getFPEXT(FromMVT, ToMVT);
315  case TargetOpcode::G_FPTRUNC:
316  return RTLIB::getFPROUND(FromMVT, ToMVT);
317  case TargetOpcode::G_FPTOSI:
318  return RTLIB::getFPTOSINT(FromMVT, ToMVT);
319  case TargetOpcode::G_FPTOUI:
320  return RTLIB::getFPTOUINT(FromMVT, ToMVT);
321  case TargetOpcode::G_SITOFP:
322  return RTLIB::getSINTTOFP(FromMVT, ToMVT);
323  case TargetOpcode::G_UITOFP:
324  return RTLIB::getUINTTOFP(FromMVT, ToMVT);
325  }
326  llvm_unreachable("Unsupported libcall function");
327 }
328 
331  Type *FromType) {
333  return createLibcall(MIRBuilder, Libcall, {MI.getOperand(0).getReg(), ToType},
334  {{MI.getOperand(1).getReg(), FromType}});
335 }
336 
339  LLT LLTy = MRI.getType(MI.getOperand(0).getReg());
340  unsigned Size = LLTy.getSizeInBits();
341  auto &Ctx = MIRBuilder.getMF().getFunction().getContext();
342 
343  MIRBuilder.setInstr(MI);
344 
345  switch (MI.getOpcode()) {
346  default:
347  return UnableToLegalize;
348  case TargetOpcode::G_SDIV:
349  case TargetOpcode::G_UDIV:
350  case TargetOpcode::G_SREM:
351  case TargetOpcode::G_UREM:
352  case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
353  Type *HLTy = IntegerType::get(Ctx, Size);
354  auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
355  if (Status != Legalized)
356  return Status;
357  break;
358  }
359  case TargetOpcode::G_FADD:
360  case TargetOpcode::G_FSUB:
361  case TargetOpcode::G_FMUL:
362  case TargetOpcode::G_FDIV:
363  case TargetOpcode::G_FMA:
364  case TargetOpcode::G_FPOW:
365  case TargetOpcode::G_FREM:
366  case TargetOpcode::G_FCOS:
367  case TargetOpcode::G_FSIN:
368  case TargetOpcode::G_FLOG10:
369  case TargetOpcode::G_FLOG:
370  case TargetOpcode::G_FLOG2:
371  case TargetOpcode::G_FEXP: {
372  if (Size > 64) {
373  LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n");
374  return UnableToLegalize;
375  }
376  Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx);
377  auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy);
378  if (Status != Legalized)
379  return Status;
380  break;
381  }
382  case TargetOpcode::G_FPEXT: {
383  // FIXME: Support other floating point types (half, fp128 etc)
384  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
385  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
386  if (ToSize != 64 || FromSize != 32)
387  return UnableToLegalize;
390  if (Status != Legalized)
391  return Status;
392  break;
393  }
394  case TargetOpcode::G_FPTRUNC: {
395  // FIXME: Support other floating point types (half, fp128 etc)
396  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
397  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
398  if (ToSize != 32 || FromSize != 64)
399  return UnableToLegalize;
402  if (Status != Legalized)
403  return Status;
404  break;
405  }
406  case TargetOpcode::G_FPTOSI:
407  case TargetOpcode::G_FPTOUI: {
408  // FIXME: Support other types
409  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
410  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
411  if (ToSize != 32 || (FromSize != 32 && FromSize != 64))
412  return UnableToLegalize;
414  MI, MIRBuilder, Type::getInt32Ty(Ctx),
415  FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx));
416  if (Status != Legalized)
417  return Status;
418  break;
419  }
420  case TargetOpcode::G_SITOFP:
421  case TargetOpcode::G_UITOFP: {
422  // FIXME: Support other types
423  unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
424  unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
425  if (FromSize != 32 || (ToSize != 32 && ToSize != 64))
426  return UnableToLegalize;
428  MI, MIRBuilder,
429  ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx),
430  Type::getInt32Ty(Ctx));
431  if (Status != Legalized)
432  return Status;
433  break;
434  }
435  }
436 
437  MI.eraseFromParent();
438  return Legalized;
439 }
440 
442  unsigned TypeIdx,
443  LLT NarrowTy) {
444  MIRBuilder.setInstr(MI);
445 
446  uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
447  uint64_t NarrowSize = NarrowTy.getSizeInBits();
448 
449  switch (MI.getOpcode()) {
450  default:
451  return UnableToLegalize;
452  case TargetOpcode::G_IMPLICIT_DEF: {
453  // FIXME: add support for when SizeOp0 isn't an exact multiple of
454  // NarrowSize.
455  if (SizeOp0 % NarrowSize != 0)
456  return UnableToLegalize;
457  int NumParts = SizeOp0 / NarrowSize;
458 
459  SmallVector<unsigned, 2> DstRegs;
460  for (int i = 0; i < NumParts; ++i)
461  DstRegs.push_back(
462  MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg());
463 
464  unsigned DstReg = MI.getOperand(0).getReg();
465  if(MRI.getType(DstReg).isVector())
466  MIRBuilder.buildBuildVector(DstReg, DstRegs);
467  else
468  MIRBuilder.buildMerge(DstReg, DstRegs);
469  MI.eraseFromParent();
470  return Legalized;
471  }
472  case TargetOpcode::G_ADD: {
473  // FIXME: add support for when SizeOp0 isn't an exact multiple of
474  // NarrowSize.
475  if (SizeOp0 % NarrowSize != 0)
476  return UnableToLegalize;
477  // Expand in terms of carry-setting/consuming G_ADDE instructions.
478  int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
479 
480  SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
481  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
482  extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
483 
484  unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1));
485  MIRBuilder.buildConstant(CarryIn, 0);
486 
487  for (int i = 0; i < NumParts; ++i) {
488  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
489  unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
490 
491  MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i],
492  Src2Regs[i], CarryIn);
493 
494  DstRegs.push_back(DstReg);
495  CarryIn = CarryOut;
496  }
497  unsigned DstReg = MI.getOperand(0).getReg();
498  if(MRI.getType(DstReg).isVector())
499  MIRBuilder.buildBuildVector(DstReg, DstRegs);
500  else
501  MIRBuilder.buildMerge(DstReg, DstRegs);
502  MI.eraseFromParent();
503  return Legalized;
504  }
505  case TargetOpcode::G_SUB: {
506  // FIXME: add support for when SizeOp0 isn't an exact multiple of
507  // NarrowSize.
508  if (SizeOp0 % NarrowSize != 0)
509  return UnableToLegalize;
510 
511  int NumParts = SizeOp0 / NarrowTy.getSizeInBits();
512 
513  SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
514  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs);
515  extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs);
516 
517  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
518  unsigned BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
519  MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut},
520  {Src1Regs[0], Src2Regs[0]});
521  DstRegs.push_back(DstReg);
522  unsigned BorrowIn = BorrowOut;
523  for (int i = 1; i < NumParts; ++i) {
524  DstReg = MRI.createGenericVirtualRegister(NarrowTy);
525  BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1));
526 
527  MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut},
528  {Src1Regs[i], Src2Regs[i], BorrowIn});
529 
530  DstRegs.push_back(DstReg);
531  BorrowIn = BorrowOut;
532  }
533  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs);
534  MI.eraseFromParent();
535  return Legalized;
536  }
537  case TargetOpcode::G_MUL:
538  case TargetOpcode::G_UMULH:
539  return narrowScalarMul(MI, NarrowTy);
540  case TargetOpcode::G_EXTRACT:
541  return narrowScalarExtract(MI, TypeIdx, NarrowTy);
542  case TargetOpcode::G_INSERT:
543  return narrowScalarInsert(MI, TypeIdx, NarrowTy);
544  case TargetOpcode::G_LOAD: {
545  const auto &MMO = **MI.memoperands_begin();
546  unsigned DstReg = MI.getOperand(0).getReg();
547  LLT DstTy = MRI.getType(DstReg);
548  if (DstTy.isVector())
549  return UnableToLegalize;
550 
551  if (8 * MMO.getSize() != DstTy.getSizeInBits()) {
552  unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
553  auto &MMO = **MI.memoperands_begin();
554  MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO);
555  MIRBuilder.buildAnyExt(DstReg, TmpReg);
556  MI.eraseFromParent();
557  return Legalized;
558  }
559 
560  return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
561  }
562  case TargetOpcode::G_ZEXTLOAD:
563  case TargetOpcode::G_SEXTLOAD: {
564  bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD;
565  unsigned DstReg = MI.getOperand(0).getReg();
566  unsigned PtrReg = MI.getOperand(1).getReg();
567 
568  unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
569  auto &MMO = **MI.memoperands_begin();
570  if (MMO.getSize() * 8 == NarrowSize) {
571  MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
572  } else {
573  unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD
574  : TargetOpcode::G_SEXTLOAD;
575  MIRBuilder.buildInstr(ExtLoad)
576  .addDef(TmpReg)
577  .addUse(PtrReg)
578  .addMemOperand(&MMO);
579  }
580 
581  if (ZExt)
582  MIRBuilder.buildZExt(DstReg, TmpReg);
583  else
584  MIRBuilder.buildSExt(DstReg, TmpReg);
585 
586  MI.eraseFromParent();
587  return Legalized;
588  }
589  case TargetOpcode::G_STORE: {
590  const auto &MMO = **MI.memoperands_begin();
591 
592  unsigned SrcReg = MI.getOperand(0).getReg();
593  LLT SrcTy = MRI.getType(SrcReg);
594  if (SrcTy.isVector())
595  return UnableToLegalize;
596 
597  int NumParts = SizeOp0 / NarrowSize;
598  unsigned HandledSize = NumParts * NarrowTy.getSizeInBits();
599  unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize;
600  if (SrcTy.isVector() && LeftoverBits != 0)
601  return UnableToLegalize;
602 
603  if (8 * MMO.getSize() != SrcTy.getSizeInBits()) {
604  unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
605  auto &MMO = **MI.memoperands_begin();
606  MIRBuilder.buildTrunc(TmpReg, SrcReg);
607  MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO);
608  MI.eraseFromParent();
609  return Legalized;
610  }
611 
612  return reduceLoadStoreWidth(MI, 0, NarrowTy);
613  }
614  case TargetOpcode::G_CONSTANT: {
615  // FIXME: add support for when SizeOp0 isn't an exact multiple of
616  // NarrowSize.
617  if (SizeOp0 % NarrowSize != 0)
618  return UnableToLegalize;
619  int NumParts = SizeOp0 / NarrowSize;
620  const APInt &Cst = MI.getOperand(1).getCImm()->getValue();
622 
623  SmallVector<unsigned, 2> DstRegs;
624  for (int i = 0; i < NumParts; ++i) {
625  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
626  ConstantInt *CI =
627  ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize));
628  MIRBuilder.buildConstant(DstReg, *CI);
629  DstRegs.push_back(DstReg);
630  }
631  unsigned DstReg = MI.getOperand(0).getReg();
632  if(MRI.getType(DstReg).isVector())
633  MIRBuilder.buildBuildVector(DstReg, DstRegs);
634  else
635  MIRBuilder.buildMerge(DstReg, DstRegs);
636  MI.eraseFromParent();
637  return Legalized;
638  }
639  case TargetOpcode::G_SELECT:
640  return narrowScalarSelect(MI, TypeIdx, NarrowTy);
641  case TargetOpcode::G_AND:
642  case TargetOpcode::G_OR:
643  case TargetOpcode::G_XOR: {
644  // Legalize bitwise operation:
645  // A = BinOp<Ty> B, C
646  // into:
647  // B1, ..., BN = G_UNMERGE_VALUES B
648  // C1, ..., CN = G_UNMERGE_VALUES C
649  // A1 = BinOp<Ty/N> B1, C2
650  // ...
651  // AN = BinOp<Ty/N> BN, CN
652  // A = G_MERGE_VALUES A1, ..., AN
653 
654  // FIXME: add support for when SizeOp0 isn't an exact multiple of
655  // NarrowSize.
656  if (SizeOp0 % NarrowSize != 0)
657  return UnableToLegalize;
658  int NumParts = SizeOp0 / NarrowSize;
659 
660  // List the registers where the destination will be scattered.
661  SmallVector<unsigned, 2> DstRegs;
662  // List the registers where the first argument will be split.
663  SmallVector<unsigned, 2> SrcsReg1;
664  // List the registers where the second argument will be split.
665  SmallVector<unsigned, 2> SrcsReg2;
666  // Create all the temporary registers.
667  for (int i = 0; i < NumParts; ++i) {
668  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
669  unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy);
670  unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy);
671 
672  DstRegs.push_back(DstReg);
673  SrcsReg1.push_back(SrcReg1);
674  SrcsReg2.push_back(SrcReg2);
675  }
676  // Explode the big arguments into smaller chunks.
677  MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg());
678  MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg());
679 
680  // Do the operation on each small part.
681  for (int i = 0; i < NumParts; ++i)
682  MIRBuilder.buildInstr(MI.getOpcode(), {DstRegs[i]},
683  {SrcsReg1[i], SrcsReg2[i]});
684 
685  // Gather the destination registers into the final destination.
686  unsigned DstReg = MI.getOperand(0).getReg();
687  if(MRI.getType(DstReg).isVector())
688  MIRBuilder.buildBuildVector(DstReg, DstRegs);
689  else
690  MIRBuilder.buildMerge(DstReg, DstRegs);
691  MI.eraseFromParent();
692  return Legalized;
693  }
694  case TargetOpcode::G_SHL:
695  case TargetOpcode::G_LSHR:
696  case TargetOpcode::G_ASHR:
697  return narrowScalarShift(MI, TypeIdx, NarrowTy);
698  case TargetOpcode::G_CTLZ:
699  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
700  case TargetOpcode::G_CTTZ:
701  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
702  case TargetOpcode::G_CTPOP:
703  if (TypeIdx != 0)
704  return UnableToLegalize; // TODO
705 
706  Observer.changingInstr(MI);
707  narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
708  Observer.changedInstr(MI);
709  return Legalized;
710  case TargetOpcode::G_INTTOPTR:
711  if (TypeIdx != 1)
712  return UnableToLegalize;
713 
714  Observer.changingInstr(MI);
715  narrowScalarSrc(MI, NarrowTy, 1);
716  Observer.changedInstr(MI);
717  return Legalized;
718  case TargetOpcode::G_PTRTOINT:
719  if (TypeIdx != 0)
720  return UnableToLegalize;
721 
722  Observer.changingInstr(MI);
723  narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT);
724  Observer.changedInstr(MI);
725  return Legalized;
726  }
727 }
728 
729 void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy,
730  unsigned OpIdx, unsigned ExtOpcode) {
731  MachineOperand &MO = MI.getOperand(OpIdx);
732  auto ExtB = MIRBuilder.buildInstr(ExtOpcode, {WideTy}, {MO.getReg()});
733  MO.setReg(ExtB->getOperand(0).getReg());
734 }
735 
736 void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy,
737  unsigned OpIdx) {
738  MachineOperand &MO = MI.getOperand(OpIdx);
739  auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy},
740  {MO.getReg()});
741  MO.setReg(ExtB->getOperand(0).getReg());
742 }
743 
744 void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy,
745  unsigned OpIdx, unsigned TruncOpcode) {
746  MachineOperand &MO = MI.getOperand(OpIdx);
747  unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
749  MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt});
750  MO.setReg(DstExt);
751 }
752 
753 void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy,
754  unsigned OpIdx, unsigned ExtOpcode) {
755  MachineOperand &MO = MI.getOperand(OpIdx);
756  unsigned DstTrunc = MRI.createGenericVirtualRegister(NarrowTy);
758  MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc});
759  MO.setReg(DstTrunc);
760 }
761 
762 void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy,
763  unsigned OpIdx) {
764  MachineOperand &MO = MI.getOperand(OpIdx);
765  unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
767  MIRBuilder.buildExtract(MO.getReg(), DstExt, 0);
768  MO.setReg(DstExt);
769 }
770 
771 void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy,
772  unsigned OpIdx) {
773  MachineOperand &MO = MI.getOperand(OpIdx);
774 
775  LLT OldTy = MRI.getType(MO.getReg());
776  unsigned OldElts = OldTy.getNumElements();
777  unsigned NewElts = MoreTy.getNumElements();
778 
779  unsigned NumParts = NewElts / OldElts;
780 
781  // Use concat_vectors if the result is a multiple of the number of elements.
782  if (NumParts * OldElts == NewElts) {
784  Parts.push_back(MO.getReg());
785 
786  unsigned ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0);
787  for (unsigned I = 1; I != NumParts; ++I)
788  Parts.push_back(ImpDef);
789 
790  auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts);
791  MO.setReg(Concat.getReg(0));
792  return;
793  }
794 
795  unsigned MoreReg = MRI.createGenericVirtualRegister(MoreTy);
796  unsigned ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0);
797  MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0);
798  MO.setReg(MoreReg);
799 }
800 
802 LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx,
803  LLT WideTy) {
804  if (TypeIdx != 1)
805  return UnableToLegalize;
806 
807  unsigned DstReg = MI.getOperand(0).getReg();
808  LLT DstTy = MRI.getType(DstReg);
809  if (!DstTy.isScalar())
810  return UnableToLegalize;
811 
812  unsigned NumOps = MI.getNumOperands();
813  unsigned NumSrc = MI.getNumOperands() - 1;
814  unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
815 
816  unsigned Src1 = MI.getOperand(1).getReg();
817  unsigned ResultReg = MIRBuilder.buildZExt(DstTy, Src1)->getOperand(0).getReg();
818 
819  for (unsigned I = 2; I != NumOps; ++I) {
820  const unsigned Offset = (I - 1) * PartSize;
821 
822  unsigned SrcReg = MI.getOperand(I).getReg();
823  assert(MRI.getType(SrcReg) == LLT::scalar(PartSize));
824 
825  auto ZextInput = MIRBuilder.buildZExt(DstTy, SrcReg);
826 
827  unsigned NextResult = I + 1 == NumOps ? DstReg :
828  MRI.createGenericVirtualRegister(DstTy);
829 
830  auto ShiftAmt = MIRBuilder.buildConstant(DstTy, Offset);
831  auto Shl = MIRBuilder.buildShl(DstTy, ZextInput, ShiftAmt);
832  MIRBuilder.buildOr(NextResult, ResultReg, Shl);
833  ResultReg = NextResult;
834  }
835 
836  MI.eraseFromParent();
837  return Legalized;
838 }
839 
841 LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx,
842  LLT WideTy) {
843  if (TypeIdx != 0)
844  return UnableToLegalize;
845 
846  unsigned NumDst = MI.getNumOperands() - 1;
847  unsigned SrcReg = MI.getOperand(NumDst).getReg();
848  LLT SrcTy = MRI.getType(SrcReg);
849  if (!SrcTy.isScalar())
850  return UnableToLegalize;
851 
852  unsigned Dst0Reg = MI.getOperand(0).getReg();
853  LLT DstTy = MRI.getType(Dst0Reg);
854  if (!DstTy.isScalar())
855  return UnableToLegalize;
856 
857  unsigned NewSrcSize = NumDst * WideTy.getSizeInBits();
858  LLT NewSrcTy = LLT::scalar(NewSrcSize);
859  unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits();
860 
861  auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg);
862 
863  for (unsigned I = 1; I != NumDst; ++I) {
864  auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I);
865  auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt);
866  WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl);
867  }
868 
869  Observer.changingInstr(MI);
870 
871  MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg());
872  for (unsigned I = 0; I != NumDst; ++I)
873  widenScalarDst(MI, WideTy, I);
874 
875  Observer.changedInstr(MI);
876 
877  return Legalized;
878 }
879 
881 LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx,
882  LLT WideTy) {
883  unsigned DstReg = MI.getOperand(0).getReg();
884  unsigned SrcReg = MI.getOperand(1).getReg();
885  LLT SrcTy = MRI.getType(SrcReg);
886 
887  LLT DstTy = MRI.getType(DstReg);
888  unsigned Offset = MI.getOperand(2).getImm();
889 
890  if (TypeIdx == 0) {
891  if (SrcTy.isVector() || DstTy.isVector())
892  return UnableToLegalize;
893 
894  SrcOp Src(SrcReg);
895  if (SrcTy.isPointer()) {
896  // Extracts from pointers can be handled only if they are really just
897  // simple integers.
898  const DataLayout &DL = MIRBuilder.getDataLayout();
900  return UnableToLegalize;
901 
902  LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits());
903  Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src);
904  SrcTy = SrcAsIntTy;
905  }
906 
907  if (DstTy.isPointer())
908  return UnableToLegalize;
909 
910  if (Offset == 0) {
911  // Avoid a shift in the degenerate case.
912  MIRBuilder.buildTrunc(DstReg,
913  MIRBuilder.buildAnyExtOrTrunc(WideTy, Src));
914  MI.eraseFromParent();
915  return Legalized;
916  }
917 
918  // Do a shift in the source type.
919  LLT ShiftTy = SrcTy;
920  if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) {
921  Src = MIRBuilder.buildAnyExt(WideTy, Src);
922  ShiftTy = WideTy;
923  } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits())
924  return UnableToLegalize;
925 
926  auto LShr = MIRBuilder.buildLShr(
927  ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset));
928  MIRBuilder.buildTrunc(DstReg, LShr);
929  MI.eraseFromParent();
930  return Legalized;
931  }
932 
933  if (!SrcTy.isVector())
934  return UnableToLegalize;
935 
936  if (DstTy != SrcTy.getElementType())
937  return UnableToLegalize;
938 
939  if (Offset % SrcTy.getScalarSizeInBits() != 0)
940  return UnableToLegalize;
941 
942  Observer.changingInstr(MI);
943  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
944 
945  MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) *
946  Offset);
947  widenScalarDst(MI, WideTy.getScalarType(), 0);
948  Observer.changedInstr(MI);
949  return Legalized;
950 }
951 
953 LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx,
954  LLT WideTy) {
955  if (TypeIdx != 0)
956  return UnableToLegalize;
957  Observer.changingInstr(MI);
958  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
959  widenScalarDst(MI, WideTy);
960  Observer.changedInstr(MI);
961  return Legalized;
962 }
963 
965 LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) {
966  MIRBuilder.setInstr(MI);
967 
968  switch (MI.getOpcode()) {
969  default:
970  return UnableToLegalize;
971  case TargetOpcode::G_EXTRACT:
972  return widenScalarExtract(MI, TypeIdx, WideTy);
973  case TargetOpcode::G_INSERT:
974  return widenScalarInsert(MI, TypeIdx, WideTy);
975  case TargetOpcode::G_MERGE_VALUES:
976  return widenScalarMergeValues(MI, TypeIdx, WideTy);
977  case TargetOpcode::G_UNMERGE_VALUES:
978  return widenScalarUnmergeValues(MI, TypeIdx, WideTy);
979  case TargetOpcode::G_UADDO:
980  case TargetOpcode::G_USUBO: {
981  if (TypeIdx == 1)
982  return UnableToLegalize; // TODO
983  auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
984  {MI.getOperand(2).getReg()});
985  auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy},
986  {MI.getOperand(3).getReg()});
987  unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO
988  ? TargetOpcode::G_ADD
989  : TargetOpcode::G_SUB;
990  // Do the arithmetic in the larger type.
991  auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext});
992  LLT OrigTy = MRI.getType(MI.getOperand(0).getReg());
994  auto AndOp = MIRBuilder.buildInstr(
995  TargetOpcode::G_AND, {WideTy},
996  {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())});
997  // There is no overflow if the AndOp is the same as NewOp.
999  AndOp);
1000  // Now trunc the NewOp to the original result.
1001  MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp);
1002  MI.eraseFromParent();
1003  return Legalized;
1004  }
1005  case TargetOpcode::G_CTTZ:
1006  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1007  case TargetOpcode::G_CTLZ:
1008  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1009  case TargetOpcode::G_CTPOP: {
1010  if (TypeIdx == 0) {
1011  Observer.changingInstr(MI);
1012  widenScalarDst(MI, WideTy, 0);
1013  Observer.changedInstr(MI);
1014  return Legalized;
1015  }
1016 
1017  unsigned SrcReg = MI.getOperand(1).getReg();
1018 
1019  // First ZEXT the input.
1020  auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg);
1021  LLT CurTy = MRI.getType(SrcReg);
1022  if (MI.getOpcode() == TargetOpcode::G_CTTZ) {
1023  // The count is the same in the larger type except if the original
1024  // value was zero. This can be handled by setting the bit just off
1025  // the top of the original type.
1026  auto TopBit =
1027  APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits());
1028  MIBSrc = MIRBuilder.buildOr(
1029  WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit));
1030  }
1031 
1032  // Perform the operation at the larger size.
1033  auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc});
1034  // This is already the correct result for CTPOP and CTTZs
1035  if (MI.getOpcode() == TargetOpcode::G_CTLZ ||
1036  MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
1037  // The correct result is NewOp - (Difference in widety and current ty).
1038  unsigned SizeDiff = WideTy.getSizeInBits() - CurTy.getSizeInBits();
1039  MIBNewOp = MIRBuilder.buildInstr(
1040  TargetOpcode::G_SUB, {WideTy},
1041  {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)});
1042  }
1043 
1044  MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp);
1045  MI.eraseFromParent();
1046  return Legalized;
1047  }
1048  case TargetOpcode::G_BSWAP: {
1049  Observer.changingInstr(MI);
1050  unsigned DstReg = MI.getOperand(0).getReg();
1051 
1052  unsigned ShrReg = MRI.createGenericVirtualRegister(WideTy);
1053  unsigned DstExt = MRI.createGenericVirtualRegister(WideTy);
1054  unsigned ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy);
1055  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1056 
1057  MI.getOperand(0).setReg(DstExt);
1058 
1060 
1061  LLT Ty = MRI.getType(DstReg);
1062  unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits();
1063  MIRBuilder.buildConstant(ShiftAmtReg, DiffBits);
1064  MIRBuilder.buildInstr(TargetOpcode::G_LSHR)
1065  .addDef(ShrReg)
1066  .addUse(DstExt)
1067  .addUse(ShiftAmtReg);
1068 
1069  MIRBuilder.buildTrunc(DstReg, ShrReg);
1070  Observer.changedInstr(MI);
1071  return Legalized;
1072  }
1073  case TargetOpcode::G_ADD:
1074  case TargetOpcode::G_AND:
1075  case TargetOpcode::G_MUL:
1076  case TargetOpcode::G_OR:
1077  case TargetOpcode::G_XOR:
1078  case TargetOpcode::G_SUB:
1079  // Perform operation at larger width (any extension is fines here, high bits
1080  // don't affect the result) and then truncate the result back to the
1081  // original type.
1082  Observer.changingInstr(MI);
1083  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1084  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1085  widenScalarDst(MI, WideTy);
1086  Observer.changedInstr(MI);
1087  return Legalized;
1088 
1089  case TargetOpcode::G_SHL:
1090  Observer.changingInstr(MI);
1091 
1092  if (TypeIdx == 0) {
1093  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT);
1094  widenScalarDst(MI, WideTy);
1095  } else {
1096  assert(TypeIdx == 1);
1097  // The "number of bits to shift" operand must preserve its value as an
1098  // unsigned integer:
1099  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1100  }
1101 
1102  Observer.changedInstr(MI);
1103  return Legalized;
1104 
1105  case TargetOpcode::G_SDIV:
1106  case TargetOpcode::G_SREM:
1107  Observer.changingInstr(MI);
1108  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1109  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1110  widenScalarDst(MI, WideTy);
1111  Observer.changedInstr(MI);
1112  return Legalized;
1113 
1114  case TargetOpcode::G_ASHR:
1115  case TargetOpcode::G_LSHR:
1116  Observer.changingInstr(MI);
1117 
1118  if (TypeIdx == 0) {
1119  unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ?
1120  TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
1121 
1122  widenScalarSrc(MI, WideTy, 1, CvtOp);
1123  widenScalarDst(MI, WideTy);
1124  } else {
1125  assert(TypeIdx == 1);
1126  // The "number of bits to shift" operand must preserve its value as an
1127  // unsigned integer:
1128  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1129  }
1130 
1131  Observer.changedInstr(MI);
1132  return Legalized;
1133  case TargetOpcode::G_UDIV:
1134  case TargetOpcode::G_UREM:
1135  Observer.changingInstr(MI);
1136  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1137  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT);
1138  widenScalarDst(MI, WideTy);
1139  Observer.changedInstr(MI);
1140  return Legalized;
1141 
1142  case TargetOpcode::G_SELECT:
1143  Observer.changingInstr(MI);
1144  if (TypeIdx == 0) {
1145  // Perform operation at larger width (any extension is fine here, high
1146  // bits don't affect the result) and then truncate the result back to the
1147  // original type.
1148  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ANYEXT);
1149  widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT);
1150  widenScalarDst(MI, WideTy);
1151  } else {
1152  bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector();
1153  // Explicit extension is required here since high bits affect the result.
1154  widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false));
1155  }
1156  Observer.changedInstr(MI);
1157  return Legalized;
1158 
1159  case TargetOpcode::G_FPTOSI:
1160  case TargetOpcode::G_FPTOUI:
1161  if (TypeIdx != 0)
1162  return UnableToLegalize;
1163  Observer.changingInstr(MI);
1164  widenScalarDst(MI, WideTy);
1165  Observer.changedInstr(MI);
1166  return Legalized;
1167 
1168  case TargetOpcode::G_SITOFP:
1169  if (TypeIdx != 1)
1170  return UnableToLegalize;
1171  Observer.changingInstr(MI);
1172  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT);
1173  Observer.changedInstr(MI);
1174  return Legalized;
1175 
1176  case TargetOpcode::G_UITOFP:
1177  if (TypeIdx != 1)
1178  return UnableToLegalize;
1179  Observer.changingInstr(MI);
1180  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1181  Observer.changedInstr(MI);
1182  return Legalized;
1183 
1184  case TargetOpcode::G_LOAD:
1185  case TargetOpcode::G_SEXTLOAD:
1186  case TargetOpcode::G_ZEXTLOAD:
1187  Observer.changingInstr(MI);
1188  widenScalarDst(MI, WideTy);
1189  Observer.changedInstr(MI);
1190  return Legalized;
1191 
1192  case TargetOpcode::G_STORE: {
1193  if (TypeIdx != 0)
1194  return UnableToLegalize;
1195 
1196  LLT Ty = MRI.getType(MI.getOperand(0).getReg());
1197  if (!isPowerOf2_32(Ty.getSizeInBits()))
1198  return UnableToLegalize;
1199 
1200  Observer.changingInstr(MI);
1201 
1202  unsigned ExtType = Ty.getScalarSizeInBits() == 1 ?
1203  TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
1204  widenScalarSrc(MI, WideTy, 0, ExtType);
1205 
1206  Observer.changedInstr(MI);
1207  return Legalized;
1208  }
1209  case TargetOpcode::G_CONSTANT: {
1210  MachineOperand &SrcMO = MI.getOperand(1);
1212  const APInt &Val = SrcMO.getCImm()->getValue().sext(WideTy.getSizeInBits());
1213  Observer.changingInstr(MI);
1214  SrcMO.setCImm(ConstantInt::get(Ctx, Val));
1215 
1216  widenScalarDst(MI, WideTy);
1217  Observer.changedInstr(MI);
1218  return Legalized;
1219  }
1220  case TargetOpcode::G_FCONSTANT: {
1221  MachineOperand &SrcMO = MI.getOperand(1);
1223  APFloat Val = SrcMO.getFPImm()->getValueAPF();
1224  bool LosesInfo;
1225  switch (WideTy.getSizeInBits()) {
1226  case 32:
1228  &LosesInfo);
1229  break;
1230  case 64:
1232  &LosesInfo);
1233  break;
1234  default:
1235  return UnableToLegalize;
1236  }
1237 
1238  assert(!LosesInfo && "extend should always be lossless");
1239 
1240  Observer.changingInstr(MI);
1241  SrcMO.setFPImm(ConstantFP::get(Ctx, Val));
1242 
1243  widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1244  Observer.changedInstr(MI);
1245  return Legalized;
1246  }
1247  case TargetOpcode::G_IMPLICIT_DEF: {
1248  Observer.changingInstr(MI);
1249  widenScalarDst(MI, WideTy);
1250  Observer.changedInstr(MI);
1251  return Legalized;
1252  }
1253  case TargetOpcode::G_BRCOND:
1254  Observer.changingInstr(MI);
1255  widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false));
1256  Observer.changedInstr(MI);
1257  return Legalized;
1258 
1259  case TargetOpcode::G_FCMP:
1260  Observer.changingInstr(MI);
1261  if (TypeIdx == 0)
1262  widenScalarDst(MI, WideTy);
1263  else {
1264  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_FPEXT);
1265  widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_FPEXT);
1266  }
1267  Observer.changedInstr(MI);
1268  return Legalized;
1269 
1270  case TargetOpcode::G_ICMP:
1271  Observer.changingInstr(MI);
1272  if (TypeIdx == 0)
1273  widenScalarDst(MI, WideTy);
1274  else {
1275  unsigned ExtOpcode = CmpInst::isSigned(static_cast<CmpInst::Predicate>(
1276  MI.getOperand(1).getPredicate()))
1277  ? TargetOpcode::G_SEXT
1278  : TargetOpcode::G_ZEXT;
1279  widenScalarSrc(MI, WideTy, 2, ExtOpcode);
1280  widenScalarSrc(MI, WideTy, 3, ExtOpcode);
1281  }
1282  Observer.changedInstr(MI);
1283  return Legalized;
1284 
1285  case TargetOpcode::G_GEP:
1286  assert(TypeIdx == 1 && "unable to legalize pointer of GEP");
1287  Observer.changingInstr(MI);
1288  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1289  Observer.changedInstr(MI);
1290  return Legalized;
1291 
1292  case TargetOpcode::G_PHI: {
1293  assert(TypeIdx == 0 && "Expecting only Idx 0");
1294 
1295  Observer.changingInstr(MI);
1296  for (unsigned I = 1; I < MI.getNumOperands(); I += 2) {
1297  MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
1298  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
1299  widenScalarSrc(MI, WideTy, I, TargetOpcode::G_ANYEXT);
1300  }
1301 
1302  MachineBasicBlock &MBB = *MI.getParent();
1303  MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
1304  widenScalarDst(MI, WideTy);
1305  Observer.changedInstr(MI);
1306  return Legalized;
1307  }
1308  case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
1309  if (TypeIdx == 0) {
1310  unsigned VecReg = MI.getOperand(1).getReg();
1311  LLT VecTy = MRI.getType(VecReg);
1312  Observer.changingInstr(MI);
1313 
1314  widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(),
1315  WideTy.getSizeInBits()),
1316  1, TargetOpcode::G_SEXT);
1317 
1318  widenScalarDst(MI, WideTy, 0);
1319  Observer.changedInstr(MI);
1320  return Legalized;
1321  }
1322 
1323  if (TypeIdx != 2)
1324  return UnableToLegalize;
1325  Observer.changingInstr(MI);
1326  widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT);
1327  Observer.changedInstr(MI);
1328  return Legalized;
1329  }
1330  case TargetOpcode::G_FADD:
1331  case TargetOpcode::G_FMUL:
1332  case TargetOpcode::G_FSUB:
1333  case TargetOpcode::G_FMA:
1334  case TargetOpcode::G_FNEG:
1335  case TargetOpcode::G_FABS:
1336  case TargetOpcode::G_FCANONICALIZE:
1337  case TargetOpcode::G_FDIV:
1338  case TargetOpcode::G_FREM:
1339  case TargetOpcode::G_FCEIL:
1340  case TargetOpcode::G_FFLOOR:
1341  case TargetOpcode::G_FCOS:
1342  case TargetOpcode::G_FSIN:
1343  case TargetOpcode::G_FLOG10:
1344  case TargetOpcode::G_FLOG:
1345  case TargetOpcode::G_FLOG2:
1346  case TargetOpcode::G_FSQRT:
1347  case TargetOpcode::G_FEXP:
1348  assert(TypeIdx == 0);
1349  Observer.changingInstr(MI);
1350 
1351  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I)
1352  widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT);
1353 
1354  widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC);
1355  Observer.changedInstr(MI);
1356  return Legalized;
1357  case TargetOpcode::G_INTTOPTR:
1358  if (TypeIdx != 1)
1359  return UnableToLegalize;
1360 
1361  Observer.changingInstr(MI);
1362  widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT);
1363  Observer.changedInstr(MI);
1364  return Legalized;
1365  case TargetOpcode::G_PTRTOINT:
1366  if (TypeIdx != 0)
1367  return UnableToLegalize;
1368 
1369  Observer.changingInstr(MI);
1370  widenScalarDst(MI, WideTy, 0);
1371  Observer.changedInstr(MI);
1372  return Legalized;
1373  }
1374 }
1375 
1377 LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
1378  using namespace TargetOpcode;
1379  MIRBuilder.setInstr(MI);
1380 
1381  switch(MI.getOpcode()) {
1382  default:
1383  return UnableToLegalize;
1384  case TargetOpcode::G_SREM:
1385  case TargetOpcode::G_UREM: {
1386  unsigned QuotReg = MRI.createGenericVirtualRegister(Ty);
1387  MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV)
1388  .addDef(QuotReg)
1389  .addUse(MI.getOperand(1).getReg())
1390  .addUse(MI.getOperand(2).getReg());
1391 
1392  unsigned ProdReg = MRI.createGenericVirtualRegister(Ty);
1393  MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg());
1395  ProdReg);
1396  MI.eraseFromParent();
1397  return Legalized;
1398  }
1399  case TargetOpcode::G_SMULO:
1400  case TargetOpcode::G_UMULO: {
1401  // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the
1402  // result.
1403  unsigned Res = MI.getOperand(0).getReg();
1404  unsigned Overflow = MI.getOperand(1).getReg();
1405  unsigned LHS = MI.getOperand(2).getReg();
1406  unsigned RHS = MI.getOperand(3).getReg();
1407 
1408  MIRBuilder.buildMul(Res, LHS, RHS);
1409 
1410  unsigned Opcode = MI.getOpcode() == TargetOpcode::G_SMULO
1411  ? TargetOpcode::G_SMULH
1412  : TargetOpcode::G_UMULH;
1413 
1414  unsigned HiPart = MRI.createGenericVirtualRegister(Ty);
1415  MIRBuilder.buildInstr(Opcode)
1416  .addDef(HiPart)
1417  .addUse(LHS)
1418  .addUse(RHS);
1419 
1420  unsigned Zero = MRI.createGenericVirtualRegister(Ty);
1421  MIRBuilder.buildConstant(Zero, 0);
1422 
1423  // For *signed* multiply, overflow is detected by checking:
1424  // (hi != (lo >> bitwidth-1))
1425  if (Opcode == TargetOpcode::G_SMULH) {
1426  unsigned Shifted = MRI.createGenericVirtualRegister(Ty);
1427  unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty);
1428  MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1);
1429  MIRBuilder.buildInstr(TargetOpcode::G_ASHR)
1430  .addDef(Shifted)
1431  .addUse(Res)
1432  .addUse(ShiftAmt);
1433  MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Shifted);
1434  } else {
1435  MIRBuilder.buildICmp(CmpInst::ICMP_NE, Overflow, HiPart, Zero);
1436  }
1437  MI.eraseFromParent();
1438  return Legalized;
1439  }
1440  case TargetOpcode::G_FNEG: {
1441  // TODO: Handle vector types once we are able to
1442  // represent them.
1443  if (Ty.isVector())
1444  return UnableToLegalize;
1445  unsigned Res = MI.getOperand(0).getReg();
1446  Type *ZeroTy;
1448  switch (Ty.getSizeInBits()) {
1449  case 16:
1450  ZeroTy = Type::getHalfTy(Ctx);
1451  break;
1452  case 32:
1453  ZeroTy = Type::getFloatTy(Ctx);
1454  break;
1455  case 64:
1456  ZeroTy = Type::getDoubleTy(Ctx);
1457  break;
1458  case 128:
1459  ZeroTy = Type::getFP128Ty(Ctx);
1460  break;
1461  default:
1462  llvm_unreachable("unexpected floating-point type");
1463  }
1464  ConstantFP &ZeroForNegation =
1465  *cast<ConstantFP>(ConstantFP::getZeroValueForNegation(ZeroTy));
1466  auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation);
1467  MIRBuilder.buildInstr(TargetOpcode::G_FSUB)
1468  .addDef(Res)
1469  .addUse(Zero->getOperand(0).getReg())
1470  .addUse(MI.getOperand(1).getReg());
1471  MI.eraseFromParent();
1472  return Legalized;
1473  }
1474  case TargetOpcode::G_FSUB: {
1475  // Lower (G_FSUB LHS, RHS) to (G_FADD LHS, (G_FNEG RHS)).
1476  // First, check if G_FNEG is marked as Lower. If so, we may
1477  // end up with an infinite loop as G_FSUB is used to legalize G_FNEG.
1478  if (LI.getAction({G_FNEG, {Ty}}).Action == Lower)
1479  return UnableToLegalize;
1480  unsigned Res = MI.getOperand(0).getReg();
1481  unsigned LHS = MI.getOperand(1).getReg();
1482  unsigned RHS = MI.getOperand(2).getReg();
1483  unsigned Neg = MRI.createGenericVirtualRegister(Ty);
1484  MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS);
1485  MIRBuilder.buildInstr(TargetOpcode::G_FADD)
1486  .addDef(Res)
1487  .addUse(LHS)
1488  .addUse(Neg);
1489  MI.eraseFromParent();
1490  return Legalized;
1491  }
1492  case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1493  unsigned OldValRes = MI.getOperand(0).getReg();
1494  unsigned SuccessRes = MI.getOperand(1).getReg();
1495  unsigned Addr = MI.getOperand(2).getReg();
1496  unsigned CmpVal = MI.getOperand(3).getReg();
1497  unsigned NewVal = MI.getOperand(4).getReg();
1498  MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal,
1499  **MI.memoperands_begin());
1500  MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal);
1501  MI.eraseFromParent();
1502  return Legalized;
1503  }
1504  case TargetOpcode::G_LOAD:
1505  case TargetOpcode::G_SEXTLOAD:
1506  case TargetOpcode::G_ZEXTLOAD: {
1507  // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT
1508  unsigned DstReg = MI.getOperand(0).getReg();
1509  unsigned PtrReg = MI.getOperand(1).getReg();
1510  LLT DstTy = MRI.getType(DstReg);
1511  auto &MMO = **MI.memoperands_begin();
1512 
1513  if (DstTy.getSizeInBits() == MMO.getSize() /* in bytes */ * 8) {
1514  // In the case of G_LOAD, this was a non-extending load already and we're
1515  // about to lower to the same instruction.
1516  if (MI.getOpcode() == TargetOpcode::G_LOAD)
1517  return UnableToLegalize;
1518  MIRBuilder.buildLoad(DstReg, PtrReg, MMO);
1519  MI.eraseFromParent();
1520  return Legalized;
1521  }
1522 
1523  if (DstTy.isScalar()) {
1524  unsigned TmpReg = MRI.createGenericVirtualRegister(
1525  LLT::scalar(MMO.getSize() /* in bytes */ * 8));
1526  MIRBuilder.buildLoad(TmpReg, PtrReg, MMO);
1527  switch (MI.getOpcode()) {
1528  default:
1529  llvm_unreachable("Unexpected opcode");
1530  case TargetOpcode::G_LOAD:
1531  MIRBuilder.buildAnyExt(DstReg, TmpReg);
1532  break;
1533  case TargetOpcode::G_SEXTLOAD:
1534  MIRBuilder.buildSExt(DstReg, TmpReg);
1535  break;
1536  case TargetOpcode::G_ZEXTLOAD:
1537  MIRBuilder.buildZExt(DstReg, TmpReg);
1538  break;
1539  }
1540  MI.eraseFromParent();
1541  return Legalized;
1542  }
1543 
1544  return UnableToLegalize;
1545  }
1546  case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1547  case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1548  case TargetOpcode::G_CTLZ:
1549  case TargetOpcode::G_CTTZ:
1550  case TargetOpcode::G_CTPOP:
1551  return lowerBitCount(MI, TypeIdx, Ty);
1552  case G_UADDO: {
1553  unsigned Res = MI.getOperand(0).getReg();
1554  unsigned CarryOut = MI.getOperand(1).getReg();
1555  unsigned LHS = MI.getOperand(2).getReg();
1556  unsigned RHS = MI.getOperand(3).getReg();
1557 
1558  MIRBuilder.buildAdd(Res, LHS, RHS);
1559  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS);
1560 
1561  MI.eraseFromParent();
1562  return Legalized;
1563  }
1564  case G_UADDE: {
1565  unsigned Res = MI.getOperand(0).getReg();
1566  unsigned CarryOut = MI.getOperand(1).getReg();
1567  unsigned LHS = MI.getOperand(2).getReg();
1568  unsigned RHS = MI.getOperand(3).getReg();
1569  unsigned CarryIn = MI.getOperand(4).getReg();
1570 
1571  unsigned TmpRes = MRI.createGenericVirtualRegister(Ty);
1572  unsigned ZExtCarryIn = MRI.createGenericVirtualRegister(Ty);
1573 
1574  MIRBuilder.buildAdd(TmpRes, LHS, RHS);
1575  MIRBuilder.buildZExt(ZExtCarryIn, CarryIn);
1576  MIRBuilder.buildAdd(Res, TmpRes, ZExtCarryIn);
1577  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, LHS);
1578 
1579  MI.eraseFromParent();
1580  return Legalized;
1581  }
1582  case G_USUBO: {
1583  unsigned Res = MI.getOperand(0).getReg();
1584  unsigned BorrowOut = MI.getOperand(1).getReg();
1585  unsigned LHS = MI.getOperand(2).getReg();
1586  unsigned RHS = MI.getOperand(3).getReg();
1587 
1588  MIRBuilder.buildSub(Res, LHS, RHS);
1589  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS);
1590 
1591  MI.eraseFromParent();
1592  return Legalized;
1593  }
1594  case G_USUBE: {
1595  unsigned Res = MI.getOperand(0).getReg();
1596  unsigned BorrowOut = MI.getOperand(1).getReg();
1597  unsigned LHS = MI.getOperand(2).getReg();
1598  unsigned RHS = MI.getOperand(3).getReg();
1599  unsigned BorrowIn = MI.getOperand(4).getReg();
1600 
1601  unsigned TmpRes = MRI.createGenericVirtualRegister(Ty);
1602  unsigned ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty);
1603  unsigned LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1604  unsigned LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1));
1605 
1606  MIRBuilder.buildSub(TmpRes, LHS, RHS);
1607  MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn);
1608  MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn);
1609  MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS);
1610  MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS);
1611  MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS);
1612 
1613  MI.eraseFromParent();
1614  return Legalized;
1615  }
1616  }
1617 }
1618 
1619 LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef(
1620  MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) {
1621  SmallVector<unsigned, 2> DstRegs;
1622 
1623  unsigned NarrowSize = NarrowTy.getSizeInBits();
1624  unsigned DstReg = MI.getOperand(0).getReg();
1625  unsigned Size = MRI.getType(DstReg).getSizeInBits();
1626  int NumParts = Size / NarrowSize;
1627  // FIXME: Don't know how to handle the situation where the small vectors
1628  // aren't all the same size yet.
1629  if (Size % NarrowSize != 0)
1630  return UnableToLegalize;
1631 
1632  for (int i = 0; i < NumParts; ++i) {
1633  unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy);
1634  MIRBuilder.buildUndef(TmpReg);
1635  DstRegs.push_back(TmpReg);
1636  }
1637 
1638  if (NarrowTy.isVector())
1639  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1640  else
1641  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1642 
1643  MI.eraseFromParent();
1644  return Legalized;
1645 }
1646 
1648 LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx,
1649  LLT NarrowTy) {
1650  const unsigned Opc = MI.getOpcode();
1651  const unsigned NumOps = MI.getNumOperands() - 1;
1652  const unsigned NarrowSize = NarrowTy.getSizeInBits();
1653  const unsigned DstReg = MI.getOperand(0).getReg();
1654  const unsigned Flags = MI.getFlags();
1655  const LLT DstTy = MRI.getType(DstReg);
1656  const unsigned Size = DstTy.getSizeInBits();
1657  const int NumParts = Size / NarrowSize;
1658  const LLT EltTy = DstTy.getElementType();
1659  const unsigned EltSize = EltTy.getSizeInBits();
1660  const unsigned BitsForNumParts = NarrowSize * NumParts;
1661 
1662  // Check if we have any leftovers. If we do, then only handle the case where
1663  // the leftover is one element.
1664  if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size)
1665  return UnableToLegalize;
1666 
1667  if (BitsForNumParts != Size) {
1668  unsigned AccumDstReg = MRI.createGenericVirtualRegister(DstTy);
1669  MIRBuilder.buildUndef(AccumDstReg);
1670 
1671  // Handle the pieces which evenly divide into the requested type with
1672  // extract/op/insert sequence.
1673  for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) {
1674  SmallVector<SrcOp, 4> SrcOps;
1675  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1676  unsigned PartOpReg = MRI.createGenericVirtualRegister(NarrowTy);
1677  MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset);
1678  SrcOps.push_back(PartOpReg);
1679  }
1680 
1681  unsigned PartDstReg = MRI.createGenericVirtualRegister(NarrowTy);
1682  MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1683 
1684  unsigned PartInsertReg = MRI.createGenericVirtualRegister(DstTy);
1685  MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset);
1686  AccumDstReg = PartInsertReg;
1687  }
1688 
1689  // Handle the remaining element sized leftover piece.
1690  SmallVector<SrcOp, 4> SrcOps;
1691  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1692  unsigned PartOpReg = MRI.createGenericVirtualRegister(EltTy);
1693  MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(),
1694  BitsForNumParts);
1695  SrcOps.push_back(PartOpReg);
1696  }
1697 
1698  unsigned PartDstReg = MRI.createGenericVirtualRegister(EltTy);
1699  MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags);
1700  MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts);
1701  MI.eraseFromParent();
1702 
1703  return Legalized;
1704  }
1705 
1706  SmallVector<unsigned, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
1707 
1708  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs);
1709 
1710  if (NumOps >= 2)
1711  extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs);
1712 
1713  if (NumOps >= 3)
1714  extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs);
1715 
1716  for (int i = 0; i < NumParts; ++i) {
1717  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
1718 
1719  if (NumOps == 1)
1720  MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags);
1721  else if (NumOps == 2) {
1722  MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags);
1723  } else if (NumOps == 3) {
1724  MIRBuilder.buildInstr(Opc, {DstReg},
1725  {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags);
1726  }
1727 
1728  DstRegs.push_back(DstReg);
1729  }
1730 
1731  if (NarrowTy.isVector())
1732  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1733  else
1734  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1735 
1736  MI.eraseFromParent();
1737  return Legalized;
1738 }
1739 
1740 // Handle splitting vector operations which need to have the same number of
1741 // elements in each type index, but each type index may have a different element
1742 // type.
1743 //
1744 // e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> ->
1745 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1746 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1747 //
1748 // Also handles some irregular breakdown cases, e.g.
1749 // e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> ->
1750 // <2 x s64> = G_SHL <2 x s64>, <2 x s32>
1751 // s64 = G_SHL s64, s32
1753 LegalizerHelper::fewerElementsVectorMultiEltType(
1754  MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) {
1755  if (TypeIdx != 0)
1756  return UnableToLegalize;
1757 
1758  const LLT NarrowTy0 = NarrowTyArg;
1759  const unsigned NewNumElts =
1760  NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1;
1761 
1762  const unsigned DstReg = MI.getOperand(0).getReg();
1763  LLT DstTy = MRI.getType(DstReg);
1764  LLT LeftoverTy0;
1765 
1766  int NumParts, NumLeftover;
1767  // All of the operands need to have the same number of elements, so if we can
1768  // determine a type breakdown for the result type, we can for all of the
1769  // source types.
1770  std::tie(NumParts, NumLeftover)
1771  = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0);
1772  if (NumParts < 0)
1773  return UnableToLegalize;
1774 
1776 
1777  SmallVector<unsigned, 4> DstRegs, LeftoverDstRegs;
1778  SmallVector<unsigned, 4> PartRegs, LeftoverRegs;
1779 
1780  for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) {
1781  LLT LeftoverTy;
1782  unsigned SrcReg = MI.getOperand(I).getReg();
1783  LLT SrcTyI = MRI.getType(SrcReg);
1784  LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType());
1785  LLT LeftoverTyI;
1786 
1787  // Split this operand into the requested typed registers, and any leftover
1788  // required to reproduce the original type.
1789  if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs,
1790  LeftoverRegs))
1791  return UnableToLegalize;
1792 
1793  if (I == 1) {
1794  // For the first operand, create an instruction for each part and setup
1795  // the result.
1796  for (unsigned PartReg : PartRegs) {
1797  unsigned PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1799  .addDef(PartDstReg)
1800  .addUse(PartReg));
1801  DstRegs.push_back(PartDstReg);
1802  }
1803 
1804  for (unsigned LeftoverReg : LeftoverRegs) {
1805  unsigned PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0);
1807  .addDef(PartDstReg)
1808  .addUse(LeftoverReg));
1809  LeftoverDstRegs.push_back(PartDstReg);
1810  }
1811  } else {
1812  assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size());
1813 
1814  // Add the newly created operand splits to the existing instructions. The
1815  // odd-sized pieces are ordered after the requested NarrowTyArg sized
1816  // pieces.
1817  unsigned InstCount = 0;
1818  for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J)
1819  NewInsts[InstCount++].addUse(PartRegs[J]);
1820  for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J)
1821  NewInsts[InstCount++].addUse(LeftoverRegs[J]);
1822  }
1823 
1824  PartRegs.clear();
1825  LeftoverRegs.clear();
1826  }
1827 
1828  // Insert the newly built operations and rebuild the result register.
1829  for (auto &MIB : NewInsts)
1830  MIRBuilder.insertInstr(MIB);
1831 
1832  insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs);
1833 
1834  MI.eraseFromParent();
1835  return Legalized;
1836 }
1837 
1839 LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx,
1840  LLT NarrowTy) {
1841  if (TypeIdx != 0)
1842  return UnableToLegalize;
1843 
1844  unsigned DstReg = MI.getOperand(0).getReg();
1845  unsigned SrcReg = MI.getOperand(1).getReg();
1846  LLT DstTy = MRI.getType(DstReg);
1847  LLT SrcTy = MRI.getType(SrcReg);
1848 
1849  LLT NarrowTy0 = NarrowTy;
1850  LLT NarrowTy1;
1851  unsigned NumParts;
1852 
1853  if (NarrowTy.isVector()) {
1854  // Uneven breakdown not handled.
1855  NumParts = DstTy.getNumElements() / NarrowTy.getNumElements();
1856  if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements())
1857  return UnableToLegalize;
1858 
1859  NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits());
1860  } else {
1861  NumParts = DstTy.getNumElements();
1862  NarrowTy1 = SrcTy.getElementType();
1863  }
1864 
1865  SmallVector<unsigned, 4> SrcRegs, DstRegs;
1866  extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs);
1867 
1868  for (unsigned I = 0; I < NumParts; ++I) {
1869  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1870  MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode())
1871  .addDef(DstReg)
1872  .addUse(SrcRegs[I]);
1873 
1874  NewInst->setFlags(MI.getFlags());
1875  DstRegs.push_back(DstReg);
1876  }
1877 
1878  if (NarrowTy.isVector())
1879  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1880  else
1881  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1882 
1883  MI.eraseFromParent();
1884  return Legalized;
1885 }
1886 
1888 LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx,
1889  LLT NarrowTy) {
1890  unsigned DstReg = MI.getOperand(0).getReg();
1891  unsigned Src0Reg = MI.getOperand(2).getReg();
1892  LLT DstTy = MRI.getType(DstReg);
1893  LLT SrcTy = MRI.getType(Src0Reg);
1894 
1895  unsigned NumParts;
1896  LLT NarrowTy0, NarrowTy1;
1897 
1898  if (TypeIdx == 0) {
1899  unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
1900  unsigned OldElts = DstTy.getNumElements();
1901 
1902  NarrowTy0 = NarrowTy;
1903  NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements();
1904  NarrowTy1 = NarrowTy.isVector() ?
1905  LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) :
1906  SrcTy.getElementType();
1907 
1908  } else {
1909  unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1;
1910  unsigned OldElts = SrcTy.getNumElements();
1911 
1912  NumParts = NarrowTy.isVector() ? (OldElts / NewElts) :
1913  NarrowTy.getNumElements();
1914  NarrowTy0 = LLT::vector(NarrowTy.getNumElements(),
1915  DstTy.getScalarSizeInBits());
1916  NarrowTy1 = NarrowTy;
1917  }
1918 
1919  // FIXME: Don't know how to handle the situation where the small vectors
1920  // aren't all the same size yet.
1921  if (NarrowTy1.isVector() &&
1922  NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements())
1923  return UnableToLegalize;
1924 
1925  CmpInst::Predicate Pred
1926  = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
1927 
1928  SmallVector<unsigned, 2> Src1Regs, Src2Regs, DstRegs;
1929  extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs);
1930  extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs);
1931 
1932  for (unsigned I = 0; I < NumParts; ++I) {
1933  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
1934  DstRegs.push_back(DstReg);
1935 
1936  if (MI.getOpcode() == TargetOpcode::G_ICMP)
1937  MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
1938  else {
1939  MachineInstr *NewCmp
1940  = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]);
1941  NewCmp->setFlags(MI.getFlags());
1942  }
1943  }
1944 
1945  if (NarrowTy1.isVector())
1946  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
1947  else
1948  MIRBuilder.buildBuildVector(DstReg, DstRegs);
1949 
1950  MI.eraseFromParent();
1951  return Legalized;
1952 }
1953 
1955 LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx,
1956  LLT NarrowTy) {
1957  unsigned DstReg = MI.getOperand(0).getReg();
1958  unsigned CondReg = MI.getOperand(1).getReg();
1959 
1960  unsigned NumParts = 0;
1961  LLT NarrowTy0, NarrowTy1;
1962 
1963  LLT DstTy = MRI.getType(DstReg);
1964  LLT CondTy = MRI.getType(CondReg);
1965  unsigned Size = DstTy.getSizeInBits();
1966 
1967  assert(TypeIdx == 0 || CondTy.isVector());
1968 
1969  if (TypeIdx == 0) {
1970  NarrowTy0 = NarrowTy;
1971  NarrowTy1 = CondTy;
1972 
1973  unsigned NarrowSize = NarrowTy0.getSizeInBits();
1974  // FIXME: Don't know how to handle the situation where the small vectors
1975  // aren't all the same size yet.
1976  if (Size % NarrowSize != 0)
1977  return UnableToLegalize;
1978 
1979  NumParts = Size / NarrowSize;
1980 
1981  // Need to break down the condition type
1982  if (CondTy.isVector()) {
1983  if (CondTy.getNumElements() == NumParts)
1984  NarrowTy1 = CondTy.getElementType();
1985  else
1986  NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts,
1987  CondTy.getScalarSizeInBits());
1988  }
1989  } else {
1990  NumParts = CondTy.getNumElements();
1991  if (NarrowTy.isVector()) {
1992  // TODO: Handle uneven breakdown.
1993  if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements())
1994  return UnableToLegalize;
1995 
1996  return UnableToLegalize;
1997  } else {
1998  NarrowTy0 = DstTy.getElementType();
1999  NarrowTy1 = NarrowTy;
2000  }
2001  }
2002 
2003  SmallVector<unsigned, 2> DstRegs, Src0Regs, Src1Regs, Src2Regs;
2004  if (CondTy.isVector())
2005  extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs);
2006 
2007  extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs);
2008  extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs);
2009 
2010  for (unsigned i = 0; i < NumParts; ++i) {
2011  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy0);
2012  MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg,
2013  Src1Regs[i], Src2Regs[i]);
2014  DstRegs.push_back(DstReg);
2015  }
2016 
2017  if (NarrowTy0.isVector())
2018  MIRBuilder.buildConcatVectors(DstReg, DstRegs);
2019  else
2020  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2021 
2022  MI.eraseFromParent();
2023  return Legalized;
2024 }
2025 
2027 LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2028  LLT NarrowTy) {
2029  const unsigned DstReg = MI.getOperand(0).getReg();
2030  LLT PhiTy = MRI.getType(DstReg);
2031  LLT LeftoverTy;
2032 
2033  // All of the operands need to have the same number of elements, so if we can
2034  // determine a type breakdown for the result type, we can for all of the
2035  // source types.
2036  int NumParts, NumLeftover;
2037  std::tie(NumParts, NumLeftover)
2038  = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy);
2039  if (NumParts < 0)
2040  return UnableToLegalize;
2041 
2042  SmallVector<unsigned, 4> DstRegs, LeftoverDstRegs;
2044 
2045  const int TotalNumParts = NumParts + NumLeftover;
2046 
2047  // Insert the new phis in the result block first.
2048  for (int I = 0; I != TotalNumParts; ++I) {
2049  LLT Ty = I < NumParts ? NarrowTy : LeftoverTy;
2050  unsigned PartDstReg = MRI.createGenericVirtualRegister(Ty);
2051  NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI)
2052  .addDef(PartDstReg));
2053  if (I < NumParts)
2054  DstRegs.push_back(PartDstReg);
2055  else
2056  LeftoverDstRegs.push_back(PartDstReg);
2057  }
2058 
2059  MachineBasicBlock *MBB = MI.getParent();
2060  MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI());
2061  insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs);
2062 
2063  SmallVector<unsigned, 4> PartRegs, LeftoverRegs;
2064 
2065  // Insert code to extract the incoming values in each predecessor block.
2066  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2067  PartRegs.clear();
2068  LeftoverRegs.clear();
2069 
2070  unsigned SrcReg = MI.getOperand(I).getReg();
2071  MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2072  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2073 
2074  LLT Unused;
2075  if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs,
2076  LeftoverRegs))
2077  return UnableToLegalize;
2078 
2079  // Add the newly created operand splits to the existing instructions. The
2080  // odd-sized pieces are ordered after the requested NarrowTyArg sized
2081  // pieces.
2082  for (int J = 0; J != TotalNumParts; ++J) {
2083  MachineInstrBuilder MIB = NewInsts[J];
2084  MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]);
2085  MIB.addMBB(&OpMBB);
2086  }
2087  }
2088 
2089  MI.eraseFromParent();
2090  return Legalized;
2091 }
2092 
2094 LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx,
2095  LLT NarrowTy) {
2096  // FIXME: Don't know how to handle secondary types yet.
2097  if (TypeIdx != 0)
2098  return UnableToLegalize;
2099 
2100  MachineMemOperand *MMO = *MI.memoperands_begin();
2101 
2102  // This implementation doesn't work for atomics. Give up instead of doing
2103  // something invalid.
2104  if (MMO->getOrdering() != AtomicOrdering::NotAtomic ||
2106  return UnableToLegalize;
2107 
2108  bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD;
2109  unsigned ValReg = MI.getOperand(0).getReg();
2110  unsigned AddrReg = MI.getOperand(1).getReg();
2111  LLT ValTy = MRI.getType(ValReg);
2112 
2113  int NumParts = -1;
2114  int NumLeftover = -1;
2115  LLT LeftoverTy;
2116  SmallVector<unsigned, 8> NarrowRegs, NarrowLeftoverRegs;
2117  if (IsLoad) {
2118  std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy);
2119  } else {
2120  if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
2121  NarrowLeftoverRegs)) {
2122  NumParts = NarrowRegs.size();
2123  NumLeftover = NarrowLeftoverRegs.size();
2124  }
2125  }
2126 
2127  if (NumParts == -1)
2128  return UnableToLegalize;
2129 
2130  const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits());
2131 
2132  unsigned TotalSize = ValTy.getSizeInBits();
2133 
2134  // Split the load/store into PartTy sized pieces starting at Offset. If this
2135  // is a load, return the new registers in ValRegs. For a store, each elements
2136  // of ValRegs should be PartTy. Returns the next offset that needs to be
2137  // handled.
2138  auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl<unsigned> &ValRegs,
2139  unsigned Offset) -> unsigned {
2141  unsigned PartSize = PartTy.getSizeInBits();
2142  for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize;
2143  Offset += PartSize, ++Idx) {
2144  unsigned ByteSize = PartSize / 8;
2145  unsigned ByteOffset = Offset / 8;
2146  unsigned NewAddrReg = 0;
2147 
2148  MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset);
2149 
2150  MachineMemOperand *NewMMO =
2151  MF.getMachineMemOperand(MMO, ByteOffset, ByteSize);
2152 
2153  if (IsLoad) {
2154  unsigned Dst = MRI.createGenericVirtualRegister(PartTy);
2155  ValRegs.push_back(Dst);
2156  MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO);
2157  } else {
2158  MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO);
2159  }
2160  }
2161 
2162  return Offset;
2163  };
2164 
2165  unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0);
2166 
2167  // Handle the rest of the register if this isn't an even type breakdown.
2168  if (LeftoverTy.isValid())
2169  splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset);
2170 
2171  if (IsLoad) {
2172  insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
2173  LeftoverTy, NarrowLeftoverRegs);
2174  }
2175 
2176  MI.eraseFromParent();
2177  return Legalized;
2178 }
2179 
2182  LLT NarrowTy) {
2183  using namespace TargetOpcode;
2184 
2185  MIRBuilder.setInstr(MI);
2186  switch (MI.getOpcode()) {
2187  case G_IMPLICIT_DEF:
2188  return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy);
2189  case G_AND:
2190  case G_OR:
2191  case G_XOR:
2192  case G_ADD:
2193  case G_SUB:
2194  case G_MUL:
2195  case G_SMULH:
2196  case G_UMULH:
2197  case G_FADD:
2198  case G_FMUL:
2199  case G_FSUB:
2200  case G_FNEG:
2201  case G_FABS:
2202  case G_FCANONICALIZE:
2203  case G_FDIV:
2204  case G_FREM:
2205  case G_FMA:
2206  case G_FPOW:
2207  case G_FEXP:
2208  case G_FEXP2:
2209  case G_FLOG:
2210  case G_FLOG2:
2211  case G_FLOG10:
2212  case G_FCEIL:
2213  case G_FFLOOR:
2214  case G_INTRINSIC_ROUND:
2215  case G_INTRINSIC_TRUNC:
2216  case G_FCOS:
2217  case G_FSIN:
2218  case G_FSQRT:
2219  case G_BSWAP:
2220  return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy);
2221  case G_SHL:
2222  case G_LSHR:
2223  case G_ASHR:
2224  case G_CTLZ:
2225  case G_CTLZ_ZERO_UNDEF:
2226  case G_CTTZ:
2227  case G_CTTZ_ZERO_UNDEF:
2228  case G_CTPOP:
2229  return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy);
2230  case G_ZEXT:
2231  case G_SEXT:
2232  case G_ANYEXT:
2233  case G_FPEXT:
2234  case G_FPTRUNC:
2235  case G_SITOFP:
2236  case G_UITOFP:
2237  case G_FPTOSI:
2238  case G_FPTOUI:
2239  case G_INTTOPTR:
2240  case G_PTRTOINT:
2241  case G_ADDRSPACE_CAST:
2242  return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy);
2243  case G_ICMP:
2244  case G_FCMP:
2245  return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy);
2246  case G_SELECT:
2247  return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy);
2248  case G_PHI:
2249  return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy);
2250  case G_LOAD:
2251  case G_STORE:
2252  return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy);
2253  default:
2254  return UnableToLegalize;
2255  }
2256 }
2257 
2259 LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt,
2260  const LLT HalfTy, const LLT AmtTy) {
2261 
2262  unsigned InL = MRI.createGenericVirtualRegister(HalfTy);
2263  unsigned InH = MRI.createGenericVirtualRegister(HalfTy);
2264  MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2265 
2266  if (Amt.isNullValue()) {
2267  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH});
2268  MI.eraseFromParent();
2269  return Legalized;
2270  }
2271 
2272  LLT NVT = HalfTy;
2273  unsigned NVTBits = HalfTy.getSizeInBits();
2274  unsigned VTBits = 2 * NVTBits;
2275 
2276  SrcOp Lo(0), Hi(0);
2277  if (MI.getOpcode() == TargetOpcode::G_SHL) {
2278  if (Amt.ugt(VTBits)) {
2279  Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2280  } else if (Amt.ugt(NVTBits)) {
2281  Lo = MIRBuilder.buildConstant(NVT, 0);
2282  Hi = MIRBuilder.buildShl(NVT, InL,
2283  MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2284  } else if (Amt == NVTBits) {
2285  Lo = MIRBuilder.buildConstant(NVT, 0);
2286  Hi = InL;
2287  } else {
2288  Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt));
2289  auto OrLHS =
2290  MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt));
2291  auto OrRHS = MIRBuilder.buildLShr(
2292  NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2293  Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2294  }
2295  } else if (MI.getOpcode() == TargetOpcode::G_LSHR) {
2296  if (Amt.ugt(VTBits)) {
2297  Lo = Hi = MIRBuilder.buildConstant(NVT, 0);
2298  } else if (Amt.ugt(NVTBits)) {
2299  Lo = MIRBuilder.buildLShr(NVT, InH,
2300  MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2301  Hi = MIRBuilder.buildConstant(NVT, 0);
2302  } else if (Amt == NVTBits) {
2303  Lo = InH;
2304  Hi = MIRBuilder.buildConstant(NVT, 0);
2305  } else {
2306  auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2307 
2308  auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2309  auto OrRHS = MIRBuilder.buildShl(
2310  NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2311 
2312  Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2313  Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst);
2314  }
2315  } else {
2316  if (Amt.ugt(VTBits)) {
2317  Hi = Lo = MIRBuilder.buildAShr(
2318  NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2319  } else if (Amt.ugt(NVTBits)) {
2320  Lo = MIRBuilder.buildAShr(NVT, InH,
2321  MIRBuilder.buildConstant(AmtTy, Amt - NVTBits));
2322  Hi = MIRBuilder.buildAShr(NVT, InH,
2323  MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2324  } else if (Amt == NVTBits) {
2325  Lo = InH;
2326  Hi = MIRBuilder.buildAShr(NVT, InH,
2327  MIRBuilder.buildConstant(AmtTy, NVTBits - 1));
2328  } else {
2329  auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt);
2330 
2331  auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst);
2332  auto OrRHS = MIRBuilder.buildShl(
2333  NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits));
2334 
2335  Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS);
2336  Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst);
2337  }
2338  }
2339 
2340  MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()});
2341  MI.eraseFromParent();
2342 
2343  return Legalized;
2344 }
2345 
2346 // TODO: Optimize if constant shift amount.
2348 LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx,
2349  LLT RequestedTy) {
2350  if (TypeIdx == 1) {
2351  Observer.changingInstr(MI);
2352  narrowScalarSrc(MI, RequestedTy, 2);
2353  Observer.changedInstr(MI);
2354  return Legalized;
2355  }
2356 
2357  unsigned DstReg = MI.getOperand(0).getReg();
2358  LLT DstTy = MRI.getType(DstReg);
2359  if (DstTy.isVector())
2360  return UnableToLegalize;
2361 
2362  unsigned Amt = MI.getOperand(2).getReg();
2363  LLT ShiftAmtTy = MRI.getType(Amt);
2364  const unsigned DstEltSize = DstTy.getScalarSizeInBits();
2365  if (DstEltSize % 2 != 0)
2366  return UnableToLegalize;
2367 
2368  // Ignore the input type. We can only go to exactly half the size of the
2369  // input. If that isn't small enough, the resulting pieces will be further
2370  // legalized.
2371  const unsigned NewBitSize = DstEltSize / 2;
2372  const LLT HalfTy = LLT::scalar(NewBitSize);
2373  const LLT CondTy = LLT::scalar(1);
2374 
2375  if (const MachineInstr *KShiftAmt =
2376  getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) {
2377  return narrowScalarShiftByConstant(
2378  MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy);
2379  }
2380 
2381  // TODO: Expand with known bits.
2382 
2383  // Handle the fully general expansion by an unknown amount.
2384  auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize);
2385 
2386  unsigned InL = MRI.createGenericVirtualRegister(HalfTy);
2387  unsigned InH = MRI.createGenericVirtualRegister(HalfTy);
2388  MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg());
2389 
2390  auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits);
2391  auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt);
2392 
2393  auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0);
2394  auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits);
2395  auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero);
2396 
2397  unsigned ResultRegs[2];
2398  switch (MI.getOpcode()) {
2399  case TargetOpcode::G_SHL: {
2400  // Short: ShAmt < NewBitSize
2401  auto LoS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2402 
2403  auto OrLHS = MIRBuilder.buildShl(HalfTy, InH, Amt);
2404  auto OrRHS = MIRBuilder.buildLShr(HalfTy, InL, AmtLack);
2405  auto HiS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2406 
2407  // Long: ShAmt >= NewBitSize
2408  auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero.
2409  auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part.
2410 
2411  auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL);
2412  auto Hi = MIRBuilder.buildSelect(
2413  HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL));
2414 
2415  ResultRegs[0] = Lo.getReg(0);
2416  ResultRegs[1] = Hi.getReg(0);
2417  break;
2418  }
2419  case TargetOpcode::G_LSHR: {
2420  // Short: ShAmt < NewBitSize
2421  auto HiS = MIRBuilder.buildLShr(HalfTy, InH, Amt);
2422 
2423  auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2424  auto OrRHS = MIRBuilder.buildShl(HalfTy, InH, AmtLack);
2425  auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2426 
2427  // Long: ShAmt >= NewBitSize
2428  auto HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero.
2429  auto LoL = MIRBuilder.buildLShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2430 
2431  auto Lo = MIRBuilder.buildSelect(
2432  HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2433  auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2434 
2435  ResultRegs[0] = Lo.getReg(0);
2436  ResultRegs[1] = Hi.getReg(0);
2437  break;
2438  }
2439  case TargetOpcode::G_ASHR: {
2440  // Short: ShAmt < NewBitSize
2441  auto HiS = MIRBuilder.buildAShr(HalfTy, InH, Amt);
2442 
2443  auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt);
2444  auto OrRHS = MIRBuilder.buildLShr(HalfTy, InH, AmtLack);
2445  auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS);
2446 
2447  // Long: ShAmt >= NewBitSize
2448 
2449  // Sign of Hi part.
2450  auto HiL = MIRBuilder.buildAShr(
2451  HalfTy, InH, MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1));
2452 
2453  auto LoL = MIRBuilder.buildAShr(HalfTy, InH, AmtExcess); // Lo from Hi part.
2454 
2455  auto Lo = MIRBuilder.buildSelect(
2456  HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL));
2457 
2458  auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL);
2459 
2460  ResultRegs[0] = Lo.getReg(0);
2461  ResultRegs[1] = Hi.getReg(0);
2462  break;
2463  }
2464  default:
2465  llvm_unreachable("not a shift");
2466  }
2467 
2468  MIRBuilder.buildMerge(DstReg, ResultRegs);
2469  MI.eraseFromParent();
2470  return Legalized;
2471 }
2472 
2474 LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx,
2475  LLT MoreTy) {
2476  assert(TypeIdx == 0 && "Expecting only Idx 0");
2477 
2478  Observer.changingInstr(MI);
2479  for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) {
2480  MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB();
2481  MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator());
2482  moreElementsVectorSrc(MI, MoreTy, I);
2483  }
2484 
2485  MachineBasicBlock &MBB = *MI.getParent();
2486  MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI());
2487  moreElementsVectorDst(MI, MoreTy, 0);
2488  Observer.changedInstr(MI);
2489  return Legalized;
2490 }
2491 
2494  LLT MoreTy) {
2495  MIRBuilder.setInstr(MI);
2496  unsigned Opc = MI.getOpcode();
2497  switch (Opc) {
2498  case TargetOpcode::G_IMPLICIT_DEF: {
2499  Observer.changingInstr(MI);
2500  moreElementsVectorDst(MI, MoreTy, 0);
2501  Observer.changedInstr(MI);
2502  return Legalized;
2503  }
2504  case TargetOpcode::G_AND:
2505  case TargetOpcode::G_OR:
2506  case TargetOpcode::G_XOR: {
2507  Observer.changingInstr(MI);
2508  moreElementsVectorSrc(MI, MoreTy, 1);
2509  moreElementsVectorSrc(MI, MoreTy, 2);
2510  moreElementsVectorDst(MI, MoreTy, 0);
2511  Observer.changedInstr(MI);
2512  return Legalized;
2513  }
2514  case TargetOpcode::G_EXTRACT:
2515  if (TypeIdx != 1)
2516  return UnableToLegalize;
2517  Observer.changingInstr(MI);
2518  moreElementsVectorSrc(MI, MoreTy, 1);
2519  Observer.changedInstr(MI);
2520  return Legalized;
2521  case TargetOpcode::G_INSERT:
2522  if (TypeIdx != 0)
2523  return UnableToLegalize;
2524  Observer.changingInstr(MI);
2525  moreElementsVectorSrc(MI, MoreTy, 1);
2526  moreElementsVectorDst(MI, MoreTy, 0);
2527  Observer.changedInstr(MI);
2528  return Legalized;
2529  case TargetOpcode::G_SELECT:
2530  if (TypeIdx != 0)
2531  return UnableToLegalize;
2532  if (MRI.getType(MI.getOperand(1).getReg()).isVector())
2533  return UnableToLegalize;
2534 
2535  Observer.changingInstr(MI);
2536  moreElementsVectorSrc(MI, MoreTy, 2);
2537  moreElementsVectorSrc(MI, MoreTy, 3);
2538  moreElementsVectorDst(MI, MoreTy, 0);
2539  Observer.changedInstr(MI);
2540  return Legalized;
2541  case TargetOpcode::G_PHI:
2542  return moreElementsVectorPhi(MI, TypeIdx, MoreTy);
2543  default:
2544  return UnableToLegalize;
2545  }
2546 }
2547 
2548 void LegalizerHelper::multiplyRegisters(SmallVectorImpl<unsigned> &DstRegs,
2549  ArrayRef<unsigned> Src1Regs,
2550  ArrayRef<unsigned> Src2Regs,
2551  LLT NarrowTy) {
2553  unsigned SrcParts = Src1Regs.size();
2554  unsigned DstParts = DstRegs.size();
2555 
2556  unsigned DstIdx = 0; // Low bits of the result.
2557  unsigned FactorSum =
2558  B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
2559  DstRegs[DstIdx] = FactorSum;
2560 
2561  unsigned CarrySumPrevDstIdx;
2562  SmallVector<unsigned, 4> Factors;
2563 
2564  for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
2565  // Collect low parts of muls for DstIdx.
2566  for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
2567  i <= std::min(DstIdx, SrcParts - 1); ++i) {
2568  MachineInstrBuilder Mul =
2569  B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
2570  Factors.push_back(Mul.getReg(0));
2571  }
2572  // Collect high parts of muls from previous DstIdx.
2573  for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
2574  i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
2575  MachineInstrBuilder Umulh =
2576  B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
2577  Factors.push_back(Umulh.getReg(0));
2578  }
2579  // Add CarrySum from additons calculated for previous DstIdx.
2580  if (DstIdx != 1) {
2581  Factors.push_back(CarrySumPrevDstIdx);
2582  }
2583 
2584  unsigned CarrySum = 0;
2585  // Add all factors and accumulate all carries into CarrySum.
2586  if (DstIdx != DstParts - 1) {
2587  MachineInstrBuilder Uaddo =
2588  B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]);
2589  FactorSum = Uaddo.getReg(0);
2590  CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0);
2591  for (unsigned i = 2; i < Factors.size(); ++i) {
2592  MachineInstrBuilder Uaddo =
2593  B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]);
2594  FactorSum = Uaddo.getReg(0);
2595  MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1));
2596  CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
2597  }
2598  } else {
2599  // Since value for the next index is not calculated, neither is CarrySum.
2600  FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
2601  for (unsigned i = 2; i < Factors.size(); ++i)
2602  FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
2603  }
2604 
2605  CarrySumPrevDstIdx = CarrySum;
2606  DstRegs[DstIdx] = FactorSum;
2607  Factors.clear();
2608  }
2609 }
2610 
2612 LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) {
2613  unsigned DstReg = MI.getOperand(0).getReg();
2614  unsigned Src1 = MI.getOperand(1).getReg();
2615  unsigned Src2 = MI.getOperand(2).getReg();
2616 
2617  LLT Ty = MRI.getType(DstReg);
2618  if (Ty.isVector())
2619  return UnableToLegalize;
2620 
2621  unsigned SrcSize = MRI.getType(Src1).getSizeInBits();
2622  unsigned DstSize = Ty.getSizeInBits();
2623  unsigned NarrowSize = NarrowTy.getSizeInBits();
2624  if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0)
2625  return UnableToLegalize;
2626 
2627  unsigned NumDstParts = DstSize / NarrowSize;
2628  unsigned NumSrcParts = SrcSize / NarrowSize;
2629  bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH;
2630  unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1);
2631 
2632  SmallVector<unsigned, 2> Src1Parts, Src2Parts, DstTmpRegs;
2633  extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts);
2634  extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts);
2635  DstTmpRegs.resize(DstTmpParts);
2636  multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
2637 
2638  // Take only high half of registers if this is high mul.
2639  ArrayRef<unsigned> DstRegs(
2640  IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts);
2641  MIRBuilder.buildMerge(DstReg, DstRegs);
2642  MI.eraseFromParent();
2643  return Legalized;
2644 }
2645 
2647 LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx,
2648  LLT NarrowTy) {
2649  if (TypeIdx != 1)
2650  return UnableToLegalize;
2651 
2652  uint64_t NarrowSize = NarrowTy.getSizeInBits();
2653 
2654  int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits();
2655  // FIXME: add support for when SizeOp1 isn't an exact multiple of
2656  // NarrowSize.
2657  if (SizeOp1 % NarrowSize != 0)
2658  return UnableToLegalize;
2659  int NumParts = SizeOp1 / NarrowSize;
2660 
2661  SmallVector<unsigned, 2> SrcRegs, DstRegs;
2662  SmallVector<uint64_t, 2> Indexes;
2663  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2664 
2665  unsigned OpReg = MI.getOperand(0).getReg();
2666  uint64_t OpStart = MI.getOperand(2).getImm();
2667  uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2668  for (int i = 0; i < NumParts; ++i) {
2669  unsigned SrcStart = i * NarrowSize;
2670 
2671  if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
2672  // No part of the extract uses this subregister, ignore it.
2673  continue;
2674  } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2675  // The entire subregister is extracted, forward the value.
2676  DstRegs.push_back(SrcRegs[i]);
2677  continue;
2678  }
2679 
2680  // OpSegStart is where this destination segment would start in OpReg if it
2681  // extended infinitely in both directions.
2682  int64_t ExtractOffset;
2683  uint64_t SegSize;
2684  if (OpStart < SrcStart) {
2685  ExtractOffset = 0;
2686  SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
2687  } else {
2688  ExtractOffset = OpStart - SrcStart;
2689  SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
2690  }
2691 
2692  unsigned SegReg = SrcRegs[i];
2693  if (ExtractOffset != 0 || SegSize != NarrowSize) {
2694  // A genuine extract is needed.
2695  SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2696  MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset);
2697  }
2698 
2699  DstRegs.push_back(SegReg);
2700  }
2701 
2702  unsigned DstReg = MI.getOperand(0).getReg();
2703  if(MRI.getType(DstReg).isVector())
2704  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2705  else
2706  MIRBuilder.buildMerge(DstReg, DstRegs);
2707  MI.eraseFromParent();
2708  return Legalized;
2709 }
2710 
2712 LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx,
2713  LLT NarrowTy) {
2714  // FIXME: Don't know how to handle secondary types yet.
2715  if (TypeIdx != 0)
2716  return UnableToLegalize;
2717 
2718  uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
2719  uint64_t NarrowSize = NarrowTy.getSizeInBits();
2720 
2721  // FIXME: add support for when SizeOp0 isn't an exact multiple of
2722  // NarrowSize.
2723  if (SizeOp0 % NarrowSize != 0)
2724  return UnableToLegalize;
2725 
2726  int NumParts = SizeOp0 / NarrowSize;
2727 
2728  SmallVector<unsigned, 2> SrcRegs, DstRegs;
2729  SmallVector<uint64_t, 2> Indexes;
2730  extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs);
2731 
2732  unsigned OpReg = MI.getOperand(2).getReg();
2733  uint64_t OpStart = MI.getOperand(3).getImm();
2734  uint64_t OpSize = MRI.getType(OpReg).getSizeInBits();
2735  for (int i = 0; i < NumParts; ++i) {
2736  unsigned DstStart = i * NarrowSize;
2737 
2738  if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
2739  // No part of the insert affects this subregister, forward the original.
2740  DstRegs.push_back(SrcRegs[i]);
2741  continue;
2742  } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) {
2743  // The entire subregister is defined by this insert, forward the new
2744  // value.
2745  DstRegs.push_back(OpReg);
2746  continue;
2747  }
2748 
2749  // OpSegStart is where this destination segment would start in OpReg if it
2750  // extended infinitely in both directions.
2751  int64_t ExtractOffset, InsertOffset;
2752  uint64_t SegSize;
2753  if (OpStart < DstStart) {
2754  InsertOffset = 0;
2755  ExtractOffset = DstStart - OpStart;
2756  SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
2757  } else {
2758  InsertOffset = OpStart - DstStart;
2759  ExtractOffset = 0;
2760  SegSize =
2761  std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
2762  }
2763 
2764  unsigned SegReg = OpReg;
2765  if (ExtractOffset != 0 || SegSize != OpSize) {
2766  // A genuine extract is needed.
2767  SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize));
2768  MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset);
2769  }
2770 
2771  unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy);
2772  MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset);
2773  DstRegs.push_back(DstReg);
2774  }
2775 
2776  assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered");
2777  unsigned DstReg = MI.getOperand(0).getReg();
2778  if(MRI.getType(DstReg).isVector())
2779  MIRBuilder.buildBuildVector(DstReg, DstRegs);
2780  else
2781  MIRBuilder.buildMerge(DstReg, DstRegs);
2782  MI.eraseFromParent();
2783  return Legalized;
2784 }
2785 
2787 LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx,
2788  LLT NarrowTy) {
2789  if (TypeIdx != 0)
2790  return UnableToLegalize;
2791 
2792  unsigned CondReg = MI.getOperand(1).getReg();
2793  LLT CondTy = MRI.getType(CondReg);
2794  if (CondTy.isVector()) // TODO: Handle vselect
2795  return UnableToLegalize;
2796 
2797  unsigned DstReg = MI.getOperand(0).getReg();
2798  LLT DstTy = MRI.getType(DstReg);
2799 
2800  SmallVector<unsigned, 4> DstRegs, DstLeftoverRegs;
2801  SmallVector<unsigned, 4> Src1Regs, Src1LeftoverRegs;
2802  SmallVector<unsigned, 4> Src2Regs, Src2LeftoverRegs;
2803  LLT LeftoverTy;
2804  if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
2805  Src1Regs, Src1LeftoverRegs))
2806  return UnableToLegalize;
2807 
2808  LLT Unused;
2809  if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
2810  Src2Regs, Src2LeftoverRegs))
2811  llvm_unreachable("inconsistent extractParts result");
2812 
2813  for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) {
2814  auto Select = MIRBuilder.buildSelect(NarrowTy,
2815  CondReg, Src1Regs[I], Src2Regs[I]);
2816  DstRegs.push_back(Select->getOperand(0).getReg());
2817  }
2818 
2819  for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) {
2820  auto Select = MIRBuilder.buildSelect(
2821  LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]);
2822  DstLeftoverRegs.push_back(Select->getOperand(0).getReg());
2823  }
2824 
2825  insertParts(DstReg, DstTy, NarrowTy, DstRegs,
2826  LeftoverTy, DstLeftoverRegs);
2827 
2828  MI.eraseFromParent();
2829  return Legalized;
2830 }
2831 
2833 LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) {
2834  unsigned Opc = MI.getOpcode();
2835  auto &TII = *MI.getMF()->getSubtarget().getInstrInfo();
2836  auto isSupported = [this](const LegalityQuery &Q) {
2837  auto QAction = LI.getAction(Q).Action;
2838  return QAction == Legal || QAction == Libcall || QAction == Custom;
2839  };
2840  switch (Opc) {
2841  default:
2842  return UnableToLegalize;
2843  case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
2844  // This trivially expands to CTLZ.
2845  Observer.changingInstr(MI);
2846  MI.setDesc(TII.get(TargetOpcode::G_CTLZ));
2847  Observer.changedInstr(MI);
2848  return Legalized;
2849  }
2850  case TargetOpcode::G_CTLZ: {
2851  unsigned SrcReg = MI.getOperand(1).getReg();
2852  unsigned Len = Ty.getSizeInBits();
2853  if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) {
2854  // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero.
2855  auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF,
2856  {Ty}, {SrcReg});
2857  auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
2858  auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
2859  auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
2860  SrcReg, MIBZero);
2861  MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
2862  MIBCtlzZU);
2863  MI.eraseFromParent();
2864  return Legalized;
2865  }
2866  // for now, we do this:
2867  // NewLen = NextPowerOf2(Len);
2868  // x = x | (x >> 1);
2869  // x = x | (x >> 2);
2870  // ...
2871  // x = x | (x >>16);
2872  // x = x | (x >>32); // for 64-bit input
2873  // Upto NewLen/2
2874  // return Len - popcount(x);
2875  //
2876  // Ref: "Hacker's Delight" by Henry Warren
2877  unsigned Op = SrcReg;
2878  unsigned NewLen = PowerOf2Ceil(Len);
2879  for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
2880  auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i);
2881  auto MIBOp = MIRBuilder.buildInstr(
2882  TargetOpcode::G_OR, {Ty},
2883  {Op, MIRBuilder.buildInstr(TargetOpcode::G_LSHR, {Ty},
2884  {Op, MIBShiftAmt})});
2885  Op = MIBOp->getOperand(0).getReg();
2886  }
2887  auto MIBPop = MIRBuilder.buildInstr(TargetOpcode::G_CTPOP, {Ty}, {Op});
2888  MIRBuilder.buildInstr(TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
2889  {MIRBuilder.buildConstant(Ty, Len), MIBPop});
2890  MI.eraseFromParent();
2891  return Legalized;
2892  }
2893  case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
2894  // This trivially expands to CTTZ.
2895  Observer.changingInstr(MI);
2896  MI.setDesc(TII.get(TargetOpcode::G_CTTZ));
2897  Observer.changedInstr(MI);
2898  return Legalized;
2899  }
2900  case TargetOpcode::G_CTTZ: {
2901  unsigned SrcReg = MI.getOperand(1).getReg();
2902  unsigned Len = Ty.getSizeInBits();
2903  if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) {
2904  // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with
2905  // zero.
2906  auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF,
2907  {Ty}, {SrcReg});
2908  auto MIBZero = MIRBuilder.buildConstant(Ty, 0);
2909  auto MIBLen = MIRBuilder.buildConstant(Ty, Len);
2910  auto MIBICmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LLT::scalar(1),
2911  SrcReg, MIBZero);
2912  MIRBuilder.buildSelect(MI.getOperand(0).getReg(), MIBICmp, MIBLen,
2913  MIBCttzZU);
2914  MI.eraseFromParent();
2915  return Legalized;
2916  }
2917  // for now, we use: { return popcount(~x & (x - 1)); }
2918  // unless the target has ctlz but not ctpop, in which case we use:
2919  // { return 32 - nlz(~x & (x-1)); }
2920  // Ref: "Hacker's Delight" by Henry Warren
2921  auto MIBCstNeg1 = MIRBuilder.buildConstant(Ty, -1);
2922  auto MIBNot =
2923  MIRBuilder.buildInstr(TargetOpcode::G_XOR, {Ty}, {SrcReg, MIBCstNeg1});
2924  auto MIBTmp = MIRBuilder.buildInstr(
2925  TargetOpcode::G_AND, {Ty},
2926  {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty},
2927  {SrcReg, MIBCstNeg1})});
2928  if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) &&
2929  isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) {
2930  auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len);
2932  TargetOpcode::G_SUB, {MI.getOperand(0).getReg()},
2933  {MIBCstLen,
2934  MIRBuilder.buildInstr(TargetOpcode::G_CTLZ, {Ty}, {MIBTmp})});
2935  MI.eraseFromParent();
2936  return Legalized;
2937  }
2938  MI.setDesc(TII.get(TargetOpcode::G_CTPOP));
2939  MI.getOperand(1).setReg(MIBTmp->getOperand(0).getReg());
2940  return Legalized;
2941  }
2942  }
2943 }
static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType)
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
static Type * getDoubleTy(LLVMContext &C)
Definition: Type.cpp:164
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1562
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:833
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ...
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:561
MachineBasicBlock * getMBB() const
const MachineFunction * getMF() const
Return the function that contains the basic block that this instruction belongs to.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
unsigned getScalarSizeInBits() const
The operation should be implemented in terms of a wider scalar base-type.
Definition: LegalizerInfo.h:57
void setFPImm(const ConstantFP *CFP)
AtomicOrdering getFailureOrdering() const
For cmpxchg atomic operations, return the atomic ordering requirements when store does not occur...
void push_back(const T &Elt)
Definition: SmallVector.h:211
The LegalityQuery object bundles together all the information that&#39;s needed to decide whether a given...
bool isScalar() const
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
unsigned getReg() const
getReg - Returns the register number.
unsigned Reg
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
Definition: LegalizerInfo.h:62
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
virtual const TargetLowering * getTargetLowering() const
unsigned less than
Definition: InstrTypes.h:671
LLT getScalarType() const
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_OR Op0, Op1.
LLT getType(unsigned Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register...
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< unsigned > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:810
bool isNonIntegralAddressSpace(unsigned AddrSpace) const
Definition: DataLayout.h:371
static uint32_t Concat[]
MachineInstrBuilder buildUAddo(const DstOp &Res, const DstOp &CarryOut, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res, CarryOut = G_UADDO Op0, Op1.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert `Res0, ...
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
The operation should be synthesized from multiple instructions acting on a narrower scalar base-type...
Definition: LegalizerInfo.h:52
MachineInstrBuilder buildStore(unsigned Val, unsigned Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
bool isVector() const
void setMF(MachineFunction &MF)
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
A description of a memory reference used in the backend.
bool isSigned() const
Definition: InstrTypes.h:816
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions. ...
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy)
Legalize an instruction by performing the operation on a wider scalar type (for example a 16-bit addi...
const HexagonInstrInfo * TII
static Type * getFloatTy(LLVMContext &C)
Definition: Type.cpp:163
const ConstantFP * getFPImm() const
unsigned getNumOperands() const
Retuns the total number of operands.
Definition: MachineInstr.h:411
MachineInstrBuilder buildUAdde(const DstOp &Res, const DstOp &CarryOut, const SrcOp &Op0, const SrcOp &Op1, const SrcOp &CarryIn)
Build and insert Res, CarryOut = G_UADDE Op0, Op1, CarryIn.
const MachineInstrBuilder & addUse(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Res = COPY Op depending on the differing sizes of Res and Op.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:408
LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
Legalize a vector instruction by increasing the number of vector elements involved and ignoring the a...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
LLT getElementType() const
Returns the vector&#39;s element type. Only valid for vector types.
LegalizerHelper::LegalizeResult createLibcall(MachineIRBuilder &MIRBuilder, RTLIB::Libcall Libcall, const CallLowering::ArgInfo &Result, ArrayRef< CallLowering::ArgInfo > Args)
Helper function that creates the given libcall.
AtomicOrdering getOrdering() const
Return the atomic ordering requirements for this memory operation.
The (vector) operation should be implemented by widening the input vector and ignoring the lanes adde...
Definition: LegalizerInfo.h:68
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:4443
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don&#39;t insert <empty> = Opcode <empty>.
MachineFunction & getMF()
Getter for the function we currently build.
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:137
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
virtual const TargetInstrInfo * getInstrInfo() const
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:122
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_SUB Op0, Op1.
MachineInstr * getOpcodeDef(unsigned Opcode, unsigned Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition: Utils.cpp:258
void setChangeObserver(GISelChangeObserver &Observer)
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &B)
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
Abstract class that contains various methods for clients to notify about changes. ...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
unsigned const MachineRegisterInfo * MRI
static LLT scalarOrVector(uint16_t NumElements, LLT ScalarTy)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:428
LegalizeResult legalizeInstrStep(MachineInstr &MI)
Replace MI by a sequence of legal instructions that can implement the same operation.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
unsigned getReg() const
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:263
Helper class to build MachineInstr.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:587
MachineInstrBuilder buildUMulH(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
bool isValid() const
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:646
void setImm(int64_t immVal)
MachineInstrBuilder buildInsert(unsigned Res, unsigned Src, unsigned Op, unsigned Index)
virtual const CallLowering * getCallLowering() const
unsigned getAddressSpace() const
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:288
void print(raw_ostream &OS, bool IsStandalone=true, bool SkipOpers=false, bool SkipDebugLoc=false, bool AddNewLine=true, const TargetInstrInfo *TII=nullptr) const
Print this MI to OS.
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function. ...
Definition: Function.cpp:192
Some kind of error has occurred and we could not legalize this instruction.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
Instruction was already legal and no change was made to the MachineFunction.
size_t size() const
Definition: SmallVector.h:52
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:970
static Type * getFP128Ty(LLVMContext &C)
Definition: Type.cpp:168
const APFloat & getValueAPF() const
Definition: Constants.h:302
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
unsigned createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
static std::pair< int, int > getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
Try to break down OrigTy into NarrowTy sized pieces.
static Type * getHalfTy(LLVMContext &C)
Definition: Type.cpp:162
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
Definition: Type.cpp:239
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:119
MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef< unsigned > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ...
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
void setDesc(const MCInstrDesc &tid)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one...
This is the shared class of boolean and integer constants.
Definition: Constants.h:83
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
LegalizeResult libcall(MachineInstr &MI)
Legalize an instruction by emiting a runtime library call instead.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
Definition: MachineInstr.h:533
void setFlags(unsigned flags)
Definition: MachineInstr.h:303
static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size)
MachineOperand class - Representation of each machine instruction operand.
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
The target wants to do something special with this combination of operand and type.
Definition: LegalizerInfo.h:81
LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by splitting it into simpler parts, hopefully understood by the target...
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
Definition: Constants.cpp:694
LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a vector instruction by splitting into multiple components, each acting on the same scalar t...
int64_t getImm() const
const Function & getFunction() const
Return the LLVM function that this machine code represents.
virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Class for arbitrary precision integers.
Definition: APInt.h:69
static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType)
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
static MachineOperand CreateES(const char *SymName, unsigned char TargetFlags=0)
unsigned getBoolExtOp(bool IsVec, bool IsFP) const
static LegalizerHelper::LegalizeResult conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, Type *FromType)
bool isPointer() const
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:253
Representation of each machine instruction.
Definition: MachineInstr.h:63
bool ugt(const APInt &RHS) const
Unsigned greather than comparison.
Definition: APInt.h:1254
Instruction has been legalized and the MachineFunction changed.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_FCMP PredOp0, Op1.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< unsigned > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
void setReg(unsigned Reg)
Change the register this operand corresponds to.
#define I(x, y, z)
Definition: MD5.cpp:58
static Constant * getZeroValueForNegation(Type *Ty)
Floating point negation must be implemented with f(x) = -0.0 - x.
Definition: Constants.cpp:780
uint32_t Size
Definition: Profile.cpp:46
void setCImm(const ConstantInt *CI)
const DataLayout & getDataLayout() const
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Optional< MachineInstrBuilder > materializeGEP(unsigned &Res, unsigned Op0, const LLT &ValueTy, uint64_t Value)
Materialize and insert Res = G_GEP Op0, (G_CONSTANT Value)
uint16_t getFlags() const
Return the MI flags bitvector.
Definition: MachineInstr.h:289
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize an instruction by reducing the width of the underlying scalar type.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const
Get the size in bits of Reg.
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
This file describes how to lower LLVM calls to machine code calls.
MachineInstrBuilder buildLoad(unsigned Res, unsigned Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, Optional< unsigned > Flags=None)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
unsigned getReg(unsigned Idx)
Get the register for the operand index.
MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, unsigned CmpVal, unsigned NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, MMO.
IRTranslator LLVM IR MI
const MachineInstrBuilder & addDef(unsigned RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
#define LLVM_DEBUG(X)
Definition: Debug.h:122
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:413
const ConstantInt * getCImm() const
The operation is expected to be selectable directly by the target, and no transformation is necessary...
Definition: LegalizerInfo.h:47
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
bool isNullValue() const
Determine if all bits are clear.
Definition: APInt.h:405
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:143
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
Definition: MathExtras.h:658
This file describes how to lower LLVM code to machine code.
unsigned getPredicate() const