LLVM  16.0.0git
PPCInstrInfo.cpp
Go to the documentation of this file.
1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "PPCInstrInfo.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/DenseSet.h"
21 #include "llvm/ADT/STLExtras.h"
22 #include "llvm/ADT/Statistic.h"
37 #include "llvm/CodeGen/StackMaps.h"
38 #include "llvm/MC/MCAsmInfo.h"
39 #include "llvm/MC/MCInst.h"
40 #include "llvm/MC/TargetRegistry.h"
42 #include "llvm/Support/Debug.h"
45 
46 using namespace llvm;
47 
48 #define DEBUG_TYPE "ppc-instr-info"
49 
50 #define GET_INSTRMAP_INFO
51 #define GET_INSTRINFO_CTOR_DTOR
52 #include "PPCGenInstrInfo.inc"
53 
54 STATISTIC(NumStoreSPILLVSRRCAsVec,
55  "Number of spillvsrrc spilled to stack as vec");
56 STATISTIC(NumStoreSPILLVSRRCAsGpr,
57  "Number of spillvsrrc spilled to stack as gpr");
58 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
59 STATISTIC(CmpIselsConverted,
60  "Number of ISELs that depend on comparison of constants converted");
61 STATISTIC(MissedConvertibleImmediateInstrs,
62  "Number of compare-immediate instructions fed by constants");
63 STATISTIC(NumRcRotatesConvertedToRcAnd,
64  "Number of record-form rotates converted to record-form andi");
65 
66 static cl::
67 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
68  cl::desc("Disable analysis for CTR loops"));
69 
70 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
71 cl::desc("Disable compare instruction optimization"), cl::Hidden);
72 
73 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
74 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
75 cl::Hidden);
76 
77 static cl::opt<bool>
78 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
79  cl::desc("Use the old (incorrect) instruction latency calculation"));
80 
81 static cl::opt<float>
82  FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
83  cl::desc("register pressure factor for the transformations."));
84 
86  "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
87  cl::desc("enable register pressure reduce in machine combiner pass."));
88 
89 // Pin the vtable to this file.
90 void PPCInstrInfo::anchor() {}
91 
93  : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
94  /* CatchRetOpcode */ -1,
95  STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
96  Subtarget(STI), RI(STI.getTargetMachine()) {}
97 
98 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
99 /// this target when scheduling the DAG.
102  const ScheduleDAG *DAG) const {
103  unsigned Directive =
104  static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
107  const InstrItineraryData *II =
108  static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
109  return new ScoreboardHazardRecognizer(II, DAG);
110  }
111 
113 }
114 
115 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
116 /// to use for this target when scheduling the DAG.
119  const ScheduleDAG *DAG) const {
120  unsigned Directive =
121  DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
122 
123  // FIXME: Leaving this as-is until we have POWER9 scheduling info
125  return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
126 
127  // Most subtargets use a PPC970 recognizer.
130  assert(DAG->TII && "No InstrInfo?");
131 
132  return new PPCHazardRecognizer970(*DAG);
133  }
134 
135  return new ScoreboardHazardRecognizer(II, DAG);
136 }
137 
139  const MachineInstr &MI,
140  unsigned *PredCost) const {
141  if (!ItinData || UseOldLatencyCalc)
142  return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
143 
144  // The default implementation of getInstrLatency calls getStageLatency, but
145  // getStageLatency does not do the right thing for us. While we have
146  // itinerary, most cores are fully pipelined, and so the itineraries only
147  // express the first part of the pipeline, not every stage. Instead, we need
148  // to use the listed output operand cycle number (using operand 0 here, which
149  // is an output).
150 
151  unsigned Latency = 1;
152  unsigned DefClass = MI.getDesc().getSchedClass();
153  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
154  const MachineOperand &MO = MI.getOperand(i);
155  if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
156  continue;
157 
158  int Cycle = ItinData->getOperandCycle(DefClass, i);
159  if (Cycle < 0)
160  continue;
161 
162  Latency = std::max(Latency, (unsigned) Cycle);
163  }
164 
165  return Latency;
166 }
167 
169  const MachineInstr &DefMI, unsigned DefIdx,
170  const MachineInstr &UseMI,
171  unsigned UseIdx) const {
172  int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
173  UseMI, UseIdx);
174 
175  if (!DefMI.getParent())
176  return Latency;
177 
178  const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
179  Register Reg = DefMO.getReg();
180 
181  bool IsRegCR;
183  const MachineRegisterInfo *MRI =
184  &DefMI.getParent()->getParent()->getRegInfo();
185  IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
186  MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
187  } else {
188  IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
189  PPC::CRBITRCRegClass.contains(Reg);
190  }
191 
192  if (UseMI.isBranch() && IsRegCR) {
193  if (Latency < 0)
194  Latency = getInstrLatency(ItinData, DefMI);
195 
196  // On some cores, there is an additional delay between writing to a condition
197  // register, and using it from a branch.
198  unsigned Directive = Subtarget.getCPUDirective();
199  switch (Directive) {
200  default: break;
201  case PPC::DIR_7400:
202  case PPC::DIR_750:
203  case PPC::DIR_970:
204  case PPC::DIR_E5500:
205  case PPC::DIR_PWR4:
206  case PPC::DIR_PWR5:
207  case PPC::DIR_PWR5X:
208  case PPC::DIR_PWR6:
209  case PPC::DIR_PWR6X:
210  case PPC::DIR_PWR7:
211  case PPC::DIR_PWR8:
212  // FIXME: Is this needed for POWER9?
213  Latency += 2;
214  break;
215  }
216  }
217 
218  return Latency;
219 }
220 
221 /// This is an architecture-specific helper function of reassociateOps.
222 /// Set special operand attributes for new instructions after reassociation.
224  MachineInstr &OldMI2,
225  MachineInstr &NewMI1,
226  MachineInstr &NewMI2) const {
227  // Propagate FP flags from the original instructions.
228  // But clear poison-generating flags because those may not be valid now.
229  uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
230  NewMI1.setFlags(IntersectedFlags);
231  NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
232  NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
233  NewMI1.clearFlag(MachineInstr::MIFlag::IsExact);
234 
235  NewMI2.setFlags(IntersectedFlags);
236  NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap);
237  NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap);
238  NewMI2.clearFlag(MachineInstr::MIFlag::IsExact);
239 }
240 
242  uint16_t Flags) const {
243  MI.setFlags(Flags);
244  MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
245  MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
246  MI.clearFlag(MachineInstr::MIFlag::IsExact);
247 }
248 
249 // This function does not list all associative and commutative operations, but
250 // only those worth feeding through the machine combiner in an attempt to
251 // reduce the critical path. Mostly, this means floating-point operations,
252 // because they have high latencies(>=5) (compared to other operations, such as
253 // and/or, which are also associative and commutative, but have low latencies).
255  switch (Inst.getOpcode()) {
256  // Floating point:
257  // FP Add:
258  case PPC::FADD:
259  case PPC::FADDS:
260  // FP Multiply:
261  case PPC::FMUL:
262  case PPC::FMULS:
263  // Altivec Add:
264  case PPC::VADDFP:
265  // VSX Add:
266  case PPC::XSADDDP:
267  case PPC::XVADDDP:
268  case PPC::XVADDSP:
269  case PPC::XSADDSP:
270  // VSX Multiply:
271  case PPC::XSMULDP:
272  case PPC::XVMULDP:
273  case PPC::XVMULSP:
274  case PPC::XSMULSP:
275  return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
276  Inst.getFlag(MachineInstr::MIFlag::FmNsz);
277  // Fixed point:
278  // Multiply:
279  case PPC::MULHD:
280  case PPC::MULLD:
281  case PPC::MULHW:
282  case PPC::MULLW:
283  return true;
284  default:
285  return false;
286  }
287 }
288 
289 #define InfoArrayIdxFMAInst 0
290 #define InfoArrayIdxFAddInst 1
291 #define InfoArrayIdxFMULInst 2
292 #define InfoArrayIdxAddOpIdx 3
293 #define InfoArrayIdxMULOpIdx 4
294 #define InfoArrayIdxFSubInst 5
295 // Array keeps info for FMA instructions:
296 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
297 // Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
298 // Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
299 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
300 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
301 // second MUL operand index is plus 1;
302 // Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
303 static const uint16_t FMAOpIdxInfo[][6] = {
304  // FIXME: Add more FMA instructions like XSNMADDADP and so on.
305  {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
306  {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
307  {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
308  {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
309  {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
310  {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
311 
312 // Check if an opcode is a FMA instruction. If it is, return the index in array
313 // FMAOpIdxInfo. Otherwise, return -1.
314 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
315  for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
316  if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
317  return I;
318  return -1;
319 }
320 
321 // On PowerPC target, we have two kinds of patterns related to FMA:
322 // 1: Improve ILP.
323 // Try to reassociate FMA chains like below:
324 //
325 // Pattern 1:
326 // A = FADD X, Y (Leaf)
327 // B = FMA A, M21, M22 (Prev)
328 // C = FMA B, M31, M32 (Root)
329 // -->
330 // A = FMA X, M21, M22
331 // B = FMA Y, M31, M32
332 // C = FADD A, B
333 //
334 // Pattern 2:
335 // A = FMA X, M11, M12 (Leaf)
336 // B = FMA A, M21, M22 (Prev)
337 // C = FMA B, M31, M32 (Root)
338 // -->
339 // A = FMUL M11, M12
340 // B = FMA X, M21, M22
341 // D = FMA A, M31, M32
342 // C = FADD B, D
343 //
344 // breaking the dependency between A and B, allowing FMA to be executed in
345 // parallel (or back-to-back in a pipeline) instead of depending on each other.
346 //
347 // 2: Reduce register pressure.
348 // Try to reassociate FMA with FSUB and a constant like below:
349 // C is a floating point const.
350 //
351 // Pattern 1:
352 // A = FSUB X, Y (Leaf)
353 // D = FMA B, C, A (Root)
354 // -->
355 // A = FMA B, Y, -C
356 // D = FMA A, X, C
357 //
358 // Pattern 2:
359 // A = FSUB X, Y (Leaf)
360 // D = FMA B, A, C (Root)
361 // -->
362 // A = FMA B, Y, -C
363 // D = FMA A, X, C
364 //
365 // Before the transformation, A must be assigned with different hardware
366 // register with D. After the transformation, A and D must be assigned with
367 // same hardware register due to TIE attribute of FMA instructions.
368 //
371  bool DoRegPressureReduce) const {
372  MachineBasicBlock *MBB = Root.getParent();
375 
376  auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
377  for (const auto &MO : Instr.explicit_operands())
378  if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg())))
379  return false;
380  return true;
381  };
382 
383  auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
384  unsigned OpType) {
385  if (Instr.getOpcode() !=
386  FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
387  return false;
388 
389  // Instruction can be reassociated.
390  // fast math flags may prohibit reassociation.
391  if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
392  Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
393  return false;
394 
395  // Instruction operands are virtual registers for reassociation.
396  if (!IsAllOpsVirtualReg(Instr))
397  return false;
398 
399  // For register pressure reassociation, the FSub must have only one use as
400  // we want to delete the sub to save its def.
401  if (OpType == InfoArrayIdxFSubInst &&
402  !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
403  return false;
404 
405  return true;
406  };
407 
408  auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
409  int16_t &MulOpIdx, bool IsLeaf) {
410  int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
411  if (Idx < 0)
412  return false;
413 
414  // Instruction can be reassociated.
415  // fast math flags may prohibit reassociation.
416  if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
417  Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
418  return false;
419 
420  // Instruction operands are virtual registers for reassociation.
421  if (!IsAllOpsVirtualReg(Instr))
422  return false;
423 
424  MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
425  if (IsLeaf)
426  return true;
427 
428  AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
429 
430  const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
431  MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
432  // If 'add' operand's def is not in current block, don't do ILP related opt.
433  if (!MIAdd || MIAdd->getParent() != MBB)
434  return false;
435 
436  // If this is not Leaf FMA Instr, its 'add' operand should only have one use
437  // as this fma will be changed later.
438  return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
439  };
440 
441  int16_t AddOpIdx = -1;
442  int16_t MulOpIdx = -1;
443 
444  bool IsUsedOnceL = false;
445  bool IsUsedOnceR = false;
446  MachineInstr *MULInstrL = nullptr;
447  MachineInstr *MULInstrR = nullptr;
448 
449  auto IsRPReductionCandidate = [&]() {
450  // Currently, we only support float and double.
451  // FIXME: add support for other types.
452  unsigned Opcode = Root.getOpcode();
453  if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
454  return false;
455 
456  // Root must be a valid FMA like instruction.
457  // Treat it as leaf as we don't care its add operand.
458  if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
459  assert((MulOpIdx >= 0) && "mul operand index not right!");
461  Root.getOperand(MulOpIdx).getReg(), MRI);
463  Root.getOperand(MulOpIdx + 1).getReg(), MRI);
464  if (!MULRegL && !MULRegR)
465  return false;
466 
467  if (MULRegL && !MULRegR) {
468  MULRegR =
469  TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
470  IsUsedOnceL = true;
471  } else if (!MULRegL && MULRegR) {
472  MULRegL =
473  TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
474  IsUsedOnceR = true;
475  } else {
476  IsUsedOnceL = true;
477  IsUsedOnceR = true;
478  }
479 
480  if (!Register::isVirtualRegister(MULRegL) ||
481  !Register::isVirtualRegister(MULRegR))
482  return false;
483 
484  MULInstrL = MRI->getVRegDef(MULRegL);
485  MULInstrR = MRI->getVRegDef(MULRegR);
486  return true;
487  }
488  return false;
489  };
490 
491  // Register pressure fma reassociation patterns.
492  if (DoRegPressureReduce && IsRPReductionCandidate()) {
493  assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
494  // Register pressure pattern 1
495  if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
496  IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
497  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
498  Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BCA);
499  return true;
500  }
501 
502  // Register pressure pattern 2
503  if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
504  IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
505  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
506  Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BAC);
507  return true;
508  }
509  }
510 
511  // ILP fma reassociation patterns.
512  // Root must be a valid FMA like instruction.
513  AddOpIdx = -1;
514  if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
515  return false;
516 
517  assert((AddOpIdx >= 0) && "add operand index not right!");
518 
519  Register RegB = Root.getOperand(AddOpIdx).getReg();
520  MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
521 
522  // Prev must be a valid FMA like instruction.
523  AddOpIdx = -1;
524  if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
525  return false;
526 
527  assert((AddOpIdx >= 0) && "add operand index not right!");
528 
529  Register RegA = Prev->getOperand(AddOpIdx).getReg();
530  MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
531  AddOpIdx = -1;
532  if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
534  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
535  return true;
536  }
537  if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
538  Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
539  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
540  return true;
541  }
542  return false;
543 }
544 
547  SmallVectorImpl<MachineInstr *> &InsInstrs) const {
548  assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
549 
550  MachineFunction *MF = Root.getMF();
554 
555  int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
556  if (Idx < 0)
557  return;
558 
559  uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
560 
561  // For now we only need to fix up placeholder for register pressure reduce
562  // patterns.
563  Register ConstReg = 0;
564  switch (P) {
566  ConstReg =
567  TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
568  break;
570  ConstReg =
571  TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
572  break;
573  default:
574  // Not register pressure reduce patterns.
575  return;
576  }
577 
578  MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
579  // Get const value from const pool.
580  const Constant *C = getConstantFromConstantPool(ConstDefInstr);
581  assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
582 
583  // Get negative fp const.
584  APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
585  F1.changeSign();
586  Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
587  Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
588 
589  // Put negative fp const into constant pool.
590  unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
591 
592  MachineOperand *Placeholder = nullptr;
593  // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
594  for (auto *Inst : InsInstrs) {
595  for (MachineOperand &Operand : Inst->explicit_operands()) {
596  assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
597  if (Operand.getReg() == PPC::ZERO8) {
598  Placeholder = &Operand;
599  break;
600  }
601  }
602  }
603 
604  assert(Placeholder && "Placeholder does not exist!");
605 
606  // Generate instructions to load the const fp from constant pool.
607  // We only support PPC64 and medium code model.
608  Register LoadNewConst =
609  generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
610 
611  // Fill the placeholder with the new load from constant pool.
612  Placeholder->setReg(LoadNewConst);
613 }
614 
616  const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
617 
619  return false;
620 
621  // Currently, we only enable register pressure reducing in machine combiner
622  // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
623  // support.
624  //
625  // So we need following instructions to access a TOC entry:
626  //
627  // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
628  // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
629  // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
630  //
631  // FIXME: add more supported targets, like Small and Large code model, PPC32,
632  // AIX.
633  if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
635  return false;
636 
638  const MachineFunction *MF = MBB->getParent();
639  const MachineRegisterInfo *MRI = &MF->getRegInfo();
640 
641  auto GetMBBPressure =
642  [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
643  RegionPressure Pressure;
644  RegPressureTracker RPTracker(Pressure);
645 
646  // Initialize the register pressure tracker.
647  RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
648  /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
649 
650  for (const auto &MI : reverse(*MBB)) {
651  if (MI.isDebugValue() || MI.isDebugLabel())
652  continue;
653  RegisterOperands RegOpers;
654  RegOpers.collect(MI, *TRI, *MRI, false, false);
655  RPTracker.recedeSkipDebugValues();
656  assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
657  RPTracker.recede(RegOpers);
658  }
659 
660  // Close the RPTracker to finalize live ins.
661  RPTracker.closeRegion();
662 
663  return RPTracker.getPressure().MaxSetPressure;
664  };
665 
666  // For now we only care about float and double type fma.
667  unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
668  *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
669 
670  // Only reduce register pressure when pressure is high.
671  return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
672  (float)VSSRCLimit * FMARPFactor;
673 }
674 
675 bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
676  // I has only one memory operand which is load from constant pool.
677  if (!I->hasOneMemOperand())
678  return false;
679 
680  MachineMemOperand *Op = I->memoperands()[0];
681  return Op->isLoad() && Op->getPseudoValue() &&
682  Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
683 }
684 
685 Register PPCInstrInfo::generateLoadForNewConst(
686  unsigned Idx, MachineInstr *MI, Type *Ty,
687  SmallVectorImpl<MachineInstr *> &InsInstrs) const {
688  // Now we only support PPC64, Medium code model and P9 with vector.
689  // We have immutable pattern to access const pool. See function
690  // shouldReduceRegisterPressure.
691  assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
692  Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
693  "Target not supported!\n");
694 
695  MachineFunction *MF = MI->getMF();
697 
698  // Generate ADDIStocHA8
699  Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
700  MachineInstrBuilder TOCOffset =
701  BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
702  .addReg(PPC::X2)
703  .addConstantPoolIndex(Idx);
704 
705  assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
706  "Only float and double are supported!");
707 
708  unsigned LoadOpcode;
709  // Should be float type or double type.
710  if (Ty->isFloatTy())
711  LoadOpcode = PPC::DFLOADf32;
712  else
713  LoadOpcode = PPC::DFLOADf64;
714 
715  const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
716  Register VReg2 = MRI->createVirtualRegister(RC);
719  Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty));
720 
721  // Generate Load from constant pool.
723  BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
725  .addReg(VReg1, getKillRegState(true))
726  .addMemOperand(MMO);
727 
728  Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
729 
730  // Insert the toc load instructions into InsInstrs.
731  InsInstrs.insert(InsInstrs.begin(), Load);
732  InsInstrs.insert(InsInstrs.begin(), TOCOffset);
733  return VReg2;
734 }
735 
736 // This function returns the const value in constant pool if the \p I is a load
737 // from constant pool.
738 const Constant *
739 PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
740  MachineFunction *MF = I->getMF();
743  assert(I->mayLoad() && "Should be a load instruction.\n");
744  for (auto MO : I->uses()) {
745  if (!MO.isReg())
746  continue;
747  Register Reg = MO.getReg();
748  if (Reg == 0 || !Register::isVirtualRegister(Reg))
749  continue;
750  // Find the toc address.
752  for (auto MO2 : DefMI->uses())
753  if (MO2.isCPI())
754  return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
755  }
756  return nullptr;
757 }
758 
761  bool DoRegPressureReduce) const {
762  // Using the machine combiner in this way is potentially expensive, so
763  // restrict to when aggressive optimizations are desired.
765  return false;
766 
767  if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
768  return true;
769 
770  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
771  DoRegPressureReduce);
772 }
773 
778  DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
779  switch (Pattern) {
784  reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
785  break;
786  default:
787  // Reassociate default patterns.
789  DelInstrs, InstrIdxForVirtReg);
790  break;
791  }
792 }
793 
794 void PPCInstrInfo::reassociateFMA(
798  DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
799  MachineFunction *MF = Root.getMF();
802  MachineOperand &OpC = Root.getOperand(0);
803  Register RegC = OpC.getReg();
804  const TargetRegisterClass *RC = MRI.getRegClass(RegC);
805  MRI.constrainRegClass(RegC, RC);
806 
807  unsigned FmaOp = Root.getOpcode();
808  int16_t Idx = getFMAOpIdxInfo(FmaOp);
809  assert(Idx >= 0 && "Root must be a FMA instruction");
810 
811  bool IsILPReassociate =
814 
815  uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
816  uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
817 
818  MachineInstr *Prev = nullptr;
819  MachineInstr *Leaf = nullptr;
820  switch (Pattern) {
821  default:
822  llvm_unreachable("not recognized pattern!");
825  Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
826  Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
827  break;
829  Register MULReg =
830  TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
831  Leaf = MRI.getVRegDef(MULReg);
832  break;
833  }
835  Register MULReg = TRI->lookThruCopyLike(
836  Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
837  Leaf = MRI.getVRegDef(MULReg);
838  break;
839  }
840  }
841 
842  uint16_t IntersectedFlags = 0;
843  if (IsILPReassociate)
844  IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
845  else
846  IntersectedFlags = Root.getFlags() & Leaf->getFlags();
847 
848  auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
849  bool &KillFlag) {
850  Reg = Operand.getReg();
852  KillFlag = Operand.isKill();
853  };
854 
855  auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
856  Register &MulOp2, Register &AddOp,
857  bool &MulOp1KillFlag, bool &MulOp2KillFlag,
858  bool &AddOpKillFlag) {
859  GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
860  GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
861  GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
862  };
863 
864  Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
865  RegA21, RegB;
866  bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
867  KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
868  KillA11 = false, KillA21 = false, KillB = false;
869 
870  GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
871 
872  if (IsILPReassociate)
873  GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
874 
876  GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
877  GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
879  GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
880  GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
881  } else {
882  // Get FSUB instruction info.
883  GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
884  GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
885  }
886 
887  // Create new virtual registers for the new results instead of
888  // recycling legacy ones because the MachineCombiner's computation of the
889  // critical path requires a new register definition rather than an existing
890  // one.
891  // For register pressure reassociation, we only need create one virtual
892  // register for the new fma.
893  Register NewVRA = MRI.createVirtualRegister(RC);
894  InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
895 
896  Register NewVRB = 0;
897  if (IsILPReassociate) {
898  NewVRB = MRI.createVirtualRegister(RC);
899  InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
900  }
901 
902  Register NewVRD = 0;
904  NewVRD = MRI.createVirtualRegister(RC);
905  InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
906  }
907 
908  auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
909  Register RegMul1, bool KillRegMul1,
910  Register RegMul2, bool KillRegMul2) {
911  MI->getOperand(AddOpIdx).setReg(RegAdd);
912  MI->getOperand(AddOpIdx).setIsKill(KillAdd);
913  MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
914  MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
915  MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
916  MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
917  };
918 
919  MachineInstrBuilder NewARegPressure, NewCRegPressure;
920  switch (Pattern) {
921  default:
922  llvm_unreachable("not recognized pattern!");
924  // Create new instructions for insertion.
925  MachineInstrBuilder MINewB =
926  BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
927  .addReg(RegX, getKillRegState(KillX))
928  .addReg(RegM21, getKillRegState(KillM21))
929  .addReg(RegM22, getKillRegState(KillM22));
930  MachineInstrBuilder MINewA =
931  BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
932  .addReg(RegY, getKillRegState(KillY))
933  .addReg(RegM31, getKillRegState(KillM31))
934  .addReg(RegM32, getKillRegState(KillM32));
935  // If AddOpIdx is not 1, adjust the order.
936  if (AddOpIdx != 1) {
937  AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
938  AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
939  }
940 
941  MachineInstrBuilder MINewC =
942  BuildMI(*MF, Root.getDebugLoc(),
944  .addReg(NewVRB, getKillRegState(true))
945  .addReg(NewVRA, getKillRegState(true));
946 
947  // Update flags for newly created instructions.
948  setSpecialOperandAttr(*MINewA, IntersectedFlags);
949  setSpecialOperandAttr(*MINewB, IntersectedFlags);
950  setSpecialOperandAttr(*MINewC, IntersectedFlags);
951 
952  // Record new instructions for insertion.
953  InsInstrs.push_back(MINewA);
954  InsInstrs.push_back(MINewB);
955  InsInstrs.push_back(MINewC);
956  break;
957  }
959  assert(NewVRD && "new FMA register not created!");
960  // Create new instructions for insertion.
961  MachineInstrBuilder MINewA =
962  BuildMI(*MF, Leaf->getDebugLoc(),
963  get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
964  .addReg(RegM11, getKillRegState(KillM11))
965  .addReg(RegM12, getKillRegState(KillM12));
966  MachineInstrBuilder MINewB =
967  BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
968  .addReg(RegX, getKillRegState(KillX))
969  .addReg(RegM21, getKillRegState(KillM21))
970  .addReg(RegM22, getKillRegState(KillM22));
971  MachineInstrBuilder MINewD =
972  BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
973  .addReg(NewVRA, getKillRegState(true))
974  .addReg(RegM31, getKillRegState(KillM31))
975  .addReg(RegM32, getKillRegState(KillM32));
976  // If AddOpIdx is not 1, adjust the order.
977  if (AddOpIdx != 1) {
978  AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
979  AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
980  KillM32);
981  }
982 
983  MachineInstrBuilder MINewC =
984  BuildMI(*MF, Root.getDebugLoc(),
986  .addReg(NewVRB, getKillRegState(true))
987  .addReg(NewVRD, getKillRegState(true));
988 
989  // Update flags for newly created instructions.
990  setSpecialOperandAttr(*MINewA, IntersectedFlags);
991  setSpecialOperandAttr(*MINewB, IntersectedFlags);
992  setSpecialOperandAttr(*MINewD, IntersectedFlags);
993  setSpecialOperandAttr(*MINewC, IntersectedFlags);
994 
995  // Record new instructions for insertion.
996  InsInstrs.push_back(MINewA);
997  InsInstrs.push_back(MINewB);
998  InsInstrs.push_back(MINewD);
999  InsInstrs.push_back(MINewC);
1000  break;
1001  }
1004  Register VarReg;
1005  bool KillVarReg = false;
1007  VarReg = RegM31;
1008  KillVarReg = KillM31;
1009  } else {
1010  VarReg = RegM32;
1011  KillVarReg = KillM32;
1012  }
1013  // We don't want to get negative const from memory pool too early, as the
1014  // created entry will not be deleted even if it has no users. Since all
1015  // operand of Leaf and Root are virtual register, we use zero register
1016  // here as a placeholder. When the InsInstrs is selected in
1017  // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1018  // with a virtual register which is a load from constant pool.
1019  NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1020  .addReg(RegB, getKillRegState(RegB))
1021  .addReg(RegY, getKillRegState(KillY))
1022  .addReg(PPC::ZERO8);
1023  NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1024  .addReg(NewVRA, getKillRegState(true))
1025  .addReg(RegX, getKillRegState(KillX))
1026  .addReg(VarReg, getKillRegState(KillVarReg));
1027  // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1028  // both at index 1, no need to adjust.
1029  // FIXME: when add more fma instructions support, like fma/fmas, adjust
1030  // the operand index here.
1031  break;
1032  }
1033  }
1034 
1035  if (!IsILPReassociate) {
1036  setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1037  setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1038 
1039  InsInstrs.push_back(NewARegPressure);
1040  InsInstrs.push_back(NewCRegPressure);
1041  }
1042 
1043  assert(!InsInstrs.empty() &&
1044  "Insertion instructions set should not be empty!");
1045 
1046  // Record old instructions for deletion.
1047  DelInstrs.push_back(Leaf);
1048  if (IsILPReassociate)
1049  DelInstrs.push_back(Prev);
1050  DelInstrs.push_back(&Root);
1051 }
1052 
1053 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1055  Register &SrcReg, Register &DstReg,
1056  unsigned &SubIdx) const {
1057  switch (MI.getOpcode()) {
1058  default: return false;
1059  case PPC::EXTSW:
1060  case PPC::EXTSW_32:
1061  case PPC::EXTSW_32_64:
1062  SrcReg = MI.getOperand(1).getReg();
1063  DstReg = MI.getOperand(0).getReg();
1064  SubIdx = PPC::sub_32;
1065  return true;
1066  }
1067 }
1068 
1070  int &FrameIndex) const {
1071  if (llvm::is_contained(getLoadOpcodesForSpillArray(), MI.getOpcode())) {
1072  // Check for the operands added by addFrameReference (the immediate is the
1073  // offset which defaults to 0).
1074  if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1075  MI.getOperand(2).isFI()) {
1076  FrameIndex = MI.getOperand(2).getIndex();
1077  return MI.getOperand(0).getReg();
1078  }
1079  }
1080  return 0;
1081 }
1082 
1083 // For opcodes with the ReMaterializable flag set, this function is called to
1084 // verify the instruction is really rematable.
1086  const MachineInstr &MI) const {
1087  switch (MI.getOpcode()) {
1088  default:
1089  // This function should only be called for opcodes with the ReMaterializable
1090  // flag set.
1091  llvm_unreachable("Unknown rematerializable operation!");
1092  break;
1093  case PPC::LI:
1094  case PPC::LI8:
1095  case PPC::PLI:
1096  case PPC::PLI8:
1097  case PPC::LIS:
1098  case PPC::LIS8:
1099  case PPC::ADDIStocHA:
1100  case PPC::ADDIStocHA8:
1101  case PPC::ADDItocL:
1102  case PPC::LOAD_STACK_GUARD:
1103  case PPC::XXLXORz:
1104  case PPC::XXLXORspz:
1105  case PPC::XXLXORdpz:
1106  case PPC::XXLEQVOnes:
1107  case PPC::XXSPLTI32DX:
1108  case PPC::XXSPLTIW:
1109  case PPC::XXSPLTIDP:
1110  case PPC::V_SET0B:
1111  case PPC::V_SET0H:
1112  case PPC::V_SET0:
1113  case PPC::V_SETALLONESB:
1114  case PPC::V_SETALLONESH:
1115  case PPC::V_SETALLONES:
1116  case PPC::CRSET:
1117  case PPC::CRUNSET:
1118  case PPC::XXSETACCZ:
1119  case PPC::XXSETACCZW:
1120  return true;
1121  }
1122  return false;
1123 }
1124 
1126  int &FrameIndex) const {
1127  if (llvm::is_contained(getStoreOpcodesForSpillArray(), MI.getOpcode())) {
1128  if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1129  MI.getOperand(2).isFI()) {
1130  FrameIndex = MI.getOperand(2).getIndex();
1131  return MI.getOperand(0).getReg();
1132  }
1133  }
1134  return 0;
1135 }
1136 
1138  unsigned OpIdx1,
1139  unsigned OpIdx2) const {
1140  MachineFunction &MF = *MI.getParent()->getParent();
1141 
1142  // Normal instructions can be commuted the obvious way.
1143  if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1144  return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1145  // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1146  // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1147  // changing the relative order of the mask operands might change what happens
1148  // to the high-bits of the mask (and, thus, the result).
1149 
1150  // Cannot commute if it has a non-zero rotate count.
1151  if (MI.getOperand(3).getImm() != 0)
1152  return nullptr;
1153 
1154  // If we have a zero rotate count, we have:
1155  // M = mask(MB,ME)
1156  // Op0 = (Op1 & ~M) | (Op2 & M)
1157  // Change this to:
1158  // M = mask((ME+1)&31, (MB-1)&31)
1159  // Op0 = (Op2 & ~M) | (Op1 & M)
1160 
1161  // Swap op1/op2
1162  assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1163  "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1164  Register Reg0 = MI.getOperand(0).getReg();
1165  Register Reg1 = MI.getOperand(1).getReg();
1166  Register Reg2 = MI.getOperand(2).getReg();
1167  unsigned SubReg1 = MI.getOperand(1).getSubReg();
1168  unsigned SubReg2 = MI.getOperand(2).getSubReg();
1169  bool Reg1IsKill = MI.getOperand(1).isKill();
1170  bool Reg2IsKill = MI.getOperand(2).isKill();
1171  bool ChangeReg0 = false;
1172  // If machine instrs are no longer in two-address forms, update
1173  // destination register as well.
1174  if (Reg0 == Reg1) {
1175  // Must be two address instruction!
1176  assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
1177  "Expecting a two-address instruction!");
1178  assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1179  Reg2IsKill = false;
1180  ChangeReg0 = true;
1181  }
1182 
1183  // Masks.
1184  unsigned MB = MI.getOperand(4).getImm();
1185  unsigned ME = MI.getOperand(5).getImm();
1186 
1187  // We can't commute a trivial mask (there is no way to represent an all-zero
1188  // mask).
1189  if (MB == 0 && ME == 31)
1190  return nullptr;
1191 
1192  if (NewMI) {
1193  // Create a new instruction.
1194  Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1195  bool Reg0IsDead = MI.getOperand(0).isDead();
1196  return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1197  .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1198  .addReg(Reg2, getKillRegState(Reg2IsKill))
1199  .addReg(Reg1, getKillRegState(Reg1IsKill))
1200  .addImm((ME + 1) & 31)
1201  .addImm((MB - 1) & 31);
1202  }
1203 
1204  if (ChangeReg0) {
1205  MI.getOperand(0).setReg(Reg2);
1206  MI.getOperand(0).setSubReg(SubReg2);
1207  }
1208  MI.getOperand(2).setReg(Reg1);
1209  MI.getOperand(1).setReg(Reg2);
1210  MI.getOperand(2).setSubReg(SubReg1);
1211  MI.getOperand(1).setSubReg(SubReg2);
1212  MI.getOperand(2).setIsKill(Reg1IsKill);
1213  MI.getOperand(1).setIsKill(Reg2IsKill);
1214 
1215  // Swap the mask around.
1216  MI.getOperand(4).setImm((ME + 1) & 31);
1217  MI.getOperand(5).setImm((MB - 1) & 31);
1218  return &MI;
1219 }
1220 
1222  unsigned &SrcOpIdx1,
1223  unsigned &SrcOpIdx2) const {
1224  // For VSX A-Type FMA instructions, it is the first two operands that can be
1225  // commuted, however, because the non-encoded tied input operand is listed
1226  // first, the operands to swap are actually the second and third.
1227 
1228  int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1229  if (AltOpc == -1)
1230  return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1231 
1232  // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1233  // and SrcOpIdx2.
1234  return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1235 }
1236 
1239  // This function is used for scheduling, and the nop wanted here is the type
1240  // that terminates dispatch groups on the POWER cores.
1241  unsigned Directive = Subtarget.getCPUDirective();
1242  unsigned Opcode;
1243  switch (Directive) {
1244  default: Opcode = PPC::NOP; break;
1245  case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1246  case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1247  case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1248  // FIXME: Update when POWER9 scheduling model is ready.
1249  case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1250  }
1251 
1252  DebugLoc DL;
1253  BuildMI(MBB, MI, DL, get(Opcode));
1254 }
1255 
1256 /// Return the noop instruction to use for a noop.
1258  MCInst Nop;
1259  Nop.setOpcode(PPC::NOP);
1260  return Nop;
1261 }
1262 
1263 // Branch analysis.
1264 // Note: If the condition register is set to CTR or CTR8 then this is a
1265 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
1268  MachineBasicBlock *&FBB,
1270  bool AllowModify) const {
1271  bool isPPC64 = Subtarget.isPPC64();
1272 
1273  // If the block has no terminators, it just falls into the block after it.
1275  if (I == MBB.end())
1276  return false;
1277 
1278  if (!isUnpredicatedTerminator(*I))
1279  return false;
1280 
1281  if (AllowModify) {
1282  // If the BB ends with an unconditional branch to the fallthrough BB,
1283  // we eliminate the branch instruction.
1284  if (I->getOpcode() == PPC::B &&
1285  MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1286  I->eraseFromParent();
1287 
1288  // We update iterator after deleting the last branch.
1290  if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1291  return false;
1292  }
1293  }
1294 
1295  // Get the last instruction in the block.
1296  MachineInstr &LastInst = *I;
1297 
1298  // If there is only one terminator instruction, process it.
1299  if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1300  if (LastInst.getOpcode() == PPC::B) {
1301  if (!LastInst.getOperand(0).isMBB())
1302  return true;
1303  TBB = LastInst.getOperand(0).getMBB();
1304  return false;
1305  } else if (LastInst.getOpcode() == PPC::BCC) {
1306  if (!LastInst.getOperand(2).isMBB())
1307  return true;
1308  // Block ends with fall-through condbranch.
1309  TBB = LastInst.getOperand(2).getMBB();
1310  Cond.push_back(LastInst.getOperand(0));
1311  Cond.push_back(LastInst.getOperand(1));
1312  return false;
1313  } else if (LastInst.getOpcode() == PPC::BC) {
1314  if (!LastInst.getOperand(1).isMBB())
1315  return true;
1316  // Block ends with fall-through condbranch.
1317  TBB = LastInst.getOperand(1).getMBB();
1319  Cond.push_back(LastInst.getOperand(0));
1320  return false;
1321  } else if (LastInst.getOpcode() == PPC::BCn) {
1322  if (!LastInst.getOperand(1).isMBB())
1323  return true;
1324  // Block ends with fall-through condbranch.
1325  TBB = LastInst.getOperand(1).getMBB();
1327  Cond.push_back(LastInst.getOperand(0));
1328  return false;
1329  } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1330  LastInst.getOpcode() == PPC::BDNZ) {
1331  if (!LastInst.getOperand(0).isMBB())
1332  return true;
1333  if (DisableCTRLoopAnal)
1334  return true;
1335  TBB = LastInst.getOperand(0).getMBB();
1336  Cond.push_back(MachineOperand::CreateImm(1));
1337  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1338  true));
1339  return false;
1340  } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1341  LastInst.getOpcode() == PPC::BDZ) {
1342  if (!LastInst.getOperand(0).isMBB())
1343  return true;
1344  if (DisableCTRLoopAnal)
1345  return true;
1346  TBB = LastInst.getOperand(0).getMBB();
1347  Cond.push_back(MachineOperand::CreateImm(0));
1348  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1349  true));
1350  return false;
1351  }
1352 
1353  // Otherwise, don't know what this is.
1354  return true;
1355  }
1356 
1357  // Get the instruction before it if it's a terminator.
1358  MachineInstr &SecondLastInst = *I;
1359 
1360  // If there are three terminators, we don't know what sort of block this is.
1361  if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1362  return true;
1363 
1364  // If the block ends with PPC::B and PPC:BCC, handle it.
1365  if (SecondLastInst.getOpcode() == PPC::BCC &&
1366  LastInst.getOpcode() == PPC::B) {
1367  if (!SecondLastInst.getOperand(2).isMBB() ||
1368  !LastInst.getOperand(0).isMBB())
1369  return true;
1370  TBB = SecondLastInst.getOperand(2).getMBB();
1371  Cond.push_back(SecondLastInst.getOperand(0));
1372  Cond.push_back(SecondLastInst.getOperand(1));
1373  FBB = LastInst.getOperand(0).getMBB();
1374  return false;
1375  } else if (SecondLastInst.getOpcode() == PPC::BC &&
1376  LastInst.getOpcode() == PPC::B) {
1377  if (!SecondLastInst.getOperand(1).isMBB() ||
1378  !LastInst.getOperand(0).isMBB())
1379  return true;
1380  TBB = SecondLastInst.getOperand(1).getMBB();
1382  Cond.push_back(SecondLastInst.getOperand(0));
1383  FBB = LastInst.getOperand(0).getMBB();
1384  return false;
1385  } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1386  LastInst.getOpcode() == PPC::B) {
1387  if (!SecondLastInst.getOperand(1).isMBB() ||
1388  !LastInst.getOperand(0).isMBB())
1389  return true;
1390  TBB = SecondLastInst.getOperand(1).getMBB();
1392  Cond.push_back(SecondLastInst.getOperand(0));
1393  FBB = LastInst.getOperand(0).getMBB();
1394  return false;
1395  } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1396  SecondLastInst.getOpcode() == PPC::BDNZ) &&
1397  LastInst.getOpcode() == PPC::B) {
1398  if (!SecondLastInst.getOperand(0).isMBB() ||
1399  !LastInst.getOperand(0).isMBB())
1400  return true;
1401  if (DisableCTRLoopAnal)
1402  return true;
1403  TBB = SecondLastInst.getOperand(0).getMBB();
1404  Cond.push_back(MachineOperand::CreateImm(1));
1405  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1406  true));
1407  FBB = LastInst.getOperand(0).getMBB();
1408  return false;
1409  } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1410  SecondLastInst.getOpcode() == PPC::BDZ) &&
1411  LastInst.getOpcode() == PPC::B) {
1412  if (!SecondLastInst.getOperand(0).isMBB() ||
1413  !LastInst.getOperand(0).isMBB())
1414  return true;
1415  if (DisableCTRLoopAnal)
1416  return true;
1417  TBB = SecondLastInst.getOperand(0).getMBB();
1418  Cond.push_back(MachineOperand::CreateImm(0));
1419  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1420  true));
1421  FBB = LastInst.getOperand(0).getMBB();
1422  return false;
1423  }
1424 
1425  // If the block ends with two PPC:Bs, handle it. The second one is not
1426  // executed, so remove it.
1427  if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1428  if (!SecondLastInst.getOperand(0).isMBB())
1429  return true;
1430  TBB = SecondLastInst.getOperand(0).getMBB();
1431  I = LastInst;
1432  if (AllowModify)
1433  I->eraseFromParent();
1434  return false;
1435  }
1436 
1437  // Otherwise, can't handle this.
1438  return true;
1439 }
1440 
1442  int *BytesRemoved) const {
1443  assert(!BytesRemoved && "code size not handled");
1444 
1446  if (I == MBB.end())
1447  return 0;
1448 
1449  if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1450  I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1451  I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1452  I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1453  return 0;
1454 
1455  // Remove the branch.
1456  I->eraseFromParent();
1457 
1458  I = MBB.end();
1459 
1460  if (I == MBB.begin()) return 1;
1461  --I;
1462  if (I->getOpcode() != PPC::BCC &&
1463  I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1464  I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1465  I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1466  return 1;
1467 
1468  // Remove the branch.
1469  I->eraseFromParent();
1470  return 2;
1471 }
1472 
1475  MachineBasicBlock *FBB,
1477  const DebugLoc &DL,
1478  int *BytesAdded) const {
1479  // Shouldn't be a fall through.
1480  assert(TBB && "insertBranch must not be told to insert a fallthrough");
1481  assert((Cond.size() == 2 || Cond.size() == 0) &&
1482  "PPC branch conditions have two components!");
1483  assert(!BytesAdded && "code size not handled");
1484 
1485  bool isPPC64 = Subtarget.isPPC64();
1486 
1487  // One-way branch.
1488  if (!FBB) {
1489  if (Cond.empty()) // Unconditional branch
1490  BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1491  else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1492  BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1493  (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1494  (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1495  else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1496  BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1497  else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1498  BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1499  else // Conditional branch
1500  BuildMI(&MBB, DL, get(PPC::BCC))
1501  .addImm(Cond[0].getImm())
1502  .add(Cond[1])
1503  .addMBB(TBB);
1504  return 1;
1505  }
1506 
1507  // Two-way Conditional Branch.
1508  if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1509  BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1510  (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1511  (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1512  else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1513  BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1514  else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1515  BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1516  else
1517  BuildMI(&MBB, DL, get(PPC::BCC))
1518  .addImm(Cond[0].getImm())
1519  .add(Cond[1])
1520  .addMBB(TBB);
1521  BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1522  return 2;
1523 }
1524 
1525 // Select analysis.
1528  Register DstReg, Register TrueReg,
1529  Register FalseReg, int &CondCycles,
1530  int &TrueCycles, int &FalseCycles) const {
1531  if (Cond.size() != 2)
1532  return false;
1533 
1534  // If this is really a bdnz-like condition, then it cannot be turned into a
1535  // select.
1536  if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1537  return false;
1538 
1539  // If the conditional branch uses a physical register, then it cannot be
1540  // turned into a select.
1542  return false;
1543 
1544  // Check register classes.
1546  const TargetRegisterClass *RC =
1547  RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1548  if (!RC)
1549  return false;
1550 
1551  // isel is for regular integer GPRs only.
1552  if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1553  !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1554  !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1555  !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1556  return false;
1557 
1558  // FIXME: These numbers are for the A2, how well they work for other cores is
1559  // an open question. On the A2, the isel instruction has a 2-cycle latency
1560  // but single-cycle throughput. These numbers are used in combination with
1561  // the MispredictPenalty setting from the active SchedMachineModel.
1562  CondCycles = 1;
1563  TrueCycles = 1;
1564  FalseCycles = 1;
1565 
1566  return true;
1567 }
1568 
1571  const DebugLoc &dl, Register DestReg,
1573  Register FalseReg) const {
1574  assert(Cond.size() == 2 &&
1575  "PPC branch conditions have two components!");
1576 
1577  // Get the register classes.
1579  const TargetRegisterClass *RC =
1580  RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1581  assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1582 
1583  bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1584  PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1585  assert((Is64Bit ||
1586  PPC::GPRCRegClass.hasSubClassEq(RC) ||
1587  PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1588  "isel is for regular integer GPRs only");
1589 
1590  unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1591  auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1592 
1593  unsigned SubIdx = 0;
1594  bool SwapOps = false;
1595  switch (SelectPred) {
1596  case PPC::PRED_EQ:
1597  case PPC::PRED_EQ_MINUS:
1598  case PPC::PRED_EQ_PLUS:
1599  SubIdx = PPC::sub_eq; SwapOps = false; break;
1600  case PPC::PRED_NE:
1601  case PPC::PRED_NE_MINUS:
1602  case PPC::PRED_NE_PLUS:
1603  SubIdx = PPC::sub_eq; SwapOps = true; break;
1604  case PPC::PRED_LT:
1605  case PPC::PRED_LT_MINUS:
1606  case PPC::PRED_LT_PLUS:
1607  SubIdx = PPC::sub_lt; SwapOps = false; break;
1608  case PPC::PRED_GE:
1609  case PPC::PRED_GE_MINUS:
1610  case PPC::PRED_GE_PLUS:
1611  SubIdx = PPC::sub_lt; SwapOps = true; break;
1612  case PPC::PRED_GT:
1613  case PPC::PRED_GT_MINUS:
1614  case PPC::PRED_GT_PLUS:
1615  SubIdx = PPC::sub_gt; SwapOps = false; break;
1616  case PPC::PRED_LE:
1617  case PPC::PRED_LE_MINUS:
1618  case PPC::PRED_LE_PLUS:
1619  SubIdx = PPC::sub_gt; SwapOps = true; break;
1620  case PPC::PRED_UN:
1621  case PPC::PRED_UN_MINUS:
1622  case PPC::PRED_UN_PLUS:
1623  SubIdx = PPC::sub_un; SwapOps = false; break;
1624  case PPC::PRED_NU:
1625  case PPC::PRED_NU_MINUS:
1626  case PPC::PRED_NU_PLUS:
1627  SubIdx = PPC::sub_un; SwapOps = true; break;
1628  case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1629  case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1630  }
1631 
1632  Register FirstReg = SwapOps ? FalseReg : TrueReg,
1633  SecondReg = SwapOps ? TrueReg : FalseReg;
1634 
1635  // The first input register of isel cannot be r0. If it is a member
1636  // of a register class that can be r0, then copy it first (the
1637  // register allocator should eliminate the copy).
1638  if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1639  MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1640  const TargetRegisterClass *FirstRC =
1641  MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1642  &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1643  Register OldFirstReg = FirstReg;
1644  FirstReg = MRI.createVirtualRegister(FirstRC);
1645  BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1646  .addReg(OldFirstReg);
1647  }
1648 
1649  BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1650  .addReg(FirstReg).addReg(SecondReg)
1651  .addReg(Cond[1].getReg(), 0, SubIdx);
1652 }
1653 
1654 static unsigned getCRBitValue(unsigned CRBit) {
1655  unsigned Ret = 4;
1656  if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1657  CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1658  CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1659  CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1660  Ret = 3;
1661  if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1662  CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1663  CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1664  CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1665  Ret = 2;
1666  if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1667  CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1668  CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1669  CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1670  Ret = 1;
1671  if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1672  CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1673  CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1674  CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1675  Ret = 0;
1676 
1677  assert(Ret != 4 && "Invalid CR bit register");
1678  return Ret;
1679 }
1680 
1683  const DebugLoc &DL, MCRegister DestReg,
1684  MCRegister SrcReg, bool KillSrc) const {
1685  // We can end up with self copies and similar things as a result of VSX copy
1686  // legalization. Promote them here.
1688  if (PPC::F8RCRegClass.contains(DestReg) &&
1689  PPC::VSRCRegClass.contains(SrcReg)) {
1690  MCRegister SuperReg =
1691  TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1692 
1693  if (VSXSelfCopyCrash && SrcReg == SuperReg)
1694  llvm_unreachable("nop VSX copy");
1695 
1696  DestReg = SuperReg;
1697  } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1698  PPC::VSRCRegClass.contains(DestReg)) {
1699  MCRegister SuperReg =
1700  TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1701 
1702  if (VSXSelfCopyCrash && DestReg == SuperReg)
1703  llvm_unreachable("nop VSX copy");
1704 
1705  SrcReg = SuperReg;
1706  }
1707 
1708  // Different class register copy
1709  if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1710  PPC::GPRCRegClass.contains(DestReg)) {
1711  MCRegister CRReg = getCRFromCRBit(SrcReg);
1712  BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1713  getKillRegState(KillSrc);
1714  // Rotate the CR bit in the CR fields to be the least significant bit and
1715  // then mask with 0x1 (MB = ME = 31).
1716  BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1717  .addReg(DestReg, RegState::Kill)
1718  .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1719  .addImm(31)
1720  .addImm(31);
1721  return;
1722  } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1723  (PPC::G8RCRegClass.contains(DestReg) ||
1724  PPC::GPRCRegClass.contains(DestReg))) {
1725  bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1726  unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1727  unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1728  unsigned CRNum = TRI->getEncodingValue(SrcReg);
1729  BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1730  getKillRegState(KillSrc);
1731  if (CRNum == 7)
1732  return;
1733  // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1734  BuildMI(MBB, I, DL, get(ShCode), DestReg)
1735  .addReg(DestReg, RegState::Kill)
1736  .addImm(CRNum * 4 + 4)
1737  .addImm(28)
1738  .addImm(31);
1739  return;
1740  } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1741  PPC::VSFRCRegClass.contains(DestReg)) {
1742  assert(Subtarget.hasDirectMove() &&
1743  "Subtarget doesn't support directmove, don't know how to copy.");
1744  BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1745  NumGPRtoVSRSpill++;
1746  getKillRegState(KillSrc);
1747  return;
1748  } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1749  PPC::G8RCRegClass.contains(DestReg)) {
1750  assert(Subtarget.hasDirectMove() &&
1751  "Subtarget doesn't support directmove, don't know how to copy.");
1752  BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1753  getKillRegState(KillSrc);
1754  return;
1755  } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1756  PPC::GPRCRegClass.contains(DestReg)) {
1757  BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1758  getKillRegState(KillSrc);
1759  return;
1760  } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1761  PPC::SPERCRegClass.contains(DestReg)) {
1762  BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1763  getKillRegState(KillSrc);
1764  return;
1765  }
1766 
1767  unsigned Opc;
1768  if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1769  Opc = PPC::OR;
1770  else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1771  Opc = PPC::OR8;
1772  else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1773  Opc = PPC::FMR;
1774  else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1775  Opc = PPC::MCRF;
1776  else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1777  Opc = PPC::VOR;
1778  else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1779  // There are two different ways this can be done:
1780  // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1781  // issue in VSU pipeline 0.
1782  // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1783  // can go to either pipeline.
1784  // We'll always use xxlor here, because in practically all cases where
1785  // copies are generated, they are close enough to some use that the
1786  // lower-latency form is preferable.
1787  Opc = PPC::XXLOR;
1788  else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1789  PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1790  Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1791  else if (Subtarget.pairedVectorMemops() &&
1792  PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1793  if (SrcReg > PPC::VSRp15)
1794  SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1795  else
1796  SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1797  if (DestReg > PPC::VSRp15)
1798  DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1799  else
1800  DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1801  BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1802  addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1803  BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1804  addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1805  return;
1806  }
1807  else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1808  Opc = PPC::CROR;
1809  else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1810  Opc = PPC::EVOR;
1811  else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1812  PPC::UACCRCRegClass.contains(DestReg)) &&
1813  (PPC::ACCRCRegClass.contains(SrcReg) ||
1814  PPC::UACCRCRegClass.contains(SrcReg))) {
1815  // If primed, de-prime the source register, copy the individual registers
1816  // and prime the destination if needed. The vector subregisters are
1817  // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1818  // source is primed, we need to re-prime it after the copy as well.
1819  PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1820  bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1821  bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1822  MCRegister VSLSrcReg =
1823  PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1824  MCRegister VSLDestReg =
1825  PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1826  if (SrcPrimed)
1827  BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1828  for (unsigned Idx = 0; Idx < 4; Idx++)
1829  BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1830  .addReg(VSLSrcReg + Idx)
1831  .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1832  if (DestPrimed)
1833  BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1834  if (SrcPrimed && !KillSrc)
1835  BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1836  return;
1837  } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1838  PPC::G8pRCRegClass.contains(SrcReg)) {
1839  // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1840  unsigned DestRegIdx = DestReg - PPC::G8p0;
1841  MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1842  MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1843  unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1844  MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1845  MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1846  BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1847  .addReg(SrcRegSub0)
1848  .addReg(SrcRegSub0, getKillRegState(KillSrc));
1849  BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1850  .addReg(SrcRegSub1)
1851  .addReg(SrcRegSub1, getKillRegState(KillSrc));
1852  return;
1853  } else
1854  llvm_unreachable("Impossible reg-to-reg copy");
1855 
1856  const MCInstrDesc &MCID = get(Opc);
1857  if (MCID.getNumOperands() == 3)
1858  BuildMI(MBB, I, DL, MCID, DestReg)
1859  .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1860  else
1861  BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1862 }
1863 
1864 unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1865  int OpcodeIndex = 0;
1866 
1867  if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1868  PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1870  } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1871  PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1873  } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1875  } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1877  } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1879  } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1881  } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1883  } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1885  } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1887  } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1889  } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1891  } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1893  } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1894  assert(Subtarget.pairedVectorMemops() &&
1895  "Register unexpected when paired memops are disabled.");
1897  } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1898  assert(Subtarget.pairedVectorMemops() &&
1899  "Register unexpected when paired memops are disabled.");
1901  } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1902  assert(Subtarget.pairedVectorMemops() &&
1903  "Register unexpected when paired memops are disabled.");
1905  } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1906  assert(Subtarget.pairedVectorMemops() &&
1907  "Register unexpected when paired memops are disabled.");
1909  } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1911  } else {
1912  llvm_unreachable("Unknown regclass!");
1913  }
1914  return OpcodeIndex;
1915 }
1916 
1917 unsigned
1919  ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1920  return OpcodesForSpill[getSpillIndex(RC)];
1921 }
1922 
1923 unsigned
1925  ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1926  return OpcodesForSpill[getSpillIndex(RC)];
1927 }
1928 
1929 void PPCInstrInfo::StoreRegToStackSlot(
1930  MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1931  const TargetRegisterClass *RC,
1932  SmallVectorImpl<MachineInstr *> &NewMIs) const {
1933  unsigned Opcode = getStoreOpcodeForSpill(RC);
1934  DebugLoc DL;
1935 
1936  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1937  FuncInfo->setHasSpills();
1938 
1939  NewMIs.push_back(addFrameReference(
1940  BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1941  FrameIdx));
1942 
1943  if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1944  PPC::CRBITRCRegClass.hasSubClassEq(RC))
1945  FuncInfo->setSpillsCR();
1946 
1947  if (isXFormMemOp(Opcode))
1948  FuncInfo->setHasNonRISpills();
1949 }
1950 
1953  bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1954  const TargetRegisterInfo *TRI) const {
1955  MachineFunction &MF = *MBB.getParent();
1957 
1958  StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1959 
1960  for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1961  MBB.insert(MI, NewMIs[i]);
1962 
1963  const MachineFrameInfo &MFI = MF.getFrameInfo();
1965  MachinePointerInfo::getFixedStack(MF, FrameIdx),
1967  MFI.getObjectAlign(FrameIdx));
1968  NewMIs.back()->addMemOperand(MF, MMO);
1969 }
1970 
1973  Register SrcReg, bool isKill,
1974  int FrameIdx,
1975  const TargetRegisterClass *RC,
1976  const TargetRegisterInfo *TRI) const {
1977  // We need to avoid a situation in which the value from a VRRC register is
1978  // spilled using an Altivec instruction and reloaded into a VSRC register
1979  // using a VSX instruction. The issue with this is that the VSX
1980  // load/store instructions swap the doublewords in the vector and the Altivec
1981  // ones don't. The register classes on the spill/reload may be different if
1982  // the register is defined using an Altivec instruction and is then used by a
1983  // VSX instruction.
1984  RC = updatedRC(RC);
1985  storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1986 }
1987 
1988 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1989  unsigned DestReg, int FrameIdx,
1990  const TargetRegisterClass *RC,
1992  const {
1993  unsigned Opcode = getLoadOpcodeForSpill(RC);
1994  NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1995  FrameIdx));
1996  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1997 
1998  if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1999  PPC::CRBITRCRegClass.hasSubClassEq(RC))
2000  FuncInfo->setSpillsCR();
2001 
2002  if (isXFormMemOp(Opcode))
2003  FuncInfo->setHasNonRISpills();
2004 }
2005 
2008  int FrameIdx, const TargetRegisterClass *RC,
2009  const TargetRegisterInfo *TRI) const {
2010  MachineFunction &MF = *MBB.getParent();
2012  DebugLoc DL;
2013  if (MI != MBB.end()) DL = MI->getDebugLoc();
2014 
2015  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2016  FuncInfo->setHasSpills();
2017 
2018  LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2019 
2020  for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
2021  MBB.insert(MI, NewMIs[i]);
2022 
2023  const MachineFrameInfo &MFI = MF.getFrameInfo();
2025  MachinePointerInfo::getFixedStack(MF, FrameIdx),
2027  MFI.getObjectAlign(FrameIdx));
2028  NewMIs.back()->addMemOperand(MF, MMO);
2029 }
2030 
2033  Register DestReg, int FrameIdx,
2034  const TargetRegisterClass *RC,
2035  const TargetRegisterInfo *TRI) const {
2036  // We need to avoid a situation in which the value from a VRRC register is
2037  // spilled using an Altivec instruction and reloaded into a VSRC register
2038  // using a VSX instruction. The issue with this is that the VSX
2039  // load/store instructions swap the doublewords in the vector and the Altivec
2040  // ones don't. The register classes on the spill/reload may be different if
2041  // the register is defined using an Altivec instruction and is then used by a
2042  // VSX instruction.
2043  RC = updatedRC(RC);
2044 
2045  loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2046 }
2047 
2048 bool PPCInstrInfo::
2050  assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2051  if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2052  Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2053  else
2054  // Leave the CR# the same, but invert the condition.
2055  Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2056  return false;
2057 }
2058 
2059 // For some instructions, it is legal to fold ZERO into the RA register field.
2060 // This function performs that fold by replacing the operand with PPC::ZERO,
2061 // it does not consider whether the load immediate zero is no longer in use.
2063  Register Reg) const {
2064  // A zero immediate should always be loaded with a single li.
2065  unsigned DefOpc = DefMI.getOpcode();
2066  if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2067  return false;
2068  if (!DefMI.getOperand(1).isImm())
2069  return false;
2070  if (DefMI.getOperand(1).getImm() != 0)
2071  return false;
2072 
2073  // Note that we cannot here invert the arguments of an isel in order to fold
2074  // a ZERO into what is presented as the second argument. All we have here
2075  // is the condition bit, and that might come from a CR-logical bit operation.
2076 
2077  const MCInstrDesc &UseMCID = UseMI.getDesc();
2078 
2079  // Only fold into real machine instructions.
2080  if (UseMCID.isPseudo())
2081  return false;
2082 
2083  // We need to find which of the User's operands is to be folded, that will be
2084  // the operand that matches the given register ID.
2085  unsigned UseIdx;
2086  for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2087  if (UseMI.getOperand(UseIdx).isReg() &&
2088  UseMI.getOperand(UseIdx).getReg() == Reg)
2089  break;
2090 
2091  assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2092  assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2093 
2094  const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
2095 
2096  // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2097  // register (which might also be specified as a pointer class kind).
2098  if (UseInfo->isLookupPtrRegClass()) {
2099  if (UseInfo->RegClass /* Kind */ != 1)
2100  return false;
2101  } else {
2102  if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2103  UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2104  return false;
2105  }
2106 
2107  // Make sure this is not tied to an output register (or otherwise
2108  // constrained). This is true for ST?UX registers, for example, which
2109  // are tied to their output registers.
2110  if (UseInfo->Constraints != 0)
2111  return false;
2112 
2113  MCRegister ZeroReg;
2114  if (UseInfo->isLookupPtrRegClass()) {
2115  bool isPPC64 = Subtarget.isPPC64();
2116  ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2117  } else {
2118  ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2119  PPC::ZERO8 : PPC::ZERO;
2120  }
2121 
2122  UseMI.getOperand(UseIdx).setReg(ZeroReg);
2123  return true;
2124 }
2125 
2126 // Folds zero into instructions which have a load immediate zero as an operand
2127 // but also recognize zero as immediate zero. If the definition of the load
2128 // has no more users it is deleted.
2130  Register Reg, MachineRegisterInfo *MRI) const {
2131  bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2132  if (MRI->use_nodbg_empty(Reg))
2133  DefMI.eraseFromParent();
2134  return Changed;
2135 }
2136 
2138  for (MachineInstr &MI : MBB)
2139  if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8))
2140  return true;
2141  return false;
2142 }
2143 
2144 // We should make sure that, if we're going to predicate both sides of a
2145 // condition (a diamond), that both sides don't define the counter register. We
2146 // can predicate counter-decrement-based branches, but while that predicates
2147 // the branching, it does not predicate the counter decrement. If we tried to
2148 // merge the triangle into one predicated block, we'd decrement the counter
2149 // twice.
2151  unsigned NumT, unsigned ExtraT,
2152  MachineBasicBlock &FMBB,
2153  unsigned NumF, unsigned ExtraF,
2154  BranchProbability Probability) const {
2155  return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2156 }
2157 
2158 
2160  // The predicated branches are identified by their type, not really by the
2161  // explicit presence of a predicate. Furthermore, some of them can be
2162  // predicated more than once. Because if conversion won't try to predicate
2163  // any instruction which already claims to be predicated (by returning true
2164  // here), always return false. In doing so, we let isPredicable() be the
2165  // final word on whether not the instruction can be (further) predicated.
2166 
2167  return false;
2168 }
2169 
2171  const MachineBasicBlock *MBB,
2172  const MachineFunction &MF) const {
2173  // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2174  // across them, since some FP operations may change content of FPSCR.
2175  // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2176  if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF)
2177  return true;
2179 }
2180 
2182  ArrayRef<MachineOperand> Pred) const {
2183  unsigned OpC = MI.getOpcode();
2184  if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2185  if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2186  bool isPPC64 = Subtarget.isPPC64();
2187  MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2188  : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2189  // Need add Def and Use for CTR implicit operand.
2190  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2191  .addReg(Pred[1].getReg(), RegState::Implicit)
2192  .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2193  } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2194  MI.setDesc(get(PPC::BCLR));
2195  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2196  } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2197  MI.setDesc(get(PPC::BCLRn));
2198  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2199  } else {
2200  MI.setDesc(get(PPC::BCCLR));
2201  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2202  .addImm(Pred[0].getImm())
2203  .add(Pred[1]);
2204  }
2205 
2206  return true;
2207  } else if (OpC == PPC::B) {
2208  if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2209  bool isPPC64 = Subtarget.isPPC64();
2210  MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2211  : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2212  // Need add Def and Use for CTR implicit operand.
2213  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2214  .addReg(Pred[1].getReg(), RegState::Implicit)
2215  .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2216  } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2217  MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2218  MI.removeOperand(0);
2219 
2220  MI.setDesc(get(PPC::BC));
2221  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2222  .add(Pred[1])
2223  .addMBB(MBB);
2224  } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2225  MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2226  MI.removeOperand(0);
2227 
2228  MI.setDesc(get(PPC::BCn));
2229  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2230  .add(Pred[1])
2231  .addMBB(MBB);
2232  } else {
2233  MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2234  MI.removeOperand(0);
2235 
2236  MI.setDesc(get(PPC::BCC));
2237  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2238  .addImm(Pred[0].getImm())
2239  .add(Pred[1])
2240  .addMBB(MBB);
2241  }
2242 
2243  return true;
2244  } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2245  OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2246  OpC == PPC::BCTRL8_RM) {
2247  if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2248  llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2249 
2250  bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2251  OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2252  bool isPPC64 = Subtarget.isPPC64();
2253 
2254  if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2255  MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2256  : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2257  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2258  } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2259  MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2260  : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2261  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2262  } else {
2263  MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2264  : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2265  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2266  .addImm(Pred[0].getImm())
2267  .add(Pred[1]);
2268  }
2269 
2270  // Need add Def and Use for LR implicit operand.
2271  if (setLR)
2272  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2273  .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2274  .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2275  if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2276  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2278 
2279  return true;
2280  }
2281 
2282  return false;
2283 }
2284 
2286  ArrayRef<MachineOperand> Pred2) const {
2287  assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2288  assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2289 
2290  if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2291  return false;
2292  if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2293  return false;
2294 
2295  // P1 can only subsume P2 if they test the same condition register.
2296  if (Pred1[1].getReg() != Pred2[1].getReg())
2297  return false;
2298 
2299  PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2300  PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2301 
2302  if (P1 == P2)
2303  return true;
2304 
2305  // Does P1 subsume P2, e.g. GE subsumes GT.
2306  if (P1 == PPC::PRED_LE &&
2307  (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2308  return true;
2309  if (P1 == PPC::PRED_GE &&
2310  (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2311  return true;
2312 
2313  return false;
2314 }
2315 
2317  std::vector<MachineOperand> &Pred,
2318  bool SkipDead) const {
2319  // Note: At the present time, the contents of Pred from this function is
2320  // unused by IfConversion. This implementation follows ARM by pushing the
2321  // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2322  // predicate, instructions defining CTR or CTR8 are also included as
2323  // predicate-defining instructions.
2324 
2325  const TargetRegisterClass *RCs[] =
2326  { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2327  &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2328 
2329  bool Found = false;
2330  for (const MachineOperand &MO : MI.operands()) {
2331  for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2332  const TargetRegisterClass *RC = RCs[c];
2333  if (MO.isReg()) {
2334  if (MO.isDef() && RC->contains(MO.getReg())) {
2335  Pred.push_back(MO);
2336  Found = true;
2337  }
2338  } else if (MO.isRegMask()) {
2339  for (MCPhysReg R : *RC)
2340  if (MO.clobbersPhysReg(R)) {
2341  Pred.push_back(MO);
2342  Found = true;
2343  }
2344  }
2345  }
2346  }
2347 
2348  return Found;
2349 }
2350 
2352  Register &SrcReg2, int64_t &Mask,
2353  int64_t &Value) const {
2354  unsigned Opc = MI.getOpcode();
2355 
2356  switch (Opc) {
2357  default: return false;
2358  case PPC::CMPWI:
2359  case PPC::CMPLWI:
2360  case PPC::CMPDI:
2361  case PPC::CMPLDI:
2362  SrcReg = MI.getOperand(1).getReg();
2363  SrcReg2 = 0;
2364  Value = MI.getOperand(2).getImm();
2365  Mask = 0xFFFF;
2366  return true;
2367  case PPC::CMPW:
2368  case PPC::CMPLW:
2369  case PPC::CMPD:
2370  case PPC::CMPLD:
2371  case PPC::FCMPUS:
2372  case PPC::FCMPUD:
2373  SrcReg = MI.getOperand(1).getReg();
2374  SrcReg2 = MI.getOperand(2).getReg();
2375  Value = 0;
2376  Mask = 0;
2377  return true;
2378  }
2379 }
2380 
2382  Register SrcReg2, int64_t Mask,
2383  int64_t Value,
2384  const MachineRegisterInfo *MRI) const {
2385  if (DisableCmpOpt)
2386  return false;
2387 
2388  int OpC = CmpInstr.getOpcode();
2389  Register CRReg = CmpInstr.getOperand(0).getReg();
2390 
2391  // FP record forms set CR1 based on the exception status bits, not a
2392  // comparison with zero.
2393  if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2394  return false;
2395 
2397  // The record forms set the condition register based on a signed comparison
2398  // with zero (so says the ISA manual). This is not as straightforward as it
2399  // seems, however, because this is always a 64-bit comparison on PPC64, even
2400  // for instructions that are 32-bit in nature (like slw for example).
2401  // So, on PPC32, for unsigned comparisons, we can use the record forms only
2402  // for equality checks (as those don't depend on the sign). On PPC64,
2403  // we are restricted to equality for unsigned 64-bit comparisons and for
2404  // signed 32-bit comparisons the applicability is more restricted.
2405  bool isPPC64 = Subtarget.isPPC64();
2406  bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2407  bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2408  bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2409 
2410  // Look through copies unless that gets us to a physical register.
2411  Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2412  if (ActualSrc.isVirtual())
2413  SrcReg = ActualSrc;
2414 
2415  // Get the unique definition of SrcReg.
2416  MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2417  if (!MI) return false;
2418 
2419  bool equalityOnly = false;
2420  bool noSub = false;
2421  if (isPPC64) {
2422  if (is32BitSignedCompare) {
2423  // We can perform this optimization only if SrcReg is sign-extending.
2424  if (isSignExtended(SrcReg, MRI))
2425  noSub = true;
2426  else
2427  return false;
2428  } else if (is32BitUnsignedCompare) {
2429  // We can perform this optimization, equality only, if SrcReg is
2430  // zero-extending.
2431  if (isZeroExtended(SrcReg, MRI)) {
2432  noSub = true;
2433  equalityOnly = true;
2434  } else
2435  return false;
2436  } else
2437  equalityOnly = is64BitUnsignedCompare;
2438  } else
2439  equalityOnly = is32BitUnsignedCompare;
2440 
2441  if (equalityOnly) {
2442  // We need to check the uses of the condition register in order to reject
2443  // non-equality comparisons.
2445  I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2446  I != IE; ++I) {
2447  MachineInstr *UseMI = &*I;
2448  if (UseMI->getOpcode() == PPC::BCC) {
2450  unsigned PredCond = PPC::getPredicateCondition(Pred);
2451  // We ignore hint bits when checking for non-equality comparisons.
2452  if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2453  return false;
2454  } else if (UseMI->getOpcode() == PPC::ISEL ||
2455  UseMI->getOpcode() == PPC::ISEL8) {
2456  unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2457  if (SubIdx != PPC::sub_eq)
2458  return false;
2459  } else
2460  return false;
2461  }
2462  }
2463 
2464  MachineBasicBlock::iterator I = CmpInstr;
2465 
2466  // Scan forward to find the first use of the compare.
2467  for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2468  ++I) {
2469  bool FoundUse = false;
2471  J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2472  J != JE; ++J)
2473  if (&*J == &*I) {
2474  FoundUse = true;
2475  break;
2476  }
2477 
2478  if (FoundUse)
2479  break;
2480  }
2481 
2484 
2485  // There are two possible candidates which can be changed to set CR[01].
2486  // One is MI, the other is a SUB instruction.
2487  // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2488  MachineInstr *Sub = nullptr;
2489  if (SrcReg2 != 0)
2490  // MI is not a candidate for CMPrr.
2491  MI = nullptr;
2492  // FIXME: Conservatively refuse to convert an instruction which isn't in the
2493  // same BB as the comparison. This is to allow the check below to avoid calls
2494  // (and other explicit clobbers); instead we should really check for these
2495  // more explicitly (in at least a few predecessors).
2496  else if (MI->getParent() != CmpInstr.getParent())
2497  return false;
2498  else if (Value != 0) {
2499  // The record-form instructions set CR bit based on signed comparison
2500  // against 0. We try to convert a compare against 1 or -1 into a compare
2501  // against 0 to exploit record-form instructions. For example, we change
2502  // the condition "greater than -1" into "greater than or equal to 0"
2503  // and "less than 1" into "less than or equal to 0".
2504 
2505  // Since we optimize comparison based on a specific branch condition,
2506  // we don't optimize if condition code is used by more than once.
2507  if (equalityOnly || !MRI->hasOneUse(CRReg))
2508  return false;
2509 
2510  MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2511  if (UseMI->getOpcode() != PPC::BCC)
2512  return false;
2513 
2515  unsigned PredCond = PPC::getPredicateCondition(Pred);
2516  unsigned PredHint = PPC::getPredicateHint(Pred);
2517  int16_t Immed = (int16_t)Value;
2518 
2519  // When modifying the condition in the predicate, we propagate hint bits
2520  // from the original predicate to the new one.
2521  if (Immed == -1 && PredCond == PPC::PRED_GT)
2522  // We convert "greater than -1" into "greater than or equal to 0",
2523  // since we are assuming signed comparison by !equalityOnly
2524  Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2525  else if (Immed == -1 && PredCond == PPC::PRED_LE)
2526  // We convert "less than or equal to -1" into "less than 0".
2527  Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2528  else if (Immed == 1 && PredCond == PPC::PRED_LT)
2529  // We convert "less than 1" into "less than or equal to 0".
2530  Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2531  else if (Immed == 1 && PredCond == PPC::PRED_GE)
2532  // We convert "greater than or equal to 1" into "greater than 0".
2533  Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2534  else
2535  return false;
2536 
2537  // Convert the comparison and its user to a compare against zero with the
2538  // appropriate predicate on the branch. Zero comparison might provide
2539  // optimization opportunities post-RA (see optimization in
2540  // PPCPreEmitPeephole.cpp).
2541  UseMI->getOperand(0).setImm(Pred);
2542  CmpInstr.getOperand(2).setImm(0);
2543  }
2544 
2545  // Search for Sub.
2546  --I;
2547 
2548  // Get ready to iterate backward from CmpInstr.
2549  MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2550 
2551  for (; I != E && !noSub; --I) {
2552  const MachineInstr &Instr = *I;
2553  unsigned IOpC = Instr.getOpcode();
2554 
2555  if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2556  Instr.readsRegister(PPC::CR0, TRI)))
2557  // This instruction modifies or uses the record condition register after
2558  // the one we want to change. While we could do this transformation, it
2559  // would likely not be profitable. This transformation removes one
2560  // instruction, and so even forcing RA to generate one move probably
2561  // makes it unprofitable.
2562  return false;
2563 
2564  // Check whether CmpInstr can be made redundant by the current instruction.
2565  if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2566  OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2567  (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2568  ((Instr.getOperand(1).getReg() == SrcReg &&
2569  Instr.getOperand(2).getReg() == SrcReg2) ||
2570  (Instr.getOperand(1).getReg() == SrcReg2 &&
2571  Instr.getOperand(2).getReg() == SrcReg))) {
2572  Sub = &*I;
2573  break;
2574  }
2575 
2576  if (I == B)
2577  // The 'and' is below the comparison instruction.
2578  return false;
2579  }
2580 
2581  // Return false if no candidates exist.
2582  if (!MI && !Sub)
2583  return false;
2584 
2585  // The single candidate is called MI.
2586  if (!MI) MI = Sub;
2587 
2588  int NewOpC = -1;
2589  int MIOpC = MI->getOpcode();
2590  if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2591  MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2592  NewOpC = MIOpC;
2593  else {
2594  NewOpC = PPC::getRecordFormOpcode(MIOpC);
2595  if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2596  NewOpC = MIOpC;
2597  }
2598 
2599  // FIXME: On the non-embedded POWER architectures, only some of the record
2600  // forms are fast, and we should use only the fast ones.
2601 
2602  // The defining instruction has a record form (or is already a record
2603  // form). It is possible, however, that we'll need to reverse the condition
2604  // code of the users.
2605  if (NewOpC == -1)
2606  return false;
2607 
2608  // This transformation should not be performed if `nsw` is missing and is not
2609  // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2610  // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2611  // CRReg can reflect if compared values are equal, this optz is still valid.
2612  if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2613  Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2614  return false;
2615 
2616  // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2617  // needs to be updated to be based on SUB. Push the condition code
2618  // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2619  // condition code of these operands will be modified.
2620  // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2621  // comparison against 0, which may modify predicate.
2622  bool ShouldSwap = false;
2623  if (Sub && Value == 0) {
2624  ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2625  Sub->getOperand(2).getReg() == SrcReg;
2626 
2627  // The operands to subf are the opposite of sub, so only in the fixed-point
2628  // case, invert the order.
2629  ShouldSwap = !ShouldSwap;
2630  }
2631 
2632  if (ShouldSwap)
2634  I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2635  I != IE; ++I) {
2636  MachineInstr *UseMI = &*I;
2637  if (UseMI->getOpcode() == PPC::BCC) {
2639  unsigned PredCond = PPC::getPredicateCondition(Pred);
2640  assert((!equalityOnly ||
2641  PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2642  "Invalid predicate for equality-only optimization");
2643  (void)PredCond; // To suppress warning in release build.
2644  PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2645  PPC::getSwappedPredicate(Pred)));
2646  } else if (UseMI->getOpcode() == PPC::ISEL ||
2647  UseMI->getOpcode() == PPC::ISEL8) {
2648  unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2649  assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2650  "Invalid CR bit for equality-only optimization");
2651 
2652  if (NewSubReg == PPC::sub_lt)
2653  NewSubReg = PPC::sub_gt;
2654  else if (NewSubReg == PPC::sub_gt)
2655  NewSubReg = PPC::sub_lt;
2656 
2657  SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2658  NewSubReg));
2659  } else // We need to abort on a user we don't understand.
2660  return false;
2661  }
2662  assert(!(Value != 0 && ShouldSwap) &&
2663  "Non-zero immediate support and ShouldSwap"
2664  "may conflict in updating predicate");
2665 
2666  // Create a new virtual register to hold the value of the CR set by the
2667  // record-form instruction. If the instruction was not previously in
2668  // record form, then set the kill flag on the CR.
2669  CmpInstr.eraseFromParent();
2670 
2672  BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2673  get(TargetOpcode::COPY), CRReg)
2674  .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2675 
2676  // Even if CR0 register were dead before, it is alive now since the
2677  // instruction we just built uses it.
2678  MI->clearRegisterDeads(PPC::CR0);
2679 
2680  if (MIOpC != NewOpC) {
2681  // We need to be careful here: we're replacing one instruction with
2682  // another, and we need to make sure that we get all of the right
2683  // implicit uses and defs. On the other hand, the caller may be holding
2684  // an iterator to this instruction, and so we can't delete it (this is
2685  // specifically the case if this is the instruction directly after the
2686  // compare).
2687 
2688  // Rotates are expensive instructions. If we're emitting a record-form
2689  // rotate that can just be an andi/andis, we should just emit that.
2690  if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2691  Register GPRRes = MI->getOperand(0).getReg();
2692  int64_t SH = MI->getOperand(2).getImm();
2693  int64_t MB = MI->getOperand(3).getImm();
2694  int64_t ME = MI->getOperand(4).getImm();
2695  // We can only do this if both the start and end of the mask are in the
2696  // same halfword.
2697  bool MBInLoHWord = MB >= 16;
2698  bool MEInLoHWord = ME >= 16;
2699  uint64_t Mask = ~0LLU;
2700 
2701  if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2702  Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2703  // The mask value needs to shift right 16 if we're emitting andis.
2704  Mask >>= MBInLoHWord ? 0 : 16;
2705  NewOpC = MIOpC == PPC::RLWINM
2706  ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2707  : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2708  } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2709  (ME - MB + 1 == SH) && (MB >= 16)) {
2710  // If we are rotating by the exact number of bits as are in the mask
2711  // and the mask is in the least significant bits of the register,
2712  // that's just an andis. (as long as the GPR result has no uses).
2713  Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2714  Mask >>= 16;
2715  NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2716  }
2717  // If we've set the mask, we can transform.
2718  if (Mask != ~0LLU) {
2719  MI->removeOperand(4);
2720  MI->removeOperand(3);
2721  MI->getOperand(2).setImm(Mask);
2722  NumRcRotatesConvertedToRcAnd++;
2723  }
2724  } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2725  int64_t MB = MI->getOperand(3).getImm();
2726  if (MB >= 48) {
2727  uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2728  NewOpC = PPC::ANDI8_rec;
2729  MI->removeOperand(3);
2730  MI->getOperand(2).setImm(Mask);
2731  NumRcRotatesConvertedToRcAnd++;
2732  }
2733  }
2734 
2735  const MCInstrDesc &NewDesc = get(NewOpC);
2736  MI->setDesc(NewDesc);
2737 
2738  if (NewDesc.ImplicitDefs)
2739  for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
2740  *ImpDefs; ++ImpDefs)
2741  if (!MI->definesRegister(*ImpDefs))
2742  MI->addOperand(*MI->getParent()->getParent(),
2743  MachineOperand::CreateReg(*ImpDefs, true, true));
2744  if (NewDesc.ImplicitUses)
2745  for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
2746  *ImpUses; ++ImpUses)
2747  if (!MI->readsRegister(*ImpUses))
2748  MI->addOperand(*MI->getParent()->getParent(),
2749  MachineOperand::CreateReg(*ImpUses, false, true));
2750  }
2751  assert(MI->definesRegister(PPC::CR0) &&
2752  "Record-form instruction does not define cr0?");
2753 
2754  // Modify the condition code of operands in OperandsToUpdate.
2755  // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2756  // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2757  for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2758  PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2759 
2760  for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2761  SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2762 
2763  return true;
2764 }
2765 
2768  if (MRI->isSSA())
2769  return false;
2770 
2771  Register SrcReg, SrcReg2;
2772  int64_t CmpMask, CmpValue;
2773  if (!analyzeCompare(CmpMI, SrcReg, SrcReg2, CmpMask, CmpValue))
2774  return false;
2775 
2776  // Try to optimize the comparison against 0.
2777  if (CmpValue || !CmpMask || SrcReg2)
2778  return false;
2779 
2780  // The record forms set the condition register based on a signed comparison
2781  // with zero (see comments in optimizeCompareInstr). Since we can't do the
2782  // equality checks in post-RA, we are more restricted on a unsigned
2783  // comparison.
2784  unsigned Opc = CmpMI.getOpcode();
2785  if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2786  return false;
2787 
2788  // The record forms are always based on a 64-bit comparison on PPC64
2789  // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2790  // comparison. Since we can't do the equality checks in post-RA, we bail out
2791  // the case.
2792  if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2793  return false;
2794 
2795  // CmpMI can't be deleted if it has implicit def.
2796  if (CmpMI.hasImplicitDef())
2797  return false;
2798 
2799  bool SrcRegHasOtherUse = false;
2800  MachineInstr *SrcMI = getDefMIPostRA(SrcReg, CmpMI, SrcRegHasOtherUse);
2801  if (!SrcMI || !SrcMI->definesRegister(SrcReg))
2802  return false;
2803 
2804  MachineOperand RegMO = CmpMI.getOperand(0);
2805  Register CRReg = RegMO.getReg();
2806  if (CRReg != PPC::CR0)
2807  return false;
2808 
2809  // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2810  bool SeenUseOfCRReg = false;
2811  bool IsCRRegKilled = false;
2812  if (!isRegElgibleForForwarding(RegMO, *SrcMI, CmpMI, false, IsCRRegKilled,
2813  SeenUseOfCRReg) ||
2814  SrcMI->definesRegister(CRReg) || SeenUseOfCRReg)
2815  return false;
2816 
2817  int SrcMIOpc = SrcMI->getOpcode();
2818  int NewOpC = PPC::getRecordFormOpcode(SrcMIOpc);
2819  if (NewOpC == -1)
2820  return false;
2821 
2822  LLVM_DEBUG(dbgs() << "Replace Instr: ");
2823  LLVM_DEBUG(SrcMI->dump());
2824 
2825  const MCInstrDesc &NewDesc = get(NewOpC);
2826  SrcMI->setDesc(NewDesc);
2827  MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2829  SrcMI->clearRegisterDeads(CRReg);
2830 
2831  // Fix up killed/dead flag for SrcReg after transformation.
2832  if (SrcRegHasOtherUse || CmpMI.getOperand(1).isKill())
2833  fixupIsDeadOrKill(SrcMI, &CmpMI, SrcReg);
2834 
2835  assert(SrcMI->definesRegister(PPC::CR0) &&
2836  "Record-form instruction does not define cr0?");
2837 
2838  LLVM_DEBUG(dbgs() << "with: ");
2839  LLVM_DEBUG(SrcMI->dump());
2840  LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2841  LLVM_DEBUG(CmpMI.dump());
2842  return true;
2843 }
2844 
2847  int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2848  const TargetRegisterInfo *TRI) const {
2849  const MachineOperand *BaseOp;
2850  OffsetIsScalable = false;
2851  if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2852  return false;
2853  BaseOps.push_back(BaseOp);
2854  return true;
2855 }
2856 
2857 static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2858  const TargetRegisterInfo *TRI) {
2859  // If this is a volatile load/store, don't mess with it.
2860  if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2861  return false;
2862 
2863  if (LdSt.getOperand(2).isFI())
2864  return true;
2865 
2866  assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2867  // Can't cluster if the instruction modifies the base register
2868  // or it is update form. e.g. ld r2,3(r2)
2869  if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2870  return false;
2871 
2872  return true;
2873 }
2874 
2875 // Only cluster instruction pair that have the same opcode, and they are
2876 // clusterable according to PowerPC specification.
2877 static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2878  const PPCSubtarget &Subtarget) {
2879  switch (FirstOpc) {
2880  default:
2881  return false;
2882  case PPC::STD:
2883  case PPC::STFD:
2884  case PPC::STXSD:
2885  case PPC::DFSTOREf64:
2886  return FirstOpc == SecondOpc;
2887  // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2888  // 32bit and 64bit instruction selection. They are clusterable pair though
2889  // they are different opcode.
2890  case PPC::STW:
2891  case PPC::STW8:
2892  return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2893  }
2894 }
2895 
2898  ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2899  unsigned NumBytes) const {
2900 
2901  assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2902  const MachineOperand &BaseOp1 = *BaseOps1.front();
2903  const MachineOperand &BaseOp2 = *BaseOps2.front();
2904  assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2905  "Only base registers and frame indices are supported.");
2906 
2907  // The NumLoads means the number of loads that has been clustered.
2908  // Don't cluster memory op if there are already two ops clustered at least.
2909  if (NumLoads > 2)
2910  return false;
2911 
2912  // Cluster the load/store only when they have the same base
2913  // register or FI.
2914  if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2915  (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2916  (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2917  return false;
2918 
2919  // Check if the load/store are clusterable according to the PowerPC
2920  // specification.
2921  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2922  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2923  unsigned FirstOpc = FirstLdSt.getOpcode();
2924  unsigned SecondOpc = SecondLdSt.getOpcode();
2926  // Cluster the load/store only when they have the same opcode, and they are
2927  // clusterable opcode according to PowerPC specification.
2928  if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2929  return false;
2930 
2931  // Can't cluster load/store that have ordered or volatile memory reference.
2932  if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2933  !isLdStSafeToCluster(SecondLdSt, TRI))
2934  return false;
2935 
2936  int64_t Offset1 = 0, Offset2 = 0;
2937  unsigned Width1 = 0, Width2 = 0;
2938  const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2939  if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2940  !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2941  Width1 != Width2)
2942  return false;
2943 
2944  assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2945  "getMemOperandWithOffsetWidth return incorrect base op");
2946  // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2947  assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2948  return Offset1 + Width1 == Offset2;
2949 }
2950 
2951 /// GetInstSize - Return the number of bytes of code the specified
2952 /// instruction may be. This returns the maximum number of bytes.
2953 ///
2955  unsigned Opcode = MI.getOpcode();
2956 
2957  if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2958  const MachineFunction *MF = MI.getParent()->getParent();
2959  const char *AsmStr = MI.getOperand(0).getSymbolName();
2960  return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2961  } else if (Opcode == TargetOpcode::STACKMAP) {
2962  StackMapOpers Opers(&MI);
2963  return Opers.getNumPatchBytes();
2964  } else if (Opcode == TargetOpcode::PATCHPOINT) {
2965  PatchPointOpers Opers(&MI);
2966  return Opers.getNumPatchBytes();
2967  } else {
2968  return get(Opcode).getSize();
2969  }
2970 }
2971 
2972 std::pair<unsigned, unsigned>
2974  const unsigned Mask = PPCII::MO_ACCESS_MASK;
2975  return std::make_pair(TF & Mask, TF & ~Mask);
2976 }
2977 
2980  using namespace PPCII;
2981  static const std::pair<unsigned, const char *> TargetFlags[] = {
2982  {MO_LO, "ppc-lo"},
2983  {MO_HA, "ppc-ha"},
2984  {MO_TPREL_LO, "ppc-tprel-lo"},
2985  {MO_TPREL_HA, "ppc-tprel-ha"},
2986  {MO_DTPREL_LO, "ppc-dtprel-lo"},
2987  {MO_TLSLD_LO, "ppc-tlsld-lo"},
2988  {MO_TOC_LO, "ppc-toc-lo"},
2989  {MO_TLS, "ppc-tls"}};
2990  return makeArrayRef(TargetFlags);
2991 }
2992 
2995  using namespace PPCII;
2996  static const std::pair<unsigned, const char *> TargetFlags[] = {
2997  {MO_PLT, "ppc-plt"},
2998  {MO_PIC_FLAG, "ppc-pic"},
2999  {MO_PCREL_FLAG, "ppc-pcrel"},
3000  {MO_GOT_FLAG, "ppc-got"},
3001  {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
3002  {MO_TLSGD_FLAG, "ppc-tlsgd"},
3003  {MO_TLSLD_FLAG, "ppc-tlsld"},
3004  {MO_TPREL_FLAG, "ppc-tprel"},
3005  {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
3006  {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
3007  {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
3008  {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}};
3009  return makeArrayRef(TargetFlags);
3010 }
3011 
3012 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
3013 // The VSX versions have the advantage of a full 64-register target whereas
3014 // the FP ones have the advantage of lower latency and higher throughput. So
3015 // what we are after is using the faster instructions in low register pressure
3016 // situations and using the larger register file in high register pressure
3017 // situations.
3019  unsigned UpperOpcode, LowerOpcode;
3020  switch (MI.getOpcode()) {
3021  case PPC::DFLOADf32:
3022  UpperOpcode = PPC::LXSSP;
3023  LowerOpcode = PPC::LFS;
3024  break;
3025  case PPC::DFLOADf64:
3026  UpperOpcode = PPC::LXSD;
3027  LowerOpcode = PPC::LFD;
3028  break;
3029  case PPC::DFSTOREf32:
3030  UpperOpcode = PPC::STXSSP;
3031  LowerOpcode = PPC::STFS;
3032  break;
3033  case PPC::DFSTOREf64:
3034  UpperOpcode = PPC::STXSD;
3035  LowerOpcode = PPC::STFD;
3036  break;
3037  case PPC::XFLOADf32:
3038  UpperOpcode = PPC::LXSSPX;
3039  LowerOpcode = PPC::LFSX;
3040  break;
3041  case PPC::XFLOADf64:
3042  UpperOpcode = PPC::LXSDX;
3043  LowerOpcode = PPC::LFDX;
3044  break;
3045  case PPC::XFSTOREf32:
3046  UpperOpcode = PPC::STXSSPX;
3047  LowerOpcode = PPC::STFSX;
3048  break;
3049  case PPC::XFSTOREf64:
3050  UpperOpcode = PPC::STXSDX;
3051  LowerOpcode = PPC::STFDX;
3052  break;
3053  case PPC::LIWAX:
3054  UpperOpcode = PPC::LXSIWAX;
3055  LowerOpcode = PPC::LFIWAX;
3056  break;
3057  case PPC::LIWZX:
3058  UpperOpcode = PPC::LXSIWZX;
3059  LowerOpcode = PPC::LFIWZX;
3060  break;
3061  case PPC::STIWX:
3062  UpperOpcode = PPC::STXSIWX;
3063  LowerOpcode = PPC::STFIWX;
3064  break;
3065  default:
3066  llvm_unreachable("Unknown Operation!");
3067  }
3068 
3069  Register TargetReg = MI.getOperand(0).getReg();
3070  unsigned Opcode;
3071  if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3072  (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3073  Opcode = LowerOpcode;
3074  else
3075  Opcode = UpperOpcode;
3076  MI.setDesc(get(Opcode));
3077  return true;
3078 }
3079 
3080 static bool isAnImmediateOperand(const MachineOperand &MO) {
3081  return MO.isCPI() || MO.isGlobal() || MO.isImm();
3082 }
3083 
3085  auto &MBB = *MI.getParent();
3086  auto DL = MI.getDebugLoc();
3087 
3088  switch (MI.getOpcode()) {
3089  case PPC::BUILD_UACC: {
3090  MCRegister ACC = MI.getOperand(0).getReg();
3091  MCRegister UACC = MI.getOperand(1).getReg();
3092  if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3093  MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3094  MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3095  // FIXME: This can easily be improved to look up to the top of the MBB
3096  // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3097  // we can just re-target any such XXLOR's to DstVSR + offset.
3098  for (int VecNo = 0; VecNo < 4; VecNo++)
3099  BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3100  .addReg(SrcVSR + VecNo)
3101  .addReg(SrcVSR + VecNo);
3102  }
3103  // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3104  // So after building the 4 copies, we can replace the BUILD_UACC instruction
3105  // with a NOP.
3106  [[fallthrough]];
3107  }
3108  case PPC::KILL_PAIR: {
3109  MI.setDesc(get(PPC::UNENCODED_NOP));
3110  MI.removeOperand(1);
3111  MI.removeOperand(0);
3112  return true;
3113  }
3114  case TargetOpcode::LOAD_STACK_GUARD: {
3115  assert(Subtarget.isTargetLinux() &&
3116  "Only Linux target is expected to contain LOAD_STACK_GUARD");
3117  const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3118  const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3119  MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3120  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3121  .addImm(Offset)
3122  .addReg(Reg);
3123  return true;
3124  }
3125  case PPC::DFLOADf32:
3126  case PPC::DFLOADf64:
3127  case PPC::DFSTOREf32:
3128  case PPC::DFSTOREf64: {
3129  assert(Subtarget.hasP9Vector() &&
3130  "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3131  assert(MI.getOperand(2).isReg() &&
3132  isAnImmediateOperand(MI.getOperand(1)) &&
3133  "D-form op must have register and immediate operands");
3134  return expandVSXMemPseudo(MI);
3135  }
3136  case PPC::XFLOADf32:
3137  case PPC::XFSTOREf32:
3138  case PPC::LIWAX:
3139  case PPC::LIWZX:
3140  case PPC::STIWX: {
3141  assert(Subtarget.hasP8Vector() &&
3142  "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3143  assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3144  "X-form op must have register and register operands");
3145  return expandVSXMemPseudo(MI);
3146  }
3147  case PPC::XFLOADf64:
3148  case PPC::XFSTOREf64: {
3149  assert(Subtarget.hasVSX() &&
3150  "Invalid X-Form Pseudo-ops on target that has no VSX.");
3151  assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3152  "X-form op must have register and register operands");
3153  return expandVSXMemPseudo(MI);
3154  }
3155  case PPC::SPILLTOVSR_LD: {
3156  Register TargetReg = MI.getOperand(0).getReg();
3157  if (PPC::VSFRCRegClass.contains(TargetReg)) {
3158  MI.setDesc(get(PPC::DFLOADf64));
3159  return expandPostRAPseudo(MI);
3160  }
3161  else
3162  MI.setDesc(get(PPC::LD));
3163  return true;
3164  }
3165  case PPC::SPILLTOVSR_ST: {
3166  Register SrcReg = MI.getOperand(0).getReg();
3167  if (PPC::VSFRCRegClass.contains(SrcReg)) {
3168  NumStoreSPILLVSRRCAsVec++;
3169  MI.setDesc(get(PPC::DFSTOREf64));
3170  return expandPostRAPseudo(MI);
3171  } else {
3172  NumStoreSPILLVSRRCAsGpr++;
3173  MI.setDesc(get(PPC::STD));
3174  }
3175  return true;
3176  }
3177  case PPC::SPILLTOVSR_LDX: {
3178  Register TargetReg = MI.getOperand(0).getReg();
3179  if (PPC::VSFRCRegClass.contains(TargetReg))
3180  MI.setDesc(get(PPC::LXSDX));
3181  else
3182  MI.setDesc(get(PPC::LDX));
3183  return true;
3184  }
3185  case PPC::SPILLTOVSR_STX: {
3186  Register SrcReg = MI.getOperand(0).getReg();
3187  if (PPC::VSFRCRegClass.contains(SrcReg)) {
3188  NumStoreSPILLVSRRCAsVec++;
3189  MI.setDesc(get(PPC::STXSDX));
3190  } else {
3191  NumStoreSPILLVSRRCAsGpr++;
3192  MI.setDesc(get(PPC::STDX));
3193  }
3194  return true;
3195  }
3196 
3197  // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3198  case PPC::CFENCE8: {
3199  auto Val = MI.getOperand(0).getReg();
3200  BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
3201  BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3203  .addReg(PPC::CR7)
3204  .addImm(1);
3205  MI.setDesc(get(PPC::ISYNC));
3206  MI.removeOperand(0);
3207  return true;
3208  }
3209  }
3210  return false;
3211 }
3212 
3213 // Essentially a compile-time implementation of a compare->isel sequence.
3214 // It takes two constants to compare, along with the true/false registers
3215 // and the comparison type (as a subreg to a CR field) and returns one
3216 // of the true/false registers, depending on the comparison results.
3217 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3218  unsigned TrueReg, unsigned FalseReg,
3219  unsigned CRSubReg) {
3220  // Signed comparisons. The immediates are assumed to be sign-extended.
3221  if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3222  switch (CRSubReg) {
3223  default: llvm_unreachable("Unknown integer comparison type.");
3224  case PPC::sub_lt:
3225  return Imm1 < Imm2 ? TrueReg : FalseReg;
3226  case PPC::sub_gt:
3227  return Imm1 > Imm2 ? TrueReg : FalseReg;
3228  case PPC::sub_eq:
3229  return Imm1 == Imm2 ? TrueReg : FalseReg;
3230  }
3231  }
3232  // Unsigned comparisons.
3233  else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3234  switch (CRSubReg) {
3235  default: llvm_unreachable("Unknown integer comparison type.");
3236  case PPC::sub_lt:
3237  return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3238  case PPC::sub_gt:
3239  return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3240  case PPC::sub_eq:
3241  return Imm1 == Imm2 ? TrueReg : FalseReg;
3242  }
3243  }
3244  return PPC::NoRegister;
3245 }
3246 
3248  unsigned OpNo,
3249  int64_t Imm) const {
3250  assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3251  // Replace the REG with the Immediate.
3252  Register InUseReg = MI.getOperand(OpNo).getReg();
3253  MI.getOperand(OpNo).ChangeToImmediate(Imm);
3254 
3255  // We need to make sure that the MI didn't have any implicit use
3256  // of this REG any more. We don't call MI.implicit_operands().empty() to
3257  // return early, since MI's MCID might be changed in calling context, as a
3258  // result its number of explicit operands may be changed, thus the begin of
3259  // implicit operand is changed.
3261  int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3262  if (UseOpIdx >= 0) {
3263  MachineOperand &MO = MI.getOperand(UseOpIdx);
3264  if (MO.isImplicit())
3265  // The operands must always be in the following order:
3266  // - explicit reg defs,
3267  // - other explicit operands (reg uses, immediates, etc.),
3268  // - implicit reg defs
3269  // - implicit reg uses
3270  // Therefore, removing the implicit operand won't change the explicit
3271  // operands layout.
3272  MI.removeOperand(UseOpIdx);
3273  }
3274 }
3275 
3276 // Replace an instruction with one that materializes a constant (and sets
3277 // CR0 if the original instruction was a record-form instruction).
3279  const LoadImmediateInfo &LII) const {
3280  // Remove existing operands.
3281  int OperandToKeep = LII.SetCR ? 1 : 0;
3282  for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3283  MI.removeOperand(i);
3284 
3285  // Replace the instruction.
3286  if (LII.SetCR) {
3287  MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3288  // Set the immediate.
3289  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3290  .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3291  return;
3292  }
3293  else
3294  MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3295 
3296  // Set the immediate.
3297  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3298  .addImm(LII.Imm);
3299 }
3300 
3302  bool &SeenIntermediateUse) const {
3303  assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3304  "Should be called after register allocation.");
3306  MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3307  It++;
3308  SeenIntermediateUse = false;
3309  for (; It != E; ++It) {
3310  if (It->modifiesRegister(Reg, TRI))
3311  return &*It;
3312  if (It->readsRegister(Reg, TRI))
3313  SeenIntermediateUse = true;
3314  }
3315  return nullptr;
3316 }
3317 
3320  const DebugLoc &DL, Register Reg,
3321  int64_t Imm) const {
3322  assert(!MBB.getParent()->getRegInfo().isSSA() &&
3323  "Register should be in non-SSA form after RA");
3324  bool isPPC64 = Subtarget.isPPC64();
3325  // FIXME: Materialization here is not optimal.
3326  // For some special bit patterns we can use less instructions.
3327  // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3328  if (isInt<16>(Imm)) {
3329  BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
3330  } else if (isInt<32>(Imm)) {
3331  BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
3332  .addImm(Imm >> 16);
3333  if (Imm & 0xFFFF)
3334  BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
3336  .addImm(Imm & 0xFFFF);
3337  } else {
3338  assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3339  "only supported in PPC64");
3340  BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
3341  if ((Imm >> 32) & 0xFFFF)
3342  BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3344  .addImm((Imm >> 32) & 0xFFFF);
3345  BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
3347  .addImm(32)
3348  .addImm(31);
3349  BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
3351  .addImm((Imm >> 16) & 0xFFFF);
3352  if (Imm & 0xFFFF)
3353  BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3355  .addImm(Imm & 0xFFFF);
3356  }
3357 }
3358 
3359 MachineInstr *PPCInstrInfo::getForwardingDefMI(
3360  MachineInstr &MI,
3361  unsigned &OpNoForForwarding,
3362  bool &SeenIntermediateUse) const {
3363  OpNoForForwarding = ~0U;
3364  MachineInstr *DefMI = nullptr;
3365  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3367  // If we're in SSA, get the defs through the MRI. Otherwise, only look
3368  // within the basic block to see if the register is defined using an
3369  // LI/LI8/ADDI/ADDI8.
3370  if (MRI->isSSA()) {
3371  for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3372  if (!MI.getOperand(i).isReg())
3373  continue;
3374  Register Reg = MI.getOperand(i).getReg();
3376  continue;
3377  Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3378  if (Register::isVirtualRegister(TrueReg)) {
3379  DefMI = MRI->getVRegDef(TrueReg);
3380  if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
3381  DefMI->getOpcode() == PPC::ADDI ||
3382  DefMI->getOpcode() == PPC::ADDI8) {
3383  OpNoForForwarding = i;
3384  // The ADDI and LI operand maybe exist in one instruction at same
3385  // time. we prefer to fold LI operand as LI only has one Imm operand
3386  // and is more possible to be converted. So if current DefMI is
3387  // ADDI/ADDI8, we continue to find possible LI/LI8.
3388  if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3389  break;
3390  }
3391  }
3392  }
3393  } else {
3394  // Looking back through the definition for each operand could be expensive,
3395  // so exit early if this isn't an instruction that either has an immediate
3396  // form or is already an immediate form that we can handle.
3397  ImmInstrInfo III;
3398  unsigned Opc = MI.getOpcode();
3399  bool ConvertibleImmForm =
3400  Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3401  Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3402  Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3403  Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3404  Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3405  Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3406  Opc == PPC::RLWINM8_rec;
3407  bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3408  ? isVFRegister(MI.getOperand(0).getReg())
3409  : false;
3410  if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3411  return nullptr;
3412 
3413  // Don't convert or %X, %Y, %Y since that's just a register move.
3414  if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3415  MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3416  return nullptr;
3417  for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3418  MachineOperand &MO = MI.getOperand(i);
3419  SeenIntermediateUse = false;
3420  if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3421  Register Reg = MI.getOperand(i).getReg();
3422  // If we see another use of this reg between the def and the MI,
3423  // we want to flat it so the def isn't deleted.
3424  MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3425  if (DefMI) {
3426  // Is this register defined by some form of add-immediate (including
3427  // load-immediate) within this basic block?
3428  switch (DefMI->getOpcode()) {
3429  default:
3430  break;
3431  case PPC::LI:
3432  case PPC::LI8:
3433  case PPC::ADDItocL:
3434  case PPC::ADDI:
3435  case PPC::ADDI8:
3436  OpNoForForwarding = i;
3437  return DefMI;
3438  }
3439  }
3440  }
3441  }
3442  }
3443  return OpNoForForwarding == ~0U ? nullptr : DefMI;
3444 }
3445 
3446 unsigned PPCInstrInfo::getSpillTarget() const {
3447  // With P10, we may need to spill paired vector registers or accumulator
3448  // registers. MMA implies paired vectors, so we can just check that.
3449  bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3450  return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3451  2 : Subtarget.hasP9Vector() ?
3452  1 : 0;
3453 }
3454 
3455 ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3456  return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3457 }
3458 
3459 ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3460  return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3461 }
3462 
3464  unsigned RegNo) const {
3465  // Conservatively clear kill flag for the register if the instructions are in
3466  // different basic blocks and in SSA form, because the kill flag may no longer
3467  // be right. There is no need to bother with dead flags since defs with no
3468  // uses will be handled by DCE.
3470  if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) {
3471  MRI.clearKillFlags(RegNo);
3472  return;
3473  }
3474 
3475  // Instructions between [StartMI, EndMI] should be in same basic block.
3476  assert((StartMI->getParent() == EndMI->getParent()) &&
3477  "Instructions are not in same basic block");
3478 
3479  // If before RA, StartMI may be def through COPY, we need to adjust it to the
3480  // real def. See function getForwardingDefMI.
3481  if (MRI.isSSA()) {
3482  bool Reads, Writes;
3483  std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo);
3484  if (!Reads && !Writes) {
3486  "Must be a virtual register");
3487  // Get real def and ignore copies.
3488  StartMI = MRI.getVRegDef(RegNo);
3489  }
3490  }
3491 
3492  bool IsKillSet = false;
3493 
3494  auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
3495  MachineOperand &MO = MI.getOperand(Index);
3496  if (MO.isReg() && MO.isUse() && MO.isKill() &&
3497  getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
3498  MO.setIsKill(false);
3499  };
3500 
3501  // Set killed flag for EndMI.
3502  // No need to do anything if EndMI defines RegNo.
3503  int UseIndex =
3504  EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
3505  if (UseIndex != -1) {
3506  EndMI->getOperand(UseIndex).setIsKill(true);
3507  IsKillSet = true;
3508  // Clear killed flag for other EndMI operands related to RegNo. In some
3509  // upexpected cases, killed may be set multiple times for same register
3510  // operand in same MI.
3511  for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i)
3512  if (i != UseIndex)
3513  clearOperandKillInfo(*EndMI, i);
3514  }
3515 
3516  // Walking the inst in reverse order (EndMI -> StartMI].
3519  // EndMI has been handled above, skip it here.
3520  It++;
3521  MachineOperand *MO = nullptr;
3522  for (; It != E; ++It) {
3523  // Skip insturctions which could not be a def/use of RegNo.
3524  if (It->isDebugInstr() || It->isPosition())
3525  continue;
3526 
3527  // Clear killed flag for all It operands related to RegNo. In some
3528  // upexpected cases, killed may be set multiple times for same register
3529  // operand in same MI.
3530  for (int i = 0, e = It->getNumOperands(); i != e; ++i)
3531  clearOperandKillInfo(*It, i);
3532 
3533  // If killed is not set, set killed for its last use or set dead for its def
3534  // if no use found.
3535  if (!IsKillSet) {
3536  if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
3537  // Use found, set it killed.
3538  IsKillSet = true;
3539  MO->setIsKill(true);
3540  continue;
3541  } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
3542  &getRegisterInfo()))) {
3543  // No use found, set dead for its def.
3544  assert(&*It == StartMI && "No new def between StartMI and EndMI.");
3545  MO->setIsDead(true);
3546  break;
3547  }
3548  }
3549 
3550  if ((&*It) == StartMI)
3551  break;
3552  }
3553  // Ensure RegMo liveness is killed after EndMI.
3554  assert((IsKillSet || (MO && MO->isDead())) &&
3555  "RegNo should be killed or dead");
3556 }
3557 
3558 // This opt tries to convert the following imm form to an index form to save an
3559 // add for stack variables.
3560 // Return false if no such pattern found.
3561 //
3562 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3563 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3564 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3565 //
3566 // can be converted to:
3567 //
3568 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3569 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3570 //
3571 // In order to eliminate ADD instr, make sure that:
3572 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3573 // new ADDI instr and ADDI can only take int16 Imm.
3574 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3575 // between ADDI and ADD instr since its original def in ADDI will be changed
3576 // in new ADDI instr. And also there should be no new def for it between
3577 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3578 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3579 // between ADD and Imm instr since ADD instr will be eliminated.
3580 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3581 // moved to Index instr.
3583  MachineFunction *MF = MI.getParent()->getParent();
3585  bool PostRA = !MRI->isSSA();
3586  // Do this opt after PEI which is after RA. The reason is stack slot expansion
3587  // in PEI may expose such opportunities since in PEI, stack slot offsets to
3588  // frame base(OffsetAddi) are determined.
3589  if (!PostRA)
3590  return false;
3591  unsigned ToBeDeletedReg = 0;
3592  int64_t OffsetImm = 0;
3593  unsigned XFormOpcode = 0;
3594  ImmInstrInfo III;
3595 
3596  // Check if Imm instr meets requirement.
3597  if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3598  III))
3599  return false;
3600 
3601  bool OtherIntermediateUse = false;
3602  MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3603 
3604  // Exit if there is other use between ADD and Imm instr or no def found.
3605  if (OtherIntermediateUse || !ADDMI)
3606  return false;
3607 
3608  // Check if ADD instr meets requirement.
3609  if (!isADDInstrEligibleForFolding(*ADDMI))
3610  return false;
3611 
3612  unsigned ScaleRegIdx = 0;
3613  int64_t OffsetAddi = 0;
3614  MachineInstr *ADDIMI = nullptr;
3615 
3616  // Check if there is a valid ToBeChangedReg in ADDMI.
3617  // 1: It must be killed.
3618  // 2: Its definition must be a valid ADDIMI.
3619  // 3: It must satify int16 offset requirement.
3620  if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3621  ScaleRegIdx = 2;
3622  else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3623  ScaleRegIdx = 1;
3624  else
3625  return false;
3626 
3627  assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3628  Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3629  Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3630  auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3632  for (auto It = ++Start; It != End; It++)
3633  if (It->modifiesRegister(Reg, &getRegisterInfo()))
3634  return true;
3635  return false;
3636  };
3637 
3638  // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3639  // treated as special zero when ScaleReg is R0/X0 register.
3640  if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3641  (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3642  return false;
3643 
3644  // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3645  // and Imm Instr.
3646  if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3647  return false;
3648 
3649  // Now start to do the transformation.
3650  LLVM_DEBUG(dbgs() << "Replace instruction: "
3651  << "\n");
3652  LLVM_DEBUG(ADDIMI->dump());
3653  LLVM_DEBUG(ADDMI->dump());
3654  LLVM_DEBUG(MI.dump());
3655  LLVM_DEBUG(dbgs() << "with: "
3656  << "\n");
3657 
3658  // Update ADDI instr.
3659  ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3660 
3661  // Update Imm instr.
3662  MI.setDesc(get(XFormOpcode));
3663  MI.getOperand(III.ImmOpNo)
3664  .ChangeToRegister(ScaleReg, false, false,
3665  ADDMI->getOperand(ScaleRegIdx).isKill());
3666 
3667  MI.getOperand(III.OpNoForForwarding)
3668  .ChangeToRegister(ToBeChangedReg, false, false, true);
3669 
3670  // Eliminate ADD instr.
3671  ADDMI->eraseFromParent();
3672 
3673  LLVM_DEBUG(ADDIMI->dump());
3674  LLVM_DEBUG(MI.dump());
3675 
3676  return true;
3677 }
3678 
3680  int64_t &Imm) const {
3681  unsigned Opc = ADDIMI.getOpcode();
3682 
3683  // Exit if the instruction is not ADDI.
3684  if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3685  return false;
3686 
3687  // The operand may not necessarily be an immediate - it could be a relocation.
3688  if (!ADDIMI.getOperand(2).isImm())
3689  return false;
3690 
3691  Imm = ADDIMI.getOperand(2).getImm();
3692 
3693  return true;
3694 }
3695 
3697  unsigned Opc = ADDMI.getOpcode();
3698 
3699  // Exit if the instruction is not ADD.
3700  return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3701 }
3702 
3704  unsigned &ToBeDeletedReg,
3705  unsigned &XFormOpcode,
3706  int64_t &OffsetImm,
3707  ImmInstrInfo &III) const {
3708  // Only handle load/store.
3709  if (!MI.mayLoadOrStore())
3710  return false;
3711 
3712  unsigned Opc = MI.getOpcode();
3713 
3714  XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3715 
3716  // Exit if instruction has no index form.
3717  if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3718  return false;
3719 
3720  // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3721  if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
3722  III, true))
3723  return false;
3724 
3725  if (!III.IsSummingOperands)
3726  return false;
3727 
3728  MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3729  MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3730  // Only support imm operands, not relocation slots or others.
3731  if (!ImmOperand.isImm())
3732  return false;
3733 
3734  assert(RegOperand.isReg() && "Instruction format is not right");
3735 
3736  // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3737  if (!RegOperand.isKill())
3738  return false;
3739 
3740  ToBeDeletedReg = RegOperand.getReg();
3741  OffsetImm = ImmOperand.getImm();
3742 
3743  return true;
3744 }
3745 
3747  MachineInstr *&ADDIMI,
3748  int64_t &OffsetAddi,
3749  int64_t OffsetImm) const {
3750  assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3751  MachineOperand &MO = ADDMI->getOperand(Index);
3752 
3753  if (!MO.isKill())
3754  return false;
3755 
3756  bool OtherIntermediateUse = false;
3757 
3758  ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3759  // Currently handle only one "add + Imminstr" pair case, exit if other
3760  // intermediate use for ToBeChangedReg found.
3761  // TODO: handle the cases where there are other "add + Imminstr" pairs
3762  // with same offset in Imminstr which is like:
3763  //
3764  // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3765  // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3766  // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3767  // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3768  // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3769  //
3770  // can be converted to:
3771  //
3772  // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3773  // (OffsetAddi + OffsetImm)
3774  // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3775  // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3776 
3777  if (OtherIntermediateUse || !ADDIMI)
3778  return false;
3779  // Check if ADDI instr meets requirement.
3780  if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3781  return false;
3782 
3783  if (isInt<16>(OffsetAddi + OffsetImm))
3784  return true;
3785  return false;
3786 }
3787 
3788 // If this instruction has an immediate form and one of its operands is a
3789 // result of a load-immediate or an add-immediate, convert it to
3790 // the immediate form if the constant is in range.
3792  MachineInstr **KilledDef) const {
3793  MachineFunction *MF = MI.getParent()->getParent();
3795  bool PostRA = !MRI->isSSA();
3796  bool SeenIntermediateUse = true;
3797  unsigned ForwardingOperand = ~0U;
3798  MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3799  SeenIntermediateUse);
3800  if (!DefMI)
3801  return false;
3802  assert(ForwardingOperand < MI.getNumOperands() &&
3803  "The forwarding operand needs to be valid at this point");
3804  bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3805  bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3806  if (KilledDef && KillFwdDefMI)
3807  *KilledDef = DefMI;
3808 
3809  // If this is a imm instruction and its register operands is produced by ADDI,
3810  // put the imm into imm inst directly.
3811  if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3812  PPC::INSTRUCTION_LIST_END &&
3813  transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3814  return true;
3815 
3816  ImmInstrInfo III;
3817  bool IsVFReg = MI.getOperand(0).isReg()
3818  ? isVFRegister(MI.getOperand(0).getReg())
3819  : false;
3820  bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3821  // If this is a reg+reg instruction that has a reg+imm form,
3822  // and one of the operands is produced by an add-immediate,
3823  // try to convert it.
3824  if (HasImmForm &&
3825  transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3826  KillFwdDefMI))
3827  return true;
3828 
3829  // If this is a reg+reg instruction that has a reg+imm form,
3830  // and one of the operands is produced by LI, convert it now.
3831  if (HasImmForm &&
3832  transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3833  return true;
3834 
3835  // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3836  // can be simpified to LI.
3837  if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3838  return true;
3839 
3840  return false;
3841 }
3842 
3844  MachineInstr **ToErase) const {
3845  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3846  Register FoldingReg = MI.getOperand(1).getReg();
3847  if (!Register::isVirtualRegister(FoldingReg))
3848  return false;
3849  MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3850  if (SrcMI->getOpcode() != PPC::RLWINM &&
3851  SrcMI->getOpcode() != PPC::RLWINM_rec &&
3852  SrcMI->getOpcode() != PPC::RLWINM8 &&
3853  SrcMI->getOpcode() != PPC::RLWINM8_rec)
3854  return false;
3855  assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3856  MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3857  SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3858  "Invalid PPC::RLWINM Instruction!");
3859  uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3860  uint64_t SHMI = MI.getOperand(2).getImm();
3861  uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3862  uint64_t MBMI = MI.getOperand(3).getImm();
3863  uint64_t MESrc = SrcMI->getOperand(4).getImm();
3864  uint64_t MEMI = MI.getOperand(4).getImm();
3865 
3866  assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3867  "Invalid PPC::RLWINM Instruction!");
3868  // If MBMI is bigger than MEMI, we always can not get run of ones.
3869  // RotatedSrcMask non-wrap:
3870  // 0........31|32........63
3871  // RotatedSrcMask: B---E B---E
3872  // MaskMI: -----------|--E B------
3873  // Result: ----- --- (Bad candidate)
3874  //
3875  // RotatedSrcMask wrap:
3876  // 0........31|32........63
3877  // RotatedSrcMask: --E B----|--E B----
3878  // MaskMI: -----------|--E B------
3879  // Result: --- -----|--- ----- (Bad candidate)
3880  //
3881  // One special case is RotatedSrcMask is a full set mask.
3882  // RotatedSrcMask full:
3883  // 0........31|32........63
3884  // RotatedSrcMask: ------EB---|-------EB---
3885  // MaskMI: -----------|--E B------
3886  // Result: -----------|--- ------- (Good candidate)
3887 
3888  // Mark special case.
3889  bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3890 
3891  // For other MBMI > MEMI cases, just return.
3892  if ((MBMI > MEMI) && !SrcMaskFull)
3893  return false;
3894 
3895  // Handle MBMI <= MEMI cases.
3896  APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3897  // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3898  // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3899  // while in PowerPC ISA, lowerest bit is at index 63.
3900  APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3901 
3902  APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3903  APInt FinalMask = RotatedSrcMask & MaskMI;
3904  uint32_t NewMB, NewME;
3905  bool Simplified = false;
3906 
3907  // If final mask is 0, MI result should be 0 too.
3908  if (FinalMask.isZero()) {
3909  bool Is64Bit =
3910  (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3911  Simplified = true;
3912  LLVM_DEBUG(dbgs() << "Replace Instr: ");
3913  LLVM_DEBUG(MI.dump());
3914 
3915  if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3916  // Replace MI with "LI 0"
3917  MI.removeOperand(4);
3918  MI.removeOperand(3);
3919  MI.removeOperand(2);
3920  MI.getOperand(1).ChangeToImmediate(0);
3921  MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3922  } else {
3923  // Replace MI with "ANDI_rec reg, 0"
3924  MI.removeOperand(4);
3925  MI.removeOperand(3);
3926  MI.getOperand(2).setImm(0);
3927  MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3928  MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3929  if (SrcMI->getOperand(1).isKill()) {
3930  MI.getOperand(1).setIsKill(true);
3931  SrcMI->getOperand(1).setIsKill(false);
3932  } else
3933  // About to replace MI.getOperand(1), clear its kill flag.
3934  MI.getOperand(1).setIsKill(false);
3935  }
3936 
3937  LLVM_DEBUG(dbgs() << "With: ");
3938  LLVM_DEBUG(MI.dump());
3939 
3940  } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3941  NewMB <= NewME) ||
3942  SrcMaskFull) {
3943  // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3944  // than NewME. Otherwise we get a 64 bit value after folding, but MI
3945  // return a 32 bit value.
3946  Simplified = true;
3947  LLVM_DEBUG(dbgs() << "Converting Instr: ");
3948  LLVM_DEBUG(MI.dump());
3949 
3950  uint16_t NewSH = (SHSrc + SHMI) % 32;
3951  MI.getOperand(2).setImm(NewSH);
3952  // If SrcMI mask is full, no need to update MBMI and MEMI.
3953  if (!SrcMaskFull) {
3954  MI.getOperand(3).setImm(NewMB);
3955  MI.getOperand(4).setImm(NewME);
3956  }
3957  MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3958  if (SrcMI->getOperand(1).isKill()) {
3959  MI.getOperand(1).setIsKill(true);
3960  SrcMI->getOperand(1).setIsKill(false);
3961  } else
3962  // About to replace MI.getOperand(1), clear its kill flag.
3963  MI.getOperand(1).setIsKill(false);
3964 
3965  LLVM_DEBUG(dbgs() << "To: ");
3966  LLVM_DEBUG(MI.dump());
3967  }
3968  if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3969  !SrcMI->hasImplicitDef()) {
3970  // If FoldingReg has no non-debug use and it has no implicit def (it
3971  // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3972  // Otherwise keep it.
3973  *ToErase = SrcMI;
3974  LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3975  LLVM_DEBUG(SrcMI->dump());
3976  }
3977  return Simplified;
3978 }
3979 
3980 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3981  ImmInstrInfo &III, bool PostRA) const {
3982  // The vast majority of the instructions would need their operand 2 replaced
3983  // with an immediate when switching to the reg+imm form. A marked exception
3984  // are the update form loads/stores for which a constant operand 2 would need
3985  // to turn into a displacement and move operand 1 to the operand 2 position.
3986  III.ImmOpNo = 2;
3987  III.OpNoForForwarding = 2;
3988  III.ImmWidth = 16;
3989  III.ImmMustBeMultipleOf = 1;
3990  III.TruncateImmTo = 0;
3991  III.IsSummingOperands = false;
3992  switch (Opc) {
3993  default: return false;
3994  case PPC::ADD4:
3995  case PPC::ADD8:
3996  III.SignedImm = true;
3997  III.ZeroIsSpecialOrig = 0;
3998  III.ZeroIsSpecialNew = 1;
3999  III.IsCommutative = true;
4000  III.IsSummingOperands = true;
4001  III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
4002  break;
4003  case PPC::ADDC:
4004  case PPC::ADDC8:
4005  III.SignedImm = true;
4006  III.ZeroIsSpecialOrig = 0;
4007  III.ZeroIsSpecialNew = 0;
4008  III.IsCommutative = true;
4009  III.IsSummingOperands = true;
4010  III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
4011  break;
4012  case PPC::ADDC_rec:
4013  III.SignedImm = true;
4014  III.ZeroIsSpecialOrig = 0;
4015  III.ZeroIsSpecialNew = 0;
4016  III.IsCommutative = true;
4017  III.IsSummingOperands = true;
4018  III.ImmOpcode = PPC::ADDIC_rec;
4019  break;
4020  case PPC::SUBFC:
4021  case PPC::SUBFC8:
4022  III.SignedImm = true;
4023  III.ZeroIsSpecialOrig = 0;
4024  III.ZeroIsSpecialNew = 0;
4025  III.IsCommutative = false;
4026  III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
4027  break;
4028  case PPC::CMPW:
4029  case PPC::CMPD:
4030  III.SignedImm = true;
4031  III.ZeroIsSpecialOrig = 0;
4032  III.ZeroIsSpecialNew = 0;
4033  III.IsCommutative = false;
4034  III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
4035  break;
4036  case PPC::CMPLW:
4037  case PPC::CMPLD:
4038  III.SignedImm = false;
4039  III.ZeroIsSpecialOrig = 0;
4040  III.ZeroIsSpecialNew = 0;
4041  III.IsCommutative = false;
4042  III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
4043  break;
4044  case PPC::AND_rec:
4045  case PPC::AND8_rec:
4046  case PPC::OR:
4047  case PPC::OR8:
4048  case PPC::XOR:
4049  case PPC::XOR8:
4050  III.SignedImm = false;
4051  III.ZeroIsSpecialOrig = 0;
4052  III.ZeroIsSpecialNew = 0;
4053  III.IsCommutative = true;
4054  switch(Opc) {
4055  default: llvm_unreachable("Unknown opcode");
4056  case PPC::AND_rec:
4057  III.ImmOpcode = PPC::ANDI_rec;
4058  break;
4059  case PPC::AND8_rec:
4060  III.ImmOpcode = PPC::ANDI8_rec;
4061  break;
4062  case PPC::OR: III.ImmOpcode = PPC::ORI; break;
4063  case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
4064  case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
4065  case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
4066  }
4067  break;
4068  case PPC::RLWNM:
4069  case PPC::RLWNM8:
4070  case PPC::RLWNM_rec:
4071  case PPC::RLWNM8_rec:
4072  case PPC::SLW:
4073  case PPC::SLW8:
4074  case PPC::SLW_rec:
4075  case PPC::SLW8_rec:
4076  case PPC::SRW:
4077  case PPC::SRW8:
4078  case PPC::SRW_rec:
4079  case PPC::SRW8_rec:
4080  case PPC::SRAW:
4081  case PPC::SRAW_rec:
4082  III.SignedImm = false;
4083  III.ZeroIsSpecialOrig = 0;
4084  III.ZeroIsSpecialNew = 0;
4085  III.IsCommutative = false;
4086  // This isn't actually true, but the instructions ignore any of the
4087  // upper bits, so any immediate loaded with an LI is acceptable.
4088  // This does not apply to shift right algebraic because a value
4089  // out of range will produce a -1/0.
4090  III.ImmWidth = 16;
4091  if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
4092  Opc == PPC::RLWNM8_rec)
4093  III.TruncateImmTo = 5;
4094  else
4095  III.TruncateImmTo = 6;
4096  switch(Opc) {
4097  default: llvm_unreachable("Unknown opcode");
4098  case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
4099  case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
4100  case PPC::RLWNM_rec:
4101  III.ImmOpcode = PPC::RLWINM_rec;
4102  break;
4103  case PPC::RLWNM8_rec:
4104  III.ImmOpcode = PPC::RLWINM8_rec;
4105  break;
4106  case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4107  case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4108  case PPC::SLW_rec:
4109  III.ImmOpcode = PPC::RLWINM_rec;
4110  break;
4111  case PPC::SLW8_rec:
4112  III.ImmOpcode = PPC::RLWINM8_rec;
4113  break;
4114  case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4115  case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4116  case PPC::SRW_rec:
4117  III.ImmOpcode = PPC::RLWINM_rec;
4118  break;
4119  case PPC::SRW8_rec:
4120  III.ImmOpcode = PPC::RLWINM8_rec;
4121  break;
4122  case PPC::SRAW:
4123  III.ImmWidth = 5;
4124  III.TruncateImmTo = 0;
4125  III.ImmOpcode = PPC::SRAWI;
4126  break;
4127  case PPC::SRAW_rec:
4128  III.ImmWidth = 5;
4129  III.TruncateImmTo = 0;
4130  III.ImmOpcode = PPC::SRAWI_rec;
4131  break;
4132  }
4133  break;
4134  case PPC::RLDCL:
4135  case PPC::RLDCL_rec:
4136  case PPC::RLDCR:
4137  case PPC::RLDCR_rec:
4138  case PPC::SLD:
4139  case PPC::SLD_rec:
4140  case PPC::SRD:
4141  case PPC::SRD_rec:
4142  case PPC::SRAD:
4143  case PPC::SRAD_rec:
4144  III.SignedImm = false;
4145  III.ZeroIsSpecialOrig = 0;
4146  III.ZeroIsSpecialNew = 0;
4147  III.IsCommutative = false;
4148  // This isn't actually true, but the instructions ignore any of the
4149  // upper bits, so any immediate loaded with an LI is acceptable.
4150  // This does not apply to shift right algebraic because a value
4151  // out of range will produce a -1/0.
4152  III.ImmWidth = 16;
4153  if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4154  Opc == PPC::RLDCR_rec)
4155  III.TruncateImmTo = 6;
4156  else
4157  III.TruncateImmTo = 7;
4158  switch(Opc) {
4159  default: llvm_unreachable("Unknown opcode");
4160  case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4161  case PPC::RLDCL_rec:
4162  III.ImmOpcode = PPC::RLDICL_rec;
4163  break;
4164  case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4165  case PPC::RLDCR_rec:
4166  III.ImmOpcode = PPC::RLDICR_rec;
4167  break;
4168  case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4169  case PPC::SLD_rec:
4170  III.ImmOpcode = PPC::RLDICR_rec;
4171  break;
4172  case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4173  case PPC::SRD_rec:
4174  III.ImmOpcode = PPC::RLDICL_rec;
4175  break;
4176  case PPC::SRAD:
4177  III.ImmWidth = 6;
4178  III.TruncateImmTo = 0;
4179  III.ImmOpcode = PPC::SRADI;
4180  break;
4181  case PPC::SRAD_rec:
4182  III.ImmWidth = 6;
4183  III.TruncateImmTo = 0;
4184  III.ImmOpcode = PPC::SRADI_rec;
4185  break;
4186  }
4187  break;
4188  // Loads and stores:
4189  case PPC::LBZX:
4190  case PPC::LBZX8:
4191  case PPC::LHZX:
4192  case PPC::LHZX8:
4193  case PPC::LHAX:
4194  case PPC::LHAX8:
4195  case PPC::LWZX:
4196  case PPC::LWZX8:
4197  case PPC::LWAX:
4198  case PPC::LDX:
4199  case PPC::LFSX:
4200  case PPC::LFDX:
4201  case PPC::STBX:
4202  case PPC::STBX8:
4203  case PPC::STHX:
4204  case PPC::STHX8:
4205  case PPC::STWX:
4206  case PPC::STWX8:
4207  case PPC::STDX:
4208  case PPC::STFSX:
4209  case PPC::STFDX:
4210  III.SignedImm = true;
4211  III.ZeroIsSpecialOrig = 1;
4212  III.ZeroIsSpecialNew = 2;
4213  III.IsCommutative = true;
4214  III.IsSummingOperands = true;
4215  III.ImmOpNo = 1;
4216  III.OpNoForForwarding = 2;
4217  switch(Opc) {
4218  default: llvm_unreachable("Unknown opcode");
4219  case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4220  case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4221  case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4222  case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4223  case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4224  case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4225  case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4226  case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4227  case PPC::LWAX:
4228  III.ImmOpcode = PPC::LWA;
4229  III.ImmMustBeMultipleOf = 4;
4230  break;
4231  case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4232  case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4233  case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4234  case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4235  case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4236  case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4237  case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4238  case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4239  case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4240  case PPC::STDX:
4241  III.ImmOpcode = PPC::STD;
4242  III.ImmMustBeMultipleOf = 4;
4243  break;
4244  case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4245  case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4246  }
4247  break;
4248  case PPC::LBZUX:
4249  case PPC::LBZUX8:
4250  case PPC::LHZUX:
4251  case PPC::LHZUX8:
4252  case PPC::LHAUX:
4253  case PPC::LHAUX8:
4254  case PPC::LWZUX:
4255  case PPC::LWZUX8:
4256  case PPC::LDUX:
4257  case PPC::LFSUX:
4258  case PPC::LFDUX:
4259  case PPC::STBUX:
4260  case PPC::STBUX8:
4261  case PPC::STHUX:
4262  case PPC::STHUX8:
4263  case PPC::STWUX:
4264  case PPC::STWUX8:
4265  case PPC::STDUX:
4266  case PPC::STFSUX:
4267  case PPC::STFDUX:
4268  III.SignedImm = true;
4269  III.ZeroIsSpecialOrig = 2;
4270  III.ZeroIsSpecialNew = 3;
4271  III.IsCommutative = false;
4272  III.IsSummingOperands = true;
4273  III.ImmOpNo = 2;
4274  III.OpNoForForwarding = 3;
4275  switch(Opc) {
4276  default: llvm_unreachable("Unknown opcode");
4277  case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4278  case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4279  case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4280  case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4281  case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4282  case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4283  case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4284  case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4285  case PPC::LDUX:
4286  III.ImmOpcode = PPC::LDU;
4287  III.ImmMustBeMultipleOf = 4;
4288  break;
4289  case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4290  case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4291  case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4292  case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4293  case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4294  case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4295  case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4296  case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4297  case PPC::STDUX:
4298  III.ImmOpcode = PPC::STDU;
4299  III.ImmMustBeMultipleOf = 4;
4300  break;
4301  case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4302  case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4303  }
4304  break;
4305  // Power9 and up only. For some of these, the X-Form version has access to all
4306  // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4307  // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4308  // into or stored from is one of the VR registers.
4309  case PPC::LXVX:
4310  case PPC::LXSSPX:
4311  case PPC::LXSDX:
4312  case PPC::STXVX:
4313  case PPC::STXSSPX:
4314  case PPC::STXSDX:
4315  case PPC::XFLOADf32:
4316  case PPC::XFLOADf64:
4317  case PPC::XFSTOREf32:
4318  case PPC::XFSTOREf64:
4319  if (!Subtarget.hasP9Vector())
4320  return false;
4321  III.SignedImm = true;
4322  III.ZeroIsSpecialOrig = 1;
4323  III.ZeroIsSpecialNew = 2;
4324  III.IsCommutative = true;
4325  III.IsSummingOperands = true;
4326  III.ImmOpNo = 1;
4327  III.OpNoForForwarding = 2;
4328  III.ImmMustBeMultipleOf = 4;
4329  switch(Opc) {
4330  default: llvm_unreachable("Unknown opcode");
4331  case PPC::LXVX:
4332  III.ImmOpcode = PPC::LXV;
4333  III.ImmMustBeMultipleOf = 16;
4334  break;
4335  case PPC::LXSSPX:
4336  if (PostRA) {
4337  if (IsVFReg)
4338  III.ImmOpcode = PPC::LXSSP;
4339  else {
4340  III.ImmOpcode = PPC::LFS;
4341  III.ImmMustBeMultipleOf = 1;
4342  }
4343  break;
4344  }
4345  [[fallthrough]];
4346  case PPC::XFLOADf32:
4347  III.ImmOpcode = PPC::DFLOADf32;
4348  break;
4349  case PPC::LXSDX:
4350  if (PostRA) {
4351  if (IsVFReg)
4352  III.ImmOpcode = PPC::LXSD;
4353  else {
4354  III.ImmOpcode = PPC::LFD;
4355  III.ImmMustBeMultipleOf = 1;
4356  }
4357  break;
4358  }
4359  [[fallthrough]];
4360  case PPC::XFLOADf64:
4361  III.ImmOpcode = PPC::DFLOADf64;
4362  break;
4363  case PPC::STXVX:
4364  III.ImmOpcode = PPC::STXV;
4365  III.ImmMustBeMultipleOf = 16;
4366  break;
4367  case PPC::STXSSPX:
4368  if (PostRA) {
4369  if (IsVFReg)
4370  III.ImmOpcode = PPC::STXSSP;
4371  else {
4372  III.ImmOpcode = PPC::STFS;
4373  III.ImmMustBeMultipleOf = 1;
4374  }
4375  break;
4376  }
4377  [[fallthrough]];
4378  case PPC::XFSTOREf32:
4379  III.ImmOpcode = PPC::DFSTOREf32;
4380  break;
4381  case PPC::STXSDX:
4382  if (PostRA) {
4383  if (IsVFReg)
4384  III.ImmOpcode = PPC::STXSD;
4385  else {
4386  III.ImmOpcode = PPC::STFD;
4387  III.ImmMustBeMultipleOf = 1;
4388  }
4389  break;
4390  }
4391  [[fallthrough]];
4392  case PPC::XFSTOREf64:
4393  III.ImmOpcode = PPC::DFSTOREf64;
4394  break;
4395  }
4396  break;
4397  }
4398  return true;
4399 }
4400 
4401 // Utility function for swaping two arbitrary operands of an instruction.
4402 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4403  assert(Op1 != Op2 && "Cannot swap operand with itself.");
4404 
4405  unsigned MaxOp = std::max(Op1, Op2);
4406  unsigned MinOp = std::min(Op1, Op2);
4407  MachineOperand MOp1 = MI.getOperand(MinOp);
4408  MachineOperand MOp2 = MI.getOperand(MaxOp);
4409  MI.removeOperand(std::max(Op1, Op2));
4410  MI.removeOperand(std::min(Op1, Op2));
4411 
4412  // If the operands we are swapping are the two at the end (the common case)
4413  // we can just remove both and add them in the opposite order.
4414  if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4415  MI.addOperand(MOp2);
4416  MI.addOperand(MOp1);
4417  } else {
4418  // Store all operands in a temporary vector, remove them and re-add in the
4419  // right order.
4421  unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4422  for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4423  MOps.push_back(MI.getOperand(i));
4424  MI.removeOperand(i);
4425  }
4426  // MOp2 needs to be added next.
4427  MI.addOperand(MOp2);
4428  // Now add the rest.
4429  for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4430  if (i == MaxOp)
4431  MI.addOperand(MOp1);
4432  else {
4433  MI.addOperand(MOps.back());
4434  MOps.pop_back();
4435  }
4436  }
4437  }
4438 }
4439 
4440 // Check if the 'MI' that has the index OpNoForForwarding
4441 // meets the requirement described in the ImmInstrInfo.
4442 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4443  const ImmInstrInfo &III,
4444  unsigned OpNoForForwarding
4445  ) const {
4446  // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4447  // would not work pre-RA, we can only do the check post RA.
4448  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4449  if (MRI.isSSA())
4450  return false;
4451 
4452  // Cannot do the transform if MI isn't summing the operands.
4453  if (!III.IsSummingOperands)
4454  return false;
4455 
4456  // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4457  if (!III.ZeroIsSpecialOrig)
4458  return false;
4459 
4460  // We cannot do the transform if the operand we are trying to replace
4461  // isn't the same as the operand the instruction allows.
4462  if (OpNoForForwarding != III.OpNoForForwarding)
4463  return false;
4464 
4465  // Check if the instruction we are trying to transform really has
4466  // the special zero register as its operand.
4467  if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4468  MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4469  return false;
4470 
4471  // This machine instruction is convertible if it is,
4472  // 1. summing the operands.
4473  // 2. one of the operands is special zero register.
4474  // 3. the operand we are trying to replace is allowed by the MI.
4475  return true;
4476 }
4477 
4478 // Check if the DefMI is the add inst and set the ImmMO and RegMO
4479 // accordingly.
4480 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4481  const ImmInstrInfo &III,
4482  MachineOperand *&ImmMO,
4483  MachineOperand *&RegMO) const {
4484  unsigned Opc = DefMI.getOpcode();
4485  if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4486  return false;
4487 
4488  assert(DefMI.getNumOperands() >= 3 &&
4489  "Add inst must have at least three operands");
4490  RegMO = &DefMI.getOperand(1);
4491  ImmMO = &DefMI.getOperand(2);
4492 
4493  // Before RA, ADDI first operand could be a frame index.
4494  if (!RegMO->isReg())
4495  return false;
4496 
4497  // This DefMI is elgible for forwarding if it is:
4498  // 1. add inst
4499  // 2. one of the operands is Imm/CPI/Global.
4500  return isAnImmediateOperand(*ImmMO);
4501 }
4502 
4503 bool PPCInstrInfo::isRegElgibleForForwarding(
4504  const MachineOperand &RegMO, const MachineInstr &DefMI,
4505  const MachineInstr &MI, bool KillDefMI,
4506  bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4507  // x = addi y, imm
4508  // ...
4509  // z = lfdx 0, x -> z = lfd imm(y)
4510  // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4511  // of "y" between the DEF of "x" and "z".
4512  // The query is only valid post RA.
4513  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4514  if (MRI.isSSA())
4515  return false;
4516 
4517  Register Reg = RegMO.getReg();
4518 
4519  // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4521  MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4522  It++;
4523  for (; It != E; ++It) {
4524  if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4525  return false;
4526  else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4527  IsFwdFeederRegKilled = true;
4528  if (It->readsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4529  SeenIntermediateUse = true;
4530  // Made it to DefMI without encountering a clobber.
4531  if ((&*It) == &DefMI)
4532  break;
4533  }
4534  assert((&*It) == &DefMI && "DefMI is missing");
4535 
4536  // If DefMI also defines the register to be forwarded, we can only forward it
4537  // if DefMI is being erased.
4538  if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4539  return KillDefMI;
4540 
4541  return true;
4542 }
4543 
4544 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4545  const MachineInstr &DefMI,
4546  const ImmInstrInfo &III,
4547  int64_t &Imm,
4548  int64_t BaseImm) const {
4549  assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4550  if (DefMI.getOpcode() == PPC::ADDItocL) {
4551  // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4552  // However, we know that, it is 16-bit width, and has the alignment of 4.
4553  // Check if the instruction met the requirement.
4554  if (III.ImmMustBeMultipleOf > 4 ||
4555  III.TruncateImmTo || III.ImmWidth != 16)
4556  return false;
4557 
4558  // Going from XForm to DForm loads means that the displacement needs to be
4559  // not just an immediate but also a multiple of 4, or 16 depending on the
4560  // load. A DForm load cannot be represented if it is a multiple of say 2.
4561  // XForm loads do not have this restriction.
4562  if (ImmMO.isGlobal()) {
4563  const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4565  return false;
4566  }
4567 
4568  return true;
4569  }
4570 
4571  if (ImmMO.isImm()) {
4572  // It is Imm, we need to check if the Imm fit the range.
4573  // Sign-extend to 64-bits.
4574  // DefMI may be folded with another imm form instruction, the result Imm is
4575  // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4576  APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4577  if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4578  return false;
4579  if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4580  return false;
4581  Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4582 
4583  if (Imm % III.ImmMustBeMultipleOf)
4584  return false;
4585  if (III.TruncateImmTo)
4586  Imm &= ((1 << III.TruncateImmTo) - 1);
4587  }
4588  else
4589  return false;
4590 
4591  // This ImmMO is forwarded if it meets the requriement describle
4592  // in ImmInstrInfo
4593  return true;
4594 }
4595 
4596 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4597  unsigned OpNoForForwarding,
4598  MachineInstr **KilledDef) const {
4599  if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4600  !DefMI.getOperand(1).isImm())
4601  return false;
4602 
4603  MachineFunction *MF = MI.getParent()->getParent();
4605  bool PostRA = !MRI->isSSA();
4606 
4607  int64_t Immediate = DefMI.getOperand(1).getImm();
4608  // Sign-extend to 64-bits.
4609  int64_t SExtImm = SignExtend64<16>(Immediate);
4610 
4611  bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
4612  Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4613 
4614  bool ReplaceWithLI = false;
4615  bool Is64BitLI = false;
4616  int64_t NewImm = 0;
4617  bool SetCR = false;
4618  unsigned Opc = MI.getOpcode();
4619  switch (Opc) {
4620  default:
4621  return false;
4622 
4623  // FIXME: Any branches conditional on such a comparison can be made
4624  // unconditional. At this time, this happens too infrequently to be worth
4625  // the implementation effort, but if that ever changes, we could convert
4626  // such a pattern here.
4627  case PPC::CMPWI:
4628  case PPC::CMPLWI:
4629  case PPC::CMPDI:
4630  case PPC::CMPLDI: {
4631  // Doing this post-RA would require dataflow analysis to reliably find uses
4632  // of the CR register set by the compare.
4633  // No need to fixup killed/dead flag since this transformation is only valid
4634  // before RA.
4635  if (PostRA)
4636  return false;
4637  // If a compare-immediate is fed by an immediate and is itself an input of
4638  // an ISEL (the most common case) into a COPY of the correct register.
4639  bool Changed = false;
4640  Register DefReg = MI.getOperand(0).getReg();
4641  int64_t Comparand = MI.getOperand(2).getImm();
4642  int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4643  ? (Comparand | 0xFFFFFFFFFFFF0000)
4644  : Comparand;
4645 
4646  for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4647  unsigned UseOpc = CompareUseMI.getOpcode();
4648  if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4649  continue;
4650  unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4651  Register TrueReg = CompareUseMI.getOperand(1).getReg();
4652  Register FalseReg = CompareUseMI.getOperand(2).getReg();
4653  unsigned RegToCopy =
4654  selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4655  if (RegToCopy == PPC::NoRegister)
4656  continue;
4657  // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4658  if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4659  CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4660  replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4661  CompareUseMI.removeOperand(3);
4662  CompareUseMI.removeOperand(2);
4663  continue;
4664  }
4665  LLVM_DEBUG(
4666  dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4667  LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4668  LLVM_DEBUG(dbgs() << "Is converted to:\n");
4669  // Convert to copy and remove unneeded operands.
4670  CompareUseMI.setDesc(get(PPC::COPY));
4671  CompareUseMI.removeOperand(3);
4672  CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
4673  CmpIselsConverted++;
4674  Changed = true;
4675  LLVM_DEBUG(CompareUseMI.dump());
4676  }
4677  if (Changed)
4678  return true;
4679  // This may end up incremented multiple times since this function is called
4680  // during a fixed-point transformation, but it is only meant to indicate the
4681  // presence of this opportunity.
4682  MissedConvertibleImmediateInstrs++;
4683  return false;
4684  }
4685 
4686  // Immediate forms - may simply be convertable to an LI.
4687  case PPC::ADDI:
4688  case PPC::ADDI8: {
4689  // Does the sum fit in a 16-bit signed field?
4690  int64_t Addend = MI.getOperand(2).getImm();
4691  if (isInt<16>(Addend + SExtImm)) {
4692  ReplaceWithLI = true;
4693  Is64BitLI = Opc == PPC::ADDI8;
4694  NewImm = Addend + SExtImm;
4695  break;
4696  }
4697  return false;
4698  }
4699  case PPC::SUBFIC:
4700  case PPC::SUBFIC8: {
4701  // Only transform this if the CARRY implicit operand is dead.
4702  if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
4703  return false;
4704  int64_t Minuend = MI.getOperand(2).getImm();
4705  if (isInt<16>(Minuend - SExtImm)) {
4706  ReplaceWithLI = true;
4707  Is64BitLI = Opc == PPC::SUBFIC8;
4708  NewImm = Minuend - SExtImm;
4709  break;
4710  }
4711  return false;
4712  }
4713  case PPC::RLDICL:
4714  case PPC::RLDICL_rec:
4715  case PPC::RLDICL_32:
4716  case PPC::RLDICL_32_64: {
4717  // Use APInt's rotate function.
4718  int64_t SH = MI.getOperand(2).getImm();
4719  int64_t MB = MI.getOperand(3).getImm();
4720  APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4721  SExtImm, true);
4722  InVal = InVal.rotl(SH);
4723  uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4724  InVal &= Mask;
4725  // Can't replace negative values with an LI as that will sign-extend
4726  // and not clear the left bits. If we're setting the CR bit, we will use
4727  // ANDI_rec which won't sign extend, so that's safe.
4728  if (isUInt<15>(InVal.getSExtValue()) ||
4729  (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
4730  ReplaceWithLI = true;
4731  Is64BitLI = Opc != PPC::RLDICL_32;
4732  NewImm = InVal.getSExtValue();
4733  SetCR = Opc == PPC::RLDICL_rec;
4734  break;
4735  }
4736  return false;
4737  }
4738  case PPC::RLWINM:
4739  case PPC::RLWINM8:
4740  case PPC::RLWINM_rec:
4741  case PPC::RLWINM8_rec: {
4742  int64_t SH = MI.getOperand(2).getImm();
4743  int64_t MB = MI.getOperand(3).getImm();
4744  int64_t ME = MI.getOperand(4).getImm();
4745  APInt InVal(32, SExtImm, true);
4746  InVal = InVal.rotl(SH);
4747  APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
4748  InVal &= Mask;
4749  // Can't replace negative values with an LI as that will sign-extend
4750  // and not clear the left bits. If we're setting the CR bit, we will use
4751  // ANDI_rec which won't sign extend, so that's safe.
4752  bool ValueFits = isUInt<15>(InVal.getSExtValue());
4753  ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4754  isUInt<16>(InVal.getSExtValue()));
4755  if (ValueFits) {
4756  ReplaceWithLI = true;
4757  Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4758  NewImm = InVal.getSExtValue();
4759  SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4760  break;
4761  }
4762  return false;
4763  }
4764  case PPC::ORI:
4765  case PPC::ORI8:
4766  case PPC::XORI:
4767  case PPC::XORI8: {
4768  int64_t LogicalImm = MI.getOperand(2).getImm();
4769  int64_t Result = 0;
4770  if (Opc == PPC::ORI || Opc == PPC::ORI8)
4771  Result = LogicalImm | SExtImm;
4772  else
4773  Result = LogicalImm ^ SExtImm;
4774  if (isInt<16>(Result)) {
4775  ReplaceWithLI = true;
4776  Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4777  NewImm = Result;
4778  break;
4779  }
4780  return false;
4781  }
4782  }
4783 
4784  if (ReplaceWithLI) {
4785  // We need to be careful with CR-setting instructions we're replacing.
4786  if (SetCR) {
4787  // We don't know anything about uses when we're out of SSA, so only
4788  // replace if the new immediate will be reproduced.
4789  bool ImmChanged = (SExtImm & NewImm) != NewImm;
4790  if (PostRA && ImmChanged)
4791  return false;
4792 
4793  if (!PostRA) {
4794  // If the defining load-immediate has no other uses, we can just replace
4795  // the immediate with the new immediate.
4796  if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
4797  DefMI.getOperand(1).setImm(NewImm);
4798 
4799  // If we're not using the GPR result of the CR-setting instruction, we
4800  // just need to and with zero/non-zero depending on the new immediate.
4801  else if (MRI->use_empty(MI.getOperand(0).getReg())) {
4802  if (NewImm) {
4803  assert(Immediate && "Transformation converted zero to non-zero?");
4804  NewImm = Immediate;
4805  }
4806  } else if (ImmChanged)
4807  return false;
4808  }
4809  }
4810 
4811  LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4812  LLVM_DEBUG(MI.dump());
4813  LLVM_DEBUG(dbgs() << "Fed by:\n");
4814  LLVM_DEBUG(DefMI.dump());
4815  LoadImmediateInfo LII;
4816  LII.Imm = NewImm;
4817  LII.Is64Bit = Is64BitLI;
4818  LII.SetCR = SetCR;
4819  // If we're setting the CR, the original load-immediate must be kept (as an
4820  // operand to ANDI_rec/ANDI8_rec).
4821  if (KilledDef && SetCR)
4822  *KilledDef = nullptr;
4823  replaceInstrWithLI(MI, LII);
4824 
4825  // Fixup killed/dead flag after transformation.
4826  // Pattern:
4827  // ForwardingOperandReg = LI imm1
4828  // y = op2 imm2, ForwardingOperandReg(killed)
4829  if (IsForwardingOperandKilled)
4830  fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg);
4831 
4832  LLVM_DEBUG(dbgs() << "With:\n");
4833  LLVM_DEBUG(MI.dump());
4834  return true;
4835  }
4836  return false;
4837 }
4838 
4839 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4840  MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4841  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4842  bool PostRA = !MRI->isSSA();
4843  // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4844  // for post-ra.
4845  if (PostRA)
4846  return false;
4847 
4848  // Only handle load/store.
4849  if (!MI.mayLoadOrStore())
4850  return false;
4851 
4852  unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
4853 
4854  assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4855  "MI must have x-form opcode");
4856 
4857  // get Imm Form info.
4858  ImmInstrInfo III;
4859  bool IsVFReg = MI.getOperand(0).isReg()
4860  ? isVFRegister(MI.getOperand(0).getReg())
4861  : false;
4862 
4863  if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
4864  return false;
4865 
4866  if (!III.IsSummingOperands)
4867  return false;
4868 
4869  if (OpNoForForwarding != III.OpNoForForwarding)
4870  return false;
4871 
4872  MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
4873  if (!ImmOperandMI.isImm())
4874  return false;
4875 
4876  // Check DefMI.
4877  MachineOperand *ImmMO = nullptr;
4878  MachineOperand *RegMO = nullptr;
4879  if (!isDefMIElgibleForForwarding(