LLVM  15.0.0git
PPCInstrInfo.cpp
Go to the documentation of this file.
1 //===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file contains the PowerPC implementation of the TargetInstrInfo class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "PPCInstrInfo.h"
15 #include "PPC.h"
16 #include "PPCHazardRecognizers.h"
17 #include "PPCInstrBuilder.h"
18 #include "PPCMachineFunctionInfo.h"
19 #include "PPCTargetMachine.h"
20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/Statistic.h"
35 #include "llvm/CodeGen/StackMaps.h"
36 #include "llvm/MC/MCAsmInfo.h"
37 #include "llvm/MC/MCInst.h"
38 #include "llvm/MC/TargetRegistry.h"
40 #include "llvm/Support/Debug.h"
43 
44 using namespace llvm;
45 
46 #define DEBUG_TYPE "ppc-instr-info"
47 
48 #define GET_INSTRMAP_INFO
49 #define GET_INSTRINFO_CTOR_DTOR
50 #include "PPCGenInstrInfo.inc"
51 
52 STATISTIC(NumStoreSPILLVSRRCAsVec,
53  "Number of spillvsrrc spilled to stack as vec");
54 STATISTIC(NumStoreSPILLVSRRCAsGpr,
55  "Number of spillvsrrc spilled to stack as gpr");
56 STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
57 STATISTIC(CmpIselsConverted,
58  "Number of ISELs that depend on comparison of constants converted");
59 STATISTIC(MissedConvertibleImmediateInstrs,
60  "Number of compare-immediate instructions fed by constants");
61 STATISTIC(NumRcRotatesConvertedToRcAnd,
62  "Number of record-form rotates converted to record-form andi");
63 
64 static cl::
65 opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
66  cl::desc("Disable analysis for CTR loops"));
67 
68 static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
69 cl::desc("Disable compare instruction optimization"), cl::Hidden);
70 
71 static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
72 cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
73 cl::Hidden);
74 
75 static cl::opt<bool>
76 UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
77  cl::desc("Use the old (incorrect) instruction latency calculation"));
78 
79 static cl::opt<float>
80  FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
81  cl::desc("register pressure factor for the transformations."));
82 
84  "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
85  cl::desc("enable register pressure reduce in machine combiner pass."));
86 
87 // Pin the vtable to this file.
88 void PPCInstrInfo::anchor() {}
89 
91  : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
92  /* CatchRetOpcode */ -1,
93  STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
94  Subtarget(STI), RI(STI.getTargetMachine()) {}
95 
96 /// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
97 /// this target when scheduling the DAG.
100  const ScheduleDAG *DAG) const {
101  unsigned Directive =
102  static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
105  const InstrItineraryData *II =
106  static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
107  return new ScoreboardHazardRecognizer(II, DAG);
108  }
109 
111 }
112 
113 /// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
114 /// to use for this target when scheduling the DAG.
117  const ScheduleDAG *DAG) const {
118  unsigned Directive =
119  DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
120 
121  // FIXME: Leaving this as-is until we have POWER9 scheduling info
123  return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
124 
125  // Most subtargets use a PPC970 recognizer.
128  assert(DAG->TII && "No InstrInfo?");
129 
130  return new PPCHazardRecognizer970(*DAG);
131  }
132 
133  return new ScoreboardHazardRecognizer(II, DAG);
134 }
135 
137  const MachineInstr &MI,
138  unsigned *PredCost) const {
139  if (!ItinData || UseOldLatencyCalc)
140  return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
141 
142  // The default implementation of getInstrLatency calls getStageLatency, but
143  // getStageLatency does not do the right thing for us. While we have
144  // itinerary, most cores are fully pipelined, and so the itineraries only
145  // express the first part of the pipeline, not every stage. Instead, we need
146  // to use the listed output operand cycle number (using operand 0 here, which
147  // is an output).
148 
149  unsigned Latency = 1;
150  unsigned DefClass = MI.getDesc().getSchedClass();
151  for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
152  const MachineOperand &MO = MI.getOperand(i);
153  if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
154  continue;
155 
156  int Cycle = ItinData->getOperandCycle(DefClass, i);
157  if (Cycle < 0)
158  continue;
159 
160  Latency = std::max(Latency, (unsigned) Cycle);
161  }
162 
163  return Latency;
164 }
165 
167  const MachineInstr &DefMI, unsigned DefIdx,
168  const MachineInstr &UseMI,
169  unsigned UseIdx) const {
170  int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
171  UseMI, UseIdx);
172 
173  if (!DefMI.getParent())
174  return Latency;
175 
176  const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
177  Register Reg = DefMO.getReg();
178 
179  bool IsRegCR;
181  const MachineRegisterInfo *MRI =
182  &DefMI.getParent()->getParent()->getRegInfo();
183  IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
184  MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
185  } else {
186  IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
187  PPC::CRBITRCRegClass.contains(Reg);
188  }
189 
190  if (UseMI.isBranch() && IsRegCR) {
191  if (Latency < 0)
192  Latency = getInstrLatency(ItinData, DefMI);
193 
194  // On some cores, there is an additional delay between writing to a condition
195  // register, and using it from a branch.
196  unsigned Directive = Subtarget.getCPUDirective();
197  switch (Directive) {
198  default: break;
199  case PPC::DIR_7400:
200  case PPC::DIR_750:
201  case PPC::DIR_970:
202  case PPC::DIR_E5500:
203  case PPC::DIR_PWR4:
204  case PPC::DIR_PWR5:
205  case PPC::DIR_PWR5X:
206  case PPC::DIR_PWR6:
207  case PPC::DIR_PWR6X:
208  case PPC::DIR_PWR7:
209  case PPC::DIR_PWR8:
210  // FIXME: Is this needed for POWER9?
211  Latency += 2;
212  break;
213  }
214  }
215 
216  return Latency;
217 }
218 
219 /// This is an architecture-specific helper function of reassociateOps.
220 /// Set special operand attributes for new instructions after reassociation.
222  MachineInstr &OldMI2,
223  MachineInstr &NewMI1,
224  MachineInstr &NewMI2) const {
225  // Propagate FP flags from the original instructions.
226  // But clear poison-generating flags because those may not be valid now.
227  uint16_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
228  NewMI1.setFlags(IntersectedFlags);
229  NewMI1.clearFlag(MachineInstr::MIFlag::NoSWrap);
230  NewMI1.clearFlag(MachineInstr::MIFlag::NoUWrap);
231  NewMI1.clearFlag(MachineInstr::MIFlag::IsExact);
232 
233  NewMI2.setFlags(IntersectedFlags);
234  NewMI2.clearFlag(MachineInstr::MIFlag::NoSWrap);
235  NewMI2.clearFlag(MachineInstr::MIFlag::NoUWrap);
236  NewMI2.clearFlag(MachineInstr::MIFlag::IsExact);
237 }
238 
240  uint16_t Flags) const {
241  MI.setFlags(Flags);
242  MI.clearFlag(MachineInstr::MIFlag::NoSWrap);
243  MI.clearFlag(MachineInstr::MIFlag::NoUWrap);
244  MI.clearFlag(MachineInstr::MIFlag::IsExact);
245 }
246 
247 // This function does not list all associative and commutative operations, but
248 // only those worth feeding through the machine combiner in an attempt to
249 // reduce the critical path. Mostly, this means floating-point operations,
250 // because they have high latencies(>=5) (compared to other operations, such as
251 // and/or, which are also associative and commutative, but have low latencies).
253  switch (Inst.getOpcode()) {
254  // Floating point:
255  // FP Add:
256  case PPC::FADD:
257  case PPC::FADDS:
258  // FP Multiply:
259  case PPC::FMUL:
260  case PPC::FMULS:
261  // Altivec Add:
262  case PPC::VADDFP:
263  // VSX Add:
264  case PPC::XSADDDP:
265  case PPC::XVADDDP:
266  case PPC::XVADDSP:
267  case PPC::XSADDSP:
268  // VSX Multiply:
269  case PPC::XSMULDP:
270  case PPC::XVMULDP:
271  case PPC::XVMULSP:
272  case PPC::XSMULSP:
273  return Inst.getFlag(MachineInstr::MIFlag::FmReassoc) &&
274  Inst.getFlag(MachineInstr::MIFlag::FmNsz);
275  // Fixed point:
276  // Multiply:
277  case PPC::MULHD:
278  case PPC::MULLD:
279  case PPC::MULHW:
280  case PPC::MULLW:
281  return true;
282  default:
283  return false;
284  }
285 }
286 
287 #define InfoArrayIdxFMAInst 0
288 #define InfoArrayIdxFAddInst 1
289 #define InfoArrayIdxFMULInst 2
290 #define InfoArrayIdxAddOpIdx 3
291 #define InfoArrayIdxMULOpIdx 4
292 #define InfoArrayIdxFSubInst 5
293 // Array keeps info for FMA instructions:
294 // Index 0(InfoArrayIdxFMAInst): FMA instruction;
295 // Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
296 // Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
297 // Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
298 // Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
299 // second MUL operand index is plus 1;
300 // Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
301 static const uint16_t FMAOpIdxInfo[][6] = {
302  // FIXME: Add more FMA instructions like XSNMADDADP and so on.
303  {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
304  {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
305  {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
306  {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
307  {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
308  {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
309 
310 // Check if an opcode is a FMA instruction. If it is, return the index in array
311 // FMAOpIdxInfo. Otherwise, return -1.
312 int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
313  for (unsigned I = 0; I < array_lengthof(FMAOpIdxInfo); I++)
314  if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
315  return I;
316  return -1;
317 }
318 
319 // On PowerPC target, we have two kinds of patterns related to FMA:
320 // 1: Improve ILP.
321 // Try to reassociate FMA chains like below:
322 //
323 // Pattern 1:
324 // A = FADD X, Y (Leaf)
325 // B = FMA A, M21, M22 (Prev)
326 // C = FMA B, M31, M32 (Root)
327 // -->
328 // A = FMA X, M21, M22
329 // B = FMA Y, M31, M32
330 // C = FADD A, B
331 //
332 // Pattern 2:
333 // A = FMA X, M11, M12 (Leaf)
334 // B = FMA A, M21, M22 (Prev)
335 // C = FMA B, M31, M32 (Root)
336 // -->
337 // A = FMUL M11, M12
338 // B = FMA X, M21, M22
339 // D = FMA A, M31, M32
340 // C = FADD B, D
341 //
342 // breaking the dependency between A and B, allowing FMA to be executed in
343 // parallel (or back-to-back in a pipeline) instead of depending on each other.
344 //
345 // 2: Reduce register pressure.
346 // Try to reassociate FMA with FSUB and a constant like below:
347 // C is a floating point const.
348 //
349 // Pattern 1:
350 // A = FSUB X, Y (Leaf)
351 // D = FMA B, C, A (Root)
352 // -->
353 // A = FMA B, Y, -C
354 // D = FMA A, X, C
355 //
356 // Pattern 2:
357 // A = FSUB X, Y (Leaf)
358 // D = FMA B, A, C (Root)
359 // -->
360 // A = FMA B, Y, -C
361 // D = FMA A, X, C
362 //
363 // Before the transformation, A must be assigned with different hardware
364 // register with D. After the transformation, A and D must be assigned with
365 // same hardware register due to TIE attribute of FMA instructions.
366 //
369  bool DoRegPressureReduce) const {
370  MachineBasicBlock *MBB = Root.getParent();
373 
374  auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
375  for (const auto &MO : Instr.explicit_operands())
376  if (!(MO.isReg() && Register::isVirtualRegister(MO.getReg())))
377  return false;
378  return true;
379  };
380 
381  auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
382  unsigned OpType) {
383  if (Instr.getOpcode() !=
384  FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
385  return false;
386 
387  // Instruction can be reassociated.
388  // fast math flags may prohibit reassociation.
389  if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
390  Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
391  return false;
392 
393  // Instruction operands are virtual registers for reassociation.
394  if (!IsAllOpsVirtualReg(Instr))
395  return false;
396 
397  // For register pressure reassociation, the FSub must have only one use as
398  // we want to delete the sub to save its def.
399  if (OpType == InfoArrayIdxFSubInst &&
400  !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
401  return false;
402 
403  return true;
404  };
405 
406  auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
407  int16_t &MulOpIdx, bool IsLeaf) {
408  int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
409  if (Idx < 0)
410  return false;
411 
412  // Instruction can be reassociated.
413  // fast math flags may prohibit reassociation.
414  if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
415  Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
416  return false;
417 
418  // Instruction operands are virtual registers for reassociation.
419  if (!IsAllOpsVirtualReg(Instr))
420  return false;
421 
422  MulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
423  if (IsLeaf)
424  return true;
425 
426  AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
427 
428  const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
429  MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
430  // If 'add' operand's def is not in current block, don't do ILP related opt.
431  if (!MIAdd || MIAdd->getParent() != MBB)
432  return false;
433 
434  // If this is not Leaf FMA Instr, its 'add' operand should only have one use
435  // as this fma will be changed later.
436  return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
437  };
438 
439  int16_t AddOpIdx = -1;
440  int16_t MulOpIdx = -1;
441 
442  bool IsUsedOnceL = false;
443  bool IsUsedOnceR = false;
444  MachineInstr *MULInstrL = nullptr;
445  MachineInstr *MULInstrR = nullptr;
446 
447  auto IsRPReductionCandidate = [&]() {
448  // Currently, we only support float and double.
449  // FIXME: add support for other types.
450  unsigned Opcode = Root.getOpcode();
451  if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
452  return false;
453 
454  // Root must be a valid FMA like instruction.
455  // Treat it as leaf as we don't care its add operand.
456  if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
457  assert((MulOpIdx >= 0) && "mul operand index not right!");
459  Root.getOperand(MulOpIdx).getReg(), MRI);
461  Root.getOperand(MulOpIdx + 1).getReg(), MRI);
462  if (!MULRegL && !MULRegR)
463  return false;
464 
465  if (MULRegL && !MULRegR) {
466  MULRegR =
467  TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
468  IsUsedOnceL = true;
469  } else if (!MULRegL && MULRegR) {
470  MULRegL =
471  TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
472  IsUsedOnceR = true;
473  } else {
474  IsUsedOnceL = true;
475  IsUsedOnceR = true;
476  }
477 
478  if (!Register::isVirtualRegister(MULRegL) ||
479  !Register::isVirtualRegister(MULRegR))
480  return false;
481 
482  MULInstrL = MRI->getVRegDef(MULRegL);
483  MULInstrR = MRI->getVRegDef(MULRegR);
484  return true;
485  }
486  return false;
487  };
488 
489  // Register pressure fma reassociation patterns.
490  if (DoRegPressureReduce && IsRPReductionCandidate()) {
491  assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
492  // Register pressure pattern 1
493  if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
494  IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
495  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
496  Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BCA);
497  return true;
498  }
499 
500  // Register pressure pattern 2
501  if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
502  IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
503  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
504  Patterns.push_back(MachineCombinerPattern::REASSOC_XY_BAC);
505  return true;
506  }
507  }
508 
509  // ILP fma reassociation patterns.
510  // Root must be a valid FMA like instruction.
511  AddOpIdx = -1;
512  if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
513  return false;
514 
515  assert((AddOpIdx >= 0) && "add operand index not right!");
516 
517  Register RegB = Root.getOperand(AddOpIdx).getReg();
518  MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
519 
520  // Prev must be a valid FMA like instruction.
521  AddOpIdx = -1;
522  if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
523  return false;
524 
525  assert((AddOpIdx >= 0) && "add operand index not right!");
526 
527  Register RegA = Prev->getOperand(AddOpIdx).getReg();
528  MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
529  AddOpIdx = -1;
530  if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
532  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
533  return true;
534  }
535  if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
536  Patterns.push_back(MachineCombinerPattern::REASSOC_XY_AMM_BMM);
537  LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
538  return true;
539  }
540  return false;
541 }
542 
545  SmallVectorImpl<MachineInstr *> &InsInstrs) const {
546  assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
547 
548  MachineFunction *MF = Root.getMF();
552 
553  int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
554  if (Idx < 0)
555  return;
556 
557  uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
558 
559  // For now we only need to fix up placeholder for register pressure reduce
560  // patterns.
561  Register ConstReg = 0;
562  switch (P) {
564  ConstReg =
565  TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
566  break;
568  ConstReg =
569  TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
570  break;
571  default:
572  // Not register pressure reduce patterns.
573  return;
574  }
575 
576  MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
577  // Get const value from const pool.
578  const Constant *C = getConstantFromConstantPool(ConstDefInstr);
579  assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
580 
581  // Get negative fp const.
582  APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
583  F1.changeSign();
584  Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
585  Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
586 
587  // Put negative fp const into constant pool.
588  unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
589 
590  MachineOperand *Placeholder = nullptr;
591  // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
592  for (auto *Inst : InsInstrs) {
593  for (MachineOperand &Operand : Inst->explicit_operands()) {
594  assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
595  if (Operand.getReg() == PPC::ZERO8) {
596  Placeholder = &Operand;
597  break;
598  }
599  }
600  }
601 
602  assert(Placeholder && "Placeholder does not exist!");
603 
604  // Generate instructions to load the const fp from constant pool.
605  // We only support PPC64 and medium code model.
606  Register LoadNewConst =
607  generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
608 
609  // Fill the placeholder with the new load from constant pool.
610  Placeholder->setReg(LoadNewConst);
611 }
612 
614  MachineBasicBlock *MBB, RegisterClassInfo *RegClassInfo) const {
615 
617  return false;
618 
619  // Currently, we only enable register pressure reducing in machine combiner
620  // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
621  // support.
622  //
623  // So we need following instructions to access a TOC entry:
624  //
625  // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
626  // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
627  // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
628  //
629  // FIXME: add more supported targets, like Small and Large code model, PPC32,
630  // AIX.
631  if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
633  return false;
634 
636  MachineFunction *MF = MBB->getParent();
638 
639  auto GetMBBPressure = [&](MachineBasicBlock *MBB) -> std::vector<unsigned> {
640  RegionPressure Pressure;
641  RegPressureTracker RPTracker(Pressure);
642 
643  // Initialize the register pressure tracker.
644  RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
645  /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
646 
648  MIE = MBB->instr_begin();
649  MII != MIE; --MII) {
650  MachineInstr &MI = *std::prev(MII);
651  if (MI.isDebugValue() || MI.isDebugLabel())
652  continue;
653  RegisterOperands RegOpers;
654  RegOpers.collect(MI, *TRI, *MRI, false, false);
655  RPTracker.recedeSkipDebugValues();
656  assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
657  RPTracker.recede(RegOpers);
658  }
659 
660  // Close the RPTracker to finalize live ins.
661  RPTracker.closeRegion();
662 
663  return RPTracker.getPressure().MaxSetPressure;
664  };
665 
666  // For now we only care about float and double type fma.
667  unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
668  *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
669 
670  // Only reduce register pressure when pressure is high.
671  return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
672  (float)VSSRCLimit * FMARPFactor;
673 }
674 
675 bool PPCInstrInfo::isLoadFromConstantPool(MachineInstr *I) const {
676  // I has only one memory operand which is load from constant pool.
677  if (!I->hasOneMemOperand())
678  return false;
679 
680  MachineMemOperand *Op = I->memoperands()[0];
681  return Op->isLoad() && Op->getPseudoValue() &&
682  Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
683 }
684 
685 Register PPCInstrInfo::generateLoadForNewConst(
686  unsigned Idx, MachineInstr *MI, Type *Ty,
687  SmallVectorImpl<MachineInstr *> &InsInstrs) const {
688  // Now we only support PPC64, Medium code model and P9 with vector.
689  // We have immutable pattern to access const pool. See function
690  // shouldReduceRegisterPressure.
691  assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
692  Subtarget.getTargetMachine().getCodeModel() == CodeModel::Medium) &&
693  "Target not supported!\n");
694 
695  MachineFunction *MF = MI->getMF();
697 
698  // Generate ADDIStocHA8
699  Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
700  MachineInstrBuilder TOCOffset =
701  BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
702  .addReg(PPC::X2)
703  .addConstantPoolIndex(Idx);
704 
705  assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
706  "Only float and double are supported!");
707 
708  unsigned LoadOpcode;
709  // Should be float type or double type.
710  if (Ty->isFloatTy())
711  LoadOpcode = PPC::DFLOADf32;
712  else
713  LoadOpcode = PPC::DFLOADf64;
714 
715  const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
716  Register VReg2 = MRI->createVirtualRegister(RC);
719  Ty->getScalarSizeInBits() / 8, MF->getDataLayout().getPrefTypeAlign(Ty));
720 
721  // Generate Load from constant pool.
723  BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
725  .addReg(VReg1, getKillRegState(true))
726  .addMemOperand(MMO);
727 
728  Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
729 
730  // Insert the toc load instructions into InsInstrs.
731  InsInstrs.insert(InsInstrs.begin(), Load);
732  InsInstrs.insert(InsInstrs.begin(), TOCOffset);
733  return VReg2;
734 }
735 
736 // This function returns the const value in constant pool if the \p I is a load
737 // from constant pool.
738 const Constant *
739 PPCInstrInfo::getConstantFromConstantPool(MachineInstr *I) const {
740  MachineFunction *MF = I->getMF();
743  assert(I->mayLoad() && "Should be a load instruction.\n");
744  for (auto MO : I->uses()) {
745  if (!MO.isReg())
746  continue;
747  Register Reg = MO.getReg();
748  if (Reg == 0 || !Register::isVirtualRegister(Reg))
749  continue;
750  // Find the toc address.
752  for (auto MO2 : DefMI->uses())
753  if (MO2.isCPI())
754  return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
755  }
756  return nullptr;
757 }
758 
761  bool DoRegPressureReduce) const {
762  // Using the machine combiner in this way is potentially expensive, so
763  // restrict to when aggressive optimizations are desired.
765  return false;
766 
767  if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
768  return true;
769 
770  return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
771  DoRegPressureReduce);
772 }
773 
778  DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
779  switch (Pattern) {
784  reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
785  break;
786  default:
787  // Reassociate default patterns.
789  DelInstrs, InstrIdxForVirtReg);
790  break;
791  }
792 }
793 
794 void PPCInstrInfo::reassociateFMA(
798  DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
799  MachineFunction *MF = Root.getMF();
802  MachineOperand &OpC = Root.getOperand(0);
803  Register RegC = OpC.getReg();
804  const TargetRegisterClass *RC = MRI.getRegClass(RegC);
805  MRI.constrainRegClass(RegC, RC);
806 
807  unsigned FmaOp = Root.getOpcode();
808  int16_t Idx = getFMAOpIdxInfo(FmaOp);
809  assert(Idx >= 0 && "Root must be a FMA instruction");
810 
811  bool IsILPReassociate =
814 
815  uint16_t AddOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxAddOpIdx];
816  uint16_t FirstMulOpIdx = FMAOpIdxInfo[Idx][InfoArrayIdxMULOpIdx];
817 
818  MachineInstr *Prev = nullptr;
819  MachineInstr *Leaf = nullptr;
820  switch (Pattern) {
821  default:
822  llvm_unreachable("not recognized pattern!");
825  Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
826  Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
827  break;
829  Register MULReg =
830  TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
831  Leaf = MRI.getVRegDef(MULReg);
832  break;
833  }
835  Register MULReg = TRI->lookThruCopyLike(
836  Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
837  Leaf = MRI.getVRegDef(MULReg);
838  break;
839  }
840  }
841 
842  uint16_t IntersectedFlags = 0;
843  if (IsILPReassociate)
844  IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
845  else
846  IntersectedFlags = Root.getFlags() & Leaf->getFlags();
847 
848  auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
849  bool &KillFlag) {
850  Reg = Operand.getReg();
852  KillFlag = Operand.isKill();
853  };
854 
855  auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
856  Register &MulOp2, Register &AddOp,
857  bool &MulOp1KillFlag, bool &MulOp2KillFlag,
858  bool &AddOpKillFlag) {
859  GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
860  GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
861  GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
862  };
863 
864  Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
865  RegA21, RegB;
866  bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
867  KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
868  KillA11 = false, KillA21 = false, KillB = false;
869 
870  GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
871 
872  if (IsILPReassociate)
873  GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
874 
876  GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
877  GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
879  GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
880  GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
881  } else {
882  // Get FSUB instruction info.
883  GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
884  GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
885  }
886 
887  // Create new virtual registers for the new results instead of
888  // recycling legacy ones because the MachineCombiner's computation of the
889  // critical path requires a new register definition rather than an existing
890  // one.
891  // For register pressure reassociation, we only need create one virtual
892  // register for the new fma.
893  Register NewVRA = MRI.createVirtualRegister(RC);
894  InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
895 
896  Register NewVRB = 0;
897  if (IsILPReassociate) {
898  NewVRB = MRI.createVirtualRegister(RC);
899  InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
900  }
901 
902  Register NewVRD = 0;
904  NewVRD = MRI.createVirtualRegister(RC);
905  InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
906  }
907 
908  auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
909  Register RegMul1, bool KillRegMul1,
910  Register RegMul2, bool KillRegMul2) {
911  MI->getOperand(AddOpIdx).setReg(RegAdd);
912  MI->getOperand(AddOpIdx).setIsKill(KillAdd);
913  MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
914  MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
915  MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
916  MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
917  };
918 
919  MachineInstrBuilder NewARegPressure, NewCRegPressure;
920  switch (Pattern) {
921  default:
922  llvm_unreachable("not recognized pattern!");
924  // Create new instructions for insertion.
925  MachineInstrBuilder MINewB =
926  BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
927  .addReg(RegX, getKillRegState(KillX))
928  .addReg(RegM21, getKillRegState(KillM21))
929  .addReg(RegM22, getKillRegState(KillM22));
930  MachineInstrBuilder MINewA =
931  BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
932  .addReg(RegY, getKillRegState(KillY))
933  .addReg(RegM31, getKillRegState(KillM31))
934  .addReg(RegM32, getKillRegState(KillM32));
935  // If AddOpIdx is not 1, adjust the order.
936  if (AddOpIdx != 1) {
937  AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
938  AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
939  }
940 
941  MachineInstrBuilder MINewC =
942  BuildMI(*MF, Root.getDebugLoc(),
944  .addReg(NewVRB, getKillRegState(true))
945  .addReg(NewVRA, getKillRegState(true));
946 
947  // Update flags for newly created instructions.
948  setSpecialOperandAttr(*MINewA, IntersectedFlags);
949  setSpecialOperandAttr(*MINewB, IntersectedFlags);
950  setSpecialOperandAttr(*MINewC, IntersectedFlags);
951 
952  // Record new instructions for insertion.
953  InsInstrs.push_back(MINewA);
954  InsInstrs.push_back(MINewB);
955  InsInstrs.push_back(MINewC);
956  break;
957  }
959  assert(NewVRD && "new FMA register not created!");
960  // Create new instructions for insertion.
961  MachineInstrBuilder MINewA =
962  BuildMI(*MF, Leaf->getDebugLoc(),
963  get(FMAOpIdxInfo[Idx][InfoArrayIdxFMULInst]), NewVRA)
964  .addReg(RegM11, getKillRegState(KillM11))
965  .addReg(RegM12, getKillRegState(KillM12));
966  MachineInstrBuilder MINewB =
967  BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
968  .addReg(RegX, getKillRegState(KillX))
969  .addReg(RegM21, getKillRegState(KillM21))
970  .addReg(RegM22, getKillRegState(KillM22));
971  MachineInstrBuilder MINewD =
972  BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
973  .addReg(NewVRA, getKillRegState(true))
974  .addReg(RegM31, getKillRegState(KillM31))
975  .addReg(RegM32, getKillRegState(KillM32));
976  // If AddOpIdx is not 1, adjust the order.
977  if (AddOpIdx != 1) {
978  AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
979  AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
980  KillM32);
981  }
982 
983  MachineInstrBuilder MINewC =
984  BuildMI(*MF, Root.getDebugLoc(),
986  .addReg(NewVRB, getKillRegState(true))
987  .addReg(NewVRD, getKillRegState(true));
988 
989  // Update flags for newly created instructions.
990  setSpecialOperandAttr(*MINewA, IntersectedFlags);
991  setSpecialOperandAttr(*MINewB, IntersectedFlags);
992  setSpecialOperandAttr(*MINewD, IntersectedFlags);
993  setSpecialOperandAttr(*MINewC, IntersectedFlags);
994 
995  // Record new instructions for insertion.
996  InsInstrs.push_back(MINewA);
997  InsInstrs.push_back(MINewB);
998  InsInstrs.push_back(MINewD);
999  InsInstrs.push_back(MINewC);
1000  break;
1001  }
1004  Register VarReg;
1005  bool KillVarReg = false;
1007  VarReg = RegM31;
1008  KillVarReg = KillM31;
1009  } else {
1010  VarReg = RegM32;
1011  KillVarReg = KillM32;
1012  }
1013  // We don't want to get negative const from memory pool too early, as the
1014  // created entry will not be deleted even if it has no users. Since all
1015  // operand of Leaf and Root are virtual register, we use zero register
1016  // here as a placeholder. When the InsInstrs is selected in
1017  // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1018  // with a virtual register which is a load from constant pool.
1019  NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1020  .addReg(RegB, getKillRegState(RegB))
1021  .addReg(RegY, getKillRegState(KillY))
1022  .addReg(PPC::ZERO8);
1023  NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1024  .addReg(NewVRA, getKillRegState(true))
1025  .addReg(RegX, getKillRegState(KillX))
1026  .addReg(VarReg, getKillRegState(KillVarReg));
1027  // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1028  // both at index 1, no need to adjust.
1029  // FIXME: when add more fma instructions support, like fma/fmas, adjust
1030  // the operand index here.
1031  break;
1032  }
1033  }
1034 
1035  if (!IsILPReassociate) {
1036  setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1037  setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1038 
1039  InsInstrs.push_back(NewARegPressure);
1040  InsInstrs.push_back(NewCRegPressure);
1041  }
1042 
1043  assert(!InsInstrs.empty() &&
1044  "Insertion instructions set should not be empty!");
1045 
1046  // Record old instructions for deletion.
1047  DelInstrs.push_back(Leaf);
1048  if (IsILPReassociate)
1049  DelInstrs.push_back(Prev);
1050  DelInstrs.push_back(&Root);
1051 }
1052 
1053 // Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1055  Register &SrcReg, Register &DstReg,
1056  unsigned &SubIdx) const {
1057  switch (MI.getOpcode()) {
1058  default: return false;
1059  case PPC::EXTSW:
1060  case PPC::EXTSW_32:
1061  case PPC::EXTSW_32_64:
1062  SrcReg = MI.getOperand(1).getReg();
1063  DstReg = MI.getOperand(0).getReg();
1064  SubIdx = PPC::sub_32;
1065  return true;
1066  }
1067 }
1068 
1070  int &FrameIndex) const {
1071  unsigned Opcode = MI.getOpcode();
1072  const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1073  const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
1074 
1075  if (End != std::find(OpcodesForSpill, End, Opcode)) {
1076  // Check for the operands added by addFrameReference (the immediate is the
1077  // offset which defaults to 0).
1078  if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1079  MI.getOperand(2).isFI()) {
1080  FrameIndex = MI.getOperand(2).getIndex();
1081  return MI.getOperand(0).getReg();
1082  }
1083  }
1084  return 0;
1085 }
1086 
1087 // For opcodes with the ReMaterializable flag set, this function is called to
1088 // verify the instruction is really rematable.
1090  AliasAnalysis *AA) const {
1091  switch (MI.getOpcode()) {
1092  default:
1093  // This function should only be called for opcodes with the ReMaterializable
1094  // flag set.
1095  llvm_unreachable("Unknown rematerializable operation!");
1096  break;
1097  case PPC::LI:
1098  case PPC::LI8:
1099  case PPC::PLI:
1100  case PPC::PLI8:
1101  case PPC::LIS:
1102  case PPC::LIS8:
1103  case PPC::ADDIStocHA:
1104  case PPC::ADDIStocHA8:
1105  case PPC::ADDItocL:
1106  case PPC::LOAD_STACK_GUARD:
1107  case PPC::XXLXORz:
1108  case PPC::XXLXORspz:
1109  case PPC::XXLXORdpz:
1110  case PPC::XXLEQVOnes:
1111  case PPC::XXSPLTI32DX:
1112  case PPC::XXSPLTIW:
1113  case PPC::XXSPLTIDP:
1114  case PPC::V_SET0B:
1115  case PPC::V_SET0H:
1116  case PPC::V_SET0:
1117  case PPC::V_SETALLONESB:
1118  case PPC::V_SETALLONESH:
1119  case PPC::V_SETALLONES:
1120  case PPC::CRSET:
1121  case PPC::CRUNSET:
1122  case PPC::XXSETACCZ:
1123  return true;
1124  }
1125  return false;
1126 }
1127 
1129  int &FrameIndex) const {
1130  unsigned Opcode = MI.getOpcode();
1131  const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1132  const unsigned *End = OpcodesForSpill + SOK_LastOpcodeSpill;
1133 
1134  if (End != std::find(OpcodesForSpill, End, Opcode)) {
1135  if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1136  MI.getOperand(2).isFI()) {
1137  FrameIndex = MI.getOperand(2).getIndex();
1138  return MI.getOperand(0).getReg();
1139  }
1140  }
1141  return 0;
1142 }
1143 
1145  unsigned OpIdx1,
1146  unsigned OpIdx2) const {
1147  MachineFunction &MF = *MI.getParent()->getParent();
1148 
1149  // Normal instructions can be commuted the obvious way.
1150  if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1151  return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1152  // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1153  // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1154  // changing the relative order of the mask operands might change what happens
1155  // to the high-bits of the mask (and, thus, the result).
1156 
1157  // Cannot commute if it has a non-zero rotate count.
1158  if (MI.getOperand(3).getImm() != 0)
1159  return nullptr;
1160 
1161  // If we have a zero rotate count, we have:
1162  // M = mask(MB,ME)
1163  // Op0 = (Op1 & ~M) | (Op2 & M)
1164  // Change this to:
1165  // M = mask((ME+1)&31, (MB-1)&31)
1166  // Op0 = (Op2 & ~M) | (Op1 & M)
1167 
1168  // Swap op1/op2
1169  assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1170  "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1171  Register Reg0 = MI.getOperand(0).getReg();
1172  Register Reg1 = MI.getOperand(1).getReg();
1173  Register Reg2 = MI.getOperand(2).getReg();
1174  unsigned SubReg1 = MI.getOperand(1).getSubReg();
1175  unsigned SubReg2 = MI.getOperand(2).getSubReg();
1176  bool Reg1IsKill = MI.getOperand(1).isKill();
1177  bool Reg2IsKill = MI.getOperand(2).isKill();
1178  bool ChangeReg0 = false;
1179  // If machine instrs are no longer in two-address forms, update
1180  // destination register as well.
1181  if (Reg0 == Reg1) {
1182  // Must be two address instruction!
1183  assert(MI.getDesc().getOperandConstraint(0, MCOI::TIED_TO) &&
1184  "Expecting a two-address instruction!");
1185  assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1186  Reg2IsKill = false;
1187  ChangeReg0 = true;
1188  }
1189 
1190  // Masks.
1191  unsigned MB = MI.getOperand(4).getImm();
1192  unsigned ME = MI.getOperand(5).getImm();
1193 
1194  // We can't commute a trivial mask (there is no way to represent an all-zero
1195  // mask).
1196  if (MB == 0 && ME == 31)
1197  return nullptr;
1198 
1199  if (NewMI) {
1200  // Create a new instruction.
1201  Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1202  bool Reg0IsDead = MI.getOperand(0).isDead();
1203  return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1204  .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1205  .addReg(Reg2, getKillRegState(Reg2IsKill))
1206  .addReg(Reg1, getKillRegState(Reg1IsKill))
1207  .addImm((ME + 1) & 31)
1208  .addImm((MB - 1) & 31);
1209  }
1210 
1211  if (ChangeReg0) {
1212  MI.getOperand(0).setReg(Reg2);
1213  MI.getOperand(0).setSubReg(SubReg2);
1214  }
1215  MI.getOperand(2).setReg(Reg1);
1216  MI.getOperand(1).setReg(Reg2);
1217  MI.getOperand(2).setSubReg(SubReg1);
1218  MI.getOperand(1).setSubReg(SubReg2);
1219  MI.getOperand(2).setIsKill(Reg1IsKill);
1220  MI.getOperand(1).setIsKill(Reg2IsKill);
1221 
1222  // Swap the mask around.
1223  MI.getOperand(4).setImm((ME + 1) & 31);
1224  MI.getOperand(5).setImm((MB - 1) & 31);
1225  return &MI;
1226 }
1227 
1229  unsigned &SrcOpIdx1,
1230  unsigned &SrcOpIdx2) const {
1231  // For VSX A-Type FMA instructions, it is the first two operands that can be
1232  // commuted, however, because the non-encoded tied input operand is listed
1233  // first, the operands to swap are actually the second and third.
1234 
1235  int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1236  if (AltOpc == -1)
1237  return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1238 
1239  // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1240  // and SrcOpIdx2.
1241  return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1242 }
1243 
1246  // This function is used for scheduling, and the nop wanted here is the type
1247  // that terminates dispatch groups on the POWER cores.
1248  unsigned Directive = Subtarget.getCPUDirective();
1249  unsigned Opcode;
1250  switch (Directive) {
1251  default: Opcode = PPC::NOP; break;
1252  case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1253  case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1254  case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1255  // FIXME: Update when POWER9 scheduling model is ready.
1256  case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1257  }
1258 
1259  DebugLoc DL;
1260  BuildMI(MBB, MI, DL, get(Opcode));
1261 }
1262 
1263 /// Return the noop instruction to use for a noop.
1265  MCInst Nop;
1266  Nop.setOpcode(PPC::NOP);
1267  return Nop;
1268 }
1269 
1270 // Branch analysis.
1271 // Note: If the condition register is set to CTR or CTR8 then this is a
1272 // BDNZ (imm == 1) or BDZ (imm == 0) branch.
1274  MachineBasicBlock *&TBB,
1275  MachineBasicBlock *&FBB,
1277  bool AllowModify) const {
1278  bool isPPC64 = Subtarget.isPPC64();
1279 
1280  // If the block has no terminators, it just falls into the block after it.
1282  if (I == MBB.end())
1283  return false;
1284 
1285  if (!isUnpredicatedTerminator(*I))
1286  return false;
1287 
1288  if (AllowModify) {
1289  // If the BB ends with an unconditional branch to the fallthrough BB,
1290  // we eliminate the branch instruction.
1291  if (I->getOpcode() == PPC::B &&
1292  MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1293  I->eraseFromParent();
1294 
1295  // We update iterator after deleting the last branch.
1297  if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1298  return false;
1299  }
1300  }
1301 
1302  // Get the last instruction in the block.
1303  MachineInstr &LastInst = *I;
1304 
1305  // If there is only one terminator instruction, process it.
1306  if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1307  if (LastInst.getOpcode() == PPC::B) {
1308  if (!LastInst.getOperand(0).isMBB())
1309  return true;
1310  TBB = LastInst.getOperand(0).getMBB();
1311  return false;
1312  } else if (LastInst.getOpcode() == PPC::BCC) {
1313  if (!LastInst.getOperand(2).isMBB())
1314  return true;
1315  // Block ends with fall-through condbranch.
1316  TBB = LastInst.getOperand(2).getMBB();
1317  Cond.push_back(LastInst.getOperand(0));
1318  Cond.push_back(LastInst.getOperand(1));
1319  return false;
1320  } else if (LastInst.getOpcode() == PPC::BC) {
1321  if (!LastInst.getOperand(1).isMBB())
1322  return true;
1323  // Block ends with fall-through condbranch.
1324  TBB = LastInst.getOperand(1).getMBB();
1326  Cond.push_back(LastInst.getOperand(0));
1327  return false;
1328  } else if (LastInst.getOpcode() == PPC::BCn) {
1329  if (!LastInst.getOperand(1).isMBB())
1330  return true;
1331  // Block ends with fall-through condbranch.
1332  TBB = LastInst.getOperand(1).getMBB();
1334  Cond.push_back(LastInst.getOperand(0));
1335  return false;
1336  } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1337  LastInst.getOpcode() == PPC::BDNZ) {
1338  if (!LastInst.getOperand(0).isMBB())
1339  return true;
1340  if (DisableCTRLoopAnal)
1341  return true;
1342  TBB = LastInst.getOperand(0).getMBB();
1343  Cond.push_back(MachineOperand::CreateImm(1));
1344  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1345  true));
1346  return false;
1347  } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1348  LastInst.getOpcode() == PPC::BDZ) {
1349  if (!LastInst.getOperand(0).isMBB())
1350  return true;
1351  if (DisableCTRLoopAnal)
1352  return true;
1353  TBB = LastInst.getOperand(0).getMBB();
1354  Cond.push_back(MachineOperand::CreateImm(0));
1355  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1356  true));
1357  return false;
1358  }
1359 
1360  // Otherwise, don't know what this is.
1361  return true;
1362  }
1363 
1364  // Get the instruction before it if it's a terminator.
1365  MachineInstr &SecondLastInst = *I;
1366 
1367  // If there are three terminators, we don't know what sort of block this is.
1368  if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1369  return true;
1370 
1371  // If the block ends with PPC::B and PPC:BCC, handle it.
1372  if (SecondLastInst.getOpcode() == PPC::BCC &&
1373  LastInst.getOpcode() == PPC::B) {
1374  if (!SecondLastInst.getOperand(2).isMBB() ||
1375  !LastInst.getOperand(0).isMBB())
1376  return true;
1377  TBB = SecondLastInst.getOperand(2).getMBB();
1378  Cond.push_back(SecondLastInst.getOperand(0));
1379  Cond.push_back(SecondLastInst.getOperand(1));
1380  FBB = LastInst.getOperand(0).getMBB();
1381  return false;
1382  } else if (SecondLastInst.getOpcode() == PPC::BC &&
1383  LastInst.getOpcode() == PPC::B) {
1384  if (!SecondLastInst.getOperand(1).isMBB() ||
1385  !LastInst.getOperand(0).isMBB())
1386  return true;
1387  TBB = SecondLastInst.getOperand(1).getMBB();
1389  Cond.push_back(SecondLastInst.getOperand(0));
1390  FBB = LastInst.getOperand(0).getMBB();
1391  return false;
1392  } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1393  LastInst.getOpcode() == PPC::B) {
1394  if (!SecondLastInst.getOperand(1).isMBB() ||
1395  !LastInst.getOperand(0).isMBB())
1396  return true;
1397  TBB = SecondLastInst.getOperand(1).getMBB();
1399  Cond.push_back(SecondLastInst.getOperand(0));
1400  FBB = LastInst.getOperand(0).getMBB();
1401  return false;
1402  } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1403  SecondLastInst.getOpcode() == PPC::BDNZ) &&
1404  LastInst.getOpcode() == PPC::B) {
1405  if (!SecondLastInst.getOperand(0).isMBB() ||
1406  !LastInst.getOperand(0).isMBB())
1407  return true;
1408  if (DisableCTRLoopAnal)
1409  return true;
1410  TBB = SecondLastInst.getOperand(0).getMBB();
1411  Cond.push_back(MachineOperand::CreateImm(1));
1412  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1413  true));
1414  FBB = LastInst.getOperand(0).getMBB();
1415  return false;
1416  } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1417  SecondLastInst.getOpcode() == PPC::BDZ) &&
1418  LastInst.getOpcode() == PPC::B) {
1419  if (!SecondLastInst.getOperand(0).isMBB() ||
1420  !LastInst.getOperand(0).isMBB())
1421  return true;
1422  if (DisableCTRLoopAnal)
1423  return true;
1424  TBB = SecondLastInst.getOperand(0).getMBB();
1425  Cond.push_back(MachineOperand::CreateImm(0));
1426  Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1427  true));
1428  FBB = LastInst.getOperand(0).getMBB();
1429  return false;
1430  }
1431 
1432  // If the block ends with two PPC:Bs, handle it. The second one is not
1433  // executed, so remove it.
1434  if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1435  if (!SecondLastInst.getOperand(0).isMBB())
1436  return true;
1437  TBB = SecondLastInst.getOperand(0).getMBB();
1438  I = LastInst;
1439  if (AllowModify)
1440  I->eraseFromParent();
1441  return false;
1442  }
1443 
1444  // Otherwise, can't handle this.
1445  return true;
1446 }
1447 
1449  int *BytesRemoved) const {
1450  assert(!BytesRemoved && "code size not handled");
1451 
1453  if (I == MBB.end())
1454  return 0;
1455 
1456  if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1457  I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1458  I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1459  I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1460  return 0;
1461 
1462  // Remove the branch.
1463  I->eraseFromParent();
1464 
1465  I = MBB.end();
1466 
1467  if (I == MBB.begin()) return 1;
1468  --I;
1469  if (I->getOpcode() != PPC::BCC &&
1470  I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1471  I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1472  I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1473  return 1;
1474 
1475  // Remove the branch.
1476  I->eraseFromParent();
1477  return 2;
1478 }
1479 
1481  MachineBasicBlock *TBB,
1482  MachineBasicBlock *FBB,
1484  const DebugLoc &DL,
1485  int *BytesAdded) const {
1486  // Shouldn't be a fall through.
1487  assert(TBB && "insertBranch must not be told to insert a fallthrough");
1488  assert((Cond.size() == 2 || Cond.size() == 0) &&
1489  "PPC branch conditions have two components!");
1490  assert(!BytesAdded && "code size not handled");
1491 
1492  bool isPPC64 = Subtarget.isPPC64();
1493 
1494  // One-way branch.
1495  if (!FBB) {
1496  if (Cond.empty()) // Unconditional branch
1497  BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1498  else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1499  BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1500  (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1501  (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1502  else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1503  BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1504  else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1505  BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1506  else // Conditional branch
1507  BuildMI(&MBB, DL, get(PPC::BCC))
1508  .addImm(Cond[0].getImm())
1509  .add(Cond[1])
1510  .addMBB(TBB);
1511  return 1;
1512  }
1513 
1514  // Two-way Conditional Branch.
1515  if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1516  BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1517  (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1518  (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1519  else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1520  BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1521  else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1522  BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1523  else
1524  BuildMI(&MBB, DL, get(PPC::BCC))
1525  .addImm(Cond[0].getImm())
1526  .add(Cond[1])
1527  .addMBB(TBB);
1528  BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1529  return 2;
1530 }
1531 
1532 // Select analysis.
1535  Register DstReg, Register TrueReg,
1536  Register FalseReg, int &CondCycles,
1537  int &TrueCycles, int &FalseCycles) const {
1538  if (Cond.size() != 2)
1539  return false;
1540 
1541  // If this is really a bdnz-like condition, then it cannot be turned into a
1542  // select.
1543  if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1544  return false;
1545 
1546  // If the conditional branch uses a physical register, then it cannot be
1547  // turned into a select.
1549  return false;
1550 
1551  // Check register classes.
1553  const TargetRegisterClass *RC =
1554  RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1555  if (!RC)
1556  return false;
1557 
1558  // isel is for regular integer GPRs only.
1559  if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1560  !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1561  !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1562  !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1563  return false;
1564 
1565  // FIXME: These numbers are for the A2, how well they work for other cores is
1566  // an open question. On the A2, the isel instruction has a 2-cycle latency
1567  // but single-cycle throughput. These numbers are used in combination with
1568  // the MispredictPenalty setting from the active SchedMachineModel.
1569  CondCycles = 1;
1570  TrueCycles = 1;
1571  FalseCycles = 1;
1572 
1573  return true;
1574 }
1575 
1578  const DebugLoc &dl, Register DestReg,
1580  Register FalseReg) const {
1581  assert(Cond.size() == 2 &&
1582  "PPC branch conditions have two components!");
1583 
1584  // Get the register classes.
1586  const TargetRegisterClass *RC =
1587  RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1588  assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1589 
1590  bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1591  PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1592  assert((Is64Bit ||
1593  PPC::GPRCRegClass.hasSubClassEq(RC) ||
1594  PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1595  "isel is for regular integer GPRs only");
1596 
1597  unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1598  auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1599 
1600  unsigned SubIdx = 0;
1601  bool SwapOps = false;
1602  switch (SelectPred) {
1603  case PPC::PRED_EQ:
1604  case PPC::PRED_EQ_MINUS:
1605  case PPC::PRED_EQ_PLUS:
1606  SubIdx = PPC::sub_eq; SwapOps = false; break;
1607  case PPC::PRED_NE:
1608  case PPC::PRED_NE_MINUS:
1609  case PPC::PRED_NE_PLUS:
1610  SubIdx = PPC::sub_eq; SwapOps = true; break;
1611  case PPC::PRED_LT:
1612  case PPC::PRED_LT_MINUS:
1613  case PPC::PRED_LT_PLUS:
1614  SubIdx = PPC::sub_lt; SwapOps = false; break;
1615  case PPC::PRED_GE:
1616  case PPC::PRED_GE_MINUS:
1617  case PPC::PRED_GE_PLUS:
1618  SubIdx = PPC::sub_lt; SwapOps = true; break;
1619  case PPC::PRED_GT:
1620  case PPC::PRED_GT_MINUS:
1621  case PPC::PRED_GT_PLUS:
1622  SubIdx = PPC::sub_gt; SwapOps = false; break;
1623  case PPC::PRED_LE:
1624  case PPC::PRED_LE_MINUS:
1625  case PPC::PRED_LE_PLUS:
1626  SubIdx = PPC::sub_gt; SwapOps = true; break;
1627  case PPC::PRED_UN:
1628  case PPC::PRED_UN_MINUS:
1629  case PPC::PRED_UN_PLUS:
1630  SubIdx = PPC::sub_un; SwapOps = false; break;
1631  case PPC::PRED_NU:
1632  case PPC::PRED_NU_MINUS:
1633  case PPC::PRED_NU_PLUS:
1634  SubIdx = PPC::sub_un; SwapOps = true; break;
1635  case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1636  case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1637  }
1638 
1639  Register FirstReg = SwapOps ? FalseReg : TrueReg,
1640  SecondReg = SwapOps ? TrueReg : FalseReg;
1641 
1642  // The first input register of isel cannot be r0. If it is a member
1643  // of a register class that can be r0, then copy it first (the
1644  // register allocator should eliminate the copy).
1645  if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1646  MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1647  const TargetRegisterClass *FirstRC =
1648  MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1649  &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1650  Register OldFirstReg = FirstReg;
1651  FirstReg = MRI.createVirtualRegister(FirstRC);
1652  BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1653  .addReg(OldFirstReg);
1654  }
1655 
1656  BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1657  .addReg(FirstReg).addReg(SecondReg)
1658  .addReg(Cond[1].getReg(), 0, SubIdx);
1659 }
1660 
1661 static unsigned getCRBitValue(unsigned CRBit) {
1662  unsigned Ret = 4;
1663  if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1664  CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1665  CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1666  CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1667  Ret = 3;
1668  if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1669  CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1670  CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1671  CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1672  Ret = 2;
1673  if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1674  CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1675  CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1676  CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1677  Ret = 1;
1678  if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1679  CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1680  CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1681  CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1682  Ret = 0;
1683 
1684  assert(Ret != 4 && "Invalid CR bit register");
1685  return Ret;
1686 }
1687 
1690  const DebugLoc &DL, MCRegister DestReg,
1691  MCRegister SrcReg, bool KillSrc) const {
1692  // We can end up with self copies and similar things as a result of VSX copy
1693  // legalization. Promote them here.
1695  if (PPC::F8RCRegClass.contains(DestReg) &&
1696  PPC::VSRCRegClass.contains(SrcReg)) {
1697  MCRegister SuperReg =
1698  TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1699 
1700  if (VSXSelfCopyCrash && SrcReg == SuperReg)
1701  llvm_unreachable("nop VSX copy");
1702 
1703  DestReg = SuperReg;
1704  } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1705  PPC::VSRCRegClass.contains(DestReg)) {
1706  MCRegister SuperReg =
1707  TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1708 
1709  if (VSXSelfCopyCrash && DestReg == SuperReg)
1710  llvm_unreachable("nop VSX copy");
1711 
1712  SrcReg = SuperReg;
1713  }
1714 
1715  // Different class register copy
1716  if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1717  PPC::GPRCRegClass.contains(DestReg)) {
1718  MCRegister CRReg = getCRFromCRBit(SrcReg);
1719  BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1720  getKillRegState(KillSrc);
1721  // Rotate the CR bit in the CR fields to be the least significant bit and
1722  // then mask with 0x1 (MB = ME = 31).
1723  BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1724  .addReg(DestReg, RegState::Kill)
1725  .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1726  .addImm(31)
1727  .addImm(31);
1728  return;
1729  } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1730  (PPC::G8RCRegClass.contains(DestReg) ||
1731  PPC::GPRCRegClass.contains(DestReg))) {
1732  bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1733  unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1734  unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1735  unsigned CRNum = TRI->getEncodingValue(SrcReg);
1736  BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1737  getKillRegState(KillSrc);
1738  if (CRNum == 7)
1739  return;
1740  // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1741  BuildMI(MBB, I, DL, get(ShCode), DestReg)
1742  .addReg(DestReg, RegState::Kill)
1743  .addImm(CRNum * 4 + 4)
1744  .addImm(28)
1745  .addImm(31);
1746  return;
1747  } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1748  PPC::VSFRCRegClass.contains(DestReg)) {
1749  assert(Subtarget.hasDirectMove() &&
1750  "Subtarget doesn't support directmove, don't know how to copy.");
1751  BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1752  NumGPRtoVSRSpill++;
1753  getKillRegState(KillSrc);
1754  return;
1755  } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1756  PPC::G8RCRegClass.contains(DestReg)) {
1757  assert(Subtarget.hasDirectMove() &&
1758  "Subtarget doesn't support directmove, don't know how to copy.");
1759  BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1760  getKillRegState(KillSrc);
1761  return;
1762  } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1763  PPC::GPRCRegClass.contains(DestReg)) {
1764  BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1765  getKillRegState(KillSrc);
1766  return;
1767  } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1768  PPC::SPERCRegClass.contains(DestReg)) {
1769  BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1770  getKillRegState(KillSrc);
1771  return;
1772  }
1773 
1774  unsigned Opc;
1775  if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1776  Opc = PPC::OR;
1777  else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1778  Opc = PPC::OR8;
1779  else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1780  Opc = PPC::FMR;
1781  else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1782  Opc = PPC::MCRF;
1783  else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1784  Opc = PPC::VOR;
1785  else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1786  // There are two different ways this can be done:
1787  // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1788  // issue in VSU pipeline 0.
1789  // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1790  // can go to either pipeline.
1791  // We'll always use xxlor here, because in practically all cases where
1792  // copies are generated, they are close enough to some use that the
1793  // lower-latency form is preferable.
1794  Opc = PPC::XXLOR;
1795  else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1796  PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1797  Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1798  else if (Subtarget.pairedVectorMemops() &&
1799  PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1800  if (SrcReg > PPC::VSRp15)
1801  SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1802  else
1803  SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1804  if (DestReg > PPC::VSRp15)
1805  DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1806  else
1807  DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1808  BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1809  addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1810  BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1811  addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1812  return;
1813  }
1814  else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1815  Opc = PPC::CROR;
1816  else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1817  Opc = PPC::EVOR;
1818  else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1819  PPC::UACCRCRegClass.contains(DestReg)) &&
1820  (PPC::ACCRCRegClass.contains(SrcReg) ||
1821  PPC::UACCRCRegClass.contains(SrcReg))) {
1822  // If primed, de-prime the source register, copy the individual registers
1823  // and prime the destination if needed. The vector subregisters are
1824  // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1825  // source is primed, we need to re-prime it after the copy as well.
1826  PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1827  bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1828  bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1829  MCRegister VSLSrcReg =
1830  PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1831  MCRegister VSLDestReg =
1832  PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1833  if (SrcPrimed)
1834  BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1835  for (unsigned Idx = 0; Idx < 4; Idx++)
1836  BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1837  .addReg(VSLSrcReg + Idx)
1838  .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1839  if (DestPrimed)
1840  BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1841  if (SrcPrimed && !KillSrc)
1842  BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1843  return;
1844  } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1845  PPC::G8pRCRegClass.contains(SrcReg)) {
1846  // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1847  unsigned DestRegIdx = DestReg - PPC::G8p0;
1848  MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1849  MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1850  unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1851  MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1852  MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1853  BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1854  .addReg(SrcRegSub0)
1855  .addReg(SrcRegSub0, getKillRegState(KillSrc));
1856  BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1857  .addReg(SrcRegSub1)
1858  .addReg(SrcRegSub1, getKillRegState(KillSrc));
1859  return;
1860  } else
1861  llvm_unreachable("Impossible reg-to-reg copy");
1862 
1863  const MCInstrDesc &MCID = get(Opc);
1864  if (MCID.getNumOperands() == 3)
1865  BuildMI(MBB, I, DL, MCID, DestReg)
1866  .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1867  else
1868  BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1869 }
1870 
1871 unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1872  int OpcodeIndex = 0;
1873 
1874  if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1875  PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1877  } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1878  PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1880  } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1882  } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1884  } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1886  } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1888  } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1890  } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1892  } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1894  } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1896  } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1898  } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1900  } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1901  assert(Subtarget.pairedVectorMemops() &&
1902  "Register unexpected when paired memops are disabled.");
1904  } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1905  assert(Subtarget.pairedVectorMemops() &&
1906  "Register unexpected when paired memops are disabled.");
1908  } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1909  assert(Subtarget.pairedVectorMemops() &&
1910  "Register unexpected when paired memops are disabled.");
1912  } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1914  } else {
1915  llvm_unreachable("Unknown regclass!");
1916  }
1917  return OpcodeIndex;
1918 }
1919 
1920 unsigned
1922  const unsigned *OpcodesForSpill = getStoreOpcodesForSpillArray();
1923  return OpcodesForSpill[getSpillIndex(RC)];
1924 }
1925 
1926 unsigned
1928  const unsigned *OpcodesForSpill = getLoadOpcodesForSpillArray();
1929  return OpcodesForSpill[getSpillIndex(RC)];
1930 }
1931 
1932 void PPCInstrInfo::StoreRegToStackSlot(
1933  MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1934  const TargetRegisterClass *RC,
1935  SmallVectorImpl<MachineInstr *> &NewMIs) const {
1936  unsigned Opcode = getStoreOpcodeForSpill(RC);
1937  DebugLoc DL;
1938 
1939  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1940  FuncInfo->setHasSpills();
1941 
1942  NewMIs.push_back(addFrameReference(
1943  BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1944  FrameIdx));
1945 
1946  if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1947  PPC::CRBITRCRegClass.hasSubClassEq(RC))
1948  FuncInfo->setSpillsCR();
1949 
1950  if (isXFormMemOp(Opcode))
1951  FuncInfo->setHasNonRISpills();
1952 }
1953 
1956  bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1957  const TargetRegisterInfo *TRI) const {
1958  MachineFunction &MF = *MBB.getParent();
1960 
1961  StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1962 
1963  for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1964  MBB.insert(MI, NewMIs[i]);
1965 
1966  const MachineFrameInfo &MFI = MF.getFrameInfo();
1968  MachinePointerInfo::getFixedStack(MF, FrameIdx),
1970  MFI.getObjectAlign(FrameIdx));
1971  NewMIs.back()->addMemOperand(MF, MMO);
1972 }
1973 
1976  Register SrcReg, bool isKill,
1977  int FrameIdx,
1978  const TargetRegisterClass *RC,
1979  const TargetRegisterInfo *TRI) const {
1980  // We need to avoid a situation in which the value from a VRRC register is
1981  // spilled using an Altivec instruction and reloaded into a VSRC register
1982  // using a VSX instruction. The issue with this is that the VSX
1983  // load/store instructions swap the doublewords in the vector and the Altivec
1984  // ones don't. The register classes on the spill/reload may be different if
1985  // the register is defined using an Altivec instruction and is then used by a
1986  // VSX instruction.
1987  RC = updatedRC(RC);
1988  storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1989 }
1990 
1991 void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1992  unsigned DestReg, int FrameIdx,
1993  const TargetRegisterClass *RC,
1995  const {
1996  unsigned Opcode = getLoadOpcodeForSpill(RC);
1997  NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1998  FrameIdx));
1999  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2000 
2001  if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
2002  PPC::CRBITRCRegClass.hasSubClassEq(RC))
2003  FuncInfo->setSpillsCR();
2004 
2005  if (isXFormMemOp(Opcode))
2006  FuncInfo->setHasNonRISpills();
2007 }
2008 
2011  int FrameIdx, const TargetRegisterClass *RC,
2012  const TargetRegisterInfo *TRI) const {
2013  MachineFunction &MF = *MBB.getParent();
2015  DebugLoc DL;
2016  if (MI != MBB.end()) DL = MI->getDebugLoc();
2017 
2018  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2019  FuncInfo->setHasSpills();
2020 
2021  LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2022 
2023  for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
2024  MBB.insert(MI, NewMIs[i]);
2025 
2026  const MachineFrameInfo &MFI = MF.getFrameInfo();
2028  MachinePointerInfo::getFixedStack(MF, FrameIdx),
2030  MFI.getObjectAlign(FrameIdx));
2031  NewMIs.back()->addMemOperand(MF, MMO);
2032 }
2033 
2036  Register DestReg, int FrameIdx,
2037  const TargetRegisterClass *RC,
2038  const TargetRegisterInfo *TRI) const {
2039  // We need to avoid a situation in which the value from a VRRC register is
2040  // spilled using an Altivec instruction and reloaded into a VSRC register
2041  // using a VSX instruction. The issue with this is that the VSX
2042  // load/store instructions swap the doublewords in the vector and the Altivec
2043  // ones don't. The register classes on the spill/reload may be different if
2044  // the register is defined using an Altivec instruction and is then used by a
2045  // VSX instruction.
2046  RC = updatedRC(RC);
2047 
2048  loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2049 }
2050 
2051 bool PPCInstrInfo::
2053  assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2054  if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2055  Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2056  else
2057  // Leave the CR# the same, but invert the condition.
2058  Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2059  return false;
2060 }
2061 
2062 // For some instructions, it is legal to fold ZERO into the RA register field.
2063 // This function performs that fold by replacing the operand with PPC::ZERO,
2064 // it does not consider whether the load immediate zero is no longer in use.
2066  Register Reg) const {
2067  // A zero immediate should always be loaded with a single li.
2068  unsigned DefOpc = DefMI.getOpcode();
2069  if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2070  return false;
2071  if (!DefMI.getOperand(1).isImm())
2072  return false;
2073  if (DefMI.getOperand(1).getImm() != 0)
2074  return false;
2075 
2076  // Note that we cannot here invert the arguments of an isel in order to fold
2077  // a ZERO into what is presented as the second argument. All we have here
2078  // is the condition bit, and that might come from a CR-logical bit operation.
2079 
2080  const MCInstrDesc &UseMCID = UseMI.getDesc();
2081 
2082  // Only fold into real machine instructions.
2083  if (UseMCID.isPseudo())
2084  return false;
2085 
2086  // We need to find which of the User's operands is to be folded, that will be
2087  // the operand that matches the given register ID.
2088  unsigned UseIdx;
2089  for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2090  if (UseMI.getOperand(UseIdx).isReg() &&
2091  UseMI.getOperand(UseIdx).getReg() == Reg)
2092  break;
2093 
2094  assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2095  assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2096 
2097  const MCOperandInfo *UseInfo = &UseMCID.OpInfo[UseIdx];
2098 
2099  // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2100  // register (which might also be specified as a pointer class kind).
2101  if (UseInfo->isLookupPtrRegClass()) {
2102  if (UseInfo->RegClass /* Kind */ != 1)
2103  return false;
2104  } else {
2105  if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2106  UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2107  return false;
2108  }
2109 
2110  // Make sure this is not tied to an output register (or otherwise
2111  // constrained). This is true for ST?UX registers, for example, which
2112  // are tied to their output registers.
2113  if (UseInfo->Constraints != 0)
2114  return false;
2115 
2116  MCRegister ZeroReg;
2117  if (UseInfo->isLookupPtrRegClass()) {
2118  bool isPPC64 = Subtarget.isPPC64();
2119  ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2120  } else {
2121  ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2122  PPC::ZERO8 : PPC::ZERO;
2123  }
2124 
2125  UseMI.getOperand(UseIdx).setReg(ZeroReg);
2126  return true;
2127 }
2128 
2129 // Folds zero into instructions which have a load immediate zero as an operand
2130 // but also recognize zero as immediate zero. If the definition of the load
2131 // has no more users it is deleted.
2133  Register Reg, MachineRegisterInfo *MRI) const {
2134  bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2135  if (MRI->use_nodbg_empty(Reg))
2136  DefMI.eraseFromParent();
2137  return Changed;
2138 }
2139 
2141  for (MachineInstr &MI : MBB)
2142  if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8))
2143  return true;
2144  return false;
2145 }
2146 
2147 // We should make sure that, if we're going to predicate both sides of a
2148 // condition (a diamond), that both sides don't define the counter register. We
2149 // can predicate counter-decrement-based branches, but while that predicates
2150 // the branching, it does not predicate the counter decrement. If we tried to
2151 // merge the triangle into one predicated block, we'd decrement the counter
2152 // twice.
2154  unsigned NumT, unsigned ExtraT,
2155  MachineBasicBlock &FMBB,
2156  unsigned NumF, unsigned ExtraF,
2157  BranchProbability Probability) const {
2158  return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2159 }
2160 
2161 
2163  // The predicated branches are identified by their type, not really by the
2164  // explicit presence of a predicate. Furthermore, some of them can be
2165  // predicated more than once. Because if conversion won't try to predicate
2166  // any instruction which already claims to be predicated (by returning true
2167  // here), always return false. In doing so, we let isPredicable() be the
2168  // final word on whether not the instruction can be (further) predicated.
2169 
2170  return false;
2171 }
2172 
2174  const MachineBasicBlock *MBB,
2175  const MachineFunction &MF) const {
2176  // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2177  // across them, since some FP operations may change content of FPSCR.
2178  // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2179  if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF)
2180  return true;
2182 }
2183 
2185  ArrayRef<MachineOperand> Pred) const {
2186  unsigned OpC = MI.getOpcode();
2187  if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2188  if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2189  bool isPPC64 = Subtarget.isPPC64();
2190  MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2191  : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2192  // Need add Def and Use for CTR implicit operand.
2193  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2194  .addReg(Pred[1].getReg(), RegState::Implicit)
2195  .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2196  } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2197  MI.setDesc(get(PPC::BCLR));
2198  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2199  } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2200  MI.setDesc(get(PPC::BCLRn));
2201  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2202  } else {
2203  MI.setDesc(get(PPC::BCCLR));
2204  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2205  .addImm(Pred[0].getImm())
2206  .add(Pred[1]);
2207  }
2208 
2209  return true;
2210  } else if (OpC == PPC::B) {
2211  if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2212  bool isPPC64 = Subtarget.isPPC64();
2213  MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2214  : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2215  // Need add Def and Use for CTR implicit operand.
2216  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2217  .addReg(Pred[1].getReg(), RegState::Implicit)
2218  .addReg(Pred[1].getReg(), RegState::ImplicitDefine);
2219  } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2220  MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2221  MI.removeOperand(0);
2222 
2223  MI.setDesc(get(PPC::BC));
2224  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2225  .add(Pred[1])
2226  .addMBB(MBB);
2227  } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2228  MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2229  MI.removeOperand(0);
2230 
2231  MI.setDesc(get(PPC::BCn));
2232  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2233  .add(Pred[1])
2234  .addMBB(MBB);
2235  } else {
2236  MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2237  MI.removeOperand(0);
2238 
2239  MI.setDesc(get(PPC::BCC));
2240  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2241  .addImm(Pred[0].getImm())
2242  .add(Pred[1])
2243  .addMBB(MBB);
2244  }
2245 
2246  return true;
2247  } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2248  OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2249  OpC == PPC::BCTRL8_RM) {
2250  if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2251  llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2252 
2253  bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2254  OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2255  bool isPPC64 = Subtarget.isPPC64();
2256 
2257  if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2258  MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2259  : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2260  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2261  } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2262  MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2263  : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2264  MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2265  } else {
2266  MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2267  : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2268  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2269  .addImm(Pred[0].getImm())
2270  .add(Pred[1]);
2271  }
2272 
2273  // Need add Def and Use for LR implicit operand.
2274  if (setLR)
2275  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2276  .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2277  .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2278  if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2279  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2281 
2282  return true;
2283  }
2284 
2285  return false;
2286 }
2287 
2289  ArrayRef<MachineOperand> Pred2) const {
2290  assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2291  assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2292 
2293  if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2294  return false;
2295  if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2296  return false;
2297 
2298  // P1 can only subsume P2 if they test the same condition register.
2299  if (Pred1[1].getReg() != Pred2[1].getReg())
2300  return false;
2301 
2302  PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2303  PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2304 
2305  if (P1 == P2)
2306  return true;
2307 
2308  // Does P1 subsume P2, e.g. GE subsumes GT.
2309  if (P1 == PPC::PRED_LE &&
2310  (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2311  return true;
2312  if (P1 == PPC::PRED_GE &&
2313  (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2314  return true;
2315 
2316  return false;
2317 }
2318 
2320  std::vector<MachineOperand> &Pred,
2321  bool SkipDead) const {
2322  // Note: At the present time, the contents of Pred from this function is
2323  // unused by IfConversion. This implementation follows ARM by pushing the
2324  // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2325  // predicate, instructions defining CTR or CTR8 are also included as
2326  // predicate-defining instructions.
2327 
2328  const TargetRegisterClass *RCs[] =
2329  { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2330  &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2331 
2332  bool Found = false;
2333  for (const MachineOperand &MO : MI.operands()) {
2334  for (unsigned c = 0; c < array_lengthof(RCs) && !Found; ++c) {
2335  const TargetRegisterClass *RC = RCs[c];
2336  if (MO.isReg()) {
2337  if (MO.isDef() && RC->contains(MO.getReg())) {
2338  Pred.push_back(MO);
2339  Found = true;
2340  }
2341  } else if (MO.isRegMask()) {
2342  for (MCPhysReg R : *RC)
2343  if (MO.clobbersPhysReg(R)) {
2344  Pred.push_back(MO);
2345  Found = true;
2346  }
2347  }
2348  }
2349  }
2350 
2351  return Found;
2352 }
2353 
2355  Register &SrcReg2, int64_t &Mask,
2356  int64_t &Value) const {
2357  unsigned Opc = MI.getOpcode();
2358 
2359  switch (Opc) {
2360  default: return false;
2361  case PPC::CMPWI:
2362  case PPC::CMPLWI:
2363  case PPC::CMPDI:
2364  case PPC::CMPLDI:
2365  SrcReg = MI.getOperand(1).getReg();
2366  SrcReg2 = 0;
2367  Value = MI.getOperand(2).getImm();
2368  Mask = 0xFFFF;
2369  return true;
2370  case PPC::CMPW:
2371  case PPC::CMPLW:
2372  case PPC::CMPD:
2373  case PPC::CMPLD:
2374  case PPC::FCMPUS:
2375  case PPC::FCMPUD:
2376  SrcReg = MI.getOperand(1).getReg();
2377  SrcReg2 = MI.getOperand(2).getReg();
2378  Value = 0;
2379  Mask = 0;
2380  return true;
2381  }
2382 }
2383 
2385  Register SrcReg2, int64_t Mask,
2386  int64_t Value,
2387  const MachineRegisterInfo *MRI) const {
2388  if (DisableCmpOpt)
2389  return false;
2390 
2391  int OpC = CmpInstr.getOpcode();
2392  Register CRReg = CmpInstr.getOperand(0).getReg();
2393 
2394  // FP record forms set CR1 based on the exception status bits, not a
2395  // comparison with zero.
2396  if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2397  return false;
2398 
2400  // The record forms set the condition register based on a signed comparison
2401  // with zero (so says the ISA manual). This is not as straightforward as it
2402  // seems, however, because this is always a 64-bit comparison on PPC64, even
2403  // for instructions that are 32-bit in nature (like slw for example).
2404  // So, on PPC32, for unsigned comparisons, we can use the record forms only
2405  // for equality checks (as those don't depend on the sign). On PPC64,
2406  // we are restricted to equality for unsigned 64-bit comparisons and for
2407  // signed 32-bit comparisons the applicability is more restricted.
2408  bool isPPC64 = Subtarget.isPPC64();
2409  bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2410  bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2411  bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2412 
2413  // Look through copies unless that gets us to a physical register.
2414  Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2415  if (ActualSrc.isVirtual())
2416  SrcReg = ActualSrc;
2417 
2418  // Get the unique definition of SrcReg.
2419  MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2420  if (!MI) return false;
2421 
2422  bool equalityOnly = false;
2423  bool noSub = false;
2424  if (isPPC64) {
2425  if (is32BitSignedCompare) {
2426  // We can perform this optimization only if MI is sign-extending.
2427  if (isSignExtended(*MI))
2428  noSub = true;
2429  else
2430  return false;
2431  } else if (is32BitUnsignedCompare) {
2432  // We can perform this optimization, equality only, if MI is
2433  // zero-extending.
2434  if (isZeroExtended(*MI)) {
2435  noSub = true;
2436  equalityOnly = true;
2437  } else
2438  return false;
2439  } else
2440  equalityOnly = is64BitUnsignedCompare;
2441  } else
2442  equalityOnly = is32BitUnsignedCompare;
2443 
2444  if (equalityOnly) {
2445  // We need to check the uses of the condition register in order to reject
2446  // non-equality comparisons.
2448  I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2449  I != IE; ++I) {
2450  MachineInstr *UseMI = &*I;
2451  if (UseMI->getOpcode() == PPC::BCC) {
2453  unsigned PredCond = PPC::getPredicateCondition(Pred);
2454  // We ignore hint bits when checking for non-equality comparisons.
2455  if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2456  return false;
2457  } else if (UseMI->getOpcode() == PPC::ISEL ||
2458  UseMI->getOpcode() == PPC::ISEL8) {
2459  unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2460  if (SubIdx != PPC::sub_eq)
2461  return false;
2462  } else
2463  return false;
2464  }
2465  }
2466 
2467  MachineBasicBlock::iterator I = CmpInstr;
2468 
2469  // Scan forward to find the first use of the compare.
2470  for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2471  ++I) {
2472  bool FoundUse = false;
2474  J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2475  J != JE; ++J)
2476  if (&*J == &*I) {
2477  FoundUse = true;
2478  break;
2479  }
2480 
2481  if (FoundUse)
2482  break;
2483  }
2484 
2487 
2488  // There are two possible candidates which can be changed to set CR[01].
2489  // One is MI, the other is a SUB instruction.
2490  // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2491  MachineInstr *Sub = nullptr;
2492  if (SrcReg2 != 0)
2493  // MI is not a candidate for CMPrr.
2494  MI = nullptr;
2495  // FIXME: Conservatively refuse to convert an instruction which isn't in the
2496  // same BB as the comparison. This is to allow the check below to avoid calls
2497  // (and other explicit clobbers); instead we should really check for these
2498  // more explicitly (in at least a few predecessors).
2499  else if (MI->getParent() != CmpInstr.getParent())
2500  return false;
2501  else if (Value != 0) {
2502  // The record-form instructions set CR bit based on signed comparison
2503  // against 0. We try to convert a compare against 1 or -1 into a compare
2504  // against 0 to exploit record-form instructions. For example, we change
2505  // the condition "greater than -1" into "greater than or equal to 0"
2506  // and "less than 1" into "less than or equal to 0".
2507 
2508  // Since we optimize comparison based on a specific branch condition,
2509  // we don't optimize if condition code is used by more than once.
2510  if (equalityOnly || !MRI->hasOneUse(CRReg))
2511  return false;
2512 
2513  MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2514  if (UseMI->getOpcode() != PPC::BCC)
2515  return false;
2516 
2518  unsigned PredCond = PPC::getPredicateCondition(Pred);
2519  unsigned PredHint = PPC::getPredicateHint(Pred);
2520  int16_t Immed = (int16_t)Value;
2521 
2522  // When modifying the condition in the predicate, we propagate hint bits
2523  // from the original predicate to the new one.
2524  if (Immed == -1 && PredCond == PPC::PRED_GT)
2525  // We convert "greater than -1" into "greater than or equal to 0",
2526  // since we are assuming signed comparison by !equalityOnly
2527  Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2528  else if (Immed == -1 && PredCond == PPC::PRED_LE)
2529  // We convert "less than or equal to -1" into "less than 0".
2530  Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2531  else if (Immed == 1 && PredCond == PPC::PRED_LT)
2532  // We convert "less than 1" into "less than or equal to 0".
2533  Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2534  else if (Immed == 1 && PredCond == PPC::PRED_GE)
2535  // We convert "greater than or equal to 1" into "greater than 0".
2536  Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2537  else
2538  return false;
2539 
2540  PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)), Pred));
2541  }
2542 
2543  // Search for Sub.
2544  --I;
2545 
2546  // Get ready to iterate backward from CmpInstr.
2547  MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2548 
2549  for (; I != E && !noSub; --I) {
2550  const MachineInstr &Instr = *I;
2551  unsigned IOpC = Instr.getOpcode();
2552 
2553  if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2554  Instr.readsRegister(PPC::CR0, TRI)))
2555  // This instruction modifies or uses the record condition register after
2556  // the one we want to change. While we could do this transformation, it
2557  // would likely not be profitable. This transformation removes one
2558  // instruction, and so even forcing RA to generate one move probably
2559  // makes it unprofitable.
2560  return false;
2561 
2562  // Check whether CmpInstr can be made redundant by the current instruction.
2563  if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2564  OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2565  (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2566  ((Instr.getOperand(1).getReg() == SrcReg &&
2567  Instr.getOperand(2).getReg() == SrcReg2) ||
2568  (Instr.getOperand(1).getReg() == SrcReg2 &&
2569  Instr.getOperand(2).getReg() == SrcReg))) {
2570  Sub = &*I;
2571  break;
2572  }
2573 
2574  if (I == B)
2575  // The 'and' is below the comparison instruction.
2576  return false;
2577  }
2578 
2579  // Return false if no candidates exist.
2580  if (!MI && !Sub)
2581  return false;
2582 
2583  // The single candidate is called MI.
2584  if (!MI) MI = Sub;
2585 
2586  int NewOpC = -1;
2587  int MIOpC = MI->getOpcode();
2588  if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2589  MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2590  NewOpC = MIOpC;
2591  else {
2592  NewOpC = PPC::getRecordFormOpcode(MIOpC);
2593  if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2594  NewOpC = MIOpC;
2595  }
2596 
2597  // FIXME: On the non-embedded POWER architectures, only some of the record
2598  // forms are fast, and we should use only the fast ones.
2599 
2600  // The defining instruction has a record form (or is already a record
2601  // form). It is possible, however, that we'll need to reverse the condition
2602  // code of the users.
2603  if (NewOpC == -1)
2604  return false;
2605 
2606  // This transformation should not be performed if `nsw` is missing and is not
2607  // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2608  // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2609  // CRReg can reflect if compared values are equal, this optz is still valid.
2610  if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2611  Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2612  return false;
2613 
2614  // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2615  // needs to be updated to be based on SUB. Push the condition code
2616  // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2617  // condition code of these operands will be modified.
2618  // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2619  // comparison against 0, which may modify predicate.
2620  bool ShouldSwap = false;
2621  if (Sub && Value == 0) {
2622  ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2623  Sub->getOperand(2).getReg() == SrcReg;
2624 
2625  // The operands to subf are the opposite of sub, so only in the fixed-point
2626  // case, invert the order.
2627  ShouldSwap = !ShouldSwap;
2628  }
2629 
2630  if (ShouldSwap)
2632  I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2633  I != IE; ++I) {
2634  MachineInstr *UseMI = &*I;
2635  if (UseMI->getOpcode() == PPC::BCC) {
2637  unsigned PredCond = PPC::getPredicateCondition(Pred);
2638  assert((!equalityOnly ||
2639  PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2640  "Invalid predicate for equality-only optimization");
2641  (void)PredCond; // To suppress warning in release build.
2642  PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2643  PPC::getSwappedPredicate(Pred)));
2644  } else if (UseMI->getOpcode() == PPC::ISEL ||
2645  UseMI->getOpcode() == PPC::ISEL8) {
2646  unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2647  assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2648  "Invalid CR bit for equality-only optimization");
2649 
2650  if (NewSubReg == PPC::sub_lt)
2651  NewSubReg = PPC::sub_gt;
2652  else if (NewSubReg == PPC::sub_gt)
2653  NewSubReg = PPC::sub_lt;
2654 
2655  SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2656  NewSubReg));
2657  } else // We need to abort on a user we don't understand.
2658  return false;
2659  }
2660  assert(!(Value != 0 && ShouldSwap) &&
2661  "Non-zero immediate support and ShouldSwap"
2662  "may conflict in updating predicate");
2663 
2664  // Create a new virtual register to hold the value of the CR set by the
2665  // record-form instruction. If the instruction was not previously in
2666  // record form, then set the kill flag on the CR.
2667  CmpInstr.eraseFromParent();
2668 
2670  BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2671  get(TargetOpcode::COPY), CRReg)
2672  .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2673 
2674  // Even if CR0 register were dead before, it is alive now since the
2675  // instruction we just built uses it.
2676  MI->clearRegisterDeads(PPC::CR0);
2677 
2678  if (MIOpC != NewOpC) {
2679  // We need to be careful here: we're replacing one instruction with
2680  // another, and we need to make sure that we get all of the right
2681  // implicit uses and defs. On the other hand, the caller may be holding
2682  // an iterator to this instruction, and so we can't delete it (this is
2683  // specifically the case if this is the instruction directly after the
2684  // compare).
2685 
2686  // Rotates are expensive instructions. If we're emitting a record-form
2687  // rotate that can just be an andi/andis, we should just emit that.
2688  if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2689  Register GPRRes = MI->getOperand(0).getReg();
2690  int64_t SH = MI->getOperand(2).getImm();
2691  int64_t MB = MI->getOperand(3).getImm();
2692  int64_t ME = MI->getOperand(4).getImm();
2693  // We can only do this if both the start and end of the mask are in the
2694  // same halfword.
2695  bool MBInLoHWord = MB >= 16;
2696  bool MEInLoHWord = ME >= 16;
2697  uint64_t Mask = ~0LLU;
2698 
2699  if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2700  Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2701  // The mask value needs to shift right 16 if we're emitting andis.
2702  Mask >>= MBInLoHWord ? 0 : 16;
2703  NewOpC = MIOpC == PPC::RLWINM
2704  ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2705  : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2706  } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2707  (ME - MB + 1 == SH) && (MB >= 16)) {
2708  // If we are rotating by the exact number of bits as are in the mask
2709  // and the mask is in the least significant bits of the register,
2710  // that's just an andis. (as long as the GPR result has no uses).
2711  Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2712  Mask >>= 16;
2713  NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2714  }
2715  // If we've set the mask, we can transform.
2716  if (Mask != ~0LLU) {
2717  MI->removeOperand(4);
2718  MI->removeOperand(3);
2719  MI->getOperand(2).setImm(Mask);
2720  NumRcRotatesConvertedToRcAnd++;
2721  }
2722  } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2723  int64_t MB = MI->getOperand(3).getImm();
2724  if (MB >= 48) {
2725  uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2726  NewOpC = PPC::ANDI8_rec;
2727  MI->removeOperand(3);
2728  MI->getOperand(2).setImm(Mask);
2729  NumRcRotatesConvertedToRcAnd++;
2730  }
2731  }
2732 
2733  const MCInstrDesc &NewDesc = get(NewOpC);
2734  MI->setDesc(NewDesc);
2735 
2736  if (NewDesc.ImplicitDefs)
2737  for (const MCPhysReg *ImpDefs = NewDesc.getImplicitDefs();
2738  *ImpDefs; ++ImpDefs)
2739  if (!MI->definesRegister(*ImpDefs))
2740  MI->addOperand(*MI->getParent()->getParent(),
2741  MachineOperand::CreateReg(*ImpDefs, true, true));
2742  if (NewDesc.ImplicitUses)
2743  for (const MCPhysReg *ImpUses = NewDesc.getImplicitUses();
2744  *ImpUses; ++ImpUses)
2745  if (!MI->readsRegister(*ImpUses))
2746  MI->addOperand(*MI->getParent()->getParent(),
2747  MachineOperand::CreateReg(*ImpUses, false, true));
2748  }
2749  assert(MI->definesRegister(PPC::CR0) &&
2750  "Record-form instruction does not define cr0?");
2751 
2752  // Modify the condition code of operands in OperandsToUpdate.
2753  // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2754  // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2755  for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2756  PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2757 
2758  for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2759  SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2760 
2761  return true;
2762 }
2763 
2766  int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2767  const TargetRegisterInfo *TRI) const {
2768  const MachineOperand *BaseOp;
2769  OffsetIsScalable = false;
2770  if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2771  return false;
2772  BaseOps.push_back(BaseOp);
2773  return true;
2774 }
2775 
2776 static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2777  const TargetRegisterInfo *TRI) {
2778  // If this is a volatile load/store, don't mess with it.
2779  if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2780  return false;
2781 
2782  if (LdSt.getOperand(2).isFI())
2783  return true;
2784 
2785  assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2786  // Can't cluster if the instruction modifies the base register
2787  // or it is update form. e.g. ld r2,3(r2)
2788  if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2789  return false;
2790 
2791  return true;
2792 }
2793 
2794 // Only cluster instruction pair that have the same opcode, and they are
2795 // clusterable according to PowerPC specification.
2796 static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2797  const PPCSubtarget &Subtarget) {
2798  switch (FirstOpc) {
2799  default:
2800  return false;
2801  case PPC::STD:
2802  case PPC::STFD:
2803  case PPC::STXSD:
2804  case PPC::DFSTOREf64:
2805  return FirstOpc == SecondOpc;
2806  // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2807  // 32bit and 64bit instruction selection. They are clusterable pair though
2808  // they are different opcode.
2809  case PPC::STW:
2810  case PPC::STW8:
2811  return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2812  }
2813 }
2814 
2817  ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2818  unsigned NumBytes) const {
2819 
2820  assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2821  const MachineOperand &BaseOp1 = *BaseOps1.front();
2822  const MachineOperand &BaseOp2 = *BaseOps2.front();
2823  assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2824  "Only base registers and frame indices are supported.");
2825 
2826  // The NumLoads means the number of loads that has been clustered.
2827  // Don't cluster memory op if there are already two ops clustered at least.
2828  if (NumLoads > 2)
2829  return false;
2830 
2831  // Cluster the load/store only when they have the same base
2832  // register or FI.
2833  if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2834  (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2835  (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2836  return false;
2837 
2838  // Check if the load/store are clusterable according to the PowerPC
2839  // specification.
2840  const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2841  const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2842  unsigned FirstOpc = FirstLdSt.getOpcode();
2843  unsigned SecondOpc = SecondLdSt.getOpcode();
2845  // Cluster the load/store only when they have the same opcode, and they are
2846  // clusterable opcode according to PowerPC specification.
2847  if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2848  return false;
2849 
2850  // Can't cluster load/store that have ordered or volatile memory reference.
2851  if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2852  !isLdStSafeToCluster(SecondLdSt, TRI))
2853  return false;
2854 
2855  int64_t Offset1 = 0, Offset2 = 0;
2856  unsigned Width1 = 0, Width2 = 0;
2857  const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2858  if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2859  !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2860  Width1 != Width2)
2861  return false;
2862 
2863  assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2864  "getMemOperandWithOffsetWidth return incorrect base op");
2865  // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2866  assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2867  return Offset1 + Width1 == Offset2;
2868 }
2869 
2870 /// GetInstSize - Return the number of bytes of code the specified
2871 /// instruction may be. This returns the maximum number of bytes.
2872 ///
2874  unsigned Opcode = MI.getOpcode();
2875 
2876  if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2877  const MachineFunction *MF = MI.getParent()->getParent();
2878  const char *AsmStr = MI.getOperand(0).getSymbolName();
2879  return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2880  } else if (Opcode == TargetOpcode::STACKMAP) {
2881  StackMapOpers Opers(&MI);
2882  return Opers.getNumPatchBytes();
2883  } else if (Opcode == TargetOpcode::PATCHPOINT) {
2884  PatchPointOpers Opers(&MI);
2885  return Opers.getNumPatchBytes();
2886  } else {
2887  return get(Opcode).getSize();
2888  }
2889 }
2890 
2891 std::pair<unsigned, unsigned>
2893  const unsigned Mask = PPCII::MO_ACCESS_MASK;
2894  return std::make_pair(TF & Mask, TF & ~Mask);
2895 }
2896 
2899  using namespace PPCII;
2900  static const std::pair<unsigned, const char *> TargetFlags[] = {
2901  {MO_LO, "ppc-lo"},
2902  {MO_HA, "ppc-ha"},
2903  {MO_TPREL_LO, "ppc-tprel-lo"},
2904  {MO_TPREL_HA, "ppc-tprel-ha"},
2905  {MO_DTPREL_LO, "ppc-dtprel-lo"},
2906  {MO_TLSLD_LO, "ppc-tlsld-lo"},
2907  {MO_TOC_LO, "ppc-toc-lo"},
2908  {MO_TLS, "ppc-tls"}};
2909  return makeArrayRef(TargetFlags);
2910 }
2911 
2914  using namespace PPCII;
2915  static const std::pair<unsigned, const char *> TargetFlags[] = {
2916  {MO_PLT, "ppc-plt"},
2917  {MO_PIC_FLAG, "ppc-pic"},
2918  {MO_PCREL_FLAG, "ppc-pcrel"},
2919  {MO_GOT_FLAG, "ppc-got"},
2920  {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
2921  {MO_TLSGD_FLAG, "ppc-tlsgd"},
2922  {MO_TLSLD_FLAG, "ppc-tlsld"},
2923  {MO_TPREL_FLAG, "ppc-tprel"},
2924  {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
2925  {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
2926  {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
2927  {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}};
2928  return makeArrayRef(TargetFlags);
2929 }
2930 
2931 // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
2932 // The VSX versions have the advantage of a full 64-register target whereas
2933 // the FP ones have the advantage of lower latency and higher throughput. So
2934 // what we are after is using the faster instructions in low register pressure
2935 // situations and using the larger register file in high register pressure
2936 // situations.
2938  unsigned UpperOpcode, LowerOpcode;
2939  switch (MI.getOpcode()) {
2940  case PPC::DFLOADf32:
2941  UpperOpcode = PPC::LXSSP;
2942  LowerOpcode = PPC::LFS;
2943  break;
2944  case PPC::DFLOADf64:
2945  UpperOpcode = PPC::LXSD;
2946  LowerOpcode = PPC::LFD;
2947  break;
2948  case PPC::DFSTOREf32:
2949  UpperOpcode = PPC::STXSSP;
2950  LowerOpcode = PPC::STFS;
2951  break;
2952  case PPC::DFSTOREf64:
2953  UpperOpcode = PPC::STXSD;
2954  LowerOpcode = PPC::STFD;
2955  break;
2956  case PPC::XFLOADf32:
2957  UpperOpcode = PPC::LXSSPX;
2958  LowerOpcode = PPC::LFSX;
2959  break;
2960  case PPC::XFLOADf64:
2961  UpperOpcode = PPC::LXSDX;
2962  LowerOpcode = PPC::LFDX;
2963  break;
2964  case PPC::XFSTOREf32:
2965  UpperOpcode = PPC::STXSSPX;
2966  LowerOpcode = PPC::STFSX;
2967  break;
2968  case PPC::XFSTOREf64:
2969  UpperOpcode = PPC::STXSDX;
2970  LowerOpcode = PPC::STFDX;
2971  break;
2972  case PPC::LIWAX:
2973  UpperOpcode = PPC::LXSIWAX;
2974  LowerOpcode = PPC::LFIWAX;
2975  break;
2976  case PPC::LIWZX:
2977  UpperOpcode = PPC::LXSIWZX;
2978  LowerOpcode = PPC::LFIWZX;
2979  break;
2980  case PPC::STIWX:
2981  UpperOpcode = PPC::STXSIWX;
2982  LowerOpcode = PPC::STFIWX;
2983  break;
2984  default:
2985  llvm_unreachable("Unknown Operation!");
2986  }
2987 
2988  Register TargetReg = MI.getOperand(0).getReg();
2989  unsigned Opcode;
2990  if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
2991  (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
2992  Opcode = LowerOpcode;
2993  else
2994  Opcode = UpperOpcode;
2995  MI.setDesc(get(Opcode));
2996  return true;
2997 }
2998 
2999 static bool isAnImmediateOperand(const MachineOperand &MO) {
3000  return MO.isCPI() || MO.isGlobal() || MO.isImm();
3001 }
3002 
3004  auto &MBB = *MI.getParent();
3005  auto DL = MI.getDebugLoc();
3006 
3007  switch (MI.getOpcode()) {
3008  case PPC::BUILD_UACC: {
3009  MCRegister ACC = MI.getOperand(0).getReg();
3010  MCRegister UACC = MI.getOperand(1).getReg();
3011  if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3012  MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3013  MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3014  // FIXME: This can easily be improved to look up to the top of the MBB
3015  // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3016  // we can just re-target any such XXLOR's to DstVSR + offset.
3017  for (int VecNo = 0; VecNo < 4; VecNo++)
3018  BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3019  .addReg(SrcVSR + VecNo)
3020  .addReg(SrcVSR + VecNo);
3021  }
3022  // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3023  // So after building the 4 copies, we can replace the BUILD_UACC instruction
3024  // with a NOP.
3026  }
3027  case PPC::KILL_PAIR: {
3028  MI.setDesc(get(PPC::UNENCODED_NOP));
3029  MI.removeOperand(1);
3030  MI.removeOperand(0);
3031  return true;
3032  }
3033  case TargetOpcode::LOAD_STACK_GUARD: {
3034  assert(Subtarget.isTargetLinux() &&
3035  "Only Linux target is expected to contain LOAD_STACK_GUARD");
3036  const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3037  const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3038  MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3039  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3040  .addImm(Offset)
3041  .addReg(Reg);
3042  return true;
3043  }
3044  case PPC::DFLOADf32:
3045  case PPC::DFLOADf64:
3046  case PPC::DFSTOREf32:
3047  case PPC::DFSTOREf64: {
3048  assert(Subtarget.hasP9Vector() &&
3049  "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3050  assert(MI.getOperand(2).isReg() &&
3051  isAnImmediateOperand(MI.getOperand(1)) &&
3052  "D-form op must have register and immediate operands");
3053  return expandVSXMemPseudo(MI);
3054  }
3055  case PPC::XFLOADf32:
3056  case PPC::XFSTOREf32:
3057  case PPC::LIWAX:
3058  case PPC::LIWZX:
3059  case PPC::STIWX: {
3060  assert(Subtarget.hasP8Vector() &&
3061  "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3062  assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3063  "X-form op must have register and register operands");
3064  return expandVSXMemPseudo(MI);
3065  }
3066  case PPC::XFLOADf64:
3067  case PPC::XFSTOREf64: {
3068  assert(Subtarget.hasVSX() &&
3069  "Invalid X-Form Pseudo-ops on target that has no VSX.");
3070  assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3071  "X-form op must have register and register operands");
3072  return expandVSXMemPseudo(MI);
3073  }
3074  case PPC::SPILLTOVSR_LD: {
3075  Register TargetReg = MI.getOperand(0).getReg();
3076  if (PPC::VSFRCRegClass.contains(TargetReg)) {
3077  MI.setDesc(get(PPC::DFLOADf64));
3078  return expandPostRAPseudo(MI);
3079  }
3080  else
3081  MI.setDesc(get(PPC::LD));
3082  return true;
3083  }
3084  case PPC::SPILLTOVSR_ST: {
3085  Register SrcReg = MI.getOperand(0).getReg();
3086  if (PPC::VSFRCRegClass.contains(SrcReg)) {
3087  NumStoreSPILLVSRRCAsVec++;
3088  MI.setDesc(get(PPC::DFSTOREf64));
3089  return expandPostRAPseudo(MI);
3090  } else {
3091  NumStoreSPILLVSRRCAsGpr++;
3092  MI.setDesc(get(PPC::STD));
3093  }
3094  return true;
3095  }
3096  case PPC::SPILLTOVSR_LDX: {
3097  Register TargetReg = MI.getOperand(0).getReg();
3098  if (PPC::VSFRCRegClass.contains(TargetReg))
3099  MI.setDesc(get(PPC::LXSDX));
3100  else
3101  MI.setDesc(get(PPC::LDX));
3102  return true;
3103  }
3104  case PPC::SPILLTOVSR_STX: {
3105  Register SrcReg = MI.getOperand(0).getReg();
3106  if (PPC::VSFRCRegClass.contains(SrcReg)) {
3107  NumStoreSPILLVSRRCAsVec++;
3108  MI.setDesc(get(PPC::STXSDX));
3109  } else {
3110  NumStoreSPILLVSRRCAsGpr++;
3111  MI.setDesc(get(PPC::STDX));
3112  }
3113  return true;
3114  }
3115 
3116  // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3117  case PPC::CFENCE8: {
3118  auto Val = MI.getOperand(0).getReg();
3119  BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
3120  BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3122  .addReg(PPC::CR7)
3123  .addImm(1);
3124  MI.setDesc(get(PPC::ISYNC));
3125  MI.removeOperand(0);
3126  return true;
3127  }
3128  }
3129  return false;
3130 }
3131 
3132 // Essentially a compile-time implementation of a compare->isel sequence.
3133 // It takes two constants to compare, along with the true/false registers
3134 // and the comparison type (as a subreg to a CR field) and returns one
3135 // of the true/false registers, depending on the comparison results.
3136 static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3137  unsigned TrueReg, unsigned FalseReg,
3138  unsigned CRSubReg) {
3139  // Signed comparisons. The immediates are assumed to be sign-extended.
3140  if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3141  switch (CRSubReg) {
3142  default: llvm_unreachable("Unknown integer comparison type.");
3143  case PPC::sub_lt:
3144  return Imm1 < Imm2 ? TrueReg : FalseReg;
3145  case PPC::sub_gt:
3146  return Imm1 > Imm2 ? TrueReg : FalseReg;
3147  case PPC::sub_eq:
3148  return Imm1 == Imm2 ? TrueReg : FalseReg;
3149  }
3150  }
3151  // Unsigned comparisons.
3152  else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3153  switch (CRSubReg) {
3154  default: llvm_unreachable("Unknown integer comparison type.");
3155  case PPC::sub_lt:
3156  return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3157  case PPC::sub_gt:
3158  return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3159  case PPC::sub_eq:
3160  return Imm1 == Imm2 ? TrueReg : FalseReg;
3161  }
3162  }
3163  return PPC::NoRegister;
3164 }
3165 
3167  unsigned OpNo,
3168  int64_t Imm) const {
3169  assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3170  // Replace the REG with the Immediate.
3171  Register InUseReg = MI.getOperand(OpNo).getReg();
3172  MI.getOperand(OpNo).ChangeToImmediate(Imm);
3173 
3174  // We need to make sure that the MI didn't have any implicit use
3175  // of this REG any more. We don't call MI.implicit_operands().empty() to
3176  // return early, since MI's MCID might be changed in calling context, as a
3177  // result its number of explicit operands may be changed, thus the begin of
3178  // implicit operand is changed.
3180  int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3181  if (UseOpIdx >= 0) {
3182  MachineOperand &MO = MI.getOperand(UseOpIdx);
3183  if (MO.isImplicit())
3184  // The operands must always be in the following order:
3185  // - explicit reg defs,
3186  // - other explicit operands (reg uses, immediates, etc.),
3187  // - implicit reg defs
3188  // - implicit reg uses
3189  // Therefore, removing the implicit operand won't change the explicit
3190  // operands layout.
3191  MI.removeOperand(UseOpIdx);
3192  }
3193 }
3194 
3195 // Replace an instruction with one that materializes a constant (and sets
3196 // CR0 if the original instruction was a record-form instruction).
3198  const LoadImmediateInfo &LII) const {
3199  // Remove existing operands.
3200  int OperandToKeep = LII.SetCR ? 1 : 0;
3201  for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3202  MI.removeOperand(i);
3203 
3204  // Replace the instruction.
3205  if (LII.SetCR) {
3206  MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3207  // Set the immediate.
3208  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3209  .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3210  return;
3211  }
3212  else
3213  MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3214 
3215  // Set the immediate.
3216  MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3217  .addImm(LII.Imm);
3218 }
3219 
3221  bool &SeenIntermediateUse) const {
3222  assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3223  "Should be called after register allocation.");
3225  MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3226  It++;
3227  SeenIntermediateUse = false;
3228  for (; It != E; ++It) {
3229  if (It->modifiesRegister(Reg, TRI))
3230  return &*It;
3231  if (It->readsRegister(Reg, TRI))
3232  SeenIntermediateUse = true;
3233  }
3234  return nullptr;
3235 }
3236 
3239  const DebugLoc &DL, Register Reg,
3240  int64_t Imm) const {
3241  assert(!MBB.getParent()->getRegInfo().isSSA() &&
3242  "Register should be in non-SSA form after RA");
3243  bool isPPC64 = Subtarget.isPPC64();
3244  // FIXME: Materialization here is not optimal.
3245  // For some special bit patterns we can use less instructions.
3246  // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3247  if (isInt<16>(Imm)) {
3248  BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
3249  } else if (isInt<32>(Imm)) {
3250  BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
3251  .addImm(Imm >> 16);
3252  if (Imm & 0xFFFF)
3253  BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
3255  .addImm(Imm & 0xFFFF);
3256  } else {
3257  assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3258  "only supported in PPC64");
3259  BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
3260  if ((Imm >> 32) & 0xFFFF)
3261  BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3263  .addImm((Imm >> 32) & 0xFFFF);
3264  BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
3266  .addImm(32)
3267  .addImm(31);
3268  BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
3270  .addImm((Imm >> 16) & 0xFFFF);
3271  if (Imm & 0xFFFF)
3272  BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3274  .addImm(Imm & 0xFFFF);
3275  }
3276 }
3277 
3278 MachineInstr *PPCInstrInfo::getForwardingDefMI(
3279  MachineInstr &MI,
3280  unsigned &OpNoForForwarding,
3281  bool &SeenIntermediateUse) const {
3282  OpNoForForwarding = ~0U;
3283  MachineInstr *DefMI = nullptr;
3284  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3286  // If we're in SSA, get the defs through the MRI. Otherwise, only look
3287  // within the basic block to see if the register is defined using an
3288  // LI/LI8/ADDI/ADDI8.
3289  if (MRI->isSSA()) {
3290  for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3291  if (!MI.getOperand(i).isReg())
3292  continue;
3293  Register Reg = MI.getOperand(i).getReg();
3295  continue;
3296  Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3297  if (Register::isVirtualRegister(TrueReg)) {
3298  DefMI = MRI->getVRegDef(TrueReg);
3299  if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8 ||
3300  DefMI->getOpcode() == PPC::ADDI ||
3301  DefMI->getOpcode() == PPC::ADDI8) {
3302  OpNoForForwarding = i;
3303  // The ADDI and LI operand maybe exist in one instruction at same
3304  // time. we prefer to fold LI operand as LI only has one Imm operand
3305  // and is more possible to be converted. So if current DefMI is
3306  // ADDI/ADDI8, we continue to find possible LI/LI8.
3307  if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3308  break;
3309  }
3310  }
3311  }
3312  } else {
3313  // Looking back through the definition for each operand could be expensive,
3314  // so exit early if this isn't an instruction that either has an immediate
3315  // form or is already an immediate form that we can handle.
3316  ImmInstrInfo III;
3317  unsigned Opc = MI.getOpcode();
3318  bool ConvertibleImmForm =
3319  Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3320  Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3321  Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3322  Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3323  Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3324  Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3325  Opc == PPC::RLWINM8_rec;
3326  bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3327  ? isVFRegister(MI.getOperand(0).getReg())
3328  : false;
3329  if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3330  return nullptr;
3331 
3332  // Don't convert or %X, %Y, %Y since that's just a register move.
3333  if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3334  MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3335  return nullptr;
3336  for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3337  MachineOperand &MO = MI.getOperand(i);
3338  SeenIntermediateUse = false;
3339  if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3340  Register Reg = MI.getOperand(i).getReg();
3341  // If we see another use of this reg between the def and the MI,
3342  // we want to flat it so the def isn't deleted.
3343  MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3344  if (DefMI) {
3345  // Is this register defined by some form of add-immediate (including
3346  // load-immediate) within this basic block?
3347  switch (DefMI->getOpcode()) {
3348  default:
3349  break;
3350  case PPC::LI:
3351  case PPC::LI8:
3352  case PPC::ADDItocL:
3353  case PPC::ADDI:
3354  case PPC::ADDI8:
3355  OpNoForForwarding = i;
3356  return DefMI;
3357  }
3358  }
3359  }
3360  }
3361  }
3362  return OpNoForForwarding == ~0U ? nullptr : DefMI;
3363 }
3364 
3365 unsigned PPCInstrInfo::getSpillTarget() const {
3366  // With P10, we may need to spill paired vector registers or accumulator
3367  // registers. MMA implies paired vectors, so we can just check that.
3368  bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3369  return IsP10Variant ? 2 : Subtarget.hasP9Vector() ? 1 : 0;
3370 }
3371 
3372 const unsigned *PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3373  return StoreSpillOpcodesArray[getSpillTarget()];
3374 }
3375 
3376 const unsigned *PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3377  return LoadSpillOpcodesArray[getSpillTarget()];
3378 }
3379 
3381  unsigned RegNo) const {
3382  // Conservatively clear kill flag for the register if the instructions are in
3383  // different basic blocks and in SSA form, because the kill flag may no longer
3384  // be right. There is no need to bother with dead flags since defs with no
3385  // uses will be handled by DCE.
3387  if (MRI.isSSA() && (StartMI->getParent() != EndMI->getParent())) {
3388  MRI.clearKillFlags(RegNo);
3389  return;
3390  }
3391 
3392  // Instructions between [StartMI, EndMI] should be in same basic block.
3393  assert((StartMI->getParent() == EndMI->getParent()) &&
3394  "Instructions are not in same basic block");
3395 
3396  // If before RA, StartMI may be def through COPY, we need to adjust it to the
3397  // real def. See function getForwardingDefMI.
3398  if (MRI.isSSA()) {
3399  bool Reads, Writes;
3400  std::tie(Reads, Writes) = StartMI->readsWritesVirtualRegister(RegNo);
3401  if (!Reads && !Writes) {
3403  "Must be a virtual register");
3404  // Get real def and ignore copies.
3405  StartMI = MRI.getVRegDef(RegNo);
3406  }
3407  }
3408 
3409  bool IsKillSet = false;
3410 
3411  auto clearOperandKillInfo = [=] (MachineInstr &MI, unsigned Index) {
3412  MachineOperand &MO = MI.getOperand(Index);
3413  if (MO.isReg() && MO.isUse() && MO.isKill() &&
3414  getRegisterInfo().regsOverlap(MO.getReg(), RegNo))
3415  MO.setIsKill(false);
3416  };
3417 
3418  // Set killed flag for EndMI.
3419  // No need to do anything if EndMI defines RegNo.
3420  int UseIndex =
3421  EndMI->findRegisterUseOperandIdx(RegNo, false, &getRegisterInfo());
3422  if (UseIndex != -1) {
3423  EndMI->getOperand(UseIndex).setIsKill(true);
3424  IsKillSet = true;
3425  // Clear killed flag for other EndMI operands related to RegNo. In some
3426  // upexpected cases, killed may be set multiple times for same register
3427  // operand in same MI.
3428  for (int i = 0, e = EndMI->getNumOperands(); i != e; ++i)
3429  if (i != UseIndex)
3430  clearOperandKillInfo(*EndMI, i);
3431  }
3432 
3433  // Walking the inst in reverse order (EndMI -> StartMI].
3436  // EndMI has been handled above, skip it here.
3437  It++;
3438  MachineOperand *MO = nullptr;
3439  for (; It != E; ++It) {
3440  // Skip insturctions which could not be a def/use of RegNo.
3441  if (It->isDebugInstr() || It->isPosition())
3442  continue;
3443 
3444  // Clear killed flag for all It operands related to RegNo. In some
3445  // upexpected cases, killed may be set multiple times for same register
3446  // operand in same MI.
3447  for (int i = 0, e = It->getNumOperands(); i != e; ++i)
3448  clearOperandKillInfo(*It, i);
3449 
3450  // If killed is not set, set killed for its last use or set dead for its def
3451  // if no use found.
3452  if (!IsKillSet) {
3453  if ((MO = It->findRegisterUseOperand(RegNo, false, &getRegisterInfo()))) {
3454  // Use found, set it killed.
3455  IsKillSet = true;
3456  MO->setIsKill(true);
3457  continue;
3458  } else if ((MO = It->findRegisterDefOperand(RegNo, false, true,
3459  &getRegisterInfo()))) {
3460  // No use found, set dead for its def.
3461  assert(&*It == StartMI && "No new def between StartMI and EndMI.");
3462  MO->setIsDead(true);
3463  break;
3464  }
3465  }
3466 
3467  if ((&*It) == StartMI)
3468  break;
3469  }
3470  // Ensure RegMo liveness is killed after EndMI.
3471  assert((IsKillSet || (MO && MO->isDead())) &&
3472  "RegNo should be killed or dead");
3473 }
3474 
3475 // This opt tries to convert the following imm form to an index form to save an
3476 // add for stack variables.
3477 // Return false if no such pattern found.
3478 //
3479 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3480 // ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3481 // Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3482 //
3483 // can be converted to:
3484 //
3485 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3486 // Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3487 //
3488 // In order to eliminate ADD instr, make sure that:
3489 // 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3490 // new ADDI instr and ADDI can only take int16 Imm.
3491 // 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3492 // between ADDI and ADD instr since its original def in ADDI will be changed
3493 // in new ADDI instr. And also there should be no new def for it between
3494 // ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3495 // 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3496 // between ADD and Imm instr since ADD instr will be eliminated.
3497 // 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3498 // moved to Index instr.
3500  MachineFunction *MF = MI.getParent()->getParent();
3502  bool PostRA = !MRI->isSSA();
3503  // Do this opt after PEI which is after RA. The reason is stack slot expansion
3504  // in PEI may expose such opportunities since in PEI, stack slot offsets to
3505  // frame base(OffsetAddi) are determined.
3506  if (!PostRA)
3507  return false;
3508  unsigned ToBeDeletedReg = 0;
3509  int64_t OffsetImm = 0;
3510  unsigned XFormOpcode = 0;
3511  ImmInstrInfo III;
3512 
3513  // Check if Imm instr meets requirement.
3514  if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3515  III))
3516  return false;
3517 
3518  bool OtherIntermediateUse = false;
3519  MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3520 
3521  // Exit if there is other use between ADD and Imm instr or no def found.
3522  if (OtherIntermediateUse || !ADDMI)
3523  return false;
3524 
3525  // Check if ADD instr meets requirement.
3526  if (!isADDInstrEligibleForFolding(*ADDMI))
3527  return false;
3528 
3529  unsigned ScaleRegIdx = 0;
3530  int64_t OffsetAddi = 0;
3531  MachineInstr *ADDIMI = nullptr;
3532 
3533  // Check if there is a valid ToBeChangedReg in ADDMI.
3534  // 1: It must be killed.
3535  // 2: Its definition must be a valid ADDIMI.
3536  // 3: It must satify int16 offset requirement.
3537  if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3538  ScaleRegIdx = 2;
3539  else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3540  ScaleRegIdx = 1;
3541  else
3542  return false;
3543 
3544  assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3545  Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3546  Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3547  auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3549  for (auto It = ++Start; It != End; It++)
3550  if (It->modifiesRegister(Reg, &getRegisterInfo()))
3551  return true;
3552  return false;
3553  };
3554 
3555  // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3556  // treated as special zero when ScaleReg is R0/X0 register.
3557  if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3558  (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3559  return false;
3560 
3561  // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3562  // and Imm Instr.
3563  if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3564  return false;
3565 
3566  // Now start to do the transformation.
3567  LLVM_DEBUG(dbgs() << "Replace instruction: "
3568  << "\n");
3569  LLVM_DEBUG(ADDIMI->dump());
3570  LLVM_DEBUG(ADDMI->dump());
3571  LLVM_DEBUG(MI.dump());
3572  LLVM_DEBUG(dbgs() << "with: "
3573  << "\n");
3574 
3575  // Update ADDI instr.
3576  ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3577 
3578  // Update Imm instr.
3579  MI.setDesc(get(XFormOpcode));
3580  MI.getOperand(III.ImmOpNo)
3581  .ChangeToRegister(ScaleReg, false, false,
3582  ADDMI->getOperand(ScaleRegIdx).isKill());
3583 
3584  MI.getOperand(III.OpNoForForwarding)
3585  .ChangeToRegister(ToBeChangedReg, false, false, true);
3586 
3587  // Eliminate ADD instr.
3588  ADDMI->eraseFromParent();
3589 
3590  LLVM_DEBUG(ADDIMI->dump());
3591  LLVM_DEBUG(MI.dump());
3592 
3593  return true;
3594 }
3595 
3597  int64_t &Imm) const {
3598  unsigned Opc = ADDIMI.getOpcode();
3599 
3600  // Exit if the instruction is not ADDI.
3601  if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3602  return false;
3603 
3604  // The operand may not necessarily be an immediate - it could be a relocation.
3605  if (!ADDIMI.getOperand(2).isImm())
3606  return false;
3607 
3608  Imm = ADDIMI.getOperand(2).getImm();
3609 
3610  return true;
3611 }
3612 
3614  unsigned Opc = ADDMI.getOpcode();
3615 
3616  // Exit if the instruction is not ADD.
3617  return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3618 }
3619 
3621  unsigned &ToBeDeletedReg,
3622  unsigned &XFormOpcode,
3623  int64_t &OffsetImm,
3624  ImmInstrInfo &III) const {
3625  // Only handle load/store.
3626  if (!MI.mayLoadOrStore())
3627  return false;
3628 
3629  unsigned Opc = MI.getOpcode();
3630 
3631  XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3632 
3633  // Exit if instruction has no index form.
3634  if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3635  return false;
3636 
3637  // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3638  if (!instrHasImmForm(XFormOpcode, isVFRegister(MI.getOperand(0).getReg()),
3639  III, true))
3640  return false;
3641 
3642  if (!III.IsSummingOperands)
3643  return false;
3644 
3645  MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3646  MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3647  // Only support imm operands, not relocation slots or others.
3648  if (!ImmOperand.isImm())
3649  return false;
3650 
3651  assert(RegOperand.isReg() && "Instruction format is not right");
3652 
3653  // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3654  if (!RegOperand.isKill())
3655  return false;
3656 
3657  ToBeDeletedReg = RegOperand.getReg();
3658  OffsetImm = ImmOperand.getImm();
3659 
3660  return true;
3661 }
3662 
3664  MachineInstr *&ADDIMI,
3665  int64_t &OffsetAddi,
3666  int64_t OffsetImm) const {
3667  assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3668  MachineOperand &MO = ADDMI->getOperand(Index);
3669 
3670  if (!MO.isKill())
3671  return false;
3672 
3673  bool OtherIntermediateUse = false;
3674 
3675  ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3676  // Currently handle only one "add + Imminstr" pair case, exit if other
3677  // intermediate use for ToBeChangedReg found.
3678  // TODO: handle the cases where there are other "add + Imminstr" pairs
3679  // with same offset in Imminstr which is like:
3680  //
3681  // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3682  // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3683  // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3684  // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3685  // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3686  //
3687  // can be converted to:
3688  //
3689  // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3690  // (OffsetAddi + OffsetImm)
3691  // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3692  // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3693 
3694  if (OtherIntermediateUse || !ADDIMI)
3695  return false;
3696  // Check if ADDI instr meets requirement.
3697  if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3698  return false;
3699 
3700  if (isInt<16>(OffsetAddi + OffsetImm))
3701  return true;
3702  return false;
3703 }
3704 
3705 // If this instruction has an immediate form and one of its operands is a
3706 // result of a load-immediate or an add-immediate, convert it to
3707 // the immediate form if the constant is in range.
3709  MachineInstr **KilledDef) const {
3710  MachineFunction *MF = MI.getParent()->getParent();
3712  bool PostRA = !MRI->isSSA();
3713  bool SeenIntermediateUse = true;
3714  unsigned ForwardingOperand = ~0U;
3715  MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3716  SeenIntermediateUse);
3717  if (!DefMI)
3718  return false;
3719  assert(ForwardingOperand < MI.getNumOperands() &&
3720  "The forwarding operand needs to be valid at this point");
3721  bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3722  bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3723  if (KilledDef && KillFwdDefMI)
3724  *KilledDef = DefMI;
3725 
3726  // If this is a imm instruction and its register operands is produced by ADDI,
3727  // put the imm into imm inst directly.
3728  if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3729  PPC::INSTRUCTION_LIST_END &&
3730  transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3731  return true;
3732 
3733  ImmInstrInfo III;
3734  bool IsVFReg = MI.getOperand(0).isReg()
3735  ? isVFRegister(MI.getOperand(0).getReg())
3736  : false;
3737  bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3738  // If this is a reg+reg instruction that has a reg+imm form,
3739  // and one of the operands is produced by an add-immediate,
3740  // try to convert it.
3741  if (HasImmForm &&
3742  transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3743  KillFwdDefMI))
3744  return true;
3745 
3746  // If this is a reg+reg instruction that has a reg+imm form,
3747  // and one of the operands is produced by LI, convert it now.
3748  if (HasImmForm &&
3749  transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3750  return true;
3751 
3752  // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3753  // can be simpified to LI.
3754  if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3755  return true;
3756 
3757  return false;
3758 }
3759 
3761  MachineInstr **ToErase) const {
3762  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3763  Register FoldingReg = MI.getOperand(1).getReg();
3764  if (!Register::isVirtualRegister(FoldingReg))
3765  return false;
3766  MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3767  if (SrcMI->getOpcode() != PPC::RLWINM &&
3768  SrcMI->getOpcode() != PPC::RLWINM_rec &&
3769  SrcMI->getOpcode() != PPC::RLWINM8 &&
3770  SrcMI->getOpcode() != PPC::RLWINM8_rec)
3771  return false;
3772  assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3773  MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3774  SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3775  "Invalid PPC::RLWINM Instruction!");
3776  uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3777  uint64_t SHMI = MI.getOperand(2).getImm();
3778  uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3779  uint64_t MBMI = MI.getOperand(3).getImm();
3780  uint64_t MESrc = SrcMI->getOperand(4).getImm();
3781  uint64_t MEMI = MI.getOperand(4).getImm();
3782 
3783  assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3784  "Invalid PPC::RLWINM Instruction!");
3785  // If MBMI is bigger than MEMI, we always can not get run of ones.
3786  // RotatedSrcMask non-wrap:
3787  // 0........31|32........63
3788  // RotatedSrcMask: B---E B---E
3789  // MaskMI: -----------|--E B------
3790  // Result: ----- --- (Bad candidate)
3791  //
3792  // RotatedSrcMask wrap:
3793  // 0........31|32........63
3794  // RotatedSrcMask: --E B----|--E B----
3795  // MaskMI: -----------|--E B------
3796  // Result: --- -----|--- ----- (Bad candidate)
3797  //
3798  // One special case is RotatedSrcMask is a full set mask.
3799  // RotatedSrcMask full:
3800  // 0........31|32........63
3801  // RotatedSrcMask: ------EB---|-------EB---
3802  // MaskMI: -----------|--E B------
3803  // Result: -----------|--- ------- (Good candidate)
3804 
3805  // Mark special case.
3806  bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3807 
3808  // For other MBMI > MEMI cases, just return.
3809  if ((MBMI > MEMI) && !SrcMaskFull)
3810  return false;
3811 
3812  // Handle MBMI <= MEMI cases.
3813  APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3814  // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3815  // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3816  // while in PowerPC ISA, lowerest bit is at index 63.
3817  APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3818 
3819  APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3820  APInt FinalMask = RotatedSrcMask & MaskMI;
3821  uint32_t NewMB, NewME;
3822  bool Simplified = false;
3823 
3824  // If final mask is 0, MI result should be 0 too.
3825  if (FinalMask.isZero()) {
3826  bool Is64Bit =
3827  (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3828  Simplified = true;
3829  LLVM_DEBUG(dbgs() << "Replace Instr: ");
3830  LLVM_DEBUG(MI.dump());
3831 
3832  if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3833  // Replace MI with "LI 0"
3834  MI.removeOperand(4);
3835  MI.removeOperand(3);
3836  MI.removeOperand(2);
3837  MI.getOperand(1).ChangeToImmediate(0);
3838  MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3839  } else {
3840  // Replace MI with "ANDI_rec reg, 0"
3841  MI.removeOperand(4);
3842  MI.removeOperand(3);
3843  MI.getOperand(2).setImm(0);
3844  MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3845  MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3846  if (SrcMI->getOperand(1).isKill()) {
3847  MI.getOperand(1).setIsKill(true);
3848  SrcMI->getOperand(1).setIsKill(false);
3849  } else
3850  // About to replace MI.getOperand(1), clear its kill flag.
3851  MI.getOperand(1).setIsKill(false);
3852  }
3853 
3854  LLVM_DEBUG(dbgs() << "With: ");
3855  LLVM_DEBUG(MI.dump());
3856 
3857  } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3858  NewMB <= NewME) ||
3859  SrcMaskFull) {
3860  // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3861  // than NewME. Otherwise we get a 64 bit value after folding, but MI
3862  // return a 32 bit value.
3863  Simplified = true;
3864  LLVM_DEBUG(dbgs() << "Converting Instr: ");
3865  LLVM_DEBUG(MI.dump());
3866 
3867  uint16_t NewSH = (SHSrc + SHMI) % 32;
3868  MI.getOperand(2).setImm(NewSH);
3869  // If SrcMI mask is full, no need to update MBMI and MEMI.
3870  if (!SrcMaskFull) {
3871  MI.getOperand(3).setImm(NewMB);
3872  MI.getOperand(4).setImm(NewME);
3873  }
3874  MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3875  if (SrcMI->getOperand(1).isKill()) {
3876  MI.getOperand(1).setIsKill(true);
3877  SrcMI->getOperand(1).setIsKill(false);
3878  } else
3879  // About to replace MI.getOperand(1), clear its kill flag.
3880  MI.getOperand(1).setIsKill(false);
3881 
3882  LLVM_DEBUG(dbgs() << "To: ");
3883  LLVM_DEBUG(MI.dump());
3884  }
3885  if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3886  !SrcMI->hasImplicitDef()) {
3887  // If FoldingReg has no non-debug use and it has no implicit def (it
3888  // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3889  // Otherwise keep it.
3890  *ToErase = SrcMI;
3891  LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3892  LLVM_DEBUG(SrcMI->dump());
3893  }
3894  return Simplified;
3895 }
3896 
3897 bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3898  ImmInstrInfo &III, bool PostRA) const {
3899  // The vast majority of the instructions would need their operand 2 replaced
3900  // with an immediate when switching to the reg+imm form. A marked exception
3901  // are the update form loads/stores for which a constant operand 2 would need
3902  // to turn into a displacement and move operand 1 to the operand 2 position.
3903  III.ImmOpNo = 2;
3904  III.OpNoForForwarding = 2;
3905  III.ImmWidth = 16;
3906  III.ImmMustBeMultipleOf = 1;
3907  III.TruncateImmTo = 0;
3908  III.IsSummingOperands = false;
3909  switch (Opc) {
3910  default: return false;
3911  case PPC::ADD4:
3912  case PPC::ADD8:
3913  III.SignedImm = true;
3914  III.ZeroIsSpecialOrig = 0;
3915  III.ZeroIsSpecialNew = 1;
3916  III.IsCommutative = true;
3917  III.IsSummingOperands = true;
3918  III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3919  break;
3920  case PPC::ADDC:
3921  case PPC::ADDC8:
3922  III.SignedImm = true;
3923  III.ZeroIsSpecialOrig = 0;
3924  III.ZeroIsSpecialNew = 0;
3925  III.IsCommutative = true;
3926  III.IsSummingOperands = true;
3927  III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3928  break;
3929  case PPC::ADDC_rec:
3930  III.SignedImm = true;
3931  III.ZeroIsSpecialOrig = 0;
3932  III.ZeroIsSpecialNew = 0;
3933  III.IsCommutative = true;
3934  III.IsSummingOperands = true;
3935  III.ImmOpcode = PPC::ADDIC_rec;
3936  break;
3937  case PPC::SUBFC:
3938  case PPC::SUBFC8:
3939  III.SignedImm = true;
3940  III.ZeroIsSpecialOrig = 0;
3941  III.ZeroIsSpecialNew = 0;
3942  III.IsCommutative = false;
3943  III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3944  break;
3945  case PPC::CMPW:
3946  case PPC::CMPD:
3947  III.SignedImm = true;
3948  III.ZeroIsSpecialOrig = 0;
3949  III.ZeroIsSpecialNew = 0;
3950  III.IsCommutative = false;
3951  III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3952  break;
3953  case PPC::CMPLW:
3954  case PPC::CMPLD:
3955  III.SignedImm = false;
3956  III.ZeroIsSpecialOrig = 0;
3957  III.ZeroIsSpecialNew = 0;
3958  III.IsCommutative = false;
3959  III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3960  break;
3961  case PPC::AND_rec:
3962  case PPC::AND8_rec:
3963  case PPC::OR:
3964  case PPC::OR8:
3965  case PPC::XOR:
3966  case PPC::XOR8:
3967  III.SignedImm = false;
3968  III.ZeroIsSpecialOrig = 0;
3969  III.ZeroIsSpecialNew = 0;
3970  III.IsCommutative = true;
3971  switch(Opc) {
3972  default: llvm_unreachable("Unknown opcode");
3973  case PPC::AND_rec:
3974  III.ImmOpcode = PPC::ANDI_rec;
3975  break;
3976  case PPC::AND8_rec:
3977  III.ImmOpcode = PPC::ANDI8_rec;
3978  break;
3979  case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3980  case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3981  case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3982  case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3983  }
3984  break;
3985  case PPC::RLWNM:
3986  case PPC::RLWNM8:
3987  case PPC::RLWNM_rec:
3988  case PPC::RLWNM8_rec:
3989  case PPC::SLW:
3990  case PPC::SLW8:
3991  case PPC::SLW_rec:
3992  case PPC::SLW8_rec:
3993  case PPC::SRW:
3994  case PPC::SRW8:
3995  case PPC::SRW_rec:
3996  case PPC::SRW8_rec:
3997  case PPC::SRAW:
3998  case PPC::SRAW_rec:
3999  III.SignedImm = false;
4000  III.ZeroIsSpecialOrig = 0;
4001  III.ZeroIsSpecialNew = 0;
4002  III.IsCommutative = false;
4003  // This isn't actually true, but the instructions ignore any of the
4004  // upper bits, so any immediate loaded with an LI is acceptable.
4005  // This does not apply to shift right algebraic because a value
4006  // out of range will produce a -1/0.
4007  III.ImmWidth = 16;
4008  if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
4009  Opc == PPC::RLWNM8_rec)
4010  III.TruncateImmTo = 5;
4011  else
4012  III.TruncateImmTo = 6;
4013  switch(Opc) {
4014  default: llvm_unreachable("Unknown opcode");
4015  case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
4016  case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
4017  case PPC::RLWNM_rec:
4018  III.ImmOpcode = PPC::RLWINM_rec;
4019  break;
4020  case PPC::RLWNM8_rec:
4021  III.ImmOpcode = PPC::RLWINM8_rec;
4022  break;
4023  case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4024  case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4025  case PPC::SLW_rec:
4026  III.ImmOpcode = PPC::RLWINM_rec;
4027  break;
4028  case PPC::SLW8_rec:
4029  III.ImmOpcode = PPC::RLWINM8_rec;
4030  break;
4031  case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4032  case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4033  case PPC::SRW_rec:
4034  III.ImmOpcode = PPC::RLWINM_rec;
4035  break;
4036  case PPC::SRW8_rec:
4037  III.ImmOpcode = PPC::RLWINM8_rec;
4038  break;
4039  case PPC::SRAW:
4040  III.ImmWidth = 5;
4041  III.TruncateImmTo = 0;
4042  III.ImmOpcode = PPC::SRAWI;
4043  break;
4044  case PPC::SRAW_rec:
4045  III.ImmWidth = 5;
4046  III.TruncateImmTo = 0;
4047  III.ImmOpcode = PPC::SRAWI_rec;
4048  break;
4049  }
4050  break;
4051  case PPC::RLDCL:
4052  case PPC::RLDCL_rec:
4053  case PPC::RLDCR:
4054  case PPC::RLDCR_rec:
4055  case PPC::SLD:
4056  case PPC::SLD_rec:
4057  case PPC::SRD:
4058  case PPC::SRD_rec:
4059  case PPC::SRAD:
4060  case PPC::SRAD_rec:
4061  III.SignedImm = false;
4062  III.ZeroIsSpecialOrig = 0;
4063  III.ZeroIsSpecialNew = 0;
4064  III.IsCommutative = false;
4065  // This isn't actually true, but the instructions ignore any of the
4066  // upper bits, so any immediate loaded with an LI is acceptable.
4067  // This does not apply to shift right algebraic because a value
4068  // out of range will produce a -1/0.
4069  III.ImmWidth = 16;
4070  if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4071  Opc == PPC::RLDCR_rec)
4072  III.TruncateImmTo = 6;
4073  else
4074  III.TruncateImmTo = 7;
4075  switch(Opc) {
4076  default: llvm_unreachable("Unknown opcode");
4077  case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4078  case PPC::RLDCL_rec:
4079  III.ImmOpcode = PPC::RLDICL_rec;
4080  break;
4081  case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4082  case PPC::RLDCR_rec:
4083  III.ImmOpcode = PPC::RLDICR_rec;
4084  break;
4085  case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4086  case PPC::SLD_rec:
4087  III.ImmOpcode = PPC::RLDICR_rec;
4088  break;
4089  case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4090  case PPC::SRD_rec:
4091  III.ImmOpcode = PPC::RLDICL_rec;
4092  break;
4093  case PPC::SRAD:
4094  III.ImmWidth = 6;
4095  III.TruncateImmTo = 0;
4096  III.ImmOpcode = PPC::SRADI;
4097  break;
4098  case PPC::SRAD_rec:
4099  III.ImmWidth = 6;
4100  III.TruncateImmTo = 0;
4101  III.ImmOpcode = PPC::SRADI_rec;
4102  break;
4103  }
4104  break;
4105  // Loads and stores:
4106  case PPC::LBZX:
4107  case PPC::LBZX8:
4108  case PPC::LHZX:
4109  case PPC::LHZX8:
4110  case PPC::LHAX:
4111  case PPC::LHAX8:
4112  case PPC::LWZX:
4113  case PPC::LWZX8:
4114  case PPC::LWAX:
4115  case PPC::LDX:
4116  case PPC::LFSX:
4117  case PPC::LFDX:
4118  case PPC::STBX:
4119  case PPC::STBX8:
4120  case PPC::STHX:
4121  case PPC::STHX8:
4122  case PPC::STWX:
4123  case PPC::STWX8:
4124  case PPC::STDX:
4125  case PPC::STFSX:
4126  case PPC::STFDX:
4127  III.SignedImm = true;
4128  III.ZeroIsSpecialOrig = 1;
4129  III.ZeroIsSpecialNew = 2;
4130  III.IsCommutative = true;
4131  III.IsSummingOperands = true;
4132  III.ImmOpNo = 1;
4133  III.OpNoForForwarding = 2;
4134  switch(Opc) {
4135  default: llvm_unreachable("Unknown opcode");
4136  case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4137  case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4138  case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4139  case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4140  case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4141  case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4142  case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4143  case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4144  case PPC::LWAX:
4145  III.ImmOpcode = PPC::LWA;
4146  III.ImmMustBeMultipleOf = 4;
4147  break;
4148  case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4149  case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4150  case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4151  case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4152  case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4153  case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4154  case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4155  case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4156  case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4157  case PPC::STDX:
4158  III.ImmOpcode = PPC::STD;
4159  III.ImmMustBeMultipleOf = 4;
4160  break;
4161  case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4162  case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4163  }
4164  break;
4165  case PPC::LBZUX:
4166  case PPC::LBZUX8:
4167  case PPC::LHZUX:
4168  case PPC::LHZUX8:
4169  case PPC::LHAUX:
4170  case PPC::LHAUX8:
4171  case PPC::LWZUX:
4172  case PPC::LWZUX8:
4173  case PPC::LDUX:
4174  case PPC::LFSUX:
4175  case PPC::LFDUX:
4176  case PPC::STBUX:
4177  case PPC::STBUX8:
4178  case PPC::STHUX:
4179  case PPC::STHUX8:
4180  case PPC::STWUX:
4181  case PPC::STWUX8:
4182  case PPC::STDUX:
4183  case PPC::STFSUX:
4184  case PPC::STFDUX:
4185  III.SignedImm = true;
4186  III.ZeroIsSpecialOrig = 2;
4187  III.ZeroIsSpecialNew = 3;
4188  III.IsCommutative = false;
4189  III.IsSummingOperands = true;
4190  III.ImmOpNo = 2;
4191  III.OpNoForForwarding = 3;
4192  switch(Opc) {
4193  default: llvm_unreachable("Unknown opcode");
4194  case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4195  case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4196  case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4197  case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4198  case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4199  case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4200  case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4201  case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4202  case PPC::LDUX:
4203  III.ImmOpcode = PPC::LDU;
4204  III.ImmMustBeMultipleOf = 4;
4205  break;
4206  case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4207  case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4208  case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4209  case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4210  case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4211  case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4212  case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4213  case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4214  case PPC::STDUX:
4215  III.ImmOpcode = PPC::STDU;
4216  III.ImmMustBeMultipleOf = 4;
4217  break;
4218  case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4219  case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4220  }
4221  break;
4222  // Power9 and up only. For some of these, the X-Form version has access to all
4223  // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4224  // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4225  // into or stored from is one of the VR registers.
4226  case PPC::LXVX:
4227  case PPC::LXSSPX:
4228  case PPC::LXSDX:
4229  case PPC::STXVX:
4230  case PPC::STXSSPX:
4231  case PPC::STXSDX:
4232  case PPC::XFLOADf32:
4233  case PPC::XFLOADf64:
4234  case PPC::XFSTOREf32:
4235  case PPC::XFSTOREf64:
4236  if (!Subtarget.hasP9Vector())
4237  return false;
4238  III.SignedImm = true;
4239  III.ZeroIsSpecialOrig = 1;
4240  III.ZeroIsSpecialNew = 2;
4241  III.IsCommutative = true;
4242  III.IsSummingOperands = true;
4243  III.ImmOpNo = 1;
4244  III.OpNoForForwarding = 2;
4245  III.ImmMustBeMultipleOf = 4;
4246  switch(Opc) {
4247  default: llvm_unreachable("Unknown opcode");
4248  case PPC::LXVX:
4249  III.ImmOpcode = PPC::LXV;
4250  III.ImmMustBeMultipleOf = 16;
4251  break;
4252  case PPC::LXSSPX:
4253  if (PostRA) {
4254  if (IsVFReg)
4255  III.ImmOpcode = PPC::LXSSP;
4256  else {
4257  III.ImmOpcode = PPC::LFS;
4258  III.ImmMustBeMultipleOf = 1;
4259  }
4260  break;
4261  }
4263  case PPC::XFLOADf32:
4264  III.ImmOpcode = PPC::DFLOADf32;
4265  break;
4266  case PPC::LXSDX:
4267  if (PostRA) {
4268  if (IsVFReg)
4269  III.ImmOpcode = PPC::LXSD;
4270  else {
4271  III.ImmOpcode = PPC::LFD;
4272  III.ImmMustBeMultipleOf = 1;
4273  }
4274  break;
4275  }
4277  case PPC::XFLOADf64:
4278  III.ImmOpcode = PPC::DFLOADf64;
4279  break;
4280  case PPC::STXVX:
4281  III.ImmOpcode = PPC::STXV;
4282  III.ImmMustBeMultipleOf = 16;
4283  break;
4284  case PPC::STXSSPX:
4285  if (PostRA) {
4286  if (IsVFReg)
4287  III.ImmOpcode = PPC::STXSSP;
4288  else {
4289  III.ImmOpcode = PPC::STFS;
4290  III.ImmMustBeMultipleOf = 1;
4291  }
4292  break;
4293  }
4295  case PPC::XFSTOREf32:
4296  III.ImmOpcode = PPC::DFSTOREf32;
4297  break;
4298  case PPC::STXSDX:
4299  if (PostRA) {
4300  if (IsVFReg)
4301  III.ImmOpcode = PPC::STXSD;
4302  else {
4303  III.ImmOpcode = PPC::STFD;
4304  III.ImmMustBeMultipleOf = 1;
4305  }
4306  break;
4307  }
4309  case PPC::XFSTOREf64:
4310  III.ImmOpcode = PPC::DFSTOREf64;
4311  break;
4312  }
4313  break;
4314  }
4315  return true;
4316 }
4317 
4318 // Utility function for swaping two arbitrary operands of an instruction.
4319 static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4320  assert(Op1 != Op2 && "Cannot swap operand with itself.");
4321 
4322  unsigned MaxOp = std::max(Op1, Op2);
4323  unsigned MinOp = std::min(Op1, Op2);
4324  MachineOperand MOp1 = MI.getOperand(MinOp);
4325  MachineOperand MOp2 = MI.getOperand(MaxOp);
4326  MI.removeOperand(std::max(Op1, Op2));
4327  MI.removeOperand(std::min(Op1, Op2));
4328 
4329  // If the operands we are swapping are the two at the end (the common case)
4330  // we can just remove both and add them in the opposite order.
4331  if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4332  MI.addOperand(MOp2);
4333  MI.addOperand(MOp1);
4334  } else {
4335  // Store all operands in a temporary vector, remove them and re-add in the
4336  // right order.
4338  unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4339  for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4340  MOps.push_back(MI.getOperand(i));
4341  MI.removeOperand(i);
4342  }
4343  // MOp2 needs to be added next.
4344  MI.addOperand(MOp2);
4345  // Now add the rest.
4346  for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4347  if (i == MaxOp)
4348  MI.addOperand(MOp1);
4349  else {
4350  MI.addOperand(MOps.back());
4351  MOps.pop_back();
4352  }
4353  }
4354  }
4355 }
4356 
4357 // Check if the 'MI' that has the index OpNoForForwarding
4358 // meets the requirement described in the ImmInstrInfo.
4359 bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4360  const ImmInstrInfo &III,
4361  unsigned OpNoForForwarding
4362  ) const {
4363  // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4364  // would not work pre-RA, we can only do the check post RA.
4365  MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4366  if (MRI.isSSA())
4367  return false;
4368 
4369  // Cannot do the transform if MI isn't summing the operands.
4370  if (!III.IsSummingOperands)
4371  return false;
4372 
4373  // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4374  if (!III.ZeroIsSpecialOrig)
4375  return false;
4376 
4377  // We cannot do the transform if the operand we are trying to replace
4378  // isn't the same as the operand the instruction allows.
4379  if (OpNoForForwarding != III.OpNoForForwarding)
4380  return false;
4381 
4382  // Check if the instruction we are trying to transform really has
4383  // the special zero register as its operand.
4384  if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4385  MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4386  return false;
4387 
4388  // This machine instruction is convertible if it is,
4389  // 1. summing the operands.
4390  // 2. one of the operands is special zero register.
4391  // 3. the operand we are trying to replace is allowed by the MI.
4392  return true;
4393 }
4394 
4395 // Check if the DefMI is the add inst and set the ImmMO and RegMO
4396 // accordingly.
4397 bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4398  const ImmInstrInfo &III,
4399  MachineOperand *&ImmMO,
4400  MachineOperand *&RegMO) const {
4401  unsigned Opc = DefMI.getOpcode();
4402  if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4403  return false;
4404 
4405  assert(DefMI.getNumOperands() >= 3 &&
4406  "Add inst must have at least three operands");
4407  RegMO = &DefMI.getOperand(1);
4408  ImmMO = &DefMI.getOperand(2);
4409 
4410  // Before RA, ADDI first operand could be a frame index.
4411  if (!RegMO->isReg())
4412  return false;
4413 
4414  // This DefMI is elgible for forwarding if it is:
4415  // 1. add inst
4416  // 2. one of the operands is Imm/CPI/Global.
4417  return isAnImmediateOperand(*ImmMO);
4418 }
4419 
4420 bool PPCInstrInfo::isRegElgibleForForwarding(
4421  const MachineOperand &RegMO, const MachineInstr &DefMI,
4422  const MachineInstr &MI, bool KillDefMI,
4423  bool &IsFwdFeederRegKilled) const {
4424  // x = addi y, imm
4425  // ...
4426  // z = lfdx 0, x -> z = lfd imm(y)
4427  // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4428  // of "y" between the DEF of "x" and "z".
4429  // The query is only valid post RA.
4430  const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4431  if (MRI.isSSA())
4432  return false;
4433 
4434  Register Reg = RegMO.getReg();
4435 
4436  // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4438  MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4439  It++;
4440  for (; It != E; ++It) {
4441  if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4442  return false;
4443  else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4444  IsFwdFeederRegKilled = true;
4445  // Made it to DefMI without encountering a clobber.
4446  if ((&*It) == &DefMI)
4447  break;
4448  }
4449  assert((&*It) == &DefMI && "DefMI is missing");
4450 
4451  // If DefMI also defines the register to be forwarded, we can only forward it
4452  // if DefMI is being erased.
4453  if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4454  return KillDefMI;
4455 
4456  return true;
4457 }
4458 
4459 bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4460  const MachineInstr &DefMI,
4461  const ImmInstrInfo &III,
4462  int64_t &Imm,
4463  int64_t BaseImm) const {
4464  assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4465  if (DefMI.getOpcode() == PPC::ADDItocL) {
4466  // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4467  // However, we know that, it is 16-bit width, and has the alignment of 4.
4468  // Check if the instruction met the requirement.
4469  if (III.ImmMustBeMultipleOf > 4 ||
4470  III.TruncateImmTo || III.ImmWidth != 16)
4471  return false;
4472 
4473  // Going from XForm to DForm loads means that the displacement needs to be
4474  // not just an immediate but also a multiple of 4, or 16 depending on the
4475  // load. A DForm load cannot be represented if it is a multiple of say 2.
4476  // XForm loads do not have this restriction.
4477  if (ImmMO.isGlobal()) {
4478  const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4480  return false;
4481  }
4482 
4483  return true;
4484  }
4485 
4486  if (ImmMO.isImm()) {
4487  // It is Imm, we need to check if the Imm fit the range.
4488  // Sign-extend to 64-bits.
4489  // DefMI may be folded with another imm form instruction, the result Imm is
4490  // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4491  APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4492  if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4493  return false;
4494  if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4495  return false;
4496  Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4497 
4498  if (Imm % III.ImmMustBeMultipleOf)
4499  return false;
4500  if (III.TruncateImmTo)
4501  Imm &= ((1 << III.TruncateImmTo) - 1);
4502  }
4503  else
4504  return false;
4505 
4506  // This ImmMO is forwarded if it meets the requriement describle
4507  // in ImmInstrInfo
4508  return true;
4509 }
4510 
4511 bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4512  unsigned OpNoForForwarding,
4513  MachineInstr **KilledDef) const {
4514  if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4515  !DefMI.getOperand(1).isImm())
4516  return false;
4517 
4518  MachineFunction *MF = MI.getParent()->getParent();
4520  bool PostRA = !MRI->isSSA();
4521 
4522  int64_t Immediate = DefMI.getOperand(1).getImm();
4523  // Sign-extend to 64-bits.
4524  int64_t SExtImm = SignExtend64<16>(Immediate);
4525 
4526  bool IsForwardingOperandKilled = MI.getOperand(OpNoForForwarding).isKill();
4527  Register ForwardingOperandReg = MI.getOperand(OpNoForForwarding).getReg();
4528 
4529  bool ReplaceWithLI = false;
4530  bool Is64BitLI = false;
4531  int64_t NewImm = 0;
4532  bool SetCR = false;
4533  unsigned Opc = MI.getOpcode();
4534  switch (Opc) {
4535  default:
4536  return false;
4537 
4538  // FIXME: Any branches conditional on such a comparison can be made
4539  // unconditional. At this time, this happens too infrequently to be worth
4540  // the implementation effort, but if that ever changes, we could convert
4541  // such a pattern here.
4542  case PPC::CMPWI:
4543  case PPC::CMPLWI:
4544  case PPC::CMPDI:
4545  case PPC::CMPLDI: {
4546  // Doing this post-RA would require dataflow analysis to reliably find uses
4547  // of the CR register set by the compare.
4548  // No need to fixup killed/dead flag since this transformation is only valid
4549  // before RA.
4550  if (PostRA)
4551  return false;
4552  // If a compare-immediate is fed by an immediate and is itself an input of
4553  // an ISEL (the most common case) into a COPY of the correct register.
4554  bool Changed = false;
4555  Register DefReg = MI.getOperand(0).getReg();
4556  int64_t Comparand = MI.getOperand(2).getImm();
4557  int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4558  ? (Comparand | 0xFFFFFFFFFFFF0000)
4559  : Comparand;
4560 
4561  for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4562  unsigned UseOpc = CompareUseMI.getOpcode();
4563  if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4564  continue;
4565  unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4566  Register TrueReg = CompareUseMI.getOperand(1).getReg();
4567  Register FalseReg = CompareUseMI.getOperand(2).getReg();
4568  unsigned RegToCopy =
4569  selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4570  if (RegToCopy == PPC::NoRegister)
4571  continue;
4572  // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4573  if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4574  CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4575  replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4576  CompareUseMI.removeOperand(3);
4577  CompareUseMI.removeOperand(2);
4578  continue;
4579  }
4580  LLVM_DEBUG(
4581  dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4582  LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4583  LLVM_DEBUG(dbgs() << "Is converted to:\n");
4584  // Convert to copy and remove unneeded operands.
4585  CompareUseMI.setDesc(get(PPC::COPY));
4586  CompareUseMI.removeOperand(3);
4587  CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
4588  CmpIselsConverted++;
4589  Changed = true;
4590  LLVM_DEBUG(CompareUseMI.dump());
4591  }
4592  if (Changed)
4593  return true;
4594  // This may end up incremented multiple times since this function is called
4595  // during a fixed-point transformation, but it is only meant to indicate the
4596  // presence of this opportunity.
4597  MissedConvertibleImmediateInstrs++;
4598  return false;
4599  }
4600 
4601  // Immediate forms - may simply be convertable to an LI.
4602  case PPC::ADDI:
4603  case PPC::ADDI8: {
4604  // Does the sum fit in a 16-bit signed field?
4605  int64_t Addend = MI.getOperand(2).getImm();
4606  if (isInt<16>(Addend + SExtImm)) {
4607  ReplaceWithLI = true;
4608  Is64BitLI = Opc == PPC::ADDI8;
4609  NewImm = Addend + SExtImm;
4610  break;
4611  }
4612  return false;
4613  }
4614  case PPC::SUBFIC:
4615  case PPC::SUBFIC8: {
4616  // Only transform this if the CARRY implicit operand is dead.
4617  if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
4618  return false;
4619  int64_t Minuend = MI.getOperand(2).getImm();
4620  if (isInt<16>(Minuend - SExtImm)) {
4621  ReplaceWithLI = true;
4622  Is64BitLI = Opc == PPC::SUBFIC8;
4623  NewImm = Minuend - SExtImm;
4624  break;
4625  }
4626  return false;
4627  }
4628  case PPC::RLDICL:
4629  case PPC::RLDICL_rec:
4630  case PPC::RLDICL_32:
4631  case PPC::RLDICL_32_64: {
4632  // Use APInt's rotate function.
4633  int64_t SH = MI.getOperand(2).getImm();
4634  int64_t MB = MI.getOperand(3).getImm();
4635  APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4636  SExtImm, true);
4637  InVal = InVal.rotl(SH);
4638  uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4639  InVal &= Mask;
4640  // Can't replace negative values with an LI as that will sign-extend
4641  // and not clear the left bits. If we're setting the CR bit, we will use
4642  // ANDI_rec which won't sign extend, so that's safe.
4643  if (isUInt<15>(InVal.getSExtValue()) ||
4644  (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
4645  ReplaceWithLI = true;
4646  Is64BitLI = Opc != PPC::RLDICL_32;
4647  NewImm = InVal.getSExtValue();
4648  SetCR = Opc == PPC::RLDICL_rec;
4649  break;
4650  }
4651  return false;
4652  }
4653  case PPC::RLWINM:
4654  case PPC::RLWINM8:
4655  case PPC::RLWINM_rec:
4656  case PPC::RLWINM8_rec: {
4657  int64_t SH = MI.getOperand(2).getImm();
4658  int64_t MB = MI.getOperand(3).getImm();
4659  int64_t ME = MI.getOperand(4).getImm();
4660  APInt InVal(32, SExtImm, true);
4661  InVal = InVal.rotl(SH);
4662  APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
4663  InVal &= Mask;
4664  // Can't replace negative values with an LI as that will sign-extend
4665  // and not clear the left bits. If we're setting the CR bit, we will use
4666  // ANDI_rec which won't sign extend, so that's safe.
4667  bool ValueFits = isUInt<15>(InVal.getSExtValue());
4668  ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4669  isUInt<16>(InVal.getSExtValue()));
4670  if (ValueFits) {
4671  ReplaceWithLI = true;
4672  Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4673  NewImm = InVal.getSExtValue();
4674  SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4675  break;
4676  }
4677  return false;
4678  }
4679  case PPC::ORI:
4680  case PPC::ORI8:
4681  case PPC::XORI:
4682  case PPC::XORI8: {
4683  int64_t LogicalImm = MI.getOperand(2).getImm();
4684  int64_t Result = 0;
4685  if (Opc == PPC::ORI || Opc == PPC::ORI8)
4686  Result = LogicalImm | SExtImm;
4687  else
4688  Result = LogicalImm ^ SExtImm;
4689  if (isInt<16>(Result)) {
4690  ReplaceWithLI = true;
4691  Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4692  NewImm = Result;
4693  break;
4694  }
4695  return false;
4696  }
4697  }
4698 
4699  if (ReplaceWithLI) {
4700  // We need to be careful with CR-setting instructions we're replacing.
4701  if (SetCR) {
4702  // We don't know anything about uses when we're out of SSA, so only
4703  // replace if the new immediate will be reproduced.
4704  bool ImmChanged = (SExtImm & NewImm) != NewImm;
4705  if (PostRA && ImmChanged)
4706  return false;
4707 
4708  if (!PostRA) {
4709  // If the defining load-immediate has no other uses, we can just replace
4710  // the immediate with the new immediate.
4711  if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
4712  DefMI.getOperand(1).setImm(NewImm);
4713 
4714  // If we're not using the GPR result of the CR-setting instruction, we
4715  // just need to and with zero/non-zero depending on the new immediate.
4716  else if (MRI->use_empty(MI.getOperand(0).getReg())) {
4717  if (NewImm) {
4718  assert(Immediate && "Transformation converted zero to non-zero?");
4719  NewImm = Immediate;
4720  }
4721  } else if (ImmChanged)
4722  return false;
4723  }
4724  }
4725 
4726  LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4727  LLVM_DEBUG(MI.dump());
4728  LLVM_DEBUG(dbgs() << "Fed by:\n");
4729  LLVM_DEBUG(DefMI.dump());
4730  LoadImmediateInfo LII;
4731  LII.Imm = NewImm;
4732  LII.Is64Bit = Is64BitLI;
4733  LII.SetCR = SetCR;
4734  // If we're setting the CR, the original load-immediate must be kept (as an
4735  // operand to ANDI_rec/ANDI8_rec).
4736  if (KilledDef && SetCR)
4737  *KilledDef = nullptr;
4738  replaceInstrWithLI(MI, LII);
4739 
4740  // Fixup killed/dead flag after transformation.
4741  // Pattern:
4742  // ForwardingOperandReg = LI imm1
4743  // y = op2 imm2, ForwardingOperandReg(killed)
4744  if (IsForwardingOperandKilled)
4745  fixupIsDeadOrKill(&DefMI, &MI, ForwardingOperandReg);
4746 
4747  LLVM_DEBUG(dbgs() << "With:\n");
4748  LLVM_DEBUG(MI.dump());
4749  return true;
4750  }
4751  return false;
4752 }
4753 
4754 bool PPCInstrInfo::transformToNewImmFormFedByAdd(
4755  MachineInstr &MI, MachineInstr &DefMI, unsigned OpNoForForwarding) const {
4756  MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
4757  bool PostRA = !MRI->isSSA();
4758  // FIXME: extend this to post-ra. Need to do some change in getForwardingDefMI
4759  // for post-ra.
4760  if (PostRA)
4761  return false;
4762 
4763  // Only handle load/store.
4764  if (!MI.mayLoadOrStore())
4765  return false;
4766 
4767  unsigned XFormOpcode = RI.getMappedIdxOpcForImmOpc(MI.getOpcode());
4768 
4769  assert((XFormOpcode != PPC::INSTRUCTION_LIST_END) &&
4770  "MI must have x-form opcode");
4771 
4772  // get Imm Form info.
4773  ImmInstrInfo III;
4774  bool IsVFReg = MI.getOperand(0).isReg()
4775  ? isVFRegister(MI.getOperand(0).getReg())
4776  : false;
4777 
4778  if (!instrHasImmForm(XFormOpcode, IsVFReg, III, PostRA))
4779  return false;
4780 
4781  if (!III.IsSummingOperands)
4782  return false;
4783 
4784  if (OpNoForForwarding != III.OpNoForForwarding)
4785  return false;
4786 
4787  MachineOperand ImmOperandMI = MI.getOperand(III.ImmOpNo);
4788  if (!ImmOperandMI.isImm())
4789  return false;
4790 
4791  // Check DefMI.
4792  MachineOperand *ImmMO = nullptr;
4793  MachineOperand *RegMO = nullptr;
4794  if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4795  return false;
4796  assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4797 
4798  // Check Imm.
4799  // Set ImmBase from imm instruction as base and get new Imm inside
4800  // isImmElgibleForForwarding.
4801  int64_t ImmBase = ImmOperandMI.getImm();
4802  int64_t Imm = 0;
4803  if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm, ImmBase))
4804  return false;
4805 
4806  // Get killed info in case fixup needed after transformation.
4807  unsigned ForwardKilledOperandReg = ~0U;
4808  if (MI.getOperand(III.OpNoForForwarding).isKill())
4809  ForwardKilledOperandReg = MI.getOperand(III.OpNoForForwarding).getReg();
4810 
4811  // Do the transform
4812  LLVM_DEBUG(dbgs() << "Replacing instruction:\n");
4813  LLVM_DEBUG(MI.dump());
4814  LLVM_DEBUG(dbgs() << "Fed by:\n");
4815  LLVM_DEBUG(DefMI.dump());
4816 
4817  MI.getOperand(III.OpNoForForwarding).setReg(RegMO->getReg());
4818  if (RegMO->isKill()) {
4819  MI.getOperand(III.OpNoForForwarding).setIsKill(true);
4820  // Clear the killed flag in RegMO. Doing this here can handle some cases
4821  // that DefMI and MI are not in same basic block.
4822  RegMO->setIsKill(false);
4823  }
4824  MI.getOperand(III.ImmOpNo).setImm(Imm);
4825 
4826  // FIXME: fix kill/dead flag if MI and DefMI are not in same basic block.
4827  if (DefMI.getParent() == MI.getParent()) {
4828  // Check if reg is killed between MI and DefMI.
4829  auto IsKilledFor = [&](unsigned Reg) {
4832  It++;
4833  for (; It != E; ++It) {
4834  if (It->killsRegister(Reg))
4835  return true;
4836  }
4837  return false;
4838  };
4839 
4840  // Update kill flag
4841  if (RegMO->isKill() || IsKilledFor(RegMO->getReg()))
4842  fixupIsDeadOrKill(&DefMI, &MI, RegMO->getReg());
4843  if (ForwardKilledOperandReg != ~0U)
4844  fixupIsDeadOrKill(&DefMI, &MI, ForwardKilledOperandReg);
4845  }
4846 
4847  LLVM_DEBUG(dbgs() << "With:\n");
4848  LLVM_DEBUG(MI.dump());
4849  return true;
4850 }
4851 
4852 // If an X-Form instruction is fed by an add-immediate and one of its operands
4853 // is the literal zero, attempt to forward the source of the add-immediate to
4854 // the corresponding D-Form instruction with the displacement coming from
4855 // the immediate being added.
4856 bool PPCInstrInfo::transformToImmFormFedByAdd(
4857  MachineInstr &MI, const ImmInstrInfo &III, unsigned OpNoForForwarding,
4858  MachineInstr &DefMI, bool KillDefMI) const {
4859  // RegMO ImmMO
4860  // | |
4861  // x = addi reg, imm <----- DefMI
4862  // y = op 0 , x <----- MI
4863  // |
4864  // OpNoForForwarding
4865  // Check if the MI meet the requirement described in the III.
4866  if (!isUseMIElgibleForForwarding(MI, III, OpNoForForwarding))
4867  return false;
4868 
4869  // Check if the DefMI meet the requirement
4870  // described in the III. If yes, set the ImmMO and RegMO accordingly.
4871  MachineOperand *ImmMO = nullptr;
4872  MachineOperand *RegMO = nullptr;
4873  if (!isDefMIElgibleForForwarding(DefMI, III, ImmMO, RegMO))
4874  return false;
4875  assert(ImmMO && RegMO && "Imm and Reg operand must have been set");
4876 
4877  // As we get the Imm operand now, we need to check if the ImmMO meet
4878  // the requirement described in the III. If yes set the Imm.
4879  int64_t Imm = 0;
4880  if (!isImmElgibleForForwarding(*ImmMO, DefMI, III, Imm))
4881  return false;
4882 
4883  bool IsFwdFeederRegKilled = false;
4884  // Check if the RegMO can be forwarded to MI.
4885  if (!isRegElgibleForForwarding(*RegMO, DefMI, MI, KillDefMI,
4886  IsFwdFeederRegKilled))
4887