LLVM 18.0.0git
PPCInstrInfo.cpp
Go to the documentation of this file.
1//===-- PPCInstrInfo.cpp - PowerPC Instruction Information ----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the PowerPC implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCInstrInfo.h"
15#include "PPC.h"
17#include "PPCInstrBuilder.h"
19#include "PPCTargetMachine.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/STLExtras.h"
22#include "llvm/ADT/Statistic.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCInst.h"
43#include "llvm/Support/Debug.h"
46
47using namespace llvm;
48
49#define DEBUG_TYPE "ppc-instr-info"
50
51#define GET_INSTRMAP_INFO
52#define GET_INSTRINFO_CTOR_DTOR
53#include "PPCGenInstrInfo.inc"
54
55STATISTIC(NumStoreSPILLVSRRCAsVec,
56 "Number of spillvsrrc spilled to stack as vec");
57STATISTIC(NumStoreSPILLVSRRCAsGpr,
58 "Number of spillvsrrc spilled to stack as gpr");
59STATISTIC(NumGPRtoVSRSpill, "Number of gpr spills to spillvsrrc");
60STATISTIC(CmpIselsConverted,
61 "Number of ISELs that depend on comparison of constants converted");
62STATISTIC(MissedConvertibleImmediateInstrs,
63 "Number of compare-immediate instructions fed by constants");
64STATISTIC(NumRcRotatesConvertedToRcAnd,
65 "Number of record-form rotates converted to record-form andi");
66
67static cl::
68opt<bool> DisableCTRLoopAnal("disable-ppc-ctrloop-analysis", cl::Hidden,
69 cl::desc("Disable analysis for CTR loops"));
70
71static cl::opt<bool> DisableCmpOpt("disable-ppc-cmp-opt",
72cl::desc("Disable compare instruction optimization"), cl::Hidden);
73
74static cl::opt<bool> VSXSelfCopyCrash("crash-on-ppc-vsx-self-copy",
75cl::desc("Causes the backend to crash instead of generating a nop VSX copy"),
77
78static cl::opt<bool>
79UseOldLatencyCalc("ppc-old-latency-calc", cl::Hidden,
80 cl::desc("Use the old (incorrect) instruction latency calculation"));
81
82static cl::opt<float>
83 FMARPFactor("ppc-fma-rp-factor", cl::Hidden, cl::init(1.5),
84 cl::desc("register pressure factor for the transformations."));
85
87 "ppc-fma-rp-reduction", cl::Hidden, cl::init(true),
88 cl::desc("enable register pressure reduce in machine combiner pass."));
89
90// Pin the vtable to this file.
91void PPCInstrInfo::anchor() {}
92
94 : PPCGenInstrInfo(PPC::ADJCALLSTACKDOWN, PPC::ADJCALLSTACKUP,
95 /* CatchRetOpcode */ -1,
96 STI.isPPC64() ? PPC::BLR8 : PPC::BLR),
97 Subtarget(STI), RI(STI.getTargetMachine()) {}
98
99/// CreateTargetHazardRecognizer - Return the hazard recognizer to use for
100/// this target when scheduling the DAG.
103 const ScheduleDAG *DAG) const {
104 unsigned Directive =
105 static_cast<const PPCSubtarget *>(STI)->getCPUDirective();
108 const InstrItineraryData *II =
109 static_cast<const PPCSubtarget *>(STI)->getInstrItineraryData();
110 return new ScoreboardHazardRecognizer(II, DAG);
111 }
112
114}
115
116/// CreateTargetPostRAHazardRecognizer - Return the postRA hazard recognizer
117/// to use for this target when scheduling the DAG.
120 const ScheduleDAG *DAG) const {
121 unsigned Directive =
122 DAG->MF.getSubtarget<PPCSubtarget>().getCPUDirective();
123
124 // FIXME: Leaving this as-is until we have POWER9 scheduling info
126 return new PPCDispatchGroupSBHazardRecognizer(II, DAG);
127
128 // Most subtargets use a PPC970 recognizer.
131 assert(DAG->TII && "No InstrInfo?");
132
133 return new PPCHazardRecognizer970(*DAG);
134 }
135
136 return new ScoreboardHazardRecognizer(II, DAG);
137}
138
140 const MachineInstr &MI,
141 unsigned *PredCost) const {
142 if (!ItinData || UseOldLatencyCalc)
143 return PPCGenInstrInfo::getInstrLatency(ItinData, MI, PredCost);
144
145 // The default implementation of getInstrLatency calls getStageLatency, but
146 // getStageLatency does not do the right thing for us. While we have
147 // itinerary, most cores are fully pipelined, and so the itineraries only
148 // express the first part of the pipeline, not every stage. Instead, we need
149 // to use the listed output operand cycle number (using operand 0 here, which
150 // is an output).
151
152 unsigned Latency = 1;
153 unsigned DefClass = MI.getDesc().getSchedClass();
154 for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
155 const MachineOperand &MO = MI.getOperand(i);
156 if (!MO.isReg() || !MO.isDef() || MO.isImplicit())
157 continue;
158
159 int Cycle = ItinData->getOperandCycle(DefClass, i);
160 if (Cycle < 0)
161 continue;
162
163 Latency = std::max(Latency, (unsigned) Cycle);
164 }
165
166 return Latency;
167}
168
170 const MachineInstr &DefMI, unsigned DefIdx,
171 const MachineInstr &UseMI,
172 unsigned UseIdx) const {
173 int Latency = PPCGenInstrInfo::getOperandLatency(ItinData, DefMI, DefIdx,
174 UseMI, UseIdx);
175
176 if (!DefMI.getParent())
177 return Latency;
178
179 const MachineOperand &DefMO = DefMI.getOperand(DefIdx);
180 Register Reg = DefMO.getReg();
181
182 bool IsRegCR;
183 if (Reg.isVirtual()) {
184 const MachineRegisterInfo *MRI =
185 &DefMI.getParent()->getParent()->getRegInfo();
186 IsRegCR = MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRRCRegClass) ||
187 MRI->getRegClass(Reg)->hasSuperClassEq(&PPC::CRBITRCRegClass);
188 } else {
189 IsRegCR = PPC::CRRCRegClass.contains(Reg) ||
190 PPC::CRBITRCRegClass.contains(Reg);
191 }
192
193 if (UseMI.isBranch() && IsRegCR) {
194 if (Latency < 0)
195 Latency = getInstrLatency(ItinData, DefMI);
196
197 // On some cores, there is an additional delay between writing to a condition
198 // register, and using it from a branch.
199 unsigned Directive = Subtarget.getCPUDirective();
200 switch (Directive) {
201 default: break;
202 case PPC::DIR_7400:
203 case PPC::DIR_750:
204 case PPC::DIR_970:
205 case PPC::DIR_E5500:
206 case PPC::DIR_PWR4:
207 case PPC::DIR_PWR5:
208 case PPC::DIR_PWR5X:
209 case PPC::DIR_PWR6:
210 case PPC::DIR_PWR6X:
211 case PPC::DIR_PWR7:
212 case PPC::DIR_PWR8:
213 // FIXME: Is this needed for POWER9?
214 Latency += 2;
215 break;
216 }
217 }
218
219 return Latency;
220}
221
222/// This is an architecture-specific helper function of reassociateOps.
223/// Set special operand attributes for new instructions after reassociation.
225 MachineInstr &OldMI2,
226 MachineInstr &NewMI1,
227 MachineInstr &NewMI2) const {
228 // Propagate FP flags from the original instructions.
229 // But clear poison-generating flags because those may not be valid now.
230 uint32_t IntersectedFlags = OldMI1.getFlags() & OldMI2.getFlags();
231 NewMI1.setFlags(IntersectedFlags);
235
236 NewMI2.setFlags(IntersectedFlags);
240}
241
243 uint32_t Flags) const {
244 MI.setFlags(Flags);
248}
249
250// This function does not list all associative and commutative operations, but
251// only those worth feeding through the machine combiner in an attempt to
252// reduce the critical path. Mostly, this means floating-point operations,
253// because they have high latencies(>=5) (compared to other operations, such as
254// and/or, which are also associative and commutative, but have low latencies).
256 bool Invert) const {
257 if (Invert)
258 return false;
259 switch (Inst.getOpcode()) {
260 // Floating point:
261 // FP Add:
262 case PPC::FADD:
263 case PPC::FADDS:
264 // FP Multiply:
265 case PPC::FMUL:
266 case PPC::FMULS:
267 // Altivec Add:
268 case PPC::VADDFP:
269 // VSX Add:
270 case PPC::XSADDDP:
271 case PPC::XVADDDP:
272 case PPC::XVADDSP:
273 case PPC::XSADDSP:
274 // VSX Multiply:
275 case PPC::XSMULDP:
276 case PPC::XVMULDP:
277 case PPC::XVMULSP:
278 case PPC::XSMULSP:
281 // Fixed point:
282 // Multiply:
283 case PPC::MULHD:
284 case PPC::MULLD:
285 case PPC::MULHW:
286 case PPC::MULLW:
287 return true;
288 default:
289 return false;
290 }
291}
292
293#define InfoArrayIdxFMAInst 0
294#define InfoArrayIdxFAddInst 1
295#define InfoArrayIdxFMULInst 2
296#define InfoArrayIdxAddOpIdx 3
297#define InfoArrayIdxMULOpIdx 4
298#define InfoArrayIdxFSubInst 5
299// Array keeps info for FMA instructions:
300// Index 0(InfoArrayIdxFMAInst): FMA instruction;
301// Index 1(InfoArrayIdxFAddInst): ADD instruction associated with FMA;
302// Index 2(InfoArrayIdxFMULInst): MUL instruction associated with FMA;
303// Index 3(InfoArrayIdxAddOpIdx): ADD operand index in FMA operands;
304// Index 4(InfoArrayIdxMULOpIdx): first MUL operand index in FMA operands;
305// second MUL operand index is plus 1;
306// Index 5(InfoArrayIdxFSubInst): SUB instruction associated with FMA.
307static const uint16_t FMAOpIdxInfo[][6] = {
308 // FIXME: Add more FMA instructions like XSNMADDADP and so on.
309 {PPC::XSMADDADP, PPC::XSADDDP, PPC::XSMULDP, 1, 2, PPC::XSSUBDP},
310 {PPC::XSMADDASP, PPC::XSADDSP, PPC::XSMULSP, 1, 2, PPC::XSSUBSP},
311 {PPC::XVMADDADP, PPC::XVADDDP, PPC::XVMULDP, 1, 2, PPC::XVSUBDP},
312 {PPC::XVMADDASP, PPC::XVADDSP, PPC::XVMULSP, 1, 2, PPC::XVSUBSP},
313 {PPC::FMADD, PPC::FADD, PPC::FMUL, 3, 1, PPC::FSUB},
314 {PPC::FMADDS, PPC::FADDS, PPC::FMULS, 3, 1, PPC::FSUBS}};
315
316// Check if an opcode is a FMA instruction. If it is, return the index in array
317// FMAOpIdxInfo. Otherwise, return -1.
318int16_t PPCInstrInfo::getFMAOpIdxInfo(unsigned Opcode) const {
319 for (unsigned I = 0; I < std::size(FMAOpIdxInfo); I++)
320 if (FMAOpIdxInfo[I][InfoArrayIdxFMAInst] == Opcode)
321 return I;
322 return -1;
323}
324
325// On PowerPC target, we have two kinds of patterns related to FMA:
326// 1: Improve ILP.
327// Try to reassociate FMA chains like below:
328//
329// Pattern 1:
330// A = FADD X, Y (Leaf)
331// B = FMA A, M21, M22 (Prev)
332// C = FMA B, M31, M32 (Root)
333// -->
334// A = FMA X, M21, M22
335// B = FMA Y, M31, M32
336// C = FADD A, B
337//
338// Pattern 2:
339// A = FMA X, M11, M12 (Leaf)
340// B = FMA A, M21, M22 (Prev)
341// C = FMA B, M31, M32 (Root)
342// -->
343// A = FMUL M11, M12
344// B = FMA X, M21, M22
345// D = FMA A, M31, M32
346// C = FADD B, D
347//
348// breaking the dependency between A and B, allowing FMA to be executed in
349// parallel (or back-to-back in a pipeline) instead of depending on each other.
350//
351// 2: Reduce register pressure.
352// Try to reassociate FMA with FSUB and a constant like below:
353// C is a floating point const.
354//
355// Pattern 1:
356// A = FSUB X, Y (Leaf)
357// D = FMA B, C, A (Root)
358// -->
359// A = FMA B, Y, -C
360// D = FMA A, X, C
361//
362// Pattern 2:
363// A = FSUB X, Y (Leaf)
364// D = FMA B, A, C (Root)
365// -->
366// A = FMA B, Y, -C
367// D = FMA A, X, C
368//
369// Before the transformation, A must be assigned with different hardware
370// register with D. After the transformation, A and D must be assigned with
371// same hardware register due to TIE attribute of FMA instructions.
372//
375 bool DoRegPressureReduce) const {
379
380 auto IsAllOpsVirtualReg = [](const MachineInstr &Instr) {
381 for (const auto &MO : Instr.explicit_operands())
382 if (!(MO.isReg() && MO.getReg().isVirtual()))
383 return false;
384 return true;
385 };
386
387 auto IsReassociableAddOrSub = [&](const MachineInstr &Instr,
388 unsigned OpType) {
389 if (Instr.getOpcode() !=
390 FMAOpIdxInfo[getFMAOpIdxInfo(Root.getOpcode())][OpType])
391 return false;
392
393 // Instruction can be reassociated.
394 // fast math flags may prohibit reassociation.
395 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
396 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
397 return false;
398
399 // Instruction operands are virtual registers for reassociation.
400 if (!IsAllOpsVirtualReg(Instr))
401 return false;
402
403 // For register pressure reassociation, the FSub must have only one use as
404 // we want to delete the sub to save its def.
405 if (OpType == InfoArrayIdxFSubInst &&
406 !MRI->hasOneNonDBGUse(Instr.getOperand(0).getReg()))
407 return false;
408
409 return true;
410 };
411
412 auto IsReassociableFMA = [&](const MachineInstr &Instr, int16_t &AddOpIdx,
413 int16_t &MulOpIdx, bool IsLeaf) {
414 int16_t Idx = getFMAOpIdxInfo(Instr.getOpcode());
415 if (Idx < 0)
416 return false;
417
418 // Instruction can be reassociated.
419 // fast math flags may prohibit reassociation.
420 if (!(Instr.getFlag(MachineInstr::MIFlag::FmReassoc) &&
421 Instr.getFlag(MachineInstr::MIFlag::FmNsz)))
422 return false;
423
424 // Instruction operands are virtual registers for reassociation.
425 if (!IsAllOpsVirtualReg(Instr))
426 return false;
427
429 if (IsLeaf)
430 return true;
431
433
434 const MachineOperand &OpAdd = Instr.getOperand(AddOpIdx);
435 MachineInstr *MIAdd = MRI->getUniqueVRegDef(OpAdd.getReg());
436 // If 'add' operand's def is not in current block, don't do ILP related opt.
437 if (!MIAdd || MIAdd->getParent() != MBB)
438 return false;
439
440 // If this is not Leaf FMA Instr, its 'add' operand should only have one use
441 // as this fma will be changed later.
442 return IsLeaf ? true : MRI->hasOneNonDBGUse(OpAdd.getReg());
443 };
444
445 int16_t AddOpIdx = -1;
446 int16_t MulOpIdx = -1;
447
448 bool IsUsedOnceL = false;
449 bool IsUsedOnceR = false;
450 MachineInstr *MULInstrL = nullptr;
451 MachineInstr *MULInstrR = nullptr;
452
453 auto IsRPReductionCandidate = [&]() {
454 // Currently, we only support float and double.
455 // FIXME: add support for other types.
456 unsigned Opcode = Root.getOpcode();
457 if (Opcode != PPC::XSMADDASP && Opcode != PPC::XSMADDADP)
458 return false;
459
460 // Root must be a valid FMA like instruction.
461 // Treat it as leaf as we don't care its add operand.
462 if (IsReassociableFMA(Root, AddOpIdx, MulOpIdx, true)) {
463 assert((MulOpIdx >= 0) && "mul operand index not right!");
464 Register MULRegL = TRI->lookThruSingleUseCopyChain(
465 Root.getOperand(MulOpIdx).getReg(), MRI);
466 Register MULRegR = TRI->lookThruSingleUseCopyChain(
467 Root.getOperand(MulOpIdx + 1).getReg(), MRI);
468 if (!MULRegL && !MULRegR)
469 return false;
470
471 if (MULRegL && !MULRegR) {
472 MULRegR =
473 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx + 1).getReg(), MRI);
474 IsUsedOnceL = true;
475 } else if (!MULRegL && MULRegR) {
476 MULRegL =
477 TRI->lookThruCopyLike(Root.getOperand(MulOpIdx).getReg(), MRI);
478 IsUsedOnceR = true;
479 } else {
480 IsUsedOnceL = true;
481 IsUsedOnceR = true;
482 }
483
484 if (!MULRegL.isVirtual() || !MULRegR.isVirtual())
485 return false;
486
487 MULInstrL = MRI->getVRegDef(MULRegL);
488 MULInstrR = MRI->getVRegDef(MULRegR);
489 return true;
490 }
491 return false;
492 };
493
494 // Register pressure fma reassociation patterns.
495 if (DoRegPressureReduce && IsRPReductionCandidate()) {
496 assert((MULInstrL && MULInstrR) && "wrong register preduction candidate!");
497 // Register pressure pattern 1
498 if (isLoadFromConstantPool(MULInstrL) && IsUsedOnceR &&
499 IsReassociableAddOrSub(*MULInstrR, InfoArrayIdxFSubInst)) {
500 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BCA\n");
502 return true;
503 }
504
505 // Register pressure pattern 2
506 if ((isLoadFromConstantPool(MULInstrR) && IsUsedOnceL &&
507 IsReassociableAddOrSub(*MULInstrL, InfoArrayIdxFSubInst))) {
508 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_BAC\n");
510 return true;
511 }
512 }
513
514 // ILP fma reassociation patterns.
515 // Root must be a valid FMA like instruction.
516 AddOpIdx = -1;
517 if (!IsReassociableFMA(Root, AddOpIdx, MulOpIdx, false))
518 return false;
519
520 assert((AddOpIdx >= 0) && "add operand index not right!");
521
522 Register RegB = Root.getOperand(AddOpIdx).getReg();
523 MachineInstr *Prev = MRI->getUniqueVRegDef(RegB);
524
525 // Prev must be a valid FMA like instruction.
526 AddOpIdx = -1;
527 if (!IsReassociableFMA(*Prev, AddOpIdx, MulOpIdx, false))
528 return false;
529
530 assert((AddOpIdx >= 0) && "add operand index not right!");
531
532 Register RegA = Prev->getOperand(AddOpIdx).getReg();
533 MachineInstr *Leaf = MRI->getUniqueVRegDef(RegA);
534 AddOpIdx = -1;
535 if (IsReassociableFMA(*Leaf, AddOpIdx, MulOpIdx, true)) {
537 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XMM_AMM_BMM\n");
538 return true;
539 }
540 if (IsReassociableAddOrSub(*Leaf, InfoArrayIdxFAddInst)) {
542 LLVM_DEBUG(dbgs() << "add pattern REASSOC_XY_AMM_BMM\n");
543 return true;
544 }
545 return false;
546}
547
550 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
551 assert(!InsInstrs.empty() && "Instructions set to be inserted is empty!");
552
553 MachineFunction *MF = Root.getMF();
557
558 int16_t Idx = getFMAOpIdxInfo(Root.getOpcode());
559 if (Idx < 0)
560 return;
561
563
564 // For now we only need to fix up placeholder for register pressure reduce
565 // patterns.
566 Register ConstReg = 0;
567 switch (P) {
569 ConstReg =
570 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), MRI);
571 break;
573 ConstReg =
574 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx + 1).getReg(), MRI);
575 break;
576 default:
577 // Not register pressure reduce patterns.
578 return;
579 }
580
581 MachineInstr *ConstDefInstr = MRI->getVRegDef(ConstReg);
582 // Get const value from const pool.
583 const Constant *C = getConstantFromConstantPool(ConstDefInstr);
584 assert(isa<llvm::ConstantFP>(C) && "not a valid constant!");
585
586 // Get negative fp const.
587 APFloat F1((dyn_cast<ConstantFP>(C))->getValueAPF());
588 F1.changeSign();
589 Constant *NegC = ConstantFP::get(dyn_cast<ConstantFP>(C)->getContext(), F1);
590 Align Alignment = MF->getDataLayout().getPrefTypeAlign(C->getType());
591
592 // Put negative fp const into constant pool.
593 unsigned ConstPoolIdx = MCP->getConstantPoolIndex(NegC, Alignment);
594
595 MachineOperand *Placeholder = nullptr;
596 // Record the placeholder PPC::ZERO8 we add in reassociateFMA.
597 for (auto *Inst : InsInstrs) {
598 for (MachineOperand &Operand : Inst->explicit_operands()) {
599 assert(Operand.isReg() && "Invalid instruction in InsInstrs!");
600 if (Operand.getReg() == PPC::ZERO8) {
601 Placeholder = &Operand;
602 break;
603 }
604 }
605 }
606
607 assert(Placeholder && "Placeholder does not exist!");
608
609 // Generate instructions to load the const fp from constant pool.
610 // We only support PPC64 and medium code model.
611 Register LoadNewConst =
612 generateLoadForNewConst(ConstPoolIdx, &Root, C->getType(), InsInstrs);
613
614 // Fill the placeholder with the new load from constant pool.
615 Placeholder->setReg(LoadNewConst);
616}
617
619 const MachineBasicBlock *MBB, const RegisterClassInfo *RegClassInfo) const {
620
622 return false;
623
624 // Currently, we only enable register pressure reducing in machine combiner
625 // for: 1: PPC64; 2: Code Model is Medium; 3: Power9 which also has vector
626 // support.
627 //
628 // So we need following instructions to access a TOC entry:
629 //
630 // %6:g8rc_and_g8rc_nox0 = ADDIStocHA8 $x2, %const.0
631 // %7:vssrc = DFLOADf32 target-flags(ppc-toc-lo) %const.0,
632 // killed %6:g8rc_and_g8rc_nox0, implicit $x2 :: (load 4 from constant-pool)
633 //
634 // FIXME: add more supported targets, like Small and Large code model, PPC32,
635 // AIX.
636 if (!(Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
638 return false;
639
641 const MachineFunction *MF = MBB->getParent();
642 const MachineRegisterInfo *MRI = &MF->getRegInfo();
643
644 auto GetMBBPressure =
645 [&](const MachineBasicBlock *MBB) -> std::vector<unsigned> {
646 RegionPressure Pressure;
647 RegPressureTracker RPTracker(Pressure);
648
649 // Initialize the register pressure tracker.
650 RPTracker.init(MBB->getParent(), RegClassInfo, nullptr, MBB, MBB->end(),
651 /*TrackLaneMasks*/ false, /*TrackUntiedDefs=*/true);
652
653 for (const auto &MI : reverse(*MBB)) {
654 if (MI.isDebugValue() || MI.isDebugLabel())
655 continue;
656 RegisterOperands RegOpers;
657 RegOpers.collect(MI, *TRI, *MRI, false, false);
658 RPTracker.recedeSkipDebugValues();
659 assert(&*RPTracker.getPos() == &MI && "RPTracker sync error!");
660 RPTracker.recede(RegOpers);
661 }
662
663 // Close the RPTracker to finalize live ins.
664 RPTracker.closeRegion();
665
666 return RPTracker.getPressure().MaxSetPressure;
667 };
668
669 // For now we only care about float and double type fma.
670 unsigned VSSRCLimit = TRI->getRegPressureSetLimit(
671 *MBB->getParent(), PPC::RegisterPressureSets::VSSRC);
672
673 // Only reduce register pressure when pressure is high.
674 return GetMBBPressure(MBB)[PPC::RegisterPressureSets::VSSRC] >
675 (float)VSSRCLimit * FMARPFactor;
676}
677
679 // I has only one memory operand which is load from constant pool.
680 if (!I->hasOneMemOperand())
681 return false;
682
683 MachineMemOperand *Op = I->memoperands()[0];
684 return Op->isLoad() && Op->getPseudoValue() &&
685 Op->getPseudoValue()->kind() == PseudoSourceValue::ConstantPool;
686}
687
688Register PPCInstrInfo::generateLoadForNewConst(
689 unsigned Idx, MachineInstr *MI, Type *Ty,
690 SmallVectorImpl<MachineInstr *> &InsInstrs) const {
691 // Now we only support PPC64, Medium code model and P9 with vector.
692 // We have immutable pattern to access const pool. See function
693 // shouldReduceRegisterPressure.
694 assert((Subtarget.isPPC64() && Subtarget.hasP9Vector() &&
696 "Target not supported!\n");
697
698 MachineFunction *MF = MI->getMF();
700
701 // Generate ADDIStocHA8
702 Register VReg1 = MRI->createVirtualRegister(&PPC::G8RC_and_G8RC_NOX0RegClass);
703 MachineInstrBuilder TOCOffset =
704 BuildMI(*MF, MI->getDebugLoc(), get(PPC::ADDIStocHA8), VReg1)
705 .addReg(PPC::X2)
707
708 assert((Ty->isFloatTy() || Ty->isDoubleTy()) &&
709 "Only float and double are supported!");
710
711 unsigned LoadOpcode;
712 // Should be float type or double type.
713 if (Ty->isFloatTy())
714 LoadOpcode = PPC::DFLOADf32;
715 else
716 LoadOpcode = PPC::DFLOADf64;
717
718 const TargetRegisterClass *RC = MRI->getRegClass(MI->getOperand(0).getReg());
719 Register VReg2 = MRI->createVirtualRegister(RC);
723
724 // Generate Load from constant pool.
726 BuildMI(*MF, MI->getDebugLoc(), get(LoadOpcode), VReg2)
728 .addReg(VReg1, getKillRegState(true))
729 .addMemOperand(MMO);
730
731 Load->getOperand(1).setTargetFlags(PPCII::MO_TOC_LO);
732
733 // Insert the toc load instructions into InsInstrs.
734 InsInstrs.insert(InsInstrs.begin(), Load);
735 InsInstrs.insert(InsInstrs.begin(), TOCOffset);
736 return VReg2;
737}
738
739// This function returns the const value in constant pool if the \p I is a load
740// from constant pool.
741const Constant *
743 MachineFunction *MF = I->getMF();
746 assert(I->mayLoad() && "Should be a load instruction.\n");
747 for (auto MO : I->uses()) {
748 if (!MO.isReg())
749 continue;
750 Register Reg = MO.getReg();
751 if (Reg == 0 || !Reg.isVirtual())
752 continue;
753 // Find the toc address.
754 MachineInstr *DefMI = MRI->getVRegDef(Reg);
755 for (auto MO2 : DefMI->uses())
756 if (MO2.isCPI())
757 return (MCP->getConstants())[MO2.getIndex()].Val.ConstVal;
758 }
759 return nullptr;
760}
761
764 bool DoRegPressureReduce) const {
765 // Using the machine combiner in this way is potentially expensive, so
766 // restrict to when aggressive optimizations are desired.
768 return false;
769
770 if (getFMAPatterns(Root, Patterns, DoRegPressureReduce))
771 return true;
772
774 DoRegPressureReduce);
775}
776
781 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
782 switch (Pattern) {
787 reassociateFMA(Root, Pattern, InsInstrs, DelInstrs, InstrIdxForVirtReg);
788 break;
789 default:
790 // Reassociate default patterns.
792 DelInstrs, InstrIdxForVirtReg);
793 break;
794 }
795}
796
797void PPCInstrInfo::reassociateFMA(
801 DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
802 MachineFunction *MF = Root.getMF();
805 MachineOperand &OpC = Root.getOperand(0);
806 Register RegC = OpC.getReg();
807 const TargetRegisterClass *RC = MRI.getRegClass(RegC);
808 MRI.constrainRegClass(RegC, RC);
809
810 unsigned FmaOp = Root.getOpcode();
811 int16_t Idx = getFMAOpIdxInfo(FmaOp);
812 assert(Idx >= 0 && "Root must be a FMA instruction");
813
814 bool IsILPReassociate =
817
820
821 MachineInstr *Prev = nullptr;
822 MachineInstr *Leaf = nullptr;
823 switch (Pattern) {
824 default:
825 llvm_unreachable("not recognized pattern!");
828 Prev = MRI.getUniqueVRegDef(Root.getOperand(AddOpIdx).getReg());
829 Leaf = MRI.getUniqueVRegDef(Prev->getOperand(AddOpIdx).getReg());
830 break;
832 Register MULReg =
833 TRI->lookThruCopyLike(Root.getOperand(FirstMulOpIdx).getReg(), &MRI);
834 Leaf = MRI.getVRegDef(MULReg);
835 break;
836 }
838 Register MULReg = TRI->lookThruCopyLike(
839 Root.getOperand(FirstMulOpIdx + 1).getReg(), &MRI);
840 Leaf = MRI.getVRegDef(MULReg);
841 break;
842 }
843 }
844
845 uint32_t IntersectedFlags = 0;
846 if (IsILPReassociate)
847 IntersectedFlags = Root.getFlags() & Prev->getFlags() & Leaf->getFlags();
848 else
849 IntersectedFlags = Root.getFlags() & Leaf->getFlags();
850
851 auto GetOperandInfo = [&](const MachineOperand &Operand, Register &Reg,
852 bool &KillFlag) {
853 Reg = Operand.getReg();
854 MRI.constrainRegClass(Reg, RC);
855 KillFlag = Operand.isKill();
856 };
857
858 auto GetFMAInstrInfo = [&](const MachineInstr &Instr, Register &MulOp1,
859 Register &MulOp2, Register &AddOp,
860 bool &MulOp1KillFlag, bool &MulOp2KillFlag,
861 bool &AddOpKillFlag) {
862 GetOperandInfo(Instr.getOperand(FirstMulOpIdx), MulOp1, MulOp1KillFlag);
863 GetOperandInfo(Instr.getOperand(FirstMulOpIdx + 1), MulOp2, MulOp2KillFlag);
864 GetOperandInfo(Instr.getOperand(AddOpIdx), AddOp, AddOpKillFlag);
865 };
866
867 Register RegM11, RegM12, RegX, RegY, RegM21, RegM22, RegM31, RegM32, RegA11,
868 RegA21, RegB;
869 bool KillX = false, KillY = false, KillM11 = false, KillM12 = false,
870 KillM21 = false, KillM22 = false, KillM31 = false, KillM32 = false,
871 KillA11 = false, KillA21 = false, KillB = false;
872
873 GetFMAInstrInfo(Root, RegM31, RegM32, RegB, KillM31, KillM32, KillB);
874
875 if (IsILPReassociate)
876 GetFMAInstrInfo(*Prev, RegM21, RegM22, RegA21, KillM21, KillM22, KillA21);
877
879 GetFMAInstrInfo(*Leaf, RegM11, RegM12, RegA11, KillM11, KillM12, KillA11);
880 GetOperandInfo(Leaf->getOperand(AddOpIdx), RegX, KillX);
882 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
883 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
884 } else {
885 // Get FSUB instruction info.
886 GetOperandInfo(Leaf->getOperand(1), RegX, KillX);
887 GetOperandInfo(Leaf->getOperand(2), RegY, KillY);
888 }
889
890 // Create new virtual registers for the new results instead of
891 // recycling legacy ones because the MachineCombiner's computation of the
892 // critical path requires a new register definition rather than an existing
893 // one.
894 // For register pressure reassociation, we only need create one virtual
895 // register for the new fma.
896 Register NewVRA = MRI.createVirtualRegister(RC);
897 InstrIdxForVirtReg.insert(std::make_pair(NewVRA, 0));
898
899 Register NewVRB = 0;
900 if (IsILPReassociate) {
901 NewVRB = MRI.createVirtualRegister(RC);
902 InstrIdxForVirtReg.insert(std::make_pair(NewVRB, 1));
903 }
904
905 Register NewVRD = 0;
907 NewVRD = MRI.createVirtualRegister(RC);
908 InstrIdxForVirtReg.insert(std::make_pair(NewVRD, 2));
909 }
910
911 auto AdjustOperandOrder = [&](MachineInstr *MI, Register RegAdd, bool KillAdd,
912 Register RegMul1, bool KillRegMul1,
913 Register RegMul2, bool KillRegMul2) {
914 MI->getOperand(AddOpIdx).setReg(RegAdd);
915 MI->getOperand(AddOpIdx).setIsKill(KillAdd);
916 MI->getOperand(FirstMulOpIdx).setReg(RegMul1);
917 MI->getOperand(FirstMulOpIdx).setIsKill(KillRegMul1);
918 MI->getOperand(FirstMulOpIdx + 1).setReg(RegMul2);
919 MI->getOperand(FirstMulOpIdx + 1).setIsKill(KillRegMul2);
920 };
921
922 MachineInstrBuilder NewARegPressure, NewCRegPressure;
923 switch (Pattern) {
924 default:
925 llvm_unreachable("not recognized pattern!");
927 // Create new instructions for insertion.
928 MachineInstrBuilder MINewB =
929 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
930 .addReg(RegX, getKillRegState(KillX))
931 .addReg(RegM21, getKillRegState(KillM21))
932 .addReg(RegM22, getKillRegState(KillM22));
933 MachineInstrBuilder MINewA =
934 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
935 .addReg(RegY, getKillRegState(KillY))
936 .addReg(RegM31, getKillRegState(KillM31))
937 .addReg(RegM32, getKillRegState(KillM32));
938 // If AddOpIdx is not 1, adjust the order.
939 if (AddOpIdx != 1) {
940 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
941 AdjustOperandOrder(MINewA, RegY, KillY, RegM31, KillM31, RegM32, KillM32);
942 }
943
944 MachineInstrBuilder MINewC =
945 BuildMI(*MF, Root.getDebugLoc(),
947 .addReg(NewVRB, getKillRegState(true))
948 .addReg(NewVRA, getKillRegState(true));
949
950 // Update flags for newly created instructions.
951 setSpecialOperandAttr(*MINewA, IntersectedFlags);
952 setSpecialOperandAttr(*MINewB, IntersectedFlags);
953 setSpecialOperandAttr(*MINewC, IntersectedFlags);
954
955 // Record new instructions for insertion.
956 InsInstrs.push_back(MINewA);
957 InsInstrs.push_back(MINewB);
958 InsInstrs.push_back(MINewC);
959 break;
960 }
962 assert(NewVRD && "new FMA register not created!");
963 // Create new instructions for insertion.
964 MachineInstrBuilder MINewA =
965 BuildMI(*MF, Leaf->getDebugLoc(),
967 .addReg(RegM11, getKillRegState(KillM11))
968 .addReg(RegM12, getKillRegState(KillM12));
969 MachineInstrBuilder MINewB =
970 BuildMI(*MF, Prev->getDebugLoc(), get(FmaOp), NewVRB)
971 .addReg(RegX, getKillRegState(KillX))
972 .addReg(RegM21, getKillRegState(KillM21))
973 .addReg(RegM22, getKillRegState(KillM22));
974 MachineInstrBuilder MINewD =
975 BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRD)
976 .addReg(NewVRA, getKillRegState(true))
977 .addReg(RegM31, getKillRegState(KillM31))
978 .addReg(RegM32, getKillRegState(KillM32));
979 // If AddOpIdx is not 1, adjust the order.
980 if (AddOpIdx != 1) {
981 AdjustOperandOrder(MINewB, RegX, KillX, RegM21, KillM21, RegM22, KillM22);
982 AdjustOperandOrder(MINewD, NewVRA, true, RegM31, KillM31, RegM32,
983 KillM32);
984 }
985
986 MachineInstrBuilder MINewC =
987 BuildMI(*MF, Root.getDebugLoc(),
989 .addReg(NewVRB, getKillRegState(true))
990 .addReg(NewVRD, getKillRegState(true));
991
992 // Update flags for newly created instructions.
993 setSpecialOperandAttr(*MINewA, IntersectedFlags);
994 setSpecialOperandAttr(*MINewB, IntersectedFlags);
995 setSpecialOperandAttr(*MINewD, IntersectedFlags);
996 setSpecialOperandAttr(*MINewC, IntersectedFlags);
997
998 // Record new instructions for insertion.
999 InsInstrs.push_back(MINewA);
1000 InsInstrs.push_back(MINewB);
1001 InsInstrs.push_back(MINewD);
1002 InsInstrs.push_back(MINewC);
1003 break;
1004 }
1007 Register VarReg;
1008 bool KillVarReg = false;
1010 VarReg = RegM31;
1011 KillVarReg = KillM31;
1012 } else {
1013 VarReg = RegM32;
1014 KillVarReg = KillM32;
1015 }
1016 // We don't want to get negative const from memory pool too early, as the
1017 // created entry will not be deleted even if it has no users. Since all
1018 // operand of Leaf and Root are virtual register, we use zero register
1019 // here as a placeholder. When the InsInstrs is selected in
1020 // MachineCombiner, we call finalizeInsInstrs to replace the zero register
1021 // with a virtual register which is a load from constant pool.
1022 NewARegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), NewVRA)
1023 .addReg(RegB, getKillRegState(RegB))
1024 .addReg(RegY, getKillRegState(KillY))
1025 .addReg(PPC::ZERO8);
1026 NewCRegPressure = BuildMI(*MF, Root.getDebugLoc(), get(FmaOp), RegC)
1027 .addReg(NewVRA, getKillRegState(true))
1028 .addReg(RegX, getKillRegState(KillX))
1029 .addReg(VarReg, getKillRegState(KillVarReg));
1030 // For now, we only support xsmaddadp/xsmaddasp, their add operand are
1031 // both at index 1, no need to adjust.
1032 // FIXME: when add more fma instructions support, like fma/fmas, adjust
1033 // the operand index here.
1034 break;
1035 }
1036 }
1037
1038 if (!IsILPReassociate) {
1039 setSpecialOperandAttr(*NewARegPressure, IntersectedFlags);
1040 setSpecialOperandAttr(*NewCRegPressure, IntersectedFlags);
1041
1042 InsInstrs.push_back(NewARegPressure);
1043 InsInstrs.push_back(NewCRegPressure);
1044 }
1045
1046 assert(!InsInstrs.empty() &&
1047 "Insertion instructions set should not be empty!");
1048
1049 // Record old instructions for deletion.
1050 DelInstrs.push_back(Leaf);
1051 if (IsILPReassociate)
1052 DelInstrs.push_back(Prev);
1053 DelInstrs.push_back(&Root);
1054}
1055
1056// Detect 32 -> 64-bit extensions where we may reuse the low sub-register.
1058 Register &SrcReg, Register &DstReg,
1059 unsigned &SubIdx) const {
1060 switch (MI.getOpcode()) {
1061 default: return false;
1062 case PPC::EXTSW:
1063 case PPC::EXTSW_32:
1064 case PPC::EXTSW_32_64:
1065 SrcReg = MI.getOperand(1).getReg();
1066 DstReg = MI.getOperand(0).getReg();
1067 SubIdx = PPC::sub_32;
1068 return true;
1069 }
1070}
1071
1073 int &FrameIndex) const {
1074 if (llvm::is_contained(getLoadOpcodesForSpillArray(), MI.getOpcode())) {
1075 // Check for the operands added by addFrameReference (the immediate is the
1076 // offset which defaults to 0).
1077 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1078 MI.getOperand(2).isFI()) {
1079 FrameIndex = MI.getOperand(2).getIndex();
1080 return MI.getOperand(0).getReg();
1081 }
1082 }
1083 return 0;
1084}
1085
1086// For opcodes with the ReMaterializable flag set, this function is called to
1087// verify the instruction is really rematable.
1089 const MachineInstr &MI) const {
1090 switch (MI.getOpcode()) {
1091 default:
1092 // This function should only be called for opcodes with the ReMaterializable
1093 // flag set.
1094 llvm_unreachable("Unknown rematerializable operation!");
1095 break;
1096 case PPC::LI:
1097 case PPC::LI8:
1098 case PPC::PLI:
1099 case PPC::PLI8:
1100 case PPC::LIS:
1101 case PPC::LIS8:
1102 case PPC::ADDIStocHA:
1103 case PPC::ADDIStocHA8:
1104 case PPC::ADDItocL:
1105 case PPC::LOAD_STACK_GUARD:
1106 case PPC::XXLXORz:
1107 case PPC::XXLXORspz:
1108 case PPC::XXLXORdpz:
1109 case PPC::XXLEQVOnes:
1110 case PPC::XXSPLTI32DX:
1111 case PPC::XXSPLTIW:
1112 case PPC::XXSPLTIDP:
1113 case PPC::V_SET0B:
1114 case PPC::V_SET0H:
1115 case PPC::V_SET0:
1116 case PPC::V_SETALLONESB:
1117 case PPC::V_SETALLONESH:
1118 case PPC::V_SETALLONES:
1119 case PPC::CRSET:
1120 case PPC::CRUNSET:
1121 case PPC::XXSETACCZ:
1122 case PPC::XXSETACCZW:
1123 return true;
1124 }
1126}
1127
1129 int &FrameIndex) const {
1130 if (llvm::is_contained(getStoreOpcodesForSpillArray(), MI.getOpcode())) {
1131 if (MI.getOperand(1).isImm() && !MI.getOperand(1).getImm() &&
1132 MI.getOperand(2).isFI()) {
1133 FrameIndex = MI.getOperand(2).getIndex();
1134 return MI.getOperand(0).getReg();
1135 }
1136 }
1137 return 0;
1138}
1139
1141 unsigned OpIdx1,
1142 unsigned OpIdx2) const {
1143 MachineFunction &MF = *MI.getParent()->getParent();
1144
1145 // Normal instructions can be commuted the obvious way.
1146 if (MI.getOpcode() != PPC::RLWIMI && MI.getOpcode() != PPC::RLWIMI_rec)
1147 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
1148 // Note that RLWIMI can be commuted as a 32-bit instruction, but not as a
1149 // 64-bit instruction (so we don't handle PPC::RLWIMI8 here), because
1150 // changing the relative order of the mask operands might change what happens
1151 // to the high-bits of the mask (and, thus, the result).
1152
1153 // Cannot commute if it has a non-zero rotate count.
1154 if (MI.getOperand(3).getImm() != 0)
1155 return nullptr;
1156
1157 // If we have a zero rotate count, we have:
1158 // M = mask(MB,ME)
1159 // Op0 = (Op1 & ~M) | (Op2 & M)
1160 // Change this to:
1161 // M = mask((ME+1)&31, (MB-1)&31)
1162 // Op0 = (Op2 & ~M) | (Op1 & M)
1163
1164 // Swap op1/op2
1165 assert(((OpIdx1 == 1 && OpIdx2 == 2) || (OpIdx1 == 2 && OpIdx2 == 1)) &&
1166 "Only the operands 1 and 2 can be swapped in RLSIMI/RLWIMI_rec.");
1167 Register Reg0 = MI.getOperand(0).getReg();
1168 Register Reg1 = MI.getOperand(1).getReg();
1169 Register Reg2 = MI.getOperand(2).getReg();
1170 unsigned SubReg1 = MI.getOperand(1).getSubReg();
1171 unsigned SubReg2 = MI.getOperand(2).getSubReg();
1172 bool Reg1IsKill = MI.getOperand(1).isKill();
1173 bool Reg2IsKill = MI.getOperand(2).isKill();
1174 bool ChangeReg0 = false;
1175 // If machine instrs are no longer in two-address forms, update
1176 // destination register as well.
1177 if (Reg0 == Reg1) {
1178 // Must be two address instruction (i.e. op1 is tied to op0).
1179 assert(MI.getDesc().getOperandConstraint(1, MCOI::TIED_TO) == 0 &&
1180 "Expecting a two-address instruction!");
1181 assert(MI.getOperand(0).getSubReg() == SubReg1 && "Tied subreg mismatch");
1182 Reg2IsKill = false;
1183 ChangeReg0 = true;
1184 }
1185
1186 // Masks.
1187 unsigned MB = MI.getOperand(4).getImm();
1188 unsigned ME = MI.getOperand(5).getImm();
1189
1190 // We can't commute a trivial mask (there is no way to represent an all-zero
1191 // mask).
1192 if (MB == 0 && ME == 31)
1193 return nullptr;
1194
1195 if (NewMI) {
1196 // Create a new instruction.
1197 Register Reg0 = ChangeReg0 ? Reg2 : MI.getOperand(0).getReg();
1198 bool Reg0IsDead = MI.getOperand(0).isDead();
1199 return BuildMI(MF, MI.getDebugLoc(), MI.getDesc())
1200 .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead))
1201 .addReg(Reg2, getKillRegState(Reg2IsKill))
1202 .addReg(Reg1, getKillRegState(Reg1IsKill))
1203 .addImm((ME + 1) & 31)
1204 .addImm((MB - 1) & 31);
1205 }
1206
1207 if (ChangeReg0) {
1208 MI.getOperand(0).setReg(Reg2);
1209 MI.getOperand(0).setSubReg(SubReg2);
1210 }
1211 MI.getOperand(2).setReg(Reg1);
1212 MI.getOperand(1).setReg(Reg2);
1213 MI.getOperand(2).setSubReg(SubReg1);
1214 MI.getOperand(1).setSubReg(SubReg2);
1215 MI.getOperand(2).setIsKill(Reg1IsKill);
1216 MI.getOperand(1).setIsKill(Reg2IsKill);
1217
1218 // Swap the mask around.
1219 MI.getOperand(4).setImm((ME + 1) & 31);
1220 MI.getOperand(5).setImm((MB - 1) & 31);
1221 return &MI;
1222}
1223
1225 unsigned &SrcOpIdx1,
1226 unsigned &SrcOpIdx2) const {
1227 // For VSX A-Type FMA instructions, it is the first two operands that can be
1228 // commuted, however, because the non-encoded tied input operand is listed
1229 // first, the operands to swap are actually the second and third.
1230
1231 int AltOpc = PPC::getAltVSXFMAOpcode(MI.getOpcode());
1232 if (AltOpc == -1)
1233 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
1234
1235 // The commutable operand indices are 2 and 3. Return them in SrcOpIdx1
1236 // and SrcOpIdx2.
1237 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 2, 3);
1238}
1239
1242 // This function is used for scheduling, and the nop wanted here is the type
1243 // that terminates dispatch groups on the POWER cores.
1244 unsigned Directive = Subtarget.getCPUDirective();
1245 unsigned Opcode;
1246 switch (Directive) {
1247 default: Opcode = PPC::NOP; break;
1248 case PPC::DIR_PWR6: Opcode = PPC::NOP_GT_PWR6; break;
1249 case PPC::DIR_PWR7: Opcode = PPC::NOP_GT_PWR7; break;
1250 case PPC::DIR_PWR8: Opcode = PPC::NOP_GT_PWR7; break; /* FIXME: Update when P8 InstrScheduling model is ready */
1251 // FIXME: Update when POWER9 scheduling model is ready.
1252 case PPC::DIR_PWR9: Opcode = PPC::NOP_GT_PWR7; break;
1253 }
1254
1255 DebugLoc DL;
1256 BuildMI(MBB, MI, DL, get(Opcode));
1257}
1258
1259/// Return the noop instruction to use for a noop.
1261 MCInst Nop;
1262 Nop.setOpcode(PPC::NOP);
1263 return Nop;
1264}
1265
1266// Branch analysis.
1267// Note: If the condition register is set to CTR or CTR8 then this is a
1268// BDNZ (imm == 1) or BDZ (imm == 0) branch.
1271 MachineBasicBlock *&FBB,
1273 bool AllowModify) const {
1274 bool isPPC64 = Subtarget.isPPC64();
1275
1276 // If the block has no terminators, it just falls into the block after it.
1278 if (I == MBB.end())
1279 return false;
1280
1281 if (!isUnpredicatedTerminator(*I))
1282 return false;
1283
1284 if (AllowModify) {
1285 // If the BB ends with an unconditional branch to the fallthrough BB,
1286 // we eliminate the branch instruction.
1287 if (I->getOpcode() == PPC::B &&
1288 MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
1289 I->eraseFromParent();
1290
1291 // We update iterator after deleting the last branch.
1293 if (I == MBB.end() || !isUnpredicatedTerminator(*I))
1294 return false;
1295 }
1296 }
1297
1298 // Get the last instruction in the block.
1299 MachineInstr &LastInst = *I;
1300
1301 // If there is only one terminator instruction, process it.
1302 if (I == MBB.begin() || !isUnpredicatedTerminator(*--I)) {
1303 if (LastInst.getOpcode() == PPC::B) {
1304 if (!LastInst.getOperand(0).isMBB())
1305 return true;
1306 TBB = LastInst.getOperand(0).getMBB();
1307 return false;
1308 } else if (LastInst.getOpcode() == PPC::BCC) {
1309 if (!LastInst.getOperand(2).isMBB())
1310 return true;
1311 // Block ends with fall-through condbranch.
1312 TBB = LastInst.getOperand(2).getMBB();
1313 Cond.push_back(LastInst.getOperand(0));
1314 Cond.push_back(LastInst.getOperand(1));
1315 return false;
1316 } else if (LastInst.getOpcode() == PPC::BC) {
1317 if (!LastInst.getOperand(1).isMBB())
1318 return true;
1319 // Block ends with fall-through condbranch.
1320 TBB = LastInst.getOperand(1).getMBB();
1322 Cond.push_back(LastInst.getOperand(0));
1323 return false;
1324 } else if (LastInst.getOpcode() == PPC::BCn) {
1325 if (!LastInst.getOperand(1).isMBB())
1326 return true;
1327 // Block ends with fall-through condbranch.
1328 TBB = LastInst.getOperand(1).getMBB();
1330 Cond.push_back(LastInst.getOperand(0));
1331 return false;
1332 } else if (LastInst.getOpcode() == PPC::BDNZ8 ||
1333 LastInst.getOpcode() == PPC::BDNZ) {
1334 if (!LastInst.getOperand(0).isMBB())
1335 return true;
1337 return true;
1338 TBB = LastInst.getOperand(0).getMBB();
1339 Cond.push_back(MachineOperand::CreateImm(1));
1340 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1341 true));
1342 return false;
1343 } else if (LastInst.getOpcode() == PPC::BDZ8 ||
1344 LastInst.getOpcode() == PPC::BDZ) {
1345 if (!LastInst.getOperand(0).isMBB())
1346 return true;
1348 return true;
1349 TBB = LastInst.getOperand(0).getMBB();
1350 Cond.push_back(MachineOperand::CreateImm(0));
1351 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1352 true));
1353 return false;
1354 }
1355
1356 // Otherwise, don't know what this is.
1357 return true;
1358 }
1359
1360 // Get the instruction before it if it's a terminator.
1361 MachineInstr &SecondLastInst = *I;
1362
1363 // If there are three terminators, we don't know what sort of block this is.
1364 if (I != MBB.begin() && isUnpredicatedTerminator(*--I))
1365 return true;
1366
1367 // If the block ends with PPC::B and PPC:BCC, handle it.
1368 if (SecondLastInst.getOpcode() == PPC::BCC &&
1369 LastInst.getOpcode() == PPC::B) {
1370 if (!SecondLastInst.getOperand(2).isMBB() ||
1371 !LastInst.getOperand(0).isMBB())
1372 return true;
1373 TBB = SecondLastInst.getOperand(2).getMBB();
1374 Cond.push_back(SecondLastInst.getOperand(0));
1375 Cond.push_back(SecondLastInst.getOperand(1));
1376 FBB = LastInst.getOperand(0).getMBB();
1377 return false;
1378 } else if (SecondLastInst.getOpcode() == PPC::BC &&
1379 LastInst.getOpcode() == PPC::B) {
1380 if (!SecondLastInst.getOperand(1).isMBB() ||
1381 !LastInst.getOperand(0).isMBB())
1382 return true;
1383 TBB = SecondLastInst.getOperand(1).getMBB();
1385 Cond.push_back(SecondLastInst.getOperand(0));
1386 FBB = LastInst.getOperand(0).getMBB();
1387 return false;
1388 } else if (SecondLastInst.getOpcode() == PPC::BCn &&
1389 LastInst.getOpcode() == PPC::B) {
1390 if (!SecondLastInst.getOperand(1).isMBB() ||
1391 !LastInst.getOperand(0).isMBB())
1392 return true;
1393 TBB = SecondLastInst.getOperand(1).getMBB();
1395 Cond.push_back(SecondLastInst.getOperand(0));
1396 FBB = LastInst.getOperand(0).getMBB();
1397 return false;
1398 } else if ((SecondLastInst.getOpcode() == PPC::BDNZ8 ||
1399 SecondLastInst.getOpcode() == PPC::BDNZ) &&
1400 LastInst.getOpcode() == PPC::B) {
1401 if (!SecondLastInst.getOperand(0).isMBB() ||
1402 !LastInst.getOperand(0).isMBB())
1403 return true;
1405 return true;
1406 TBB = SecondLastInst.getOperand(0).getMBB();
1407 Cond.push_back(MachineOperand::CreateImm(1));
1408 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1409 true));
1410 FBB = LastInst.getOperand(0).getMBB();
1411 return false;
1412 } else if ((SecondLastInst.getOpcode() == PPC::BDZ8 ||
1413 SecondLastInst.getOpcode() == PPC::BDZ) &&
1414 LastInst.getOpcode() == PPC::B) {
1415 if (!SecondLastInst.getOperand(0).isMBB() ||
1416 !LastInst.getOperand(0).isMBB())
1417 return true;
1419 return true;
1420 TBB = SecondLastInst.getOperand(0).getMBB();
1421 Cond.push_back(MachineOperand::CreateImm(0));
1422 Cond.push_back(MachineOperand::CreateReg(isPPC64 ? PPC::CTR8 : PPC::CTR,
1423 true));
1424 FBB = LastInst.getOperand(0).getMBB();
1425 return false;
1426 }
1427
1428 // If the block ends with two PPC:Bs, handle it. The second one is not
1429 // executed, so remove it.
1430 if (SecondLastInst.getOpcode() == PPC::B && LastInst.getOpcode() == PPC::B) {
1431 if (!SecondLastInst.getOperand(0).isMBB())
1432 return true;
1433 TBB = SecondLastInst.getOperand(0).getMBB();
1434 I = LastInst;
1435 if (AllowModify)
1436 I->eraseFromParent();
1437 return false;
1438 }
1439
1440 // Otherwise, can't handle this.
1441 return true;
1442}
1443
1445 int *BytesRemoved) const {
1446 assert(!BytesRemoved && "code size not handled");
1447
1449 if (I == MBB.end())
1450 return 0;
1451
1452 if (I->getOpcode() != PPC::B && I->getOpcode() != PPC::BCC &&
1453 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1454 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1455 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1456 return 0;
1457
1458 // Remove the branch.
1459 I->eraseFromParent();
1460
1461 I = MBB.end();
1462
1463 if (I == MBB.begin()) return 1;
1464 --I;
1465 if (I->getOpcode() != PPC::BCC &&
1466 I->getOpcode() != PPC::BC && I->getOpcode() != PPC::BCn &&
1467 I->getOpcode() != PPC::BDNZ8 && I->getOpcode() != PPC::BDNZ &&
1468 I->getOpcode() != PPC::BDZ8 && I->getOpcode() != PPC::BDZ)
1469 return 1;
1470
1471 // Remove the branch.
1472 I->eraseFromParent();
1473 return 2;
1474}
1475
1478 MachineBasicBlock *FBB,
1480 const DebugLoc &DL,
1481 int *BytesAdded) const {
1482 // Shouldn't be a fall through.
1483 assert(TBB && "insertBranch must not be told to insert a fallthrough");
1484 assert((Cond.size() == 2 || Cond.size() == 0) &&
1485 "PPC branch conditions have two components!");
1486 assert(!BytesAdded && "code size not handled");
1487
1488 bool isPPC64 = Subtarget.isPPC64();
1489
1490 // One-way branch.
1491 if (!FBB) {
1492 if (Cond.empty()) // Unconditional branch
1493 BuildMI(&MBB, DL, get(PPC::B)).addMBB(TBB);
1494 else if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1495 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1496 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1497 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1498 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1499 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1500 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1501 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1502 else // Conditional branch
1503 BuildMI(&MBB, DL, get(PPC::BCC))
1504 .addImm(Cond[0].getImm())
1505 .add(Cond[1])
1506 .addMBB(TBB);
1507 return 1;
1508 }
1509
1510 // Two-way Conditional Branch.
1511 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1512 BuildMI(&MBB, DL, get(Cond[0].getImm() ?
1513 (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ) :
1514 (isPPC64 ? PPC::BDZ8 : PPC::BDZ))).addMBB(TBB);
1515 else if (Cond[0].getImm() == PPC::PRED_BIT_SET)
1516 BuildMI(&MBB, DL, get(PPC::BC)).add(Cond[1]).addMBB(TBB);
1517 else if (Cond[0].getImm() == PPC::PRED_BIT_UNSET)
1518 BuildMI(&MBB, DL, get(PPC::BCn)).add(Cond[1]).addMBB(TBB);
1519 else
1520 BuildMI(&MBB, DL, get(PPC::BCC))
1521 .addImm(Cond[0].getImm())
1522 .add(Cond[1])
1523 .addMBB(TBB);
1524 BuildMI(&MBB, DL, get(PPC::B)).addMBB(FBB);
1525 return 2;
1526}
1527
1528// Select analysis.
1531 Register DstReg, Register TrueReg,
1532 Register FalseReg, int &CondCycles,
1533 int &TrueCycles, int &FalseCycles) const {
1534 if (Cond.size() != 2)
1535 return false;
1536
1537 // If this is really a bdnz-like condition, then it cannot be turned into a
1538 // select.
1539 if (Cond[1].getReg() == PPC::CTR || Cond[1].getReg() == PPC::CTR8)
1540 return false;
1541
1542 // If the conditional branch uses a physical register, then it cannot be
1543 // turned into a select.
1544 if (Cond[1].getReg().isPhysical())
1545 return false;
1546
1547 // Check register classes.
1549 const TargetRegisterClass *RC =
1550 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1551 if (!RC)
1552 return false;
1553
1554 // isel is for regular integer GPRs only.
1555 if (!PPC::GPRCRegClass.hasSubClassEq(RC) &&
1556 !PPC::GPRC_NOR0RegClass.hasSubClassEq(RC) &&
1557 !PPC::G8RCRegClass.hasSubClassEq(RC) &&
1558 !PPC::G8RC_NOX0RegClass.hasSubClassEq(RC))
1559 return false;
1560
1561 // FIXME: These numbers are for the A2, how well they work for other cores is
1562 // an open question. On the A2, the isel instruction has a 2-cycle latency
1563 // but single-cycle throughput. These numbers are used in combination with
1564 // the MispredictPenalty setting from the active SchedMachineModel.
1565 CondCycles = 1;
1566 TrueCycles = 1;
1567 FalseCycles = 1;
1568
1569 return true;
1570}
1571
1574 const DebugLoc &dl, Register DestReg,
1576 Register FalseReg) const {
1577 assert(Cond.size() == 2 &&
1578 "PPC branch conditions have two components!");
1579
1580 // Get the register classes.
1582 const TargetRegisterClass *RC =
1583 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
1584 assert(RC && "TrueReg and FalseReg must have overlapping register classes");
1585
1586 bool Is64Bit = PPC::G8RCRegClass.hasSubClassEq(RC) ||
1587 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC);
1588 assert((Is64Bit ||
1589 PPC::GPRCRegClass.hasSubClassEq(RC) ||
1590 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) &&
1591 "isel is for regular integer GPRs only");
1592
1593 unsigned OpCode = Is64Bit ? PPC::ISEL8 : PPC::ISEL;
1594 auto SelectPred = static_cast<PPC::Predicate>(Cond[0].getImm());
1595
1596 unsigned SubIdx = 0;
1597 bool SwapOps = false;
1598 switch (SelectPred) {
1599 case PPC::PRED_EQ:
1600 case PPC::PRED_EQ_MINUS:
1601 case PPC::PRED_EQ_PLUS:
1602 SubIdx = PPC::sub_eq; SwapOps = false; break;
1603 case PPC::PRED_NE:
1604 case PPC::PRED_NE_MINUS:
1605 case PPC::PRED_NE_PLUS:
1606 SubIdx = PPC::sub_eq; SwapOps = true; break;
1607 case PPC::PRED_LT:
1608 case PPC::PRED_LT_MINUS:
1609 case PPC::PRED_LT_PLUS:
1610 SubIdx = PPC::sub_lt; SwapOps = false; break;
1611 case PPC::PRED_GE:
1612 case PPC::PRED_GE_MINUS:
1613 case PPC::PRED_GE_PLUS:
1614 SubIdx = PPC::sub_lt; SwapOps = true; break;
1615 case PPC::PRED_GT:
1616 case PPC::PRED_GT_MINUS:
1617 case PPC::PRED_GT_PLUS:
1618 SubIdx = PPC::sub_gt; SwapOps = false; break;
1619 case PPC::PRED_LE:
1620 case PPC::PRED_LE_MINUS:
1621 case PPC::PRED_LE_PLUS:
1622 SubIdx = PPC::sub_gt; SwapOps = true; break;
1623 case PPC::PRED_UN:
1624 case PPC::PRED_UN_MINUS:
1625 case PPC::PRED_UN_PLUS:
1626 SubIdx = PPC::sub_un; SwapOps = false; break;
1627 case PPC::PRED_NU:
1628 case PPC::PRED_NU_MINUS:
1629 case PPC::PRED_NU_PLUS:
1630 SubIdx = PPC::sub_un; SwapOps = true; break;
1631 case PPC::PRED_BIT_SET: SubIdx = 0; SwapOps = false; break;
1632 case PPC::PRED_BIT_UNSET: SubIdx = 0; SwapOps = true; break;
1633 }
1634
1635 Register FirstReg = SwapOps ? FalseReg : TrueReg,
1636 SecondReg = SwapOps ? TrueReg : FalseReg;
1637
1638 // The first input register of isel cannot be r0. If it is a member
1639 // of a register class that can be r0, then copy it first (the
1640 // register allocator should eliminate the copy).
1641 if (MRI.getRegClass(FirstReg)->contains(PPC::R0) ||
1642 MRI.getRegClass(FirstReg)->contains(PPC::X0)) {
1643 const TargetRegisterClass *FirstRC =
1644 MRI.getRegClass(FirstReg)->contains(PPC::X0) ?
1645 &PPC::G8RC_NOX0RegClass : &PPC::GPRC_NOR0RegClass;
1646 Register OldFirstReg = FirstReg;
1647 FirstReg = MRI.createVirtualRegister(FirstRC);
1648 BuildMI(MBB, MI, dl, get(TargetOpcode::COPY), FirstReg)
1649 .addReg(OldFirstReg);
1650 }
1651
1652 BuildMI(MBB, MI, dl, get(OpCode), DestReg)
1653 .addReg(FirstReg).addReg(SecondReg)
1654 .addReg(Cond[1].getReg(), 0, SubIdx);
1655}
1656
1657static unsigned getCRBitValue(unsigned CRBit) {
1658 unsigned Ret = 4;
1659 if (CRBit == PPC::CR0LT || CRBit == PPC::CR1LT ||
1660 CRBit == PPC::CR2LT || CRBit == PPC::CR3LT ||
1661 CRBit == PPC::CR4LT || CRBit == PPC::CR5LT ||
1662 CRBit == PPC::CR6LT || CRBit == PPC::CR7LT)
1663 Ret = 3;
1664 if (CRBit == PPC::CR0GT || CRBit == PPC::CR1GT ||
1665 CRBit == PPC::CR2GT || CRBit == PPC::CR3GT ||
1666 CRBit == PPC::CR4GT || CRBit == PPC::CR5GT ||
1667 CRBit == PPC::CR6GT || CRBit == PPC::CR7GT)
1668 Ret = 2;
1669 if (CRBit == PPC::CR0EQ || CRBit == PPC::CR1EQ ||
1670 CRBit == PPC::CR2EQ || CRBit == PPC::CR3EQ ||
1671 CRBit == PPC::CR4EQ || CRBit == PPC::CR5EQ ||
1672 CRBit == PPC::CR6EQ || CRBit == PPC::CR7EQ)
1673 Ret = 1;
1674 if (CRBit == PPC::CR0UN || CRBit == PPC::CR1UN ||
1675 CRBit == PPC::CR2UN || CRBit == PPC::CR3UN ||
1676 CRBit == PPC::CR4UN || CRBit == PPC::CR5UN ||
1677 CRBit == PPC::CR6UN || CRBit == PPC::CR7UN)
1678 Ret = 0;
1679
1680 assert(Ret != 4 && "Invalid CR bit register");
1681 return Ret;
1682}
1683
1686 const DebugLoc &DL, MCRegister DestReg,
1687 MCRegister SrcReg, bool KillSrc) const {
1688 // We can end up with self copies and similar things as a result of VSX copy
1689 // legalization. Promote them here.
1691 if (PPC::F8RCRegClass.contains(DestReg) &&
1692 PPC::VSRCRegClass.contains(SrcReg)) {
1693 MCRegister SuperReg =
1694 TRI->getMatchingSuperReg(DestReg, PPC::sub_64, &PPC::VSRCRegClass);
1695
1696 if (VSXSelfCopyCrash && SrcReg == SuperReg)
1697 llvm_unreachable("nop VSX copy");
1698
1699 DestReg = SuperReg;
1700 } else if (PPC::F8RCRegClass.contains(SrcReg) &&
1701 PPC::VSRCRegClass.contains(DestReg)) {
1702 MCRegister SuperReg =
1703 TRI->getMatchingSuperReg(SrcReg, PPC::sub_64, &PPC::VSRCRegClass);
1704
1705 if (VSXSelfCopyCrash && DestReg == SuperReg)
1706 llvm_unreachable("nop VSX copy");
1707
1708 SrcReg = SuperReg;
1709 }
1710
1711 // Different class register copy
1712 if (PPC::CRBITRCRegClass.contains(SrcReg) &&
1713 PPC::GPRCRegClass.contains(DestReg)) {
1714 MCRegister CRReg = getCRFromCRBit(SrcReg);
1715 BuildMI(MBB, I, DL, get(PPC::MFOCRF), DestReg).addReg(CRReg);
1716 getKillRegState(KillSrc);
1717 // Rotate the CR bit in the CR fields to be the least significant bit and
1718 // then mask with 0x1 (MB = ME = 31).
1719 BuildMI(MBB, I, DL, get(PPC::RLWINM), DestReg)
1720 .addReg(DestReg, RegState::Kill)
1721 .addImm(TRI->getEncodingValue(CRReg) * 4 + (4 - getCRBitValue(SrcReg)))
1722 .addImm(31)
1723 .addImm(31);
1724 return;
1725 } else if (PPC::CRRCRegClass.contains(SrcReg) &&
1726 (PPC::G8RCRegClass.contains(DestReg) ||
1727 PPC::GPRCRegClass.contains(DestReg))) {
1728 bool Is64Bit = PPC::G8RCRegClass.contains(DestReg);
1729 unsigned MvCode = Is64Bit ? PPC::MFOCRF8 : PPC::MFOCRF;
1730 unsigned ShCode = Is64Bit ? PPC::RLWINM8 : PPC::RLWINM;
1731 unsigned CRNum = TRI->getEncodingValue(SrcReg);
1732 BuildMI(MBB, I, DL, get(MvCode), DestReg).addReg(SrcReg);
1733 getKillRegState(KillSrc);
1734 if (CRNum == 7)
1735 return;
1736 // Shift the CR bits to make the CR field in the lowest 4 bits of GRC.
1737 BuildMI(MBB, I, DL, get(ShCode), DestReg)
1738 .addReg(DestReg, RegState::Kill)
1739 .addImm(CRNum * 4 + 4)
1740 .addImm(28)
1741 .addImm(31);
1742 return;
1743 } else if (PPC::G8RCRegClass.contains(SrcReg) &&
1744 PPC::VSFRCRegClass.contains(DestReg)) {
1745 assert(Subtarget.hasDirectMove() &&
1746 "Subtarget doesn't support directmove, don't know how to copy.");
1747 BuildMI(MBB, I, DL, get(PPC::MTVSRD), DestReg).addReg(SrcReg);
1748 NumGPRtoVSRSpill++;
1749 getKillRegState(KillSrc);
1750 return;
1751 } else if (PPC::VSFRCRegClass.contains(SrcReg) &&
1752 PPC::G8RCRegClass.contains(DestReg)) {
1753 assert(Subtarget.hasDirectMove() &&
1754 "Subtarget doesn't support directmove, don't know how to copy.");
1755 BuildMI(MBB, I, DL, get(PPC::MFVSRD), DestReg).addReg(SrcReg);
1756 getKillRegState(KillSrc);
1757 return;
1758 } else if (PPC::SPERCRegClass.contains(SrcReg) &&
1759 PPC::GPRCRegClass.contains(DestReg)) {
1760 BuildMI(MBB, I, DL, get(PPC::EFSCFD), DestReg).addReg(SrcReg);
1761 getKillRegState(KillSrc);
1762 return;
1763 } else if (PPC::GPRCRegClass.contains(SrcReg) &&
1764 PPC::SPERCRegClass.contains(DestReg)) {
1765 BuildMI(MBB, I, DL, get(PPC::EFDCFS), DestReg).addReg(SrcReg);
1766 getKillRegState(KillSrc);
1767 return;
1768 }
1769
1770 unsigned Opc;
1771 if (PPC::GPRCRegClass.contains(DestReg, SrcReg))
1772 Opc = PPC::OR;
1773 else if (PPC::G8RCRegClass.contains(DestReg, SrcReg))
1774 Opc = PPC::OR8;
1775 else if (PPC::F4RCRegClass.contains(DestReg, SrcReg))
1776 Opc = PPC::FMR;
1777 else if (PPC::CRRCRegClass.contains(DestReg, SrcReg))
1778 Opc = PPC::MCRF;
1779 else if (PPC::VRRCRegClass.contains(DestReg, SrcReg))
1780 Opc = PPC::VOR;
1781 else if (PPC::VSRCRegClass.contains(DestReg, SrcReg))
1782 // There are two different ways this can be done:
1783 // 1. xxlor : This has lower latency (on the P7), 2 cycles, but can only
1784 // issue in VSU pipeline 0.
1785 // 2. xmovdp/xmovsp: This has higher latency (on the P7), 6 cycles, but
1786 // can go to either pipeline.
1787 // We'll always use xxlor here, because in practically all cases where
1788 // copies are generated, they are close enough to some use that the
1789 // lower-latency form is preferable.
1790 Opc = PPC::XXLOR;
1791 else if (PPC::VSFRCRegClass.contains(DestReg, SrcReg) ||
1792 PPC::VSSRCRegClass.contains(DestReg, SrcReg))
1793 Opc = (Subtarget.hasP9Vector()) ? PPC::XSCPSGNDP : PPC::XXLORf;
1794 else if (Subtarget.pairedVectorMemops() &&
1795 PPC::VSRpRCRegClass.contains(DestReg, SrcReg)) {
1796 if (SrcReg > PPC::VSRp15)
1797 SrcReg = PPC::V0 + (SrcReg - PPC::VSRp16) * 2;
1798 else
1799 SrcReg = PPC::VSL0 + (SrcReg - PPC::VSRp0) * 2;
1800 if (DestReg > PPC::VSRp15)
1801 DestReg = PPC::V0 + (DestReg - PPC::VSRp16) * 2;
1802 else
1803 DestReg = PPC::VSL0 + (DestReg - PPC::VSRp0) * 2;
1804 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg).
1805 addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1806 BuildMI(MBB, I, DL, get(PPC::XXLOR), DestReg + 1).
1807 addReg(SrcReg + 1).addReg(SrcReg + 1, getKillRegState(KillSrc));
1808 return;
1809 }
1810 else if (PPC::CRBITRCRegClass.contains(DestReg, SrcReg))
1811 Opc = PPC::CROR;
1812 else if (PPC::SPERCRegClass.contains(DestReg, SrcReg))
1813 Opc = PPC::EVOR;
1814 else if ((PPC::ACCRCRegClass.contains(DestReg) ||
1815 PPC::UACCRCRegClass.contains(DestReg)) &&
1816 (PPC::ACCRCRegClass.contains(SrcReg) ||
1817 PPC::UACCRCRegClass.contains(SrcReg))) {
1818 // If primed, de-prime the source register, copy the individual registers
1819 // and prime the destination if needed. The vector subregisters are
1820 // vs[(u)acc * 4] - vs[(u)acc * 4 + 3]. If the copy is not a kill and the
1821 // source is primed, we need to re-prime it after the copy as well.
1822 PPCRegisterInfo::emitAccCopyInfo(MBB, DestReg, SrcReg);
1823 bool DestPrimed = PPC::ACCRCRegClass.contains(DestReg);
1824 bool SrcPrimed = PPC::ACCRCRegClass.contains(SrcReg);
1825 MCRegister VSLSrcReg =
1826 PPC::VSL0 + (SrcReg - (SrcPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1827 MCRegister VSLDestReg =
1828 PPC::VSL0 + (DestReg - (DestPrimed ? PPC::ACC0 : PPC::UACC0)) * 4;
1829 if (SrcPrimed)
1830 BuildMI(MBB, I, DL, get(PPC::XXMFACC), SrcReg).addReg(SrcReg);
1831 for (unsigned Idx = 0; Idx < 4; Idx++)
1832 BuildMI(MBB, I, DL, get(PPC::XXLOR), VSLDestReg + Idx)
1833 .addReg(VSLSrcReg + Idx)
1834 .addReg(VSLSrcReg + Idx, getKillRegState(KillSrc));
1835 if (DestPrimed)
1836 BuildMI(MBB, I, DL, get(PPC::XXMTACC), DestReg).addReg(DestReg);
1837 if (SrcPrimed && !KillSrc)
1838 BuildMI(MBB, I, DL, get(PPC::XXMTACC), SrcReg).addReg(SrcReg);
1839 return;
1840 } else if (PPC::G8pRCRegClass.contains(DestReg) &&
1841 PPC::G8pRCRegClass.contains(SrcReg)) {
1842 // TODO: Handle G8RC to G8pRC (and vice versa) copy.
1843 unsigned DestRegIdx = DestReg - PPC::G8p0;
1844 MCRegister DestRegSub0 = PPC::X0 + 2 * DestRegIdx;
1845 MCRegister DestRegSub1 = PPC::X0 + 2 * DestRegIdx + 1;
1846 unsigned SrcRegIdx = SrcReg - PPC::G8p0;
1847 MCRegister SrcRegSub0 = PPC::X0 + 2 * SrcRegIdx;
1848 MCRegister SrcRegSub1 = PPC::X0 + 2 * SrcRegIdx + 1;
1849 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub0)
1850 .addReg(SrcRegSub0)
1851 .addReg(SrcRegSub0, getKillRegState(KillSrc));
1852 BuildMI(MBB, I, DL, get(PPC::OR8), DestRegSub1)
1853 .addReg(SrcRegSub1)
1854 .addReg(SrcRegSub1, getKillRegState(KillSrc));
1855 return;
1856 } else
1857 llvm_unreachable("Impossible reg-to-reg copy");
1858
1859 const MCInstrDesc &MCID = get(Opc);
1860 if (MCID.getNumOperands() == 3)
1861 BuildMI(MBB, I, DL, MCID, DestReg)
1862 .addReg(SrcReg).addReg(SrcReg, getKillRegState(KillSrc));
1863 else
1864 BuildMI(MBB, I, DL, MCID, DestReg).addReg(SrcReg, getKillRegState(KillSrc));
1865}
1866
1867unsigned PPCInstrInfo::getSpillIndex(const TargetRegisterClass *RC) const {
1868 int OpcodeIndex = 0;
1869
1870 if (PPC::GPRCRegClass.hasSubClassEq(RC) ||
1871 PPC::GPRC_NOR0RegClass.hasSubClassEq(RC)) {
1873 } else if (PPC::G8RCRegClass.hasSubClassEq(RC) ||
1874 PPC::G8RC_NOX0RegClass.hasSubClassEq(RC)) {
1876 } else if (PPC::F8RCRegClass.hasSubClassEq(RC)) {
1878 } else if (PPC::F4RCRegClass.hasSubClassEq(RC)) {
1880 } else if (PPC::SPERCRegClass.hasSubClassEq(RC)) {
1882 } else if (PPC::CRRCRegClass.hasSubClassEq(RC)) {
1884 } else if (PPC::CRBITRCRegClass.hasSubClassEq(RC)) {
1886 } else if (PPC::VRRCRegClass.hasSubClassEq(RC)) {
1888 } else if (PPC::VSRCRegClass.hasSubClassEq(RC)) {
1890 } else if (PPC::VSFRCRegClass.hasSubClassEq(RC)) {
1892 } else if (PPC::VSSRCRegClass.hasSubClassEq(RC)) {
1894 } else if (PPC::SPILLTOVSRRCRegClass.hasSubClassEq(RC)) {
1896 } else if (PPC::ACCRCRegClass.hasSubClassEq(RC)) {
1897 assert(Subtarget.pairedVectorMemops() &&
1898 "Register unexpected when paired memops are disabled.");
1900 } else if (PPC::UACCRCRegClass.hasSubClassEq(RC)) {
1901 assert(Subtarget.pairedVectorMemops() &&
1902 "Register unexpected when paired memops are disabled.");
1904 } else if (PPC::WACCRCRegClass.hasSubClassEq(RC)) {
1905 assert(Subtarget.pairedVectorMemops() &&
1906 "Register unexpected when paired memops are disabled.");
1908 } else if (PPC::VSRpRCRegClass.hasSubClassEq(RC)) {
1909 assert(Subtarget.pairedVectorMemops() &&
1910 "Register unexpected when paired memops are disabled.");
1912 } else if (PPC::G8pRCRegClass.hasSubClassEq(RC)) {
1914 } else {
1915 llvm_unreachable("Unknown regclass!");
1916 }
1917 return OpcodeIndex;
1918}
1919
1920unsigned
1922 ArrayRef<unsigned> OpcodesForSpill = getStoreOpcodesForSpillArray();
1923 return OpcodesForSpill[getSpillIndex(RC)];
1924}
1925
1926unsigned
1928 ArrayRef<unsigned> OpcodesForSpill = getLoadOpcodesForSpillArray();
1929 return OpcodesForSpill[getSpillIndex(RC)];
1930}
1931
1932void PPCInstrInfo::StoreRegToStackSlot(
1933 MachineFunction &MF, unsigned SrcReg, bool isKill, int FrameIdx,
1934 const TargetRegisterClass *RC,
1935 SmallVectorImpl<MachineInstr *> &NewMIs) const {
1936 unsigned Opcode = getStoreOpcodeForSpill(RC);
1937 DebugLoc DL;
1938
1939 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1940 FuncInfo->setHasSpills();
1941
1943 BuildMI(MF, DL, get(Opcode)).addReg(SrcReg, getKillRegState(isKill)),
1944 FrameIdx));
1945
1946 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
1947 PPC::CRBITRCRegClass.hasSubClassEq(RC))
1948 FuncInfo->setSpillsCR();
1949
1950 if (isXFormMemOp(Opcode))
1951 FuncInfo->setHasNonRISpills();
1952}
1953
1956 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1957 const TargetRegisterInfo *TRI) const {
1958 MachineFunction &MF = *MBB.getParent();
1960
1961 StoreRegToStackSlot(MF, SrcReg, isKill, FrameIdx, RC, NewMIs);
1962
1963 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
1964 MBB.insert(MI, NewMIs[i]);
1965
1966 const MachineFrameInfo &MFI = MF.getFrameInfo();
1970 MFI.getObjectAlign(FrameIdx));
1971 NewMIs.back()->addMemOperand(MF, MMO);
1972}
1973
1976 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
1977 const TargetRegisterInfo *TRI, Register VReg) const {
1978 // We need to avoid a situation in which the value from a VRRC register is
1979 // spilled using an Altivec instruction and reloaded into a VSRC register
1980 // using a VSX instruction. The issue with this is that the VSX
1981 // load/store instructions swap the doublewords in the vector and the Altivec
1982 // ones don't. The register classes on the spill/reload may be different if
1983 // the register is defined using an Altivec instruction and is then used by a
1984 // VSX instruction.
1985 RC = updatedRC(RC);
1986 storeRegToStackSlotNoUpd(MBB, MI, SrcReg, isKill, FrameIdx, RC, TRI);
1987}
1988
1989void PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, const DebugLoc &DL,
1990 unsigned DestReg, int FrameIdx,
1991 const TargetRegisterClass *RC,
1993 const {
1994 unsigned Opcode = getLoadOpcodeForSpill(RC);
1995 NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Opcode), DestReg),
1996 FrameIdx));
1997 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
1998
1999 if (PPC::CRRCRegClass.hasSubClassEq(RC) ||
2000 PPC::CRBITRCRegClass.hasSubClassEq(RC))
2001 FuncInfo->setSpillsCR();
2002
2003 if (isXFormMemOp(Opcode))
2004 FuncInfo->setHasNonRISpills();
2005}
2006
2009 int FrameIdx, const TargetRegisterClass *RC,
2010 const TargetRegisterInfo *TRI) const {
2011 MachineFunction &MF = *MBB.getParent();
2013 DebugLoc DL;
2014 if (MI != MBB.end()) DL = MI->getDebugLoc();
2015
2016 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2017 FuncInfo->setHasSpills();
2018
2019 LoadRegFromStackSlot(MF, DL, DestReg, FrameIdx, RC, NewMIs);
2020
2021 for (unsigned i = 0, e = NewMIs.size(); i != e; ++i)
2022 MBB.insert(MI, NewMIs[i]);
2023
2024 const MachineFrameInfo &MFI = MF.getFrameInfo();
2028 MFI.getObjectAlign(FrameIdx));
2029 NewMIs.back()->addMemOperand(MF, MMO);
2030}
2031
2034 Register DestReg, int FrameIdx,
2035 const TargetRegisterClass *RC,
2036 const TargetRegisterInfo *TRI,
2037 Register VReg) const {
2038 // We need to avoid a situation in which the value from a VRRC register is
2039 // spilled using an Altivec instruction and reloaded into a VSRC register
2040 // using a VSX instruction. The issue with this is that the VSX
2041 // load/store instructions swap the doublewords in the vector and the Altivec
2042 // ones don't. The register classes on the spill/reload may be different if
2043 // the register is defined using an Altivec instruction and is then used by a
2044 // VSX instruction.
2045 RC = updatedRC(RC);
2046
2047 loadRegFromStackSlotNoUpd(MBB, MI, DestReg, FrameIdx, RC, TRI);
2048}
2049
2052 assert(Cond.size() == 2 && "Invalid PPC branch opcode!");
2053 if (Cond[1].getReg() == PPC::CTR8 || Cond[1].getReg() == PPC::CTR)
2054 Cond[0].setImm(Cond[0].getImm() == 0 ? 1 : 0);
2055 else
2056 // Leave the CR# the same, but invert the condition.
2057 Cond[0].setImm(PPC::InvertPredicate((PPC::Predicate)Cond[0].getImm()));
2058 return false;
2059}
2060
2061// For some instructions, it is legal to fold ZERO into the RA register field.
2062// This function performs that fold by replacing the operand with PPC::ZERO,
2063// it does not consider whether the load immediate zero is no longer in use.
2065 Register Reg) const {
2066 // A zero immediate should always be loaded with a single li.
2067 unsigned DefOpc = DefMI.getOpcode();
2068 if (DefOpc != PPC::LI && DefOpc != PPC::LI8)
2069 return false;
2070 if (!DefMI.getOperand(1).isImm())
2071 return false;
2072 if (DefMI.getOperand(1).getImm() != 0)
2073 return false;
2074
2075 // Note that we cannot here invert the arguments of an isel in order to fold
2076 // a ZERO into what is presented as the second argument. All we have here
2077 // is the condition bit, and that might come from a CR-logical bit operation.
2078
2079 const MCInstrDesc &UseMCID = UseMI.getDesc();
2080
2081 // Only fold into real machine instructions.
2082 if (UseMCID.isPseudo())
2083 return false;
2084
2085 // We need to find which of the User's operands is to be folded, that will be
2086 // the operand that matches the given register ID.
2087 unsigned UseIdx;
2088 for (UseIdx = 0; UseIdx < UseMI.getNumOperands(); ++UseIdx)
2089 if (UseMI.getOperand(UseIdx).isReg() &&
2090 UseMI.getOperand(UseIdx).getReg() == Reg)
2091 break;
2092
2093 assert(UseIdx < UseMI.getNumOperands() && "Cannot find Reg in UseMI");
2094 assert(UseIdx < UseMCID.getNumOperands() && "No operand description for Reg");
2095
2096 const MCOperandInfo *UseInfo = &UseMCID.operands()[UseIdx];
2097
2098 // We can fold the zero if this register requires a GPRC_NOR0/G8RC_NOX0
2099 // register (which might also be specified as a pointer class kind).
2100 if (UseInfo->isLookupPtrRegClass()) {
2101 if (UseInfo->RegClass /* Kind */ != 1)
2102 return false;
2103 } else {
2104 if (UseInfo->RegClass != PPC::GPRC_NOR0RegClassID &&
2105 UseInfo->RegClass != PPC::G8RC_NOX0RegClassID)
2106 return false;
2107 }
2108
2109 // Make sure this is not tied to an output register (or otherwise
2110 // constrained). This is true for ST?UX registers, for example, which
2111 // are tied to their output registers.
2112 if (UseInfo->Constraints != 0)
2113 return false;
2114
2115 MCRegister ZeroReg;
2116 if (UseInfo->isLookupPtrRegClass()) {
2117 bool isPPC64 = Subtarget.isPPC64();
2118 ZeroReg = isPPC64 ? PPC::ZERO8 : PPC::ZERO;
2119 } else {
2120 ZeroReg = UseInfo->RegClass == PPC::G8RC_NOX0RegClassID ?
2121 PPC::ZERO8 : PPC::ZERO;
2122 }
2123
2124 LLVM_DEBUG(dbgs() << "Folded immediate zero for: ");
2125 LLVM_DEBUG(UseMI.dump());
2126 UseMI.getOperand(UseIdx).setReg(ZeroReg);
2127 LLVM_DEBUG(dbgs() << "Into: ");
2128 LLVM_DEBUG(UseMI.dump());
2129 return true;
2130}
2131
2132// Folds zero into instructions which have a load immediate zero as an operand
2133// but also recognize zero as immediate zero. If the definition of the load
2134// has no more users it is deleted.
2136 Register Reg, MachineRegisterInfo *MRI) const {
2137 bool Changed = onlyFoldImmediate(UseMI, DefMI, Reg);
2138 if (MRI->use_nodbg_empty(Reg))
2139 DefMI.eraseFromParent();
2140 return Changed;
2141}
2142
2144 for (MachineInstr &MI : MBB)
2145 if (MI.definesRegister(PPC::CTR) || MI.definesRegister(PPC::CTR8))
2146 return true;
2147 return false;
2148}
2149
2150// We should make sure that, if we're going to predicate both sides of a
2151// condition (a diamond), that both sides don't define the counter register. We
2152// can predicate counter-decrement-based branches, but while that predicates
2153// the branching, it does not predicate the counter decrement. If we tried to
2154// merge the triangle into one predicated block, we'd decrement the counter
2155// twice.
2157 unsigned NumT, unsigned ExtraT,
2158 MachineBasicBlock &FMBB,
2159 unsigned NumF, unsigned ExtraF,
2160 BranchProbability Probability) const {
2161 return !(MBBDefinesCTR(TMBB) && MBBDefinesCTR(FMBB));
2162}
2163
2164
2166 // The predicated branches are identified by their type, not really by the
2167 // explicit presence of a predicate. Furthermore, some of them can be
2168 // predicated more than once. Because if conversion won't try to predicate
2169 // any instruction which already claims to be predicated (by returning true
2170 // here), always return false. In doing so, we let isPredicable() be the
2171 // final word on whether not the instruction can be (further) predicated.
2172
2173 return false;
2174}
2175
2177 const MachineBasicBlock *MBB,
2178 const MachineFunction &MF) const {
2179 // Set MFFS and MTFSF as scheduling boundary to avoid unexpected code motion
2180 // across them, since some FP operations may change content of FPSCR.
2181 // TODO: Model FPSCR in PPC instruction definitions and remove the workaround
2182 if (MI.getOpcode() == PPC::MFFS || MI.getOpcode() == PPC::MTFSF)
2183 return true;
2185}
2186
2188 ArrayRef<MachineOperand> Pred) const {
2189 unsigned OpC = MI.getOpcode();
2190 if (OpC == PPC::BLR || OpC == PPC::BLR8) {
2191 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2192 bool isPPC64 = Subtarget.isPPC64();
2193 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZLR8 : PPC::BDNZLR)
2194 : (isPPC64 ? PPC::BDZLR8 : PPC::BDZLR)));
2195 // Need add Def and Use for CTR implicit operand.
2196 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2197 .addReg(Pred[1].getReg(), RegState::Implicit)
2199 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2200 MI.setDesc(get(PPC::BCLR));
2201 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2202 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2203 MI.setDesc(get(PPC::BCLRn));
2204 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2205 } else {
2206 MI.setDesc(get(PPC::BCCLR));
2207 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2208 .addImm(Pred[0].getImm())
2209 .add(Pred[1]);
2210 }
2211
2212 return true;
2213 } else if (OpC == PPC::B) {
2214 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR) {
2215 bool isPPC64 = Subtarget.isPPC64();
2216 MI.setDesc(get(Pred[0].getImm() ? (isPPC64 ? PPC::BDNZ8 : PPC::BDNZ)
2217 : (isPPC64 ? PPC::BDZ8 : PPC::BDZ)));
2218 // Need add Def and Use for CTR implicit operand.
2219 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2220 .addReg(Pred[1].getReg(), RegState::Implicit)
2222 } else if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2223 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2224 MI.removeOperand(0);
2225
2226 MI.setDesc(get(PPC::BC));
2227 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2228 .add(Pred[1])
2229 .addMBB(MBB);
2230 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2231 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2232 MI.removeOperand(0);
2233
2234 MI.setDesc(get(PPC::BCn));
2235 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2236 .add(Pred[1])
2237 .addMBB(MBB);
2238 } else {
2239 MachineBasicBlock *MBB = MI.getOperand(0).getMBB();
2240 MI.removeOperand(0);
2241
2242 MI.setDesc(get(PPC::BCC));
2243 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2244 .addImm(Pred[0].getImm())
2245 .add(Pred[1])
2246 .addMBB(MBB);
2247 }
2248
2249 return true;
2250 } else if (OpC == PPC::BCTR || OpC == PPC::BCTR8 || OpC == PPC::BCTRL ||
2251 OpC == PPC::BCTRL8 || OpC == PPC::BCTRL_RM ||
2252 OpC == PPC::BCTRL8_RM) {
2253 if (Pred[1].getReg() == PPC::CTR8 || Pred[1].getReg() == PPC::CTR)
2254 llvm_unreachable("Cannot predicate bctr[l] on the ctr register");
2255
2256 bool setLR = OpC == PPC::BCTRL || OpC == PPC::BCTRL8 ||
2257 OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM;
2258 bool isPPC64 = Subtarget.isPPC64();
2259
2260 if (Pred[0].getImm() == PPC::PRED_BIT_SET) {
2261 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8 : PPC::BCCTR8)
2262 : (setLR ? PPC::BCCTRL : PPC::BCCTR)));
2263 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2264 } else if (Pred[0].getImm() == PPC::PRED_BIT_UNSET) {
2265 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCTRL8n : PPC::BCCTR8n)
2266 : (setLR ? PPC::BCCTRLn : PPC::BCCTRn)));
2267 MachineInstrBuilder(*MI.getParent()->getParent(), MI).add(Pred[1]);
2268 } else {
2269 MI.setDesc(get(isPPC64 ? (setLR ? PPC::BCCCTRL8 : PPC::BCCCTR8)
2270 : (setLR ? PPC::BCCCTRL : PPC::BCCCTR)));
2271 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2272 .addImm(Pred[0].getImm())
2273 .add(Pred[1]);
2274 }
2275
2276 // Need add Def and Use for LR implicit operand.
2277 if (setLR)
2278 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2279 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::Implicit)
2280 .addReg(isPPC64 ? PPC::LR8 : PPC::LR, RegState::ImplicitDefine);
2281 if (OpC == PPC::BCTRL_RM || OpC == PPC::BCTRL8_RM)
2282 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
2284
2285 return true;
2286 }
2287
2288 return false;
2289}
2290
2292 ArrayRef<MachineOperand> Pred2) const {
2293 assert(Pred1.size() == 2 && "Invalid PPC first predicate");
2294 assert(Pred2.size() == 2 && "Invalid PPC second predicate");
2295
2296 if (Pred1[1].getReg() == PPC::CTR8 || Pred1[1].getReg() == PPC::CTR)
2297 return false;
2298 if (Pred2[1].getReg() == PPC::CTR8 || Pred2[1].getReg() == PPC::CTR)
2299 return false;
2300
2301 // P1 can only subsume P2 if they test the same condition register.
2302 if (Pred1[1].getReg() != Pred2[1].getReg())
2303 return false;
2304
2305 PPC::Predicate P1 = (PPC::Predicate) Pred1[0].getImm();
2306 PPC::Predicate P2 = (PPC::Predicate) Pred2[0].getImm();
2307
2308 if (P1 == P2)
2309 return true;
2310
2311 // Does P1 subsume P2, e.g. GE subsumes GT.
2312 if (P1 == PPC::PRED_LE &&
2313 (P2 == PPC::PRED_LT || P2 == PPC::PRED_EQ))
2314 return true;
2315 if (P1 == PPC::PRED_GE &&
2316 (P2 == PPC::PRED_GT || P2 == PPC::PRED_EQ))
2317 return true;
2318
2319 return false;
2320}
2321
2323 std::vector<MachineOperand> &Pred,
2324 bool SkipDead) const {
2325 // Note: At the present time, the contents of Pred from this function is
2326 // unused by IfConversion. This implementation follows ARM by pushing the
2327 // CR-defining operand. Because the 'DZ' and 'DNZ' count as types of
2328 // predicate, instructions defining CTR or CTR8 are also included as
2329 // predicate-defining instructions.
2330
2331 const TargetRegisterClass *RCs[] =
2332 { &PPC::CRRCRegClass, &PPC::CRBITRCRegClass,
2333 &PPC::CTRRCRegClass, &PPC::CTRRC8RegClass };
2334
2335 bool Found = false;
2336 for (const MachineOperand &MO : MI.operands()) {
2337 for (unsigned c = 0; c < std::size(RCs) && !Found; ++c) {
2338 const TargetRegisterClass *RC = RCs[c];
2339 if (MO.isReg()) {
2340 if (MO.isDef() && RC->contains(MO.getReg())) {
2341 Pred.push_back(MO);
2342 Found = true;
2343 }
2344 } else if (MO.isRegMask()) {
2345 for (MCPhysReg R : *RC)
2346 if (MO.clobbersPhysReg(R)) {
2347 Pred.push_back(MO);
2348 Found = true;
2349 }
2350 }
2351 }
2352 }
2353
2354 return Found;
2355}
2356
2358 Register &SrcReg2, int64_t &Mask,
2359 int64_t &Value) const {
2360 unsigned Opc = MI.getOpcode();
2361
2362 switch (Opc) {
2363 default: return false;
2364 case PPC::CMPWI:
2365 case PPC::CMPLWI:
2366 case PPC::CMPDI:
2367 case PPC::CMPLDI:
2368 SrcReg = MI.getOperand(1).getReg();
2369 SrcReg2 = 0;
2370 Value = MI.getOperand(2).getImm();
2371 Mask = 0xFFFF;
2372 return true;
2373 case PPC::CMPW:
2374 case PPC::CMPLW:
2375 case PPC::CMPD:
2376 case PPC::CMPLD:
2377 case PPC::FCMPUS:
2378 case PPC::FCMPUD:
2379 SrcReg = MI.getOperand(1).getReg();
2380 SrcReg2 = MI.getOperand(2).getReg();
2381 Value = 0;
2382 Mask = 0;
2383 return true;
2384 }
2385}
2386
2388 Register SrcReg2, int64_t Mask,
2389 int64_t Value,
2390 const MachineRegisterInfo *MRI) const {
2391 if (DisableCmpOpt)
2392 return false;
2393
2394 int OpC = CmpInstr.getOpcode();
2395 Register CRReg = CmpInstr.getOperand(0).getReg();
2396
2397 // FP record forms set CR1 based on the exception status bits, not a
2398 // comparison with zero.
2399 if (OpC == PPC::FCMPUS || OpC == PPC::FCMPUD)
2400 return false;
2401
2403 // The record forms set the condition register based on a signed comparison
2404 // with zero (so says the ISA manual). This is not as straightforward as it
2405 // seems, however, because this is always a 64-bit comparison on PPC64, even
2406 // for instructions that are 32-bit in nature (like slw for example).
2407 // So, on PPC32, for unsigned comparisons, we can use the record forms only
2408 // for equality checks (as those don't depend on the sign). On PPC64,
2409 // we are restricted to equality for unsigned 64-bit comparisons and for
2410 // signed 32-bit comparisons the applicability is more restricted.
2411 bool isPPC64 = Subtarget.isPPC64();
2412 bool is32BitSignedCompare = OpC == PPC::CMPWI || OpC == PPC::CMPW;
2413 bool is32BitUnsignedCompare = OpC == PPC::CMPLWI || OpC == PPC::CMPLW;
2414 bool is64BitUnsignedCompare = OpC == PPC::CMPLDI || OpC == PPC::CMPLD;
2415
2416 // Look through copies unless that gets us to a physical register.
2417 Register ActualSrc = TRI->lookThruCopyLike(SrcReg, MRI);
2418 if (ActualSrc.isVirtual())
2419 SrcReg = ActualSrc;
2420
2421 // Get the unique definition of SrcReg.
2422 MachineInstr *MI = MRI->getUniqueVRegDef(SrcReg);
2423 if (!MI) return false;
2424
2425 bool equalityOnly = false;
2426 bool noSub = false;
2427 if (isPPC64) {
2428 if (is32BitSignedCompare) {
2429 // We can perform this optimization only if SrcReg is sign-extending.
2430 if (isSignExtended(SrcReg, MRI))
2431 noSub = true;
2432 else
2433 return false;
2434 } else if (is32BitUnsignedCompare) {
2435 // We can perform this optimization, equality only, if SrcReg is
2436 // zero-extending.
2437 if (isZeroExtended(SrcReg, MRI)) {
2438 noSub = true;
2439 equalityOnly = true;
2440 } else
2441 return false;
2442 } else
2443 equalityOnly = is64BitUnsignedCompare;
2444 } else
2445 equalityOnly = is32BitUnsignedCompare;
2446
2447 if (equalityOnly) {
2448 // We need to check the uses of the condition register in order to reject
2449 // non-equality comparisons.
2451 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2452 I != IE; ++I) {
2453 MachineInstr *UseMI = &*I;
2454 if (UseMI->getOpcode() == PPC::BCC) {
2456 unsigned PredCond = PPC::getPredicateCondition(Pred);
2457 // We ignore hint bits when checking for non-equality comparisons.
2458 if (PredCond != PPC::PRED_EQ && PredCond != PPC::PRED_NE)
2459 return false;
2460 } else if (UseMI->getOpcode() == PPC::ISEL ||
2461 UseMI->getOpcode() == PPC::ISEL8) {
2462 unsigned SubIdx = UseMI->getOperand(3).getSubReg();
2463 if (SubIdx != PPC::sub_eq)
2464 return false;
2465 } else
2466 return false;
2467 }
2468 }
2469
2470 MachineBasicBlock::iterator I = CmpInstr;
2471
2472 // Scan forward to find the first use of the compare.
2473 for (MachineBasicBlock::iterator EL = CmpInstr.getParent()->end(); I != EL;
2474 ++I) {
2475 bool FoundUse = false;
2477 J = MRI->use_instr_begin(CRReg), JE = MRI->use_instr_end();
2478 J != JE; ++J)
2479 if (&*J == &*I) {
2480 FoundUse = true;
2481 break;
2482 }
2483
2484 if (FoundUse)
2485 break;
2486 }
2487
2490
2491 // There are two possible candidates which can be changed to set CR[01].
2492 // One is MI, the other is a SUB instruction.
2493 // For CMPrr(r1,r2), we are looking for SUB(r1,r2) or SUB(r2,r1).
2494 MachineInstr *Sub = nullptr;
2495 if (SrcReg2 != 0)
2496 // MI is not a candidate for CMPrr.
2497 MI = nullptr;
2498 // FIXME: Conservatively refuse to convert an instruction which isn't in the
2499 // same BB as the comparison. This is to allow the check below to avoid calls
2500 // (and other explicit clobbers); instead we should really check for these
2501 // more explicitly (in at least a few predecessors).
2502 else if (MI->getParent() != CmpInstr.getParent())
2503 return false;
2504 else if (Value != 0) {
2505 // The record-form instructions set CR bit based on signed comparison
2506 // against 0. We try to convert a compare against 1 or -1 into a compare
2507 // against 0 to exploit record-form instructions. For example, we change
2508 // the condition "greater than -1" into "greater than or equal to 0"
2509 // and "less than 1" into "less than or equal to 0".
2510
2511 // Since we optimize comparison based on a specific branch condition,
2512 // we don't optimize if condition code is used by more than once.
2513 if (equalityOnly || !MRI->hasOneUse(CRReg))
2514 return false;
2515
2516 MachineInstr *UseMI = &*MRI->use_instr_begin(CRReg);
2517 if (UseMI->getOpcode() != PPC::BCC)
2518 return false;
2519
2521 unsigned PredCond = PPC::getPredicateCondition(Pred);
2522 unsigned PredHint = PPC::getPredicateHint(Pred);
2523 int16_t Immed = (int16_t)Value;
2524
2525 // When modifying the condition in the predicate, we propagate hint bits
2526 // from the original predicate to the new one.
2527 if (Immed == -1 && PredCond == PPC::PRED_GT)
2528 // We convert "greater than -1" into "greater than or equal to 0",
2529 // since we are assuming signed comparison by !equalityOnly
2530 Pred = PPC::getPredicate(PPC::PRED_GE, PredHint);
2531 else if (Immed == -1 && PredCond == PPC::PRED_LE)
2532 // We convert "less than or equal to -1" into "less than 0".
2533 Pred = PPC::getPredicate(PPC::PRED_LT, PredHint);
2534 else if (Immed == 1 && PredCond == PPC::PRED_LT)
2535 // We convert "less than 1" into "less than or equal to 0".
2536 Pred = PPC::getPredicate(PPC::PRED_LE, PredHint);
2537 else if (Immed == 1 && PredCond == PPC::PRED_GE)
2538 // We convert "greater than or equal to 1" into "greater than 0".
2539 Pred = PPC::getPredicate(PPC::PRED_GT, PredHint);
2540 else
2541 return false;
2542
2543 // Convert the comparison and its user to a compare against zero with the
2544 // appropriate predicate on the branch. Zero comparison might provide
2545 // optimization opportunities post-RA (see optimization in
2546 // PPCPreEmitPeephole.cpp).
2547 UseMI->getOperand(0).setImm(Pred);
2548 CmpInstr.getOperand(2).setImm(0);
2549 }
2550
2551 // Search for Sub.
2552 --I;
2553
2554 // Get ready to iterate backward from CmpInstr.
2555 MachineBasicBlock::iterator E = MI, B = CmpInstr.getParent()->begin();
2556
2557 for (; I != E && !noSub; --I) {
2558 const MachineInstr &Instr = *I;
2559 unsigned IOpC = Instr.getOpcode();
2560
2561 if (&*I != &CmpInstr && (Instr.modifiesRegister(PPC::CR0, TRI) ||
2562 Instr.readsRegister(PPC::CR0, TRI)))
2563 // This instruction modifies or uses the record condition register after
2564 // the one we want to change. While we could do this transformation, it
2565 // would likely not be profitable. This transformation removes one
2566 // instruction, and so even forcing RA to generate one move probably
2567 // makes it unprofitable.
2568 return false;
2569
2570 // Check whether CmpInstr can be made redundant by the current instruction.
2571 if ((OpC == PPC::CMPW || OpC == PPC::CMPLW ||
2572 OpC == PPC::CMPD || OpC == PPC::CMPLD) &&
2573 (IOpC == PPC::SUBF || IOpC == PPC::SUBF8) &&
2574 ((Instr.getOperand(1).getReg() == SrcReg &&
2575 Instr.getOperand(2).getReg() == SrcReg2) ||
2576 (Instr.getOperand(1).getReg() == SrcReg2 &&
2577 Instr.getOperand(2).getReg() == SrcReg))) {
2578 Sub = &*I;
2579 break;
2580 }
2581
2582 if (I == B)
2583 // The 'and' is below the comparison instruction.
2584 return false;
2585 }
2586
2587 // Return false if no candidates exist.
2588 if (!MI && !Sub)
2589 return false;
2590
2591 // The single candidate is called MI.
2592 if (!MI) MI = Sub;
2593
2594 int NewOpC = -1;
2595 int MIOpC = MI->getOpcode();
2596 if (MIOpC == PPC::ANDI_rec || MIOpC == PPC::ANDI8_rec ||
2597 MIOpC == PPC::ANDIS_rec || MIOpC == PPC::ANDIS8_rec)
2598 NewOpC = MIOpC;
2599 else {
2600 NewOpC = PPC::getRecordFormOpcode(MIOpC);
2601 if (NewOpC == -1 && PPC::getNonRecordFormOpcode(MIOpC) != -1)
2602 NewOpC = MIOpC;
2603 }
2604
2605 // FIXME: On the non-embedded POWER architectures, only some of the record
2606 // forms are fast, and we should use only the fast ones.
2607
2608 // The defining instruction has a record form (or is already a record
2609 // form). It is possible, however, that we'll need to reverse the condition
2610 // code of the users.
2611 if (NewOpC == -1)
2612 return false;
2613
2614 // This transformation should not be performed if `nsw` is missing and is not
2615 // `equalityOnly` comparison. Since if there is overflow, sub_lt, sub_gt in
2616 // CRReg do not reflect correct order. If `equalityOnly` is true, sub_eq in
2617 // CRReg can reflect if compared values are equal, this optz is still valid.
2618 if (!equalityOnly && (NewOpC == PPC::SUBF_rec || NewOpC == PPC::SUBF8_rec) &&
2619 Sub && !Sub->getFlag(MachineInstr::NoSWrap))
2620 return false;
2621
2622 // If we have SUB(r1, r2) and CMP(r2, r1), the condition code based on CMP
2623 // needs to be updated to be based on SUB. Push the condition code
2624 // operands to OperandsToUpdate. If it is safe to remove CmpInstr, the
2625 // condition code of these operands will be modified.
2626 // Here, Value == 0 means we haven't converted comparison against 1 or -1 to
2627 // comparison against 0, which may modify predicate.
2628 bool ShouldSwap = false;
2629 if (Sub && Value == 0) {
2630 ShouldSwap = SrcReg2 != 0 && Sub->getOperand(1).getReg() == SrcReg2 &&
2631 Sub->getOperand(2).getReg() == SrcReg;
2632
2633 // The operands to subf are the opposite of sub, so only in the fixed-point
2634 // case, invert the order.
2635 ShouldSwap = !ShouldSwap;
2636 }
2637
2638 if (ShouldSwap)
2640 I = MRI->use_instr_begin(CRReg), IE = MRI->use_instr_end();
2641 I != IE; ++I) {
2642 MachineInstr *UseMI = &*I;
2643 if (UseMI->getOpcode() == PPC::BCC) {
2645 unsigned PredCond = PPC::getPredicateCondition(Pred);
2646 assert((!equalityOnly ||
2647 PredCond == PPC::PRED_EQ || PredCond == PPC::PRED_NE) &&
2648 "Invalid predicate for equality-only optimization");
2649 (void)PredCond; // To suppress warning in release build.
2650 PredsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(0)),
2652 } else if (UseMI->getOpcode() == PPC::ISEL ||
2653 UseMI->getOpcode() == PPC::ISEL8) {
2654 unsigned NewSubReg = UseMI->getOperand(3).getSubReg();
2655 assert((!equalityOnly || NewSubReg == PPC::sub_eq) &&
2656 "Invalid CR bit for equality-only optimization");
2657
2658 if (NewSubReg == PPC::sub_lt)
2659 NewSubReg = PPC::sub_gt;
2660 else if (NewSubReg == PPC::sub_gt)
2661 NewSubReg = PPC::sub_lt;
2662
2663 SubRegsToUpdate.push_back(std::make_pair(&(UseMI->getOperand(3)),
2664 NewSubReg));
2665 } else // We need to abort on a user we don't understand.
2666 return false;
2667 }
2668 assert(!(Value != 0 && ShouldSwap) &&
2669 "Non-zero immediate support and ShouldSwap"
2670 "may conflict in updating predicate");
2671
2672 // Create a new virtual register to hold the value of the CR set by the
2673 // record-form instruction. If the instruction was not previously in
2674 // record form, then set the kill flag on the CR.
2675 CmpInstr.eraseFromParent();
2676
2678 BuildMI(*MI->getParent(), std::next(MII), MI->getDebugLoc(),
2679 get(TargetOpcode::COPY), CRReg)
2680 .addReg(PPC::CR0, MIOpC != NewOpC ? RegState::Kill : 0);
2681
2682 // Even if CR0 register were dead before, it is alive now since the
2683 // instruction we just built uses it.
2684 MI->clearRegisterDeads(PPC::CR0);
2685
2686 if (MIOpC != NewOpC) {
2687 // We need to be careful here: we're replacing one instruction with
2688 // another, and we need to make sure that we get all of the right
2689 // implicit uses and defs. On the other hand, the caller may be holding
2690 // an iterator to this instruction, and so we can't delete it (this is
2691 // specifically the case if this is the instruction directly after the
2692 // compare).
2693
2694 // Rotates are expensive instructions. If we're emitting a record-form
2695 // rotate that can just be an andi/andis, we should just emit that.
2696 if (MIOpC == PPC::RLWINM || MIOpC == PPC::RLWINM8) {
2697 Register GPRRes = MI->getOperand(0).getReg();
2698 int64_t SH = MI->getOperand(2).getImm();
2699 int64_t MB = MI->getOperand(3).getImm();
2700 int64_t ME = MI->getOperand(4).getImm();
2701 // We can only do this if both the start and end of the mask are in the
2702 // same halfword.
2703 bool MBInLoHWord = MB >= 16;
2704 bool MEInLoHWord = ME >= 16;
2705 uint64_t Mask = ~0LLU;
2706
2707 if (MB <= ME && MBInLoHWord == MEInLoHWord && SH == 0) {
2708 Mask = ((1LLU << (32 - MB)) - 1) & ~((1LLU << (31 - ME)) - 1);
2709 // The mask value needs to shift right 16 if we're emitting andis.
2710 Mask >>= MBInLoHWord ? 0 : 16;
2711 NewOpC = MIOpC == PPC::RLWINM
2712 ? (MBInLoHWord ? PPC::ANDI_rec : PPC::ANDIS_rec)
2713 : (MBInLoHWord ? PPC::ANDI8_rec : PPC::ANDIS8_rec);
2714 } else if (MRI->use_empty(GPRRes) && (ME == 31) &&
2715 (ME - MB + 1 == SH) && (MB >= 16)) {
2716 // If we are rotating by the exact number of bits as are in the mask
2717 // and the mask is in the least significant bits of the register,
2718 // that's just an andis. (as long as the GPR result has no uses).
2719 Mask = ((1LLU << 32) - 1) & ~((1LLU << (32 - SH)) - 1);
2720 Mask >>= 16;
2721 NewOpC = MIOpC == PPC::RLWINM ? PPC::ANDIS_rec : PPC::ANDIS8_rec;
2722 }
2723 // If we've set the mask, we can transform.
2724 if (Mask != ~0LLU) {
2725 MI->removeOperand(4);
2726 MI->removeOperand(3);
2727 MI->getOperand(2).setImm(Mask);
2728 NumRcRotatesConvertedToRcAnd++;
2729 }
2730 } else if (MIOpC == PPC::RLDICL && MI->getOperand(2).getImm() == 0) {
2731 int64_t MB = MI->getOperand(3).getImm();
2732 if (MB >= 48) {
2733 uint64_t Mask = (1LLU << (63 - MB + 1)) - 1;
2734 NewOpC = PPC::ANDI8_rec;
2735 MI->removeOperand(3);
2736 MI->getOperand(2).setImm(Mask);
2737 NumRcRotatesConvertedToRcAnd++;
2738 }
2739 }
2740
2741 const MCInstrDesc &NewDesc = get(NewOpC);
2742 MI->setDesc(NewDesc);
2743
2744 for (MCPhysReg ImpDef : NewDesc.implicit_defs()) {
2745 if (!MI->definesRegister(ImpDef)) {
2746 MI->addOperand(*MI->getParent()->getParent(),
2747 MachineOperand::CreateReg(ImpDef, true, true));
2748 }
2749 }
2750 for (MCPhysReg ImpUse : NewDesc.implicit_uses()) {
2751 if (!MI->readsRegister(ImpUse)) {
2752 MI->addOperand(*MI->getParent()->getParent(),
2753 MachineOperand::CreateReg(ImpUse, false, true));
2754 }
2755 }
2756 }
2757 assert(MI->definesRegister(PPC::CR0) &&
2758 "Record-form instruction does not define cr0?");
2759
2760 // Modify the condition code of operands in OperandsToUpdate.
2761 // Since we have SUB(r1, r2) and CMP(r2, r1), the condition code needs to
2762 // be changed from r2 > r1 to r1 < r2, from r2 < r1 to r1 > r2, etc.
2763 for (unsigned i = 0, e = PredsToUpdate.size(); i < e; i++)
2764 PredsToUpdate[i].first->setImm(PredsToUpdate[i].second);
2765
2766 for (unsigned i = 0, e = SubRegsToUpdate.size(); i < e; i++)
2767 SubRegsToUpdate[i].first->setSubReg(SubRegsToUpdate[i].second);
2768
2769 return true;
2770}
2771
2774 if (MRI->isSSA())
2775 return false;
2776
2777 Register SrcReg, SrcReg2;
2778 int64_t CmpMask, CmpValue;
2779 if (!analyzeCompare(CmpMI, SrcReg, SrcReg2, CmpMask, CmpValue))
2780 return false;
2781
2782 // Try to optimize the comparison against 0.
2783 if (CmpValue || !CmpMask || SrcReg2)
2784 return false;
2785
2786 // The record forms set the condition register based on a signed comparison
2787 // with zero (see comments in optimizeCompareInstr). Since we can't do the
2788 // equality checks in post-RA, we are more restricted on a unsigned
2789 // comparison.
2790 unsigned Opc = CmpMI.getOpcode();
2791 if (Opc == PPC::CMPLWI || Opc == PPC::CMPLDI)
2792 return false;
2793
2794 // The record forms are always based on a 64-bit comparison on PPC64
2795 // (similary, a 32-bit comparison on PPC32), while the CMPWI is a 32-bit
2796 // comparison. Since we can't do the equality checks in post-RA, we bail out
2797 // the case.
2798 if (Subtarget.isPPC64() && Opc == PPC::CMPWI)
2799 return false;
2800
2801 // CmpMI can't be deleted if it has implicit def.
2802 if (CmpMI.hasImplicitDef())
2803 return false;
2804
2805 bool SrcRegHasOtherUse = false;
2806 MachineInstr *SrcMI = getDefMIPostRA(SrcReg, CmpMI, SrcRegHasOtherUse);
2807 if (!SrcMI || !SrcMI->definesRegister(SrcReg))
2808 return false;
2809
2810 MachineOperand RegMO = CmpMI.getOperand(0);
2811 Register CRReg = RegMO.getReg();
2812 if (CRReg != PPC::CR0)
2813 return false;
2814
2815 // Make sure there is no def/use of CRReg between SrcMI and CmpMI.
2816 bool SeenUseOfCRReg = false;
2817 bool IsCRRegKilled = false;
2818 if (!isRegElgibleForForwarding(RegMO, *SrcMI, CmpMI, false, IsCRRegKilled,
2819 SeenUseOfCRReg) ||
2820 SrcMI->definesRegister(CRReg) || SeenUseOfCRReg)
2821 return false;
2822
2823 int SrcMIOpc = SrcMI->getOpcode();
2824 int NewOpC = PPC::getRecordFormOpcode(SrcMIOpc);
2825 if (NewOpC == -1)
2826 return false;
2827
2828 LLVM_DEBUG(dbgs() << "Replace Instr: ");
2829 LLVM_DEBUG(SrcMI->dump());
2830
2831 const MCInstrDesc &NewDesc = get(NewOpC);
2832 SrcMI->setDesc(NewDesc);
2833 MachineInstrBuilder(*SrcMI->getParent()->getParent(), SrcMI)
2835 SrcMI->clearRegisterDeads(CRReg);
2836
2837 assert(SrcMI->definesRegister(PPC::CR0) &&
2838 "Record-form instruction does not define cr0?");
2839
2840 LLVM_DEBUG(dbgs() << "with: ");
2841 LLVM_DEBUG(SrcMI->dump());
2842 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
2843 LLVM_DEBUG(CmpMI.dump());
2844 return true;
2845}
2846
2849 int64_t &Offset, bool &OffsetIsScalable, unsigned &Width,
2850 const TargetRegisterInfo *TRI) const {
2851 const MachineOperand *BaseOp;
2852 OffsetIsScalable = false;
2853 if (!getMemOperandWithOffsetWidth(LdSt, BaseOp, Offset, Width, TRI))
2854 return false;
2855 BaseOps.push_back(BaseOp);
2856 return true;
2857}
2858
2859static bool isLdStSafeToCluster(const MachineInstr &LdSt,
2860 const TargetRegisterInfo *TRI) {
2861 // If this is a volatile load/store, don't mess with it.
2862 if (LdSt.hasOrderedMemoryRef() || LdSt.getNumExplicitOperands() != 3)
2863 return false;
2864
2865 if (LdSt.getOperand(2).isFI())
2866 return true;
2867
2868 assert(LdSt.getOperand(2).isReg() && "Expected a reg operand.");
2869 // Can't cluster if the instruction modifies the base register
2870 // or it is update form. e.g. ld r2,3(r2)
2871 if (LdSt.modifiesRegister(LdSt.getOperand(2).getReg(), TRI))
2872 return false;
2873
2874 return true;
2875}
2876
2877// Only cluster instruction pair that have the same opcode, and they are
2878// clusterable according to PowerPC specification.
2879static bool isClusterableLdStOpcPair(unsigned FirstOpc, unsigned SecondOpc,
2880 const PPCSubtarget &Subtarget) {
2881 switch (FirstOpc) {
2882 default:
2883 return false;
2884 case PPC::STD:
2885 case PPC::STFD:
2886 case PPC::STXSD:
2887 case PPC::DFSTOREf64:
2888 return FirstOpc == SecondOpc;
2889 // PowerPC backend has opcode STW/STW8 for instruction "stw" to deal with
2890 // 32bit and 64bit instruction selection. They are clusterable pair though
2891 // they are different opcode.
2892 case PPC::STW:
2893 case PPC::STW8:
2894 return SecondOpc == PPC::STW || SecondOpc == PPC::STW8;
2895 }
2896}
2897
2900 ArrayRef<const MachineOperand *> BaseOps2, unsigned NumLoads,
2901 unsigned NumBytes) const {
2902
2903 assert(BaseOps1.size() == 1 && BaseOps2.size() == 1);
2904 const MachineOperand &BaseOp1 = *BaseOps1.front();
2905 const MachineOperand &BaseOp2 = *BaseOps2.front();
2906 assert((BaseOp1.isReg() || BaseOp1.isFI()) &&
2907 "Only base registers and frame indices are supported.");
2908
2909 // The NumLoads means the number of loads that has been clustered.
2910 // Don't cluster memory op if there are already two ops clustered at least.
2911 if (NumLoads > 2)
2912 return false;
2913
2914 // Cluster the load/store only when they have the same base
2915 // register or FI.
2916 if ((BaseOp1.isReg() != BaseOp2.isReg()) ||
2917 (BaseOp1.isReg() && BaseOp1.getReg() != BaseOp2.getReg()) ||
2918 (BaseOp1.isFI() && BaseOp1.getIndex() != BaseOp2.getIndex()))
2919 return false;
2920
2921 // Check if the load/store are clusterable according to the PowerPC
2922 // specification.
2923 const MachineInstr &FirstLdSt = *BaseOp1.getParent();
2924 const MachineInstr &SecondLdSt = *BaseOp2.getParent();
2925 unsigned FirstOpc = FirstLdSt.getOpcode();
2926 unsigned SecondOpc = SecondLdSt.getOpcode();
2928 // Cluster the load/store only when they have the same opcode, and they are
2929 // clusterable opcode according to PowerPC specification.
2930 if (!isClusterableLdStOpcPair(FirstOpc, SecondOpc, Subtarget))
2931 return false;
2932
2933 // Can't cluster load/store that have ordered or volatile memory reference.
2934 if (!isLdStSafeToCluster(FirstLdSt, TRI) ||
2935 !isLdStSafeToCluster(SecondLdSt, TRI))
2936 return false;
2937
2938 int64_t Offset1 = 0, Offset2 = 0;
2939 unsigned Width1 = 0, Width2 = 0;
2940 const MachineOperand *Base1 = nullptr, *Base2 = nullptr;
2941 if (!getMemOperandWithOffsetWidth(FirstLdSt, Base1, Offset1, Width1, TRI) ||
2942 !getMemOperandWithOffsetWidth(SecondLdSt, Base2, Offset2, Width2, TRI) ||
2943 Width1 != Width2)
2944 return false;
2945
2946 assert(Base1 == &BaseOp1 && Base2 == &BaseOp2 &&
2947 "getMemOperandWithOffsetWidth return incorrect base op");
2948 // The caller should already have ordered FirstMemOp/SecondMemOp by offset.
2949 assert(Offset1 <= Offset2 && "Caller should have ordered offsets.");
2950 return Offset1 + Width1 == Offset2;
2951}
2952
2953/// GetInstSize - Return the number of bytes of code the specified
2954/// instruction may be. This returns the maximum number of bytes.
2955///
2957 unsigned Opcode = MI.getOpcode();
2958
2959 if (Opcode == PPC::INLINEASM || Opcode == PPC::INLINEASM_BR) {
2960 const MachineFunction *MF = MI.getParent()->getParent();
2961 const char *AsmStr = MI.getOperand(0).getSymbolName();
2962 return getInlineAsmLength(AsmStr, *MF->getTarget().getMCAsmInfo());
2963 } else if (Opcode == TargetOpcode::STACKMAP) {
2964 StackMapOpers Opers(&MI);
2965 return Opers.getNumPatchBytes();
2966 } else if (Opcode == TargetOpcode::PATCHPOINT) {
2967 PatchPointOpers Opers(&MI);
2968 return Opers.getNumPatchBytes();
2969 } else {
2970 return get(Opcode).getSize();
2971 }
2972}
2973
2974std::pair<unsigned, unsigned>
2976 const unsigned Mask = PPCII::MO_ACCESS_MASK;
2977 return std::make_pair(TF & Mask, TF & ~Mask);
2978}
2979
2982 using namespace PPCII;
2983 static const std::pair<unsigned, const char *> TargetFlags[] = {
2984 {MO_LO, "ppc-lo"},
2985 {MO_HA, "ppc-ha"},
2986 {MO_TPREL_LO, "ppc-tprel-lo"},
2987 {MO_TPREL_HA, "ppc-tprel-ha"},
2988 {MO_DTPREL_LO, "ppc-dtprel-lo"},
2989 {MO_TLSLD_LO, "ppc-tlsld-lo"},
2990 {MO_TOC_LO, "ppc-toc-lo"},
2991 {MO_TLS, "ppc-tls"}};
2992 return ArrayRef(TargetFlags);
2993}
2994
2997 using namespace PPCII;
2998 static const std::pair<unsigned, const char *> TargetFlags[] = {
2999 {MO_PLT, "ppc-plt"},
3000 {MO_PIC_FLAG, "ppc-pic"},
3001 {MO_PCREL_FLAG, "ppc-pcrel"},
3002 {MO_GOT_FLAG, "ppc-got"},
3003 {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"},
3004 {MO_TLSGD_FLAG, "ppc-tlsgd"},
3005 {MO_TLSLD_FLAG, "ppc-tlsld"},
3006 {MO_TPREL_FLAG, "ppc-tprel"},
3007 {MO_TLSGDM_FLAG, "ppc-tlsgdm"},
3008 {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"},
3009 {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"},
3010 {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}};
3011 return ArrayRef(TargetFlags);
3012}
3013
3014// Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction.
3015// The VSX versions have the advantage of a full 64-register target whereas
3016// the FP ones have the advantage of lower latency and higher throughput. So
3017// what we are after is using the faster instructions in low register pressure
3018// situations and using the larger register file in high register pressure
3019// situations.
3021 unsigned UpperOpcode, LowerOpcode;
3022 switch (MI.getOpcode()) {
3023 case PPC::DFLOADf32:
3024 UpperOpcode = PPC::LXSSP;
3025 LowerOpcode = PPC::LFS;
3026 break;
3027 case PPC::DFLOADf64:
3028 UpperOpcode = PPC::LXSD;
3029 LowerOpcode = PPC::LFD;
3030 break;
3031 case PPC::DFSTOREf32:
3032 UpperOpcode = PPC::STXSSP;
3033 LowerOpcode = PPC::STFS;
3034 break;
3035 case PPC::DFSTOREf64:
3036 UpperOpcode = PPC::STXSD;
3037 LowerOpcode = PPC::STFD;
3038 break;
3039 case PPC::XFLOADf32:
3040 UpperOpcode = PPC::LXSSPX;
3041 LowerOpcode = PPC::LFSX;
3042 break;
3043 case PPC::XFLOADf64:
3044 UpperOpcode = PPC::LXSDX;
3045 LowerOpcode = PPC::LFDX;
3046 break;
3047 case PPC::XFSTOREf32:
3048 UpperOpcode = PPC::STXSSPX;
3049 LowerOpcode = PPC::STFSX;
3050 break;
3051 case PPC::XFSTOREf64:
3052 UpperOpcode = PPC::STXSDX;
3053 LowerOpcode = PPC::STFDX;
3054 break;
3055 case PPC::LIWAX:
3056 UpperOpcode = PPC::LXSIWAX;
3057 LowerOpcode = PPC::LFIWAX;
3058 break;
3059 case PPC::LIWZX:
3060 UpperOpcode = PPC::LXSIWZX;
3061 LowerOpcode = PPC::LFIWZX;
3062 break;
3063 case PPC::STIWX:
3064 UpperOpcode = PPC::STXSIWX;
3065 LowerOpcode = PPC::STFIWX;
3066 break;
3067 default:
3068 llvm_unreachable("Unknown Operation!");
3069 }
3070
3071 Register TargetReg = MI.getOperand(0).getReg();
3072 unsigned Opcode;
3073 if ((TargetReg >= PPC::F0 && TargetReg <= PPC::F31) ||
3074 (TargetReg >= PPC::VSL0 && TargetReg <= PPC::VSL31))
3075 Opcode = LowerOpcode;
3076 else
3077 Opcode = UpperOpcode;
3078 MI.setDesc(get(Opcode));
3079 return true;
3080}
3081
3082static bool isAnImmediateOperand(const MachineOperand &MO) {
3083 return MO.isCPI() || MO.isGlobal() || MO.isImm();
3084}
3085
3087 auto &MBB = *MI.getParent();
3088 auto DL = MI.getDebugLoc();
3089
3090 switch (MI.getOpcode()) {
3091 case PPC::BUILD_UACC: {
3092 MCRegister ACC = MI.getOperand(0).getReg();
3093 MCRegister UACC = MI.getOperand(1).getReg();
3094 if (ACC - PPC::ACC0 != UACC - PPC::UACC0) {
3095 MCRegister SrcVSR = PPC::VSL0 + (UACC - PPC::UACC0) * 4;
3096 MCRegister DstVSR = PPC::VSL0 + (ACC - PPC::ACC0) * 4;
3097 // FIXME: This can easily be improved to look up to the top of the MBB
3098 // to see if the inputs are XXLOR's. If they are and SrcReg is killed,
3099 // we can just re-target any such XXLOR's to DstVSR + offset.
3100 for (int VecNo = 0; VecNo < 4; VecNo++)
3101 BuildMI(MBB, MI, DL, get(PPC::XXLOR), DstVSR + VecNo)
3102 .addReg(SrcVSR + VecNo)
3103 .addReg(SrcVSR + VecNo);
3104 }
3105 // BUILD_UACC is expanded to 4 copies of the underlying vsx registers.
3106 // So after building the 4 copies, we can replace the BUILD_UACC instruction
3107 // with a NOP.
3108 [[fallthrough]];
3109 }
3110 case PPC::KILL_PAIR: {
3111 MI.setDesc(get(PPC::UNENCODED_NOP));
3112 MI.removeOperand(1);
3113 MI.removeOperand(0);
3114 return true;
3115 }
3116 case TargetOpcode::LOAD_STACK_GUARD: {
3117 assert(Subtarget.isTargetLinux() &&
3118 "Only Linux target is expected to contain LOAD_STACK_GUARD");
3119 const int64_t Offset = Subtarget.isPPC64() ? -0x7010 : -0x7008;
3120 const unsigned Reg = Subtarget.isPPC64() ? PPC::X13 : PPC::R2;
3121 MI.setDesc(get(Subtarget.isPPC64() ? PPC::LD : PPC::LWZ));
3122 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3123 .addImm(Offset)
3124 .addReg(Reg);
3125 return true;
3126 }
3127 case PPC::DFLOADf32:
3128 case PPC::DFLOADf64:
3129 case PPC::DFSTOREf32:
3130 case PPC::DFSTOREf64: {
3131 assert(Subtarget.hasP9Vector() &&
3132 "Invalid D-Form Pseudo-ops on Pre-P9 target.");
3133 assert(MI.getOperand(2).isReg() &&
3134 isAnImmediateOperand(MI.getOperand(1)) &&
3135 "D-form op must have register and immediate operands");
3136 return expandVSXMemPseudo(MI);
3137 }
3138 case PPC::XFLOADf32:
3139 case PPC::XFSTOREf32:
3140 case PPC::LIWAX:
3141 case PPC::LIWZX:
3142 case PPC::STIWX: {
3143 assert(Subtarget.hasP8Vector() &&
3144 "Invalid X-Form Pseudo-ops on Pre-P8 target.");
3145 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3146 "X-form op must have register and register operands");
3147 return expandVSXMemPseudo(MI);
3148 }
3149 case PPC::XFLOADf64:
3150 case PPC::XFSTOREf64: {
3151 assert(Subtarget.hasVSX() &&
3152 "Invalid X-Form Pseudo-ops on target that has no VSX.");
3153 assert(MI.getOperand(2).isReg() && MI.getOperand(1).isReg() &&
3154 "X-form op must have register and register operands");
3155 return expandVSXMemPseudo(MI);
3156 }
3157 case PPC::SPILLTOVSR_LD: {
3158 Register TargetReg = MI.getOperand(0).getReg();
3159 if (PPC::VSFRCRegClass.contains(TargetReg)) {
3160 MI.setDesc(get(PPC::DFLOADf64));
3161 return expandPostRAPseudo(MI);
3162 }
3163 else
3164 MI.setDesc(get(PPC::LD));
3165 return true;
3166 }
3167 case PPC::SPILLTOVSR_ST: {
3168 Register SrcReg = MI.getOperand(0).getReg();
3169 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3170 NumStoreSPILLVSRRCAsVec++;
3171 MI.setDesc(get(PPC::DFSTOREf64));
3172 return expandPostRAPseudo(MI);
3173 } else {
3174 NumStoreSPILLVSRRCAsGpr++;
3175 MI.setDesc(get(PPC::STD));
3176 }
3177 return true;
3178 }
3179 case PPC::SPILLTOVSR_LDX: {
3180 Register TargetReg = MI.getOperand(0).getReg();
3181 if (PPC::VSFRCRegClass.contains(TargetReg))
3182 MI.setDesc(get(PPC::LXSDX));
3183 else
3184 MI.setDesc(get(PPC::LDX));
3185 return true;
3186 }
3187 case PPC::SPILLTOVSR_STX: {
3188 Register SrcReg = MI.getOperand(0).getReg();
3189 if (PPC::VSFRCRegClass.contains(SrcReg)) {
3190 NumStoreSPILLVSRRCAsVec++;
3191 MI.setDesc(get(PPC::STXSDX));
3192 } else {
3193 NumStoreSPILLVSRRCAsGpr++;
3194 MI.setDesc(get(PPC::STDX));
3195 }
3196 return true;
3197 }
3198
3199 // FIXME: Maybe we can expand it in 'PowerPC Expand Atomic' pass.
3200 case PPC::CFENCE8: {
3201 auto Val = MI.getOperand(0).getReg();
3202 BuildMI(MBB, MI, DL, get(PPC::CMPD), PPC::CR7).addReg(Val).addReg(Val);
3203 BuildMI(MBB, MI, DL, get(PPC::CTRL_DEP))
3205 .addReg(PPC::CR7)
3206 .addImm(1);
3207 MI.setDesc(get(PPC::ISYNC));
3208 MI.removeOperand(0);
3209 return true;
3210 }
3211 }
3212 return false;
3213}
3214
3215// Essentially a compile-time implementation of a compare->isel sequence.
3216// It takes two constants to compare, along with the true/false registers
3217// and the comparison type (as a subreg to a CR field) and returns one
3218// of the true/false registers, depending on the comparison results.
3219static unsigned selectReg(int64_t Imm1, int64_t Imm2, unsigned CompareOpc,
3220 unsigned TrueReg, unsigned FalseReg,
3221 unsigned CRSubReg) {
3222 // Signed comparisons. The immediates are assumed to be sign-extended.
3223 if (CompareOpc == PPC::CMPWI || CompareOpc == PPC::CMPDI) {
3224 switch (CRSubReg) {
3225 default: llvm_unreachable("Unknown integer comparison type.");
3226 case PPC::sub_lt:
3227 return Imm1 < Imm2 ? TrueReg : FalseReg;
3228 case PPC::sub_gt:
3229 return Imm1 > Imm2 ? TrueReg : FalseReg;
3230 case PPC::sub_eq:
3231 return Imm1 == Imm2 ? TrueReg : FalseReg;
3232 }
3233 }
3234 // Unsigned comparisons.
3235 else if (CompareOpc == PPC::CMPLWI || CompareOpc == PPC::CMPLDI) {
3236 switch (CRSubReg) {
3237 default: llvm_unreachable("Unknown integer comparison type.");
3238 case PPC::sub_lt:
3239 return (uint64_t)Imm1 < (uint64_t)Imm2 ? TrueReg : FalseReg;
3240 case PPC::sub_gt:
3241 return (uint64_t)Imm1 > (uint64_t)Imm2 ? TrueReg : FalseReg;
3242 case PPC::sub_eq:
3243 return Imm1 == Imm2 ? TrueReg : FalseReg;
3244 }
3245 }
3246 return PPC::NoRegister;
3247}
3248
3250 unsigned OpNo,
3251 int64_t Imm) const {
3252 assert(MI.getOperand(OpNo).isReg() && "Operand must be a REG");
3253 // Replace the REG with the Immediate.
3254 Register InUseReg = MI.getOperand(OpNo).getReg();
3255 MI.getOperand(OpNo).ChangeToImmediate(Imm);
3256
3257 // We need to make sure that the MI didn't have any implicit use
3258 // of this REG any more. We don't call MI.implicit_operands().empty() to
3259 // return early, since MI's MCID might be changed in calling context, as a
3260 // result its number of explicit operands may be changed, thus the begin of
3261 // implicit operand is changed.
3263 int UseOpIdx = MI.findRegisterUseOperandIdx(InUseReg, false, TRI);
3264 if (UseOpIdx >= 0) {
3265 MachineOperand &MO = MI.getOperand(UseOpIdx);
3266 if (MO.isImplicit())
3267 // The operands must always be in the following order:
3268 // - explicit reg defs,
3269 // - other explicit operands (reg uses, immediates, etc.),
3270 // - implicit reg defs
3271 // - implicit reg uses
3272 // Therefore, removing the implicit operand won't change the explicit
3273 // operands layout.
3274 MI.removeOperand(UseOpIdx);
3275 }
3276}
3277
3278// Replace an instruction with one that materializes a constant (and sets
3279// CR0 if the original instruction was a record-form instruction).
3281 const LoadImmediateInfo &LII) const {
3282 // Remove existing operands.
3283 int OperandToKeep = LII.SetCR ? 1 : 0;
3284 for (int i = MI.getNumOperands() - 1; i > OperandToKeep; i--)
3285 MI.removeOperand(i);
3286
3287 // Replace the instruction.
3288 if (LII.SetCR) {
3289 MI.setDesc(get(LII.Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3290 // Set the immediate.
3291 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3292 .addImm(LII.Imm).addReg(PPC::CR0, RegState::ImplicitDefine);
3293 return;
3294 }
3295 else
3296 MI.setDesc(get(LII.Is64Bit ? PPC::LI8 : PPC::LI));
3297
3298 // Set the immediate.
3299 MachineInstrBuilder(*MI.getParent()->getParent(), MI)
3300 .addImm(LII.Imm);
3301}
3302
3304 bool &SeenIntermediateUse) const {
3305 assert(!MI.getParent()->getParent()->getRegInfo().isSSA() &&
3306 "Should be called after register allocation.");
3308 MachineBasicBlock::reverse_iterator E = MI.getParent()->rend(), It = MI;
3309 It++;
3310 SeenIntermediateUse = false;
3311 for (; It != E; ++It) {
3312 if (It->modifiesRegister(Reg, TRI))
3313 return &*It;
3314 if (It->readsRegister(Reg, TRI))
3315 SeenIntermediateUse = true;
3316 }
3317 return nullptr;
3318}
3319
3322 const DebugLoc &DL, Register Reg,
3323 int64_t Imm) const {
3325 "Register should be in non-SSA form after RA");
3326 bool isPPC64 = Subtarget.isPPC64();
3327 // FIXME: Materialization here is not optimal.
3328 // For some special bit patterns we can use less instructions.
3329 // See `selectI64ImmDirect` in PPCISelDAGToDAG.cpp.
3330 if (isInt<16>(Imm)) {
3331 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LI8 : PPC::LI), Reg).addImm(Imm);
3332 } else if (isInt<32>(Imm)) {
3333 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::LIS8 : PPC::LIS), Reg)
3334 .addImm(Imm >> 16);
3335 if (Imm & 0xFFFF)
3336 BuildMI(MBB, MBBI, DL, get(isPPC64 ? PPC::ORI8 : PPC::ORI), Reg)
3337 .addReg(Reg, RegState::Kill)
3338 .addImm(Imm & 0xFFFF);
3339 } else {
3340 assert(isPPC64 && "Materializing 64-bit immediate to single register is "
3341 "only supported in PPC64");
3342 BuildMI(MBB, MBBI, DL, get(PPC::LIS8), Reg).addImm(Imm >> 48);
3343 if ((Imm >> 32) & 0xFFFF)
3344 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3345 .addReg(Reg, RegState::Kill)
3346 .addImm((Imm >> 32) & 0xFFFF);
3347 BuildMI(MBB, MBBI, DL, get(PPC::RLDICR), Reg)
3348 .addReg(Reg, RegState::Kill)
3349 .addImm(32)
3350 .addImm(31);
3351 BuildMI(MBB, MBBI, DL, get(PPC::ORIS8), Reg)
3352 .addReg(Reg, RegState::Kill)
3353 .addImm((Imm >> 16) & 0xFFFF);
3354 if (Imm & 0xFFFF)
3355 BuildMI(MBB, MBBI, DL, get(PPC::ORI8), Reg)
3356 .addReg(Reg, RegState::Kill)
3357 .addImm(Imm & 0xFFFF);
3358 }
3359}
3360
3361MachineInstr *PPCInstrInfo::getForwardingDefMI(
3363 unsigned &OpNoForForwarding,
3364 bool &SeenIntermediateUse) const {
3365 OpNoForForwarding = ~0U;
3366 MachineInstr *DefMI = nullptr;
3367 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3369 // If we're in SSA, get the defs through the MRI. Otherwise, only look
3370 // within the basic block to see if the register is defined using an
3371 // LI/LI8/ADDI/ADDI8.
3372 if (MRI->isSSA()) {
3373 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3374 if (!MI.getOperand(i).isReg())
3375 continue;
3376 Register Reg = MI.getOperand(i).getReg();
3377 if (!Reg.isVirtual())
3378 continue;
3379 Register TrueReg = TRI->lookThruCopyLike(Reg, MRI);
3380 if (TrueReg.isVirtual()) {
3381 MachineInstr *DefMIForTrueReg = MRI->getVRegDef(TrueReg);
3382 if (DefMIForTrueReg->getOpcode() == PPC::LI ||
3383 DefMIForTrueReg->getOpcode() == PPC::LI8 ||
3384 DefMIForTrueReg->getOpcode() == PPC::ADDI ||
3385 DefMIForTrueReg->getOpcode() == PPC::ADDI8) {
3386 OpNoForForwarding = i;
3387 DefMI = DefMIForTrueReg;
3388 // The ADDI and LI operand maybe exist in one instruction at same
3389 // time. we prefer to fold LI operand as LI only has one Imm operand
3390 // and is more possible to be converted. So if current DefMI is
3391 // ADDI/ADDI8, we continue to find possible LI/LI8.
3392 if (DefMI->getOpcode() == PPC::LI || DefMI->getOpcode() == PPC::LI8)
3393 break;
3394 }
3395 }
3396 }
3397 } else {
3398 // Looking back through the definition for each operand could be expensive,
3399 // so exit early if this isn't an instruction that either has an immediate
3400 // form or is already an immediate form that we can handle.
3401 ImmInstrInfo III;
3402 unsigned Opc = MI.getOpcode();
3403 bool ConvertibleImmForm =
3404 Opc == PPC::CMPWI || Opc == PPC::CMPLWI || Opc == PPC::CMPDI ||
3405 Opc == PPC::CMPLDI || Opc == PPC::ADDI || Opc == PPC::ADDI8 ||
3406 Opc == PPC::ORI || Opc == PPC::ORI8 || Opc == PPC::XORI ||
3407 Opc == PPC::XORI8 || Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec ||
3408 Opc == PPC::RLDICL_32 || Opc == PPC::RLDICL_32_64 ||
3409 Opc == PPC::RLWINM || Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8 ||
3410 Opc == PPC::RLWINM8_rec;
3411 bool IsVFReg = (MI.getNumOperands() && MI.getOperand(0).isReg())
3412 ? PPC::isVFRegister(MI.getOperand(0).getReg())
3413 : false;
3414 if (!ConvertibleImmForm && !instrHasImmForm(Opc, IsVFReg, III, true))
3415 return nullptr;
3416
3417 // Don't convert or %X, %Y, %Y since that's just a register move.
3418 if ((Opc == PPC::OR || Opc == PPC::OR8) &&
3419 MI.getOperand(1).getReg() == MI.getOperand(2).getReg())
3420 return nullptr;
3421 for (int i = 1, e = MI.getNumOperands(); i < e; i++) {
3422 MachineOperand &MO = MI.getOperand(i);
3423 SeenIntermediateUse = false;
3424 if (MO.isReg() && MO.isUse() && !MO.isImplicit()) {
3425 Register Reg = MI.getOperand(i).getReg();
3426 // If we see another use of this reg between the def and the MI,
3427 // we want to flag it so the def isn't deleted.
3428 MachineInstr *DefMI = getDefMIPostRA(Reg, MI, SeenIntermediateUse);
3429 if (DefMI) {
3430 // Is this register defined by some form of add-immediate (including
3431 // load-immediate) within this basic block?
3432 switch (DefMI->getOpcode()) {
3433 default:
3434 break;
3435 case PPC::LI:
3436 case PPC::LI8:
3437 case PPC::ADDItocL:
3438 case PPC::ADDI:
3439 case PPC::ADDI8:
3440 OpNoForForwarding = i;
3441 return DefMI;
3442 }
3443 }
3444 }
3445 }
3446 }
3447 return OpNoForForwarding == ~0U ? nullptr : DefMI;
3448}
3449
3450unsigned PPCInstrInfo::getSpillTarget() const {
3451 // With P10, we may need to spill paired vector registers or accumulator
3452 // registers. MMA implies paired vectors, so we can just check that.
3453 bool IsP10Variant = Subtarget.isISA3_1() || Subtarget.pairedVectorMemops();
3454 return Subtarget.isISAFuture() ? 3 : IsP10Variant ?
3455 2 : Subtarget.hasP9Vector() ?
3456 1 : 0;
3457}
3458
3459ArrayRef<unsigned> PPCInstrInfo::getStoreOpcodesForSpillArray() const {
3460 return {StoreSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3461}
3462
3463ArrayRef<unsigned> PPCInstrInfo::getLoadOpcodesForSpillArray() const {
3464 return {LoadSpillOpcodesArray[getSpillTarget()], SOK_LastOpcodeSpill};
3465}
3466
3467// This opt tries to convert the following imm form to an index form to save an
3468// add for stack variables.
3469// Return false if no such pattern found.
3470//
3471// ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3472// ADD instr: ToBeDeletedReg = ADD ToBeChangedReg(killed), ScaleReg
3473// Imm instr: Reg = op OffsetImm, ToBeDeletedReg(killed)
3474//
3475// can be converted to:
3476//
3477// new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, (OffsetAddi + OffsetImm)
3478// Index instr: Reg = opx ScaleReg, ToBeChangedReg(killed)
3479//
3480// In order to eliminate ADD instr, make sure that:
3481// 1: (OffsetAddi + OffsetImm) must be int16 since this offset will be used in
3482// new ADDI instr and ADDI can only take int16 Imm.
3483// 2: ToBeChangedReg must be killed in ADD instr and there is no other use
3484// between ADDI and ADD instr since its original def in ADDI will be changed
3485// in new ADDI instr. And also there should be no new def for it between
3486// ADD and Imm instr as ToBeChangedReg will be used in Index instr.
3487// 3: ToBeDeletedReg must be killed in Imm instr and there is no other use
3488// between ADD and Imm instr since ADD instr will be eliminated.
3489// 4: ScaleReg must not be redefined between ADD and Imm instr since it will be
3490// moved to Index instr.
3492 MachineFunction *MF = MI.getParent()->getParent();
3494 bool PostRA = !MRI->isSSA();
3495 // Do this opt after PEI which is after RA. The reason is stack slot expansion
3496 // in PEI may expose such opportunities since in PEI, stack slot offsets to
3497 // frame base(OffsetAddi) are determined.
3498 if (!PostRA)
3499 return false;
3500 unsigned ToBeDeletedReg = 0;
3501 int64_t OffsetImm = 0;
3502 unsigned XFormOpcode = 0;
3503 ImmInstrInfo III;
3504
3505 // Check if Imm instr meets requirement.
3506 if (!isImmInstrEligibleForFolding(MI, ToBeDeletedReg, XFormOpcode, OffsetImm,
3507 III))
3508 return false;
3509
3510 bool OtherIntermediateUse = false;
3511 MachineInstr *ADDMI = getDefMIPostRA(ToBeDeletedReg, MI, OtherIntermediateUse);
3512
3513 // Exit if there is other use between ADD and Imm instr or no def found.
3514 if (OtherIntermediateUse || !ADDMI)
3515 return false;
3516
3517 // Check if ADD instr meets requirement.
3518 if (!isADDInstrEligibleForFolding(*ADDMI))
3519 return false;
3520
3521 unsigned ScaleRegIdx = 0;
3522 int64_t OffsetAddi = 0;
3523 MachineInstr *ADDIMI = nullptr;
3524
3525 // Check if there is a valid ToBeChangedReg in ADDMI.
3526 // 1: It must be killed.
3527 // 2: Its definition must be a valid ADDIMI.
3528 // 3: It must satify int16 offset requirement.
3529 if (isValidToBeChangedReg(ADDMI, 1, ADDIMI, OffsetAddi, OffsetImm))
3530 ScaleRegIdx = 2;
3531 else if (isValidToBeChangedReg(ADDMI, 2, ADDIMI, OffsetAddi, OffsetImm))
3532 ScaleRegIdx = 1;
3533 else
3534 return false;
3535
3536 assert(ADDIMI && "There should be ADDIMI for valid ToBeChangedReg.");
3537 Register ToBeChangedReg = ADDIMI->getOperand(0).getReg();
3538 Register ScaleReg = ADDMI->getOperand(ScaleRegIdx).getReg();
3539 auto NewDefFor = [&](unsigned Reg, MachineBasicBlock::iterator Start,
3541 for (auto It = ++Start; It != End; It++)
3542 if (It->modifiesRegister(Reg, &getRegisterInfo()))
3543 return true;
3544 return false;
3545 };
3546
3547 // We are trying to replace the ImmOpNo with ScaleReg. Give up if it is
3548 // treated as special zero when ScaleReg is R0/X0 register.
3549 if (III.ZeroIsSpecialOrig == III.ImmOpNo &&
3550 (ScaleReg == PPC::R0 || ScaleReg == PPC::X0))
3551 return false;
3552
3553 // Make sure no other def for ToBeChangedReg and ScaleReg between ADD Instr
3554 // and Imm Instr.
3555 if (NewDefFor(ToBeChangedReg, *ADDMI, MI) || NewDefFor(ScaleReg, *ADDMI, MI))
3556 return false;
3557
3558 // Now start to do the transformation.
3559 LLVM_DEBUG(dbgs() << "Replace instruction: "
3560 << "\n");
3561 LLVM_DEBUG(ADDIMI->dump());
3562 LLVM_DEBUG(ADDMI->dump());
3563 LLVM_DEBUG(MI.dump());
3564 LLVM_DEBUG(dbgs() << "with: "
3565 << "\n");
3566
3567 // Update ADDI instr.
3568 ADDIMI->getOperand(2).setImm(OffsetAddi + OffsetImm);
3569
3570 // Update Imm instr.
3571 MI.setDesc(get(XFormOpcode));
3572 MI.getOperand(III.ImmOpNo)
3573 .ChangeToRegister(ScaleReg, false, false,
3574 ADDMI->getOperand(ScaleRegIdx).isKill());
3575
3576 MI.getOperand(III.OpNoForForwarding)
3577 .ChangeToRegister(ToBeChangedReg, false, false, true);
3578
3579 // Eliminate ADD instr.
3580 ADDMI->eraseFromParent();
3581
3582 LLVM_DEBUG(ADDIMI->dump());
3583 LLVM_DEBUG(MI.dump());
3584
3585 return true;
3586}
3587
3589 int64_t &Imm) const {
3590 unsigned Opc = ADDIMI.getOpcode();
3591
3592 // Exit if the instruction is not ADDI.
3593 if (Opc != PPC::ADDI && Opc != PPC::ADDI8)
3594 return false;
3595
3596 // The operand may not necessarily be an immediate - it could be a relocation.
3597 if (!ADDIMI.getOperand(2).isImm())
3598 return false;
3599
3600 Imm = ADDIMI.getOperand(2).getImm();
3601
3602 return true;
3603}
3604
3606 unsigned Opc = ADDMI.getOpcode();
3607
3608 // Exit if the instruction is not ADD.
3609 return Opc == PPC::ADD4 || Opc == PPC::ADD8;
3610}
3611
3613 unsigned &ToBeDeletedReg,
3614 unsigned &XFormOpcode,
3615 int64_t &OffsetImm,
3616 ImmInstrInfo &III) const {
3617 // Only handle load/store.
3618 if (!MI.mayLoadOrStore())
3619 return false;
3620
3621 unsigned Opc = MI.getOpcode();
3622
3623 XFormOpcode = RI.getMappedIdxOpcForImmOpc(Opc);
3624
3625 // Exit if instruction has no index form.
3626 if (XFormOpcode == PPC::INSTRUCTION_LIST_END)
3627 return false;
3628
3629 // TODO: sync the logic between instrHasImmForm() and ImmToIdxMap.
3630 if (!instrHasImmForm(XFormOpcode,
3631 PPC::isVFRegister(MI.getOperand(0).getReg()), III, true))
3632 return false;
3633
3634 if (!III.IsSummingOperands)
3635 return false;
3636
3637 MachineOperand ImmOperand = MI.getOperand(III.ImmOpNo);
3638 MachineOperand RegOperand = MI.getOperand(III.OpNoForForwarding);
3639 // Only support imm operands, not relocation slots or others.
3640 if (!ImmOperand.isImm())
3641 return false;
3642
3643 assert(RegOperand.isReg() && "Instruction format is not right");
3644
3645 // There are other use for ToBeDeletedReg after Imm instr, can not delete it.
3646 if (!RegOperand.isKill())
3647 return false;
3648
3649 ToBeDeletedReg = RegOperand.getReg();
3650 OffsetImm = ImmOperand.getImm();
3651
3652 return true;
3653}
3654
3656 MachineInstr *&ADDIMI,
3657 int64_t &OffsetAddi,
3658 int64_t OffsetImm) const {
3659 assert((Index == 1 || Index == 2) && "Invalid operand index for add.");
3660 MachineOperand &MO = ADDMI->getOperand(Index);
3661
3662 if (!MO.isKill())
3663 return false;
3664
3665 bool OtherIntermediateUse = false;
3666
3667 ADDIMI = getDefMIPostRA(MO.getReg(), *ADDMI, OtherIntermediateUse);
3668 // Currently handle only one "add + Imminstr" pair case, exit if other
3669 // intermediate use for ToBeChangedReg found.
3670 // TODO: handle the cases where there are other "add + Imminstr" pairs
3671 // with same offset in Imminstr which is like:
3672 //
3673 // ADDI instr: ToBeChangedReg = ADDI FrameBaseReg, OffsetAddi
3674 // ADD instr1: ToBeDeletedReg1 = ADD ToBeChangedReg, ScaleReg1
3675 // Imm instr1: Reg1 = op1 OffsetImm, ToBeDeletedReg1(killed)
3676 // ADD instr2: ToBeDeletedReg2 = ADD ToBeChangedReg(killed), ScaleReg2
3677 // Imm instr2: Reg2 = op2 OffsetImm, ToBeDeletedReg2(killed)
3678 //
3679 // can be converted to:
3680 //
3681 // new ADDI instr: ToBeChangedReg = ADDI FrameBaseReg,
3682 // (OffsetAddi + OffsetImm)
3683 // Index instr1: Reg1 = opx1 ScaleReg1, ToBeChangedReg
3684 // Index instr2: Reg2 = opx2 ScaleReg2, ToBeChangedReg(killed)
3685
3686 if (OtherIntermediateUse || !ADDIMI)
3687 return false;
3688 // Check if ADDI instr meets requirement.
3689 if (!isADDIInstrEligibleForFolding(*ADDIMI, OffsetAddi))
3690 return false;
3691
3692 if (isInt<16>(OffsetAddi + OffsetImm))
3693 return true;
3694 return false;
3695}
3696
3697// If this instruction has an immediate form and one of its operands is a
3698// result of a load-immediate or an add-immediate, convert it to
3699// the immediate form if the constant is in range.
3701 SmallSet<Register, 4> &RegsToUpdate,
3702 MachineInstr **KilledDef) const {
3703 MachineFunction *MF = MI.getParent()->getParent();
3705 bool PostRA = !MRI->isSSA();
3706 bool SeenIntermediateUse = true;
3707 unsigned ForwardingOperand = ~0U;
3708 MachineInstr *DefMI = getForwardingDefMI(MI, ForwardingOperand,
3709 SeenIntermediateUse);
3710 if (!DefMI)
3711 return false;
3712 assert(ForwardingOperand < MI.getNumOperands() &&
3713 "The forwarding operand needs to be valid at this point");
3714 bool IsForwardingOperandKilled = MI.getOperand(ForwardingOperand).isKill();
3715 bool KillFwdDefMI = !SeenIntermediateUse && IsForwardingOperandKilled;
3716 if (KilledDef && KillFwdDefMI)
3717 *KilledDef = DefMI;
3718
3719 // Conservatively add defs from DefMI and defs/uses from MI to the set of
3720 // registers that need their kill flags updated.
3721 for (const MachineOperand &MO : DefMI->operands())
3722 if (MO.isReg() && MO.isDef())
3723 RegsToUpdate.insert(MO.getReg());
3724 for (const MachineOperand &MO : MI.operands())
3725 if (MO.isReg())
3726 RegsToUpdate.insert(MO.getReg());
3727
3728 // If this is a imm instruction and its register operands is produced by ADDI,
3729 // put the imm into imm inst directly.
3730 if (RI.getMappedIdxOpcForImmOpc(MI.getOpcode()) !=
3731 PPC::INSTRUCTION_LIST_END &&
3732 transformToNewImmFormFedByAdd(MI, *DefMI, ForwardingOperand))
3733 return true;
3734
3735 ImmInstrInfo III;
3736 bool IsVFReg = MI.getOperand(0).isReg()
3737 ? PPC::isVFRegister(MI.getOperand(0).getReg())
3738 : false;
3739 bool HasImmForm = instrHasImmForm(MI.getOpcode(), IsVFReg, III, PostRA);
3740 // If this is a reg+reg instruction that has a reg+imm form,
3741 // and one of the operands is produced by an add-immediate,
3742 // try to convert it.
3743 if (HasImmForm &&
3744 transformToImmFormFedByAdd(MI, III, ForwardingOperand, *DefMI,
3745 KillFwdDefMI))
3746 return true;
3747
3748 // If this is a reg+reg instruction that has a reg+imm form,
3749 // and one of the operands is produced by LI, convert it now.
3750 if (HasImmForm &&
3751 transformToImmFormFedByLI(MI, III, ForwardingOperand, *DefMI))
3752 return true;
3753
3754 // If this is not a reg+reg, but the DefMI is LI/LI8, check if its user MI
3755 // can be simpified to LI.
3756 if (!HasImmForm && simplifyToLI(MI, *DefMI, ForwardingOperand, KilledDef))
3757 return true;
3758
3759 return false;
3760}
3761
3763 MachineInstr **ToErase) const {
3764 MachineRegisterInfo *MRI = &MI.getParent()->getParent()->getRegInfo();
3765 Register FoldingReg = MI.getOperand(1).getReg();
3766 if (!FoldingReg.isVirtual())
3767 return false;
3768 MachineInstr *SrcMI = MRI->getVRegDef(FoldingReg);
3769 if (SrcMI->getOpcode() != PPC::RLWINM &&
3770 SrcMI->getOpcode() != PPC::RLWINM_rec &&
3771 SrcMI->getOpcode() != PPC::RLWINM8 &&
3772 SrcMI->getOpcode() != PPC::RLWINM8_rec)
3773 return false;
3774 assert((MI.getOperand(2).isImm() && MI.getOperand(3).isImm() &&
3775 MI.getOperand(4).isImm() && SrcMI->getOperand(2).isImm() &&
3776 SrcMI->getOperand(3).isImm() && SrcMI->getOperand(4).isImm()) &&
3777 "Invalid PPC::RLWINM Instruction!");
3778 uint64_t SHSrc = SrcMI->getOperand(2).getImm();
3779 uint64_t SHMI = MI.getOperand(2).getImm();
3780 uint64_t MBSrc = SrcMI->getOperand(3).getImm();
3781 uint64_t MBMI = MI.getOperand(3).getImm();
3782 uint64_t MESrc = SrcMI->getOperand(4).getImm();
3783 uint64_t MEMI = MI.getOperand(4).getImm();
3784
3785 assert((MEMI < 32 && MESrc < 32 && MBMI < 32 && MBSrc < 32) &&
3786 "Invalid PPC::RLWINM Instruction!");
3787 // If MBMI is bigger than MEMI, we always can not get run of ones.
3788 // RotatedSrcMask non-wrap:
3789 // 0........31|32........63
3790 // RotatedSrcMask: B---E B---E
3791 // MaskMI: -----------|--E B------
3792 // Result: ----- --- (Bad candidate)
3793 //
3794 // RotatedSrcMask wrap:
3795 // 0........31|32........63
3796 // RotatedSrcMask: --E B----|--E B----
3797 // MaskMI: -----------|--E B------
3798 // Result: --- -----|--- ----- (Bad candidate)
3799 //
3800 // One special case is RotatedSrcMask is a full set mask.
3801 // RotatedSrcMask full:
3802 // 0........31|32........63
3803 // RotatedSrcMask: ------EB---|-------EB---
3804 // MaskMI: -----------|--E B------
3805 // Result: -----------|--- ------- (Good candidate)
3806
3807 // Mark special case.
3808 bool SrcMaskFull = (MBSrc - MESrc == 1) || (MBSrc == 0 && MESrc == 31);
3809
3810 // For other MBMI > MEMI cases, just return.
3811 if ((MBMI > MEMI) && !SrcMaskFull)
3812 return false;
3813
3814 // Handle MBMI <= MEMI cases.
3815 APInt MaskMI = APInt::getBitsSetWithWrap(32, 32 - MEMI - 1, 32 - MBMI);
3816 // In MI, we only need low 32 bits of SrcMI, just consider about low 32
3817 // bit of SrcMI mask. Note that in APInt, lowerest bit is at index 0,
3818 // while in PowerPC ISA, lowerest bit is at index 63.
3819 APInt MaskSrc = APInt::getBitsSetWithWrap(32, 32 - MESrc - 1, 32 - MBSrc);
3820
3821 APInt RotatedSrcMask = MaskSrc.rotl(SHMI);
3822 APInt FinalMask = RotatedSrcMask & MaskMI;
3823 uint32_t NewMB, NewME;
3824 bool Simplified = false;
3825
3826 // If final mask is 0, MI result should be 0 too.
3827 if (FinalMask.isZero()) {
3828 bool Is64Bit =
3829 (MI.getOpcode() == PPC::RLWINM8 || MI.getOpcode() == PPC::RLWINM8_rec);
3830 Simplified = true;
3831 LLVM_DEBUG(dbgs() << "Replace Instr: ");
3832 LLVM_DEBUG(MI.dump());
3833
3834 if (MI.getOpcode() == PPC::RLWINM || MI.getOpcode() == PPC::RLWINM8) {
3835 // Replace MI with "LI 0"
3836 MI.removeOperand(4);
3837 MI.removeOperand(3);
3838 MI.removeOperand(2);
3839 MI.getOperand(1).ChangeToImmediate(0);
3840 MI.setDesc(get(Is64Bit ? PPC::LI8 : PPC::LI));
3841 } else {
3842 // Replace MI with "ANDI_rec reg, 0"
3843 MI.removeOperand(4);
3844 MI.removeOperand(3);
3845 MI.getOperand(2).setImm(0);
3846 MI.setDesc(get(Is64Bit ? PPC::ANDI8_rec : PPC::ANDI_rec));
3847 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3848 if (SrcMI->getOperand(1).isKill()) {
3849 MI.getOperand(1).setIsKill(true);
3850 SrcMI->getOperand(1).setIsKill(false);
3851 } else
3852 // About to replace MI.getOperand(1), clear its kill flag.
3853 MI.getOperand(1).setIsKill(false);
3854 }
3855
3856 LLVM_DEBUG(dbgs() << "With: ");
3857 LLVM_DEBUG(MI.dump());
3858
3859 } else if ((isRunOfOnes((unsigned)(FinalMask.getZExtValue()), NewMB, NewME) &&
3860 NewMB <= NewME) ||
3861 SrcMaskFull) {
3862 // Here we only handle MBMI <= MEMI case, so NewMB must be no bigger
3863 // than NewME. Otherwise we get a 64 bit value after folding, but MI
3864 // return a 32 bit value.
3865 Simplified = true;
3866 LLVM_DEBUG(dbgs() << "Converting Instr: ");
3867 LLVM_DEBUG(MI.dump());
3868
3869 uint16_t NewSH = (SHSrc + SHMI) % 32;
3870 MI.getOperand(2).setImm(NewSH);
3871 // If SrcMI mask is full, no need to update MBMI and MEMI.
3872 if (!SrcMaskFull) {
3873 MI.getOperand(3).setImm(NewMB);
3874 MI.getOperand(4).setImm(NewME);
3875 }
3876 MI.getOperand(1).setReg(SrcMI->getOperand(1).getReg());
3877 if (SrcMI->getOperand(1).isKill()) {
3878 MI.getOperand(1).setIsKill(true);
3879 SrcMI->getOperand(1).setIsKill(false);
3880 } else
3881 // About to replace MI.getOperand(1), clear its kill flag.
3882 MI.getOperand(1).setIsKill(false);
3883
3884 LLVM_DEBUG(dbgs() << "To: ");
3885 LLVM_DEBUG(MI.dump());
3886 }
3887 if (Simplified & MRI->use_nodbg_empty(FoldingReg) &&
3888 !SrcMI->hasImplicitDef()) {
3889 // If FoldingReg has no non-debug use and it has no implicit def (it
3890 // is not RLWINMO or RLWINM8o), it's safe to delete its def SrcMI.
3891 // Otherwise keep it.
3892 *ToErase = SrcMI;
3893 LLVM_DEBUG(dbgs() << "Delete dead instruction: ");
3894 LLVM_DEBUG(SrcMI->dump());
3895 }
3896 return Simplified;
3897}
3898
3899bool PPCInstrInfo::instrHasImmForm(unsigned Opc, bool IsVFReg,
3900 ImmInstrInfo &III, bool PostRA) const {
3901 // The vast majority of the instructions would need their operand 2 replaced
3902 // with an immediate when switching to the reg+imm form. A marked exception
3903 // are the update form loads/stores for which a constant operand 2 would need
3904 // to turn into a displacement and move operand 1 to the operand 2 position.
3905 III.ImmOpNo = 2;
3906 III.OpNoForForwarding = 2;
3907 III.ImmWidth = 16;
3908 III.ImmMustBeMultipleOf = 1;
3909 III.TruncateImmTo = 0;
3910 III.IsSummingOperands = false;
3911 switch (Opc) {
3912 default: return false;
3913 case PPC::ADD4:
3914 case PPC::ADD8:
3915 III.SignedImm = true;
3916 III.ZeroIsSpecialOrig = 0;
3917 III.ZeroIsSpecialNew = 1;
3918 III.IsCommutative = true;
3919 III.IsSummingOperands = true;
3920 III.ImmOpcode = Opc == PPC::ADD4 ? PPC::ADDI : PPC::ADDI8;
3921 break;
3922 case PPC::ADDC:
3923 case PPC::ADDC8:
3924 III.SignedImm = true;
3925 III.ZeroIsSpecialOrig = 0;
3926 III.ZeroIsSpecialNew = 0;
3927 III.IsCommutative = true;
3928 III.IsSummingOperands = true;
3929 III.ImmOpcode = Opc == PPC::ADDC ? PPC::ADDIC : PPC::ADDIC8;
3930 break;
3931 case PPC::ADDC_rec:
3932 III.SignedImm = true;
3933 III.ZeroIsSpecialOrig = 0;
3934 III.ZeroIsSpecialNew = 0;
3935 III.IsCommutative = true;
3936 III.IsSummingOperands = true;
3937 III.ImmOpcode = PPC::ADDIC_rec;
3938 break;
3939 case PPC::SUBFC:
3940 case PPC::SUBFC8:
3941 III.SignedImm = true;
3942 III.ZeroIsSpecialOrig = 0;
3943 III.ZeroIsSpecialNew = 0;
3944 III.IsCommutative = false;
3945 III.ImmOpcode = Opc == PPC::SUBFC ? PPC::SUBFIC : PPC::SUBFIC8;
3946 break;
3947 case PPC::CMPW:
3948 case PPC::CMPD:
3949 III.SignedImm = true;
3950 III.ZeroIsSpecialOrig = 0;
3951 III.ZeroIsSpecialNew = 0;
3952 III.IsCommutative = false;
3953 III.ImmOpcode = Opc == PPC::CMPW ? PPC::CMPWI : PPC::CMPDI;
3954 break;
3955 case PPC::CMPLW:
3956 case PPC::CMPLD:
3957 III.SignedImm = false;
3958 III.ZeroIsSpecialOrig = 0;
3959 III.ZeroIsSpecialNew = 0;
3960 III.IsCommutative = false;
3961 III.ImmOpcode = Opc == PPC::CMPLW ? PPC::CMPLWI : PPC::CMPLDI;
3962 break;
3963 case PPC::AND_rec:
3964 case PPC::AND8_rec:
3965 case PPC::OR:
3966 case PPC::OR8:
3967 case PPC::XOR:
3968 case PPC::XOR8:
3969 III.SignedImm = false;
3970 III.ZeroIsSpecialOrig = 0;
3971 III.ZeroIsSpecialNew = 0;
3972 III.IsCommutative = true;
3973 switch(Opc) {
3974 default: llvm_unreachable("Unknown opcode");
3975 case PPC::AND_rec:
3976 III.ImmOpcode = PPC::ANDI_rec;
3977 break;
3978 case PPC::AND8_rec:
3979 III.ImmOpcode = PPC::ANDI8_rec;
3980 break;
3981 case PPC::OR: III.ImmOpcode = PPC::ORI; break;
3982 case PPC::OR8: III.ImmOpcode = PPC::ORI8; break;
3983 case PPC::XOR: III.ImmOpcode = PPC::XORI; break;
3984 case PPC::XOR8: III.ImmOpcode = PPC::XORI8; break;
3985 }
3986 break;
3987 case PPC::RLWNM:
3988 case PPC::RLWNM8:
3989 case PPC::RLWNM_rec:
3990 case PPC::RLWNM8_rec:
3991 case PPC::SLW:
3992 case PPC::SLW8:
3993 case PPC::SLW_rec:
3994 case PPC::SLW8_rec:
3995 case PPC::SRW:
3996 case PPC::SRW8:
3997 case PPC::SRW_rec:
3998 case PPC::SRW8_rec:
3999 case PPC::SRAW:
4000 case PPC::SRAW_rec:
4001 III.SignedImm = false;
4002 III.ZeroIsSpecialOrig = 0;
4003 III.ZeroIsSpecialNew = 0;
4004 III.IsCommutative = false;
4005 // This isn't actually true, but the instructions ignore any of the
4006 // upper bits, so any immediate loaded with an LI is acceptable.
4007 // This does not apply to shift right algebraic because a value
4008 // out of range will produce a -1/0.
4009 III.ImmWidth = 16;
4010 if (Opc == PPC::RLWNM || Opc == PPC::RLWNM8 || Opc == PPC::RLWNM_rec ||
4011 Opc == PPC::RLWNM8_rec)
4012 III.TruncateImmTo = 5;
4013 else
4014 III.TruncateImmTo = 6;
4015 switch(Opc) {
4016 default: llvm_unreachable("Unknown opcode");
4017 case PPC::RLWNM: III.ImmOpcode = PPC::RLWINM; break;
4018 case PPC::RLWNM8: III.ImmOpcode = PPC::RLWINM8; break;
4019 case PPC::RLWNM_rec:
4020 III.ImmOpcode = PPC::RLWINM_rec;
4021 break;
4022 case PPC::RLWNM8_rec:
4023 III.ImmOpcode = PPC::RLWINM8_rec;
4024 break;
4025 case PPC::SLW: III.ImmOpcode = PPC::RLWINM; break;
4026 case PPC::SLW8: III.ImmOpcode = PPC::RLWINM8; break;
4027 case PPC::SLW_rec:
4028 III.ImmOpcode = PPC::RLWINM_rec;
4029 break;
4030 case PPC::SLW8_rec:
4031 III.ImmOpcode = PPC::RLWINM8_rec;
4032 break;
4033 case PPC::SRW: III.ImmOpcode = PPC::RLWINM; break;
4034 case PPC::SRW8: III.ImmOpcode = PPC::RLWINM8; break;
4035 case PPC::SRW_rec:
4036 III.ImmOpcode = PPC::RLWINM_rec;
4037 break;
4038 case PPC::SRW8_rec:
4039 III.ImmOpcode = PPC::RLWINM8_rec;
4040 break;
4041 case PPC::SRAW:
4042 III.ImmWidth = 5;
4043 III.TruncateImmTo = 0;
4044 III.ImmOpcode = PPC::SRAWI;
4045 break;
4046 case PPC::SRAW_rec:
4047 III.ImmWidth = 5;
4048 III.TruncateImmTo = 0;
4049 III.ImmOpcode = PPC::SRAWI_rec;
4050 break;
4051 }
4052 break;
4053 case PPC::RLDCL:
4054 case PPC::RLDCL_rec:
4055 case PPC::RLDCR:
4056 case PPC::RLDCR_rec:
4057 case PPC::SLD:
4058 case PPC::SLD_rec:
4059 case PPC::SRD:
4060 case PPC::SRD_rec:
4061 case PPC::SRAD:
4062 case PPC::SRAD_rec:
4063 III.SignedImm = false;
4064 III.ZeroIsSpecialOrig = 0;
4065 III.ZeroIsSpecialNew = 0;
4066 III.IsCommutative = false;
4067 // This isn't actually true, but the instructions ignore any of the
4068 // upper bits, so any immediate loaded with an LI is acceptable.
4069 // This does not apply to shift right algebraic because a value
4070 // out of range will produce a -1/0.
4071 III.ImmWidth = 16;
4072 if (Opc == PPC::RLDCL || Opc == PPC::RLDCL_rec || Opc == PPC::RLDCR ||
4073 Opc == PPC::RLDCR_rec)
4074 III.TruncateImmTo = 6;
4075 else
4076 III.TruncateImmTo = 7;
4077 switch(Opc) {
4078 default: llvm_unreachable("Unknown opcode");
4079 case PPC::RLDCL: III.ImmOpcode = PPC::RLDICL; break;
4080 case PPC::RLDCL_rec:
4081 III.ImmOpcode = PPC::RLDICL_rec;
4082 break;
4083 case PPC::RLDCR: III.ImmOpcode = PPC::RLDICR; break;
4084 case PPC::RLDCR_rec:
4085 III.ImmOpcode = PPC::RLDICR_rec;
4086 break;
4087 case PPC::SLD: III.ImmOpcode = PPC::RLDICR; break;
4088 case PPC::SLD_rec:
4089 III.ImmOpcode = PPC::RLDICR_rec;
4090 break;
4091 case PPC::SRD: III.ImmOpcode = PPC::RLDICL; break;
4092 case PPC::SRD_rec:
4093 III.ImmOpcode = PPC::RLDICL_rec;
4094 break;
4095 case PPC::SRAD:
4096 III.ImmWidth = 6;
4097 III.TruncateImmTo = 0;
4098 III.ImmOpcode = PPC::SRADI;
4099 break;
4100 case PPC::SRAD_rec:
4101 III.ImmWidth = 6;
4102 III.TruncateImmTo = 0;
4103 III.ImmOpcode = PPC::SRADI_rec;
4104 break;
4105 }
4106 break;
4107 // Loads and stores:
4108 case PPC::LBZX:
4109 case PPC::LBZX8:
4110 case PPC::LHZX:
4111 case PPC::LHZX8:
4112 case PPC::LHAX:
4113 case PPC::LHAX8:
4114 case PPC::LWZX:
4115 case PPC::LWZX8:
4116 case PPC::LWAX:
4117 case PPC::LDX:
4118 case PPC::LFSX:
4119 case PPC::LFDX:
4120 case PPC::STBX:
4121 case PPC::STBX8:
4122 case PPC::STHX:
4123 case PPC::STHX8:
4124 case PPC::STWX:
4125 case PPC::STWX8:
4126 case PPC::STDX:
4127 case PPC::STFSX:
4128 case PPC::STFDX:
4129 III.SignedImm = true;
4130 III.ZeroIsSpecialOrig = 1;
4131 III.ZeroIsSpecialNew = 2;
4132 III.IsCommutative = true;
4133 III.IsSummingOperands = true;
4134 III.ImmOpNo = 1;
4135 III.OpNoForForwarding = 2;
4136 switch(Opc) {
4137 default: llvm_unreachable("Unknown opcode");
4138 case PPC::LBZX: III.ImmOpcode = PPC::LBZ; break;
4139 case PPC::LBZX8: III.ImmOpcode = PPC::LBZ8; break;
4140 case PPC::LHZX: III.ImmOpcode = PPC::LHZ; break;
4141 case PPC::LHZX8: III.ImmOpcode = PPC::LHZ8; break;
4142 case PPC::LHAX: III.ImmOpcode = PPC::LHA; break;
4143 case PPC::LHAX8: III.ImmOpcode = PPC::LHA8; break;
4144 case PPC::LWZX: III.ImmOpcode = PPC::LWZ; break;
4145 case PPC::LWZX8: III.ImmOpcode = PPC::LWZ8; break;
4146 case PPC::LWAX:
4147 III.ImmOpcode = PPC::LWA;
4148 III.ImmMustBeMultipleOf = 4;
4149 break;
4150 case PPC::LDX: III.ImmOpcode = PPC::LD; III.ImmMustBeMultipleOf = 4; break;
4151 case PPC::LFSX: III.ImmOpcode = PPC::LFS; break;
4152 case PPC::LFDX: III.ImmOpcode = PPC::LFD; break;
4153 case PPC::STBX: III.ImmOpcode = PPC::STB; break;
4154 case PPC::STBX8: III.ImmOpcode = PPC::STB8; break;
4155 case PPC::STHX: III.ImmOpcode = PPC::STH; break;
4156 case PPC::STHX8: III.ImmOpcode = PPC::STH8; break;
4157 case PPC::STWX: III.ImmOpcode = PPC::STW; break;
4158 case PPC::STWX8: III.ImmOpcode = PPC::STW8; break;
4159 case PPC::STDX:
4160 III.ImmOpcode = PPC::STD;
4161 III.ImmMustBeMultipleOf = 4;
4162 break;
4163 case PPC::STFSX: III.ImmOpcode = PPC::STFS; break;
4164 case PPC::STFDX: III.ImmOpcode = PPC::STFD; break;
4165 }
4166 break;
4167 case PPC::LBZUX:
4168 case PPC::LBZUX8:
4169 case PPC::LHZUX:
4170 case PPC::LHZUX8:
4171 case PPC::LHAUX:
4172 case PPC::LHAUX8:
4173 case PPC::LWZUX:
4174 case PPC::LWZUX8:
4175 case PPC::LDUX:
4176 case PPC::LFSUX:
4177 case PPC::LFDUX:
4178 case PPC::STBUX:
4179 case PPC::STBUX8:
4180 case PPC::STHUX:
4181 case PPC::STHUX8:
4182 case PPC::STWUX:
4183 case PPC::STWUX8:
4184 case PPC::STDUX:
4185 case PPC::STFSUX:
4186 case PPC::STFDUX:
4187 III.SignedImm = true;
4188 III.ZeroIsSpecialOrig = 2;
4189 III.ZeroIsSpecialNew = 3;
4190 III.IsCommutative = false;
4191 III.IsSummingOperands = true;
4192 III.ImmOpNo = 2;
4193 III.OpNoForForwarding = 3;
4194 switch(Opc) {
4195 default: llvm_unreachable("Unknown opcode");
4196 case PPC::LBZUX: III.ImmOpcode = PPC::LBZU; break;
4197 case PPC::LBZUX8: III.ImmOpcode = PPC::LBZU8; break;
4198 case PPC::LHZUX: III.ImmOpcode = PPC::LHZU; break;
4199 case PPC::LHZUX8: III.ImmOpcode = PPC::LHZU8; break;
4200 case PPC::LHAUX: III.ImmOpcode = PPC::LHAU; break;
4201 case PPC::LHAUX8: III.ImmOpcode = PPC::LHAU8; break;
4202 case PPC::LWZUX: III.ImmOpcode = PPC::LWZU; break;
4203 case PPC::LWZUX8: III.ImmOpcode = PPC::LWZU8; break;
4204 case PPC::LDUX:
4205 III.ImmOpcode = PPC::LDU;
4206 III.ImmMustBeMultipleOf = 4;
4207 break;
4208 case PPC::LFSUX: III.ImmOpcode = PPC::LFSU; break;
4209 case PPC::LFDUX: III.ImmOpcode = PPC::LFDU; break;
4210 case PPC::STBUX: III.ImmOpcode = PPC::STBU; break;
4211 case PPC::STBUX8: III.ImmOpcode = PPC::STBU8; break;
4212 case PPC::STHUX: III.ImmOpcode = PPC::STHU; break;
4213 case PPC::STHUX8: III.ImmOpcode = PPC::STHU8; break;
4214 case PPC::STWUX: III.ImmOpcode = PPC::STWU; break;
4215 case PPC::STWUX8: III.ImmOpcode = PPC::STWU8; break;
4216 case PPC::STDUX:
4217 III.ImmOpcode = PPC::STDU;
4218 III.ImmMustBeMultipleOf = 4;
4219 break;
4220 case PPC::STFSUX: III.ImmOpcode = PPC::STFSU; break;
4221 case PPC::STFDUX: III.ImmOpcode = PPC::STFDU; break;
4222 }
4223 break;
4224 // Power9 and up only. For some of these, the X-Form version has access to all
4225 // 64 VSR's whereas the D-Form only has access to the VR's. We replace those
4226 // with pseudo-ops pre-ra and for post-ra, we check that the register loaded
4227 // into or stored from is one of the VR registers.
4228 case PPC::LXVX:
4229 case PPC::LXSSPX:
4230 case PPC::LXSDX:
4231 case PPC::STXVX:
4232 case PPC::STXSSPX:
4233 case PPC::STXSDX:
4234 case PPC::XFLOADf32:
4235 case PPC::XFLOADf64:
4236 case PPC::XFSTOREf32:
4237 case PPC::XFSTOREf64:
4238 if (!Subtarget.hasP9Vector())
4239 return false;
4240 III.SignedImm = true;
4241 III.ZeroIsSpecialOrig = 1;
4242 III.ZeroIsSpecialNew = 2;
4243 III.IsCommutative = true;
4244 III.IsSummingOperands = true;
4245 III.ImmOpNo = 1;
4246 III.OpNoForForwarding = 2;
4247 III.ImmMustBeMultipleOf = 4;
4248 switch(Opc) {
4249 default: llvm_unreachable("Unknown opcode");
4250 case PPC::LXVX:
4251 III.ImmOpcode = PPC::LXV;
4252 III.ImmMustBeMultipleOf = 16;
4253 break;
4254 case PPC::LXSSPX:
4255 if (PostRA) {
4256 if (IsVFReg)
4257 III.ImmOpcode = PPC::LXSSP;
4258 else {
4259 III.ImmOpcode = PPC::LFS;
4260 III.ImmMustBeMultipleOf = 1;
4261 }
4262 break;
4263 }
4264 [[fallthrough]];
4265 case PPC::XFLOADf32:
4266 III.ImmOpcode = PPC::DFLOADf32;
4267 break;
4268 case PPC::LXSDX:
4269 if (PostRA) {
4270 if (IsVFReg)
4271 III.ImmOpcode = PPC::LXSD;
4272 else {
4273 III.ImmOpcode = PPC::LFD;
4274 III.ImmMustBeMultipleOf = 1;
4275 }
4276 break;
4277 }
4278 [[fallthrough]];
4279 case PPC::XFLOADf64:
4280 III.ImmOpcode = PPC::DFLOADf64;
4281 break;
4282 case PPC::STXVX:
4283 III.ImmOpcode = PPC::STXV;
4284 III.ImmMustBeMultipleOf = 16;
4285 break;
4286 case PPC::STXSSPX:
4287 if (PostRA) {
4288 if (IsVFReg)
4289 III.ImmOpcode = PPC::STXSSP;
4290 else {
4291 III.ImmOpcode = PPC::STFS;
4292 III.ImmMustBeMultipleOf = 1;
4293 }
4294 break;
4295 }
4296 [[fallthrough]];
4297 case PPC::XFSTOREf32:
4298 III.ImmOpcode = PPC::DFSTOREf32;
4299 break;
4300 case PPC::STXSDX:
4301 if (PostRA) {
4302 if (IsVFReg)
4303 III.ImmOpcode = PPC::STXSD;
4304 else {
4305 III.ImmOpcode = PPC::STFD;
4306 III.ImmMustBeMultipleOf = 1;
4307 }
4308 break;
4309 }
4310 [[fallthrough]];
4311 case PPC::XFSTOREf64:
4312 III.ImmOpcode = PPC::DFSTOREf64;
4313 break;
4314 }
4315 break;
4316 }
4317 return true;
4318}
4319
4320// Utility function for swaping two arbitrary operands of an instruction.
4321static void swapMIOperands(MachineInstr &MI, unsigned Op1, unsigned Op2) {
4322 assert(Op1 != Op2 && "Cannot swap operand with itself.");
4323
4324 unsigned MaxOp = std::max(Op1, Op2);
4325 unsigned MinOp = std::min(Op1, Op2);
4326 MachineOperand MOp1 = MI.getOperand(MinOp);
4327 MachineOperand MOp2 = MI.getOperand(MaxOp);
4328 MI.removeOperand(std::max(Op1, Op2));
4329 MI.removeOperand(std::min(Op1, Op2));
4330
4331 // If the operands we are swapping are the two at the end (the common case)
4332 // we can just remove both and add them in the opposite order.
4333 if (MaxOp - MinOp == 1 && MI.getNumOperands() == MinOp) {
4334 MI.addOperand(MOp2);
4335 MI.addOperand(MOp1);
4336 } else {
4337 // Store all operands in a temporary vector, remove them and re-add in the
4338 // right order.
4340 unsigned TotalOps = MI.getNumOperands() + 2; // We've already removed 2 ops.
4341 for (unsigned i = MI.getNumOperands() - 1; i >= MinOp; i--) {
4342 MOps.push_back(MI.getOperand(i));
4343 MI.removeOperand(i);
4344 }
4345 // MOp2 needs to be added next.
4346 MI.addOperand(MOp2);
4347 // Now add the rest.
4348 for (unsigned i = MI.getNumOperands(); i < TotalOps; i++) {
4349 if (i == MaxOp)
4350 MI.addOperand(MOp1);
4351 else {
4352 MI.addOperand(MOps.back());
4353 MOps.pop_back();
4354 }
4355 }
4356 }
4357}
4358
4359// Check if the 'MI' that has the index OpNoForForwarding
4360// meets the requirement described in the ImmInstrInfo.
4361bool PPCInstrInfo::isUseMIElgibleForForwarding(MachineInstr &MI,
4362 const ImmInstrInfo &III,
4363 unsigned OpNoForForwarding
4364 ) const {
4365 // As the algorithm of checking for PPC::ZERO/PPC::ZERO8
4366 // would not work pre-RA, we can only do the check post RA.
4367 MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4368 if (MRI.isSSA())
4369 return false;
4370
4371 // Cannot do the transform if MI isn't summing the operands.
4372 if (!III.IsSummingOperands)
4373 return false;
4374
4375 // The instruction we are trying to replace must have the ZeroIsSpecialOrig set.
4376 if (!III.ZeroIsSpecialOrig)
4377 return false;
4378
4379 // We cannot do the transform if the operand we are trying to replace
4380 // isn't the same as the operand the instruction allows.
4381 if (OpNoForForwarding != III.OpNoForForwarding)
4382 return false;
4383
4384 // Check if the instruction we are trying to transform really has
4385 // the special zero register as its operand.
4386 if (MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO &&
4387 MI.getOperand(III.ZeroIsSpecialOrig).getReg() != PPC::ZERO8)
4388 return false;
4389
4390 // This machine instruction is convertible if it is,
4391 // 1. summing the operands.
4392 // 2. one of the operands is special zero register.
4393 // 3. the operand we are trying to replace is allowed by the MI.
4394 return true;
4395}
4396
4397// Check if the DefMI is the add inst and set the ImmMO and RegMO
4398// accordingly.
4399bool PPCInstrInfo::isDefMIElgibleForForwarding(MachineInstr &DefMI,
4400 const ImmInstrInfo &III,
4401 MachineOperand *&ImmMO,
4402 MachineOperand *&RegMO) const {
4403 unsigned Opc = DefMI.getOpcode();
4404 if (Opc != PPC::ADDItocL && Opc != PPC::ADDI && Opc != PPC::ADDI8)
4405 return false;
4406
4407 assert(DefMI.getNumOperands() >= 3 &&
4408 "Add inst must have at least three operands");
4409 RegMO = &DefMI.getOperand(1);
4410 ImmMO = &DefMI.getOperand(2);
4411
4412 // Before RA, ADDI first operand could be a frame index.
4413 if (!RegMO->isReg())
4414 return false;
4415
4416 // This DefMI is elgible for forwarding if it is:
4417 // 1. add inst
4418 // 2. one of the operands is Imm/CPI/Global.
4419 return isAnImmediateOperand(*ImmMO);
4420}
4421
4422bool PPCInstrInfo::isRegElgibleForForwarding(
4423 const MachineOperand &RegMO, const MachineInstr &DefMI,
4424 const MachineInstr &MI, bool KillDefMI,
4425 bool &IsFwdFeederRegKilled, bool &SeenIntermediateUse) const {
4426 // x = addi y, imm
4427 // ...
4428 // z = lfdx 0, x -> z = lfd imm(y)
4429 // The Reg "y" can be forwarded to the MI(z) only when there is no DEF
4430 // of "y" between the DEF of "x" and "z".
4431 // The query is only valid post RA.
4432 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4433 if (MRI.isSSA())
4434 return false;
4435
4436 Register Reg = RegMO.getReg();
4437
4438 // Walking the inst in reverse(MI-->DefMI) to get the last DEF of the Reg.
4440 MachineBasicBlock::const_reverse_iterator E = MI.getParent()->rend();
4441 It++;
4442 for (; It != E; ++It) {
4443 if (It->modifiesRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4444 return false;
4445 else if (It->killsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4446 IsFwdFeederRegKilled = true;
4447 if (It->readsRegister(Reg, &getRegisterInfo()) && (&*It) != &DefMI)
4448 SeenIntermediateUse = true;
4449 // Made it to DefMI without encountering a clobber.
4450 if ((&*It) == &DefMI)
4451 break;
4452 }
4453 assert((&*It) == &DefMI && "DefMI is missing");
4454
4455 // If DefMI also defines the register to be forwarded, we can only forward it
4456 // if DefMI is being erased.
4457 if (DefMI.modifiesRegister(Reg, &getRegisterInfo()))
4458 return KillDefMI;
4459
4460 return true;
4461}
4462
4463bool PPCInstrInfo::isImmElgibleForForwarding(const MachineOperand &ImmMO,
4464 const MachineInstr &DefMI,
4465 const ImmInstrInfo &III,
4466 int64_t &Imm,
4467 int64_t BaseImm) const {
4468 assert(isAnImmediateOperand(ImmMO) && "ImmMO is NOT an immediate");
4469 if (DefMI.getOpcode() == PPC::ADDItocL) {
4470 // The operand for ADDItocL is CPI, which isn't imm at compiling time,
4471 // However, we know that, it is 16-bit width, and has the alignment of 4.
4472 // Check if the instruction met the requirement.
4473 if (III.ImmMustBeMultipleOf > 4 ||
4474 III.TruncateImmTo || III.ImmWidth != 16)
4475 return false;
4476
4477 // Going from XForm to DForm loads means that the displacement needs to be
4478 // not just an immediate but also a multiple of 4, or 16 depending on the
4479 // load. A DForm load cannot be represented if it is a multiple of say 2.
4480 // XForm loads do not have this restriction.
4481 if (ImmMO.isGlobal()) {
4482 const DataLayout &DL = ImmMO.getGlobal()->getParent()->getDataLayout();
4484 return false;
4485 }
4486
4487 return true;
4488 }
4489
4490 if (ImmMO.isImm()) {
4491 // It is Imm, we need to check if the Imm fit the range.
4492 // Sign-extend to 64-bits.
4493 // DefMI may be folded with another imm form instruction, the result Imm is
4494 // the sum of Imm of DefMI and BaseImm which is from imm form instruction.
4495 APInt ActualValue(64, ImmMO.getImm() + BaseImm, true);
4496 if (III.SignedImm && !ActualValue.isSignedIntN(III.ImmWidth))
4497 return false;
4498 if (!III.SignedImm && !ActualValue.isIntN(III.ImmWidth))
4499 return false;
4500 Imm = SignExtend64<16>(ImmMO.getImm() + BaseImm);
4501
4502 if (Imm % III.ImmMustBeMultipleOf)
4503 return false;
4504 if (III.TruncateImmTo)
4505 Imm &= ((1 << III.TruncateImmTo) - 1);
4506 }
4507 else
4508 return false;
4509
4510 // This ImmMO is forwarded if it meets the requriement describle
4511 // in ImmInstrInfo
4512 return true;
4513}
4514
4515bool PPCInstrInfo::simplifyToLI(MachineInstr &MI, MachineInstr &DefMI,
4516 unsigned OpNoForForwarding,
4517 MachineInstr **KilledDef) const {
4518 if ((DefMI.getOpcode() != PPC::LI && DefMI.getOpcode() != PPC::LI8) ||
4519 !DefMI.getOperand(1).isImm())
4520 return false;
4521
4522 MachineFunction *MF = MI.getParent()->getParent();
4524 bool PostRA = !MRI->isSSA();
4525
4526 int64_t Immediate = DefMI.getOperand(1).getImm();
4527 // Sign-extend to 64-bits.
4528 int64_t SExtImm = SignExtend64<16>(Immediate);
4529
4530 bool ReplaceWithLI = false;
4531 bool Is64BitLI = false;
4532 int64_t NewImm = 0;
4533 bool SetCR = false;
4534 unsigned Opc = MI.getOpcode();
4535 switch (Opc) {
4536 default:
4537 return false;
4538
4539 // FIXME: Any branches conditional on such a comparison can be made
4540 // unconditional. At this time, this happens too infrequently to be worth
4541 // the implementation effort, but if that ever changes, we could convert
4542 // such a pattern here.
4543 case PPC::CMPWI:
4544 case PPC::CMPLWI:
4545 case PPC::CMPDI:
4546 case PPC::CMPLDI: {
4547 // Doing this post-RA would require dataflow analysis to reliably find uses
4548 // of the CR register set by the compare.
4549 // No need to fixup killed/dead flag since this transformation is only valid
4550 // before RA.
4551 if (PostRA)
4552 return false;
4553 // If a compare-immediate is fed by an immediate and is itself an input of
4554 // an ISEL (the most common case) into a COPY of the correct register.
4555 bool Changed = false;
4556 Register DefReg = MI.getOperand(0).getReg();
4557 int64_t Comparand = MI.getOperand(2).getImm();
4558 int64_t SExtComparand = ((uint64_t)Comparand & ~0x7FFFuLL) != 0
4559 ? (Comparand | 0xFFFFFFFFFFFF0000)
4560 : Comparand;
4561
4562 for (auto &CompareUseMI : MRI->use_instructions(DefReg)) {
4563 unsigned UseOpc = CompareUseMI.getOpcode();
4564 if (UseOpc != PPC::ISEL && UseOpc != PPC::ISEL8)
4565 continue;
4566 unsigned CRSubReg = CompareUseMI.getOperand(3).getSubReg();
4567 Register TrueReg = CompareUseMI.getOperand(1).getReg();
4568 Register FalseReg = CompareUseMI.getOperand(2).getReg();
4569 unsigned RegToCopy =
4570 selectReg(SExtImm, SExtComparand, Opc, TrueReg, FalseReg, CRSubReg);
4571 if (RegToCopy == PPC::NoRegister)
4572 continue;
4573 // Can't use PPC::COPY to copy PPC::ZERO[8]. Convert it to LI[8] 0.
4574 if (RegToCopy == PPC::ZERO || RegToCopy == PPC::ZERO8) {
4575 CompareUseMI.setDesc(get(UseOpc == PPC::ISEL8 ? PPC::LI8 : PPC::LI));
4576 replaceInstrOperandWithImm(CompareUseMI, 1, 0);
4577 CompareUseMI.removeOperand(3);
4578 CompareUseMI.removeOperand(2);
4579 continue;
4580 }
4581 LLVM_DEBUG(
4582 dbgs() << "Found LI -> CMPI -> ISEL, replacing with a copy.\n");
4583 LLVM_DEBUG(DefMI.dump(); MI.dump(); CompareUseMI.dump());
4584 LLVM_DEBUG(dbgs() << "Is converted to:\n");
4585 // Convert to copy and remove unneeded operands.
4586 CompareUseMI.setDesc(get(PPC::COPY));
4587 CompareUseMI.removeOperand(3);
4588 CompareUseMI.removeOperand(RegToCopy == TrueReg ? 2 : 1);
4589 CmpIselsConverted++;
4590 Changed = true;
4591 LLVM_DEBUG(CompareUseMI.dump());
4592 }
4593 if (Changed)
4594 return true;
4595 // This may end up incremented multiple times since this function is called
4596 // during a fixed-point transformation, but it is only meant to indicate the
4597 // presence of this opportunity.
4598 MissedConvertibleImmediateInstrs++;
4599 return false;
4600 }
4601
4602 // Immediate forms - may simply be convertable to an LI.
4603 case PPC::ADDI:
4604 case PPC::ADDI8: {
4605 // Does the sum fit in a 16-bit signed field?
4606 int64_t Addend = MI.getOperand(2).getImm();
4607 if (isInt<16>(Addend + SExtImm)) {
4608 ReplaceWithLI = true;
4609 Is64BitLI = Opc == PPC::ADDI8;
4610 NewImm = Addend + SExtImm;
4611 break;
4612 }
4613 return false;
4614 }
4615 case PPC::SUBFIC:
4616 case PPC::SUBFIC8: {
4617 // Only transform this if the CARRY implicit operand is dead.
4618 if (MI.getNumOperands() > 3 && !MI.getOperand(3).isDead())
4619 return false;
4620 int64_t Minuend = MI.getOperand(2).getImm();
4621 if (isInt<16>(Minuend - SExtImm)) {
4622 ReplaceWithLI = true;
4623 Is64BitLI = Opc == PPC::SUBFIC8;
4624 NewImm = Minuend - SExtImm;
4625 break;
4626 }
4627 return false;
4628 }
4629 case PPC::RLDICL:
4630 case PPC::RLDICL_rec:
4631 case PPC::RLDICL_32:
4632 case PPC::RLDICL_32_64: {
4633 // Use APInt's rotate function.
4634 int64_t SH = MI.getOperand(2).getImm();
4635 int64_t MB = MI.getOperand(3).getImm();
4636 APInt InVal((Opc == PPC::RLDICL || Opc == PPC::RLDICL_rec) ? 64 : 32,
4637 SExtImm, true);
4638 InVal = InVal.rotl(SH);
4639 uint64_t Mask = MB == 0 ? -1LLU : (1LLU << (63 - MB + 1)) - 1;
4640 InVal &= Mask;
4641 // Can't replace negative values with an LI as that will sign-extend
4642 // and not clear the left bits. If we're setting the CR bit, we will use
4643 // ANDI_rec which won't sign extend, so that's safe.
4644 if (isUInt<15>(InVal.getSExtValue()) ||
4645 (Opc == PPC::RLDICL_rec && isUInt<16>(InVal.getSExtValue()))) {
4646 ReplaceWithLI = true;
4647 Is64BitLI = Opc != PPC::RLDICL_32;
4648 NewImm = InVal.getSExtValue();
4649 SetCR = Opc == PPC::RLDICL_rec;
4650 break;
4651 }
4652 return false;
4653 }
4654 case PPC::RLWINM:
4655 case PPC::RLWINM8:
4656 case PPC::RLWINM_rec:
4657 case PPC::RLWINM8_rec: {
4658 int64_t SH = MI.getOperand(2).getImm();
4659 int64_t MB = MI.getOperand(3).getImm();
4660 int64_t ME = MI.getOperand(4).getImm();
4661 APInt InVal(32, SExtImm, true);
4662 InVal = InVal.rotl(SH);
4663 APInt Mask = APInt::getBitsSetWithWrap(32, 32 - ME - 1, 32 - MB);
4664 InVal &= Mask;
4665 // Can't replace negative values with an LI as that will sign-extend
4666 // and not clear the left bits. If we're setting the CR bit, we will use
4667 // ANDI_rec which won't sign extend, so that's safe.
4668 bool ValueFits = isUInt<15>(InVal.getSExtValue());
4669 ValueFits |= ((Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec) &&
4670 isUInt<16>(InVal.getSExtValue()));
4671 if (ValueFits) {
4672 ReplaceWithLI = true;
4673 Is64BitLI = Opc == PPC::RLWINM8 || Opc == PPC::RLWINM8_rec;
4674 NewImm = InVal.getSExtValue();
4675 SetCR = Opc == PPC::RLWINM_rec || Opc == PPC::RLWINM8_rec;
4676 break;
4677 }
4678 return false;
4679 }
4680 case PPC::ORI:
4681 case PPC::ORI8:
4682 case PPC::XORI:
4683 case PPC::XORI8: {
4684 int64_t LogicalImm = MI.getOperand(2).getImm();
4685 int64_t Result = 0;
4686 if (Opc == PPC::ORI || Opc == PPC::ORI8)
4687 Result = LogicalImm | SExtImm;
4688 else
4689 Result = LogicalImm ^ SExtImm;
4690 if (isInt<16>(Result)) {
4691 ReplaceWithLI = true;
4692 Is64BitLI = Opc == PPC::ORI8 || Opc == PPC::XORI8;
4693 NewImm = Result;
4694 break;
4695 }
4696 return false;
4697 }
4698 }
4699
4700 if (ReplaceWithLI) {
4701 // We need to be careful with CR-setting instructions we're replacing.
4702 if (SetCR) {
4703 // We don't know anything about uses when we're out of SSA, so only
4704 // replace if the new immediate will be reproduced.
4705 bool ImmChanged = (SExtImm & NewImm) != NewImm;
4706 if (PostRA && ImmChanged)
4707 return false;
4708
4709 if (!PostRA) {
4710 // If the defining load-immediate has no other uses, we can just replace
4711 // the immediate with the new immediate.
4712 if (MRI->hasOneUse(DefMI.getOperand(0).getReg()))
4713 DefMI.getOperand(1).setImm(NewImm);
4714
4715 // If we're not using the GPR result of the CR-setting instruction, we
4716 // just need to and with zero/non-zero depending on the new immediate.
4717 else if (MRI->use_empty(