Go to the documentation of this file.
30 #define DEBUG_TYPE "x86-pseudo"
31 #define X86_EXPAND_PSEUDO_NAME "X86 pseudo instruction expansion pass"
60 return "X86 pseudo instruction expansion pass";
79 void ExpandVastartSaveXmmRegs(
90 void X86ExpandPseudo::ExpandICallBranchFunnel(
99 std::vector<std::pair<MachineBasicBlock *, unsigned>> TargetMBBs;
104 auto CmpTarget = [&](
unsigned Target) {
105 if (Selector.
isReg())
119 auto CreateMBB = [&]() {
130 auto *ElseMBB = CreateMBB();
131 MF->
insert(InsPt, ElseMBB);
136 auto EmitCondJumpTarget = [&](
unsigned CC,
unsigned Target) {
137 auto *ThenMBB = CreateMBB();
138 TargetMBBs.push_back({ThenMBB,
Target});
139 EmitCondJump(CC, ThenMBB);
142 auto EmitTailCall = [&](
unsigned Target) {
149 if (NumTargets == 1) {
154 if (NumTargets == 2) {
161 if (NumTargets < 6) {
169 auto *ThenMBB = CreateMBB();
173 EmitBranchFunnel(
FirstTarget + (NumTargets / 2) + 1,
174 NumTargets - (NumTargets / 2) - 1);
176 MF->
insert(InsPt, ThenMBB);
183 for (
auto P : TargetMBBs) {
188 JTMBB->
erase(JTInst);
198 assert((
MI.getOperand(1).isGlobal() ||
MI.getOperand(1).isReg()) &&
199 "invalid operand for regular call");
201 if (
MI.getOpcode() == X86::CALL64m_RVMARKER)
203 else if (
MI.getOpcode() == X86::CALL64r_RVMARKER)
205 else if (
MI.getOpcode() == X86::CALL64pcrel32_RVMARKER)
206 Opc = X86::CALL64pcrel32;
211 bool RAXImplicitDead =
false;
215 if (
Op.isReg() &&
Op.isImplicit() &&
Op.isDead() &&
219 RAXImplicitDead =
true;
232 if (
MI.shouldUpdateCallSiteInfo())
247 MI.eraseFromParent();
252 if (
TM.getTargetTriple().isOSDarwin())
263 unsigned Opcode =
MI.getOpcode();
268 case X86::TCRETURNdi:
269 case X86::TCRETURNdicc:
270 case X86::TCRETURNri:
271 case X86::TCRETURNmi:
272 case X86::TCRETURNdi64:
273 case X86::TCRETURNdi64cc:
274 case X86::TCRETURNri64:
275 case X86::TCRETURNmi64: {
276 bool isMem = Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64;
280 assert(StackAdjust.
isImm() &&
"Expecting immediate value.");
283 int StackAdj = StackAdjust.
getImm();
284 int MaxTCDelta = X86FI->getTCReturnAddrDelta();
286 assert(MaxTCDelta <= 0 &&
"MaxTCDelta should never be positive");
289 Offset = StackAdj - MaxTCDelta;
290 assert(Offset >= 0 &&
"Offset should never be negative");
292 if (Opcode == X86::TCRETURNdicc || Opcode == X86::TCRETURNdi64cc) {
293 assert(Offset == 0 &&
"Conditional tail call cannot adjust the stack.");
299 X86FL->emitSPUpdate(
MBB,
MBBI,
DL, Offset,
true);
303 bool IsWin64 = STI->isTargetWin64();
304 if (Opcode == X86::TCRETURNdi || Opcode == X86::TCRETURNdicc ||
305 Opcode == X86::TCRETURNdi64 || Opcode == X86::TCRETURNdi64cc) {
308 case X86::TCRETURNdi:
311 case X86::TCRETURNdicc:
312 Op = X86::TAILJMPd_CC;
314 case X86::TCRETURNdi64cc:
316 "Conditional tail calls confuse "
317 "the Win64 unwinder.");
318 Op = X86::TAILJMPd64_CC;
323 Op = X86::TAILJMPd64;
335 if (
Op == X86::TAILJMPd_CC ||
Op == X86::TAILJMPd64_CC) {
339 }
else if (Opcode == X86::TCRETURNmi || Opcode == X86::TCRETURNmi64) {
340 unsigned Op = (Opcode == X86::TCRETURNmi)
342 : (IsWin64 ? X86::TAILJMPm64_REX : X86::TAILJMPm64);
346 }
else if (Opcode == X86::TCRETURNri64) {
349 TII->get(IsWin64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
361 if (
MBBI->isCandidateForCallSiteEntry())
370 case X86::EH_RETURN64: {
372 assert(DestAddr.
isReg() &&
"Offset should be in register!");
373 const bool Uses64BitFramePtr =
374 STI->isTarget64BitLP64() || STI->isTargetNaCl64();
377 TII->get(Uses64BitFramePtr ? X86::MOV64rr : X86::MOV32rr), StackPtr)
384 int64_t StackAdj =
MBBI->getOperand(0).getImm();
385 X86FL->emitSPUpdate(
MBB,
MBBI,
DL, StackAdj,
true);
387 unsigned RetOp = STI->is64Bit() ? X86::IRET64 : X86::IRET32;
389 if (STI->is64Bit() && STI->hasUINTR() &&
398 int64_t StackAdj =
MBBI->getOperand(0).getImm();
402 TII->get(STI->is64Bit() ? X86::RET64 : X86::RET32));
405 TII->get(STI->is64Bit() ? X86::RETI64 : X86::RETI32))
409 "shouldn't need to do this for x86_64 targets!");
413 X86FL->emitSPUpdate(
MBB,
MBBI,
DL, StackAdj,
true);
417 for (
unsigned I = 1,
E =
MBBI->getNumOperands();
I !=
E; ++
I)
422 case X86::LCMPXCHG16B_SAVE_RBX: {
440 for (
unsigned Idx = 1; Idx < 6; ++Idx)
458 case X86::MASKPAIR16LOAD: {
460 assert(Disp >= 0 && Disp <= INT32_MAX - 2 &&
"Unexpected displacement");
462 bool DstIsDead =
MBBI->getOperand(0).isDead();
485 MIBLo.setMemRefs(MMOLo);
486 MIBHi.setMemRefs(MMOHi);
492 case X86::MASKPAIR16STORE: {
494 assert(Disp >= 0 && Disp <= INT32_MAX - 2 &&
"Unexpected displacement");
504 MIBLo.add(
MBBI->getOperand(
i));
506 MIBHi.addImm(Disp + 2);
508 MIBHi.add(
MBBI->getOperand(
i));
519 MIBLo.setMemRefs(MMOLo);
520 MIBHi.setMemRefs(MMOHi);
526 case X86::MWAITX_SAVE_RBX: {
546 case TargetOpcode::ICALL_BRANCH_FUNNEL:
547 ExpandICallBranchFunnel(&
MBB,
MBBI);
549 case X86::PLDTILECFGV: {
550 MI.setDesc(
TII->get(X86::LDTILECFG));
553 case X86::PTILELOADDV:
554 case X86::PTILELOADDT1V: {
555 for (
unsigned i = 2;
i > 0; --
i)
558 Opcode == X86::PTILELOADDV ? X86::TILELOADD : X86::TILELOADDT1;
559 MI.setDesc(
TII->get(Opc));
566 case X86::PTDPBF16PSV: {
567 MI.untieRegOperand(4);
568 for (
unsigned i = 3;
i > 0; --
i)
572 case X86::PTDPBSSDV: Opc = X86::TDPBSSD;
break;
573 case X86::PTDPBSUDV: Opc = X86::TDPBSUD;
break;
574 case X86::PTDPBUSDV: Opc = X86::TDPBUSD;
break;
575 case X86::PTDPBUUDV: Opc = X86::TDPBUUD;
break;
576 case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS;
break;
579 MI.setDesc(
TII->get(Opc));
580 MI.tieOperands(0, 1);
583 case X86::PTILESTOREDV: {
584 for (
int i = 1;
i >= 0; --
i)
586 MI.setDesc(
TII->get(X86::TILESTORED));
589 case X86::PTILEZEROV: {
590 for (
int i = 2;
i > 0; --
i)
592 MI.setDesc(
TII->get(X86::TILEZERO));
595 case X86::CALL64pcrel32_RVMARKER:
596 case X86::CALL64r_RVMARKER:
597 case X86::CALL64m_RVMARKER:
598 expandCALL_RVMARKER(
MBB,
MBBI);
618 void X86ExpandPseudo::ExpandVastartSaveXmmRegs(
625 const DebugLoc &
DL = VAStartPseudoInstr->getDebugLoc();
626 Register CountReg = VAStartPseudoInstr->getOperand(0).getReg();
632 LiveRegs.addLiveIns(*EntryBlk);
634 if (
MI.getOpcode() == VAStartPseudoInstr->getOpcode())
637 LiveRegs.stepForward(
MI, Clobbers);
647 Func->insert(EntryBlkIter, GuardedRegsBlk);
648 Func->insert(EntryBlkIter, TailBlk);
656 uint64_t FrameOffset = VAStartPseudoInstr->getOperand(4).getImm();
657 uint64_t VarArgsRegsOffset = VAStartPseudoInstr->getOperand(6).getImm();
660 unsigned MOVOpc = STI->hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
663 for (int64_t OpndIdx = 7, RegIdx = 0;
664 OpndIdx < VAStartPseudoInstr->getNumOperands() - 1;
665 OpndIdx++, RegIdx++) {
666 auto NewMI =
BuildMI(GuardedRegsBlk,
DL,
TII->get(MOVOpc));
669 NewMI.addImm(FrameOffset + VarArgsRegsOffset + RegIdx * 16);
671 NewMI.add(VAStartPseudoInstr->getOperand(
i + 1));
673 NewMI.addReg(VAStartPseudoInstr->getOperand(OpndIdx).getReg());
675 VAStartPseudoInstr->getOperand(OpndIdx).getReg()));
683 if (!STI->isCallingConvWin64(
Func->getFunction().getCallingConv())) {
699 VAStartPseudoInstr->eraseFromParent();
718 bool X86ExpandPseudo::ExpandPseudosWhichAffectControlFlow(
MachineFunction &MF) {
724 ExpandVastartSaveXmmRegs(&(MF.
front()), Instr);
734 TII = STI->getInstrInfo();
735 TRI = STI->getRegisterInfo();
737 X86FL = STI->getFrameLowering();
739 bool Modified = ExpandPseudosWhichAffectControlFlow(MF);
748 return new X86ExpandPseudo();
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool isLiveIn(MCPhysReg Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const
Return true if the specified register is in the live in set.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
@ Define
Register definition.
const GlobalValue * getGlobal() const
const MachineInstrBuilder & add(const MachineOperand &MO) const
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Target - Wrapper for Target specific information.
void setIsKill(bool Val=true)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Reg
All possible values of the reg field in the ModR/M byte.
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
A set of physical registers with utility functions to track liveness when walking backward/forward th...
void moveCallSiteInfo(const MachineInstr *Old, const MachineInstr *New)
Move the call site info from Old to \New call site info.
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
A description of a memory reference used in the backend.
void insert(iterator MBBI, MachineBasicBlock *MBB)
unsigned getDeadRegState(bool B)
Properties which a MachineFunction may have at a given point in time.
void addLiveIns(MachineBasicBlock &MBB, const LivePhysRegs &LiveRegs)
Adds registers contained in LiveRegs to the block live-in list of MBB.
int64_t getOffset() const
Return the offset from the symbol in this operand.
unsigned const TargetRegisterInfo * TRI
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
BasicBlockListType::iterator iterator
LLVM Basic Block Representation.
instr_iterator erase(instr_iterator I)
Remove an instruction from the instruction list and delete it.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...
const MachineBasicBlock & front() const
TargetInstrInfo - Interface to description of machine instruction set.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI)
Copy implicit register operands from specified instruction to this instruction.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineOperand & getOperand(unsigned i) const
static bool isPhysicalRegister(unsigned Reg)
Return true if the specified register number is in the physical register namespace.
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
Represent the analysis usage information of a pass.
INITIALIZE_PASS(X86ExpandPseudo, DEBUG_TYPE, X86_EXPAND_PSEUDO_NAME, false, false) void X86ExpandPseudo
const HexagonInstrInfo * TII
MachineOperand class - Representation of each machine instruction operand.
MachineFunctionProperties & set(Property P)
@ IRET
Return from interrupt. Operand 0 is the number of bytes to pop.
const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
@ Implicit
Not emitted register (e.g. carry, or temporary result).
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
char & MachineLoopInfoID
MachineLoopInfo - This pass is a loop analysis pass.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
bool regsOverlap(Register RegA, Register RegB) const
Returns true if the two registers are equal or alias each other.
constexpr bool isUInt< 16 >(uint64_t x)
unsigned getTargetFlags() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
Representation of each machine instruction.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...
print Print MemDeps of function
FunctionPass * createX86ExpandPseudoPass()
Return a Machine IR pass that expands X86-specific pseudo instructions into a sequence of actual inst...
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Register getReg() const
getReg - Returns the register number.
static Target * FirstTarget
const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const
StringRef - Represent a constant reference to a string, i.e.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
AnalysisUsage & addPreservedID(const void *ID)
MachineBasicBlock MachineBasicBlock::iterator MBBI
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
C - The default llvm calling convention, compatible with C.
self_iterator getIterator()
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Wrapper class representing virtual and physical registers.
void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())
Adds the specified register as a live in.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
Iterator for intrusive lists based on ilist_node.
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
CodeModel::Model getCodeModel() const
Returns the code model.
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
const MachineInstrBuilder & addGlobalAddress(const GlobalValue *GV, int64_t Offset=0, unsigned TargetFlags=0) const
unsigned getKillRegState(bool B)
#define X86_EXPAND_PSEUDO_NAME
const char * getSymbolName() const
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
unsigned getNumOperands() const
Retuns the total number of operands.
char & MachineDominatorsID
MachineDominators - This pass is a machine dominators analysis pass.
void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
static bool isMem(const MachineInstr &MI, unsigned Op)
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
MCRegister getSubReg(MCRegister Reg, unsigned Idx) const
Returns the physical register number of sub-register "Index" for physical register RegNo.
const char LLVMTargetMachineRef TM
FunctionPass class - This class is used to implement most global optimizations.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override
Emit instructions to copy a pair of physical registers.
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function.
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.