26#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME \
27 "RISC-V atomic pseudo instruction expansion pass"
52 bool IsMasked,
int Width,
72char RISCVExpandAtomicPseudo::ID = 0;
74bool RISCVExpandAtomicPseudo::runOnMachineFunction(
MachineFunction &MF) {
87 assert(OldSize >= NewSize);
111 switch (
MBBI->getOpcode()) {
112 case RISCV::PseudoAtomicLoadNand32:
115 case RISCV::PseudoAtomicLoadNand64:
118 case RISCV::PseudoMaskedAtomicSwap32:
121 case RISCV::PseudoMaskedAtomicLoadAdd32:
123 case RISCV::PseudoMaskedAtomicLoadSub32:
125 case RISCV::PseudoMaskedAtomicLoadNand32:
128 case RISCV::PseudoMaskedAtomicLoadMax32:
131 case RISCV::PseudoMaskedAtomicLoadMin32:
134 case RISCV::PseudoMaskedAtomicLoadUMax32:
137 case RISCV::PseudoMaskedAtomicLoadUMin32:
140 case RISCV::PseudoCmpXchg32:
141 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 32, NextMBBI);
142 case RISCV::PseudoCmpXchg64:
143 return expandAtomicCmpXchg(
MBB,
MBBI,
false, 64, NextMBBI);
144 case RISCV::PseudoMaskedCmpXchg32:
145 return expandAtomicCmpXchg(
MBB,
MBBI,
true, 32, NextMBBI);
155 case AtomicOrdering::Monotonic:
157 case AtomicOrdering::Acquire:
158 return RISCV::LR_W_AQ;
159 case AtomicOrdering::Release:
161 case AtomicOrdering::AcquireRelease:
162 return RISCV::LR_W_AQ;
163 case AtomicOrdering::SequentiallyConsistent:
164 return RISCV::LR_W_AQ_RL;
172 case AtomicOrdering::Monotonic:
174 case AtomicOrdering::Acquire:
176 case AtomicOrdering::Release:
177 return RISCV::SC_W_RL;
178 case AtomicOrdering::AcquireRelease:
179 return RISCV::SC_W_RL;
180 case AtomicOrdering::SequentiallyConsistent:
181 return RISCV::SC_W_RL;
189 case AtomicOrdering::Monotonic:
191 case AtomicOrdering::Acquire:
192 return RISCV::LR_D_AQ;
193 case AtomicOrdering::Release:
195 case AtomicOrdering::AcquireRelease:
196 return RISCV::LR_D_AQ;
197 case AtomicOrdering::SequentiallyConsistent:
198 return RISCV::LR_D_AQ_RL;
206 case AtomicOrdering::Monotonic:
208 case AtomicOrdering::Acquire:
210 case AtomicOrdering::Release:
211 return RISCV::SC_D_RL;
212 case AtomicOrdering::AcquireRelease:
213 return RISCV::SC_D_RL;
214 case AtomicOrdering::SequentiallyConsistent:
215 return RISCV::SC_D_RL;
221 return getLRForRMW32(Ordering);
223 return getLRForRMW64(Ordering);
229 return getSCForRMW32(Ordering);
231 return getSCForRMW64(Ordering);
241 Register ScratchReg =
MI.getOperand(1).getReg();
252 BuildMI(LoopMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
266 BuildMI(LoopMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
279 assert(OldValReg != ScratchReg &&
"OldValReg and ScratchReg must be unique");
280 assert(OldValReg != MaskReg &&
"OldValReg and MaskReg must be unique");
281 assert(ScratchReg != MaskReg &&
"ScratchReg and MaskReg must be unique");
297static void doMaskedAtomicBinOpExpansion(
301 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
303 Register ScratchReg =
MI.getOperand(1).getReg();
318 BuildMI(LoopMBB,
DL,
TII->get(getLRForRMW32(Ordering)), DestReg)
348 insertMaskedMerge(
TII,
DL, LoopMBB, ScratchReg, DestReg, ScratchReg, MaskReg,
351 BuildMI(LoopMBB,
DL,
TII->get(getSCForRMW32(Ordering)), ScratchReg)
360bool RISCVExpandAtomicPseudo::expandAtomicBinOp(
383 doAtomicBinOpExpansion(
TII,
MI,
DL, &
MBB, LoopMBB, DoneMBB, BinOp, Width);
385 doMaskedAtomicBinOpExpansion(
TII,
MI,
DL, &
MBB, LoopMBB, DoneMBB, BinOp,
389 MI.eraseFromParent();
409bool RISCVExpandAtomicPseudo::expandAtomicMinMaxOp(
413 assert(IsMasked ==
true &&
414 "Should only need to expand masked atomic max/min");
415 assert(Width == 32 &&
"Should never need to expand masked 64-bit operations");
427 MF->
insert(++LoopHeadMBB->getIterator(), LoopIfBodyMBB);
428 MF->
insert(++LoopIfBodyMBB->getIterator(), LoopTailMBB);
429 MF->
insert(++LoopTailMBB->getIterator(), DoneMBB);
432 LoopHeadMBB->addSuccessor(LoopIfBodyMBB);
433 LoopHeadMBB->addSuccessor(LoopTailMBB);
434 LoopIfBodyMBB->addSuccessor(LoopTailMBB);
435 LoopTailMBB->addSuccessor(LoopHeadMBB);
436 LoopTailMBB->addSuccessor(DoneMBB);
442 Register Scratch1Reg =
MI.getOperand(1).getReg();
443 Register Scratch2Reg =
MI.getOperand(2).getReg();
458 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW32(Ordering)), DestReg)
460 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::AND), Scratch2Reg)
463 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::ADDI), Scratch1Reg)
471 insertSext(
TII,
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
479 insertSext(
TII,
DL, LoopHeadMBB, Scratch2Reg,
MI.getOperand(6).getReg());
504 insertMaskedMerge(
TII,
DL, LoopIfBodyMBB, Scratch1Reg, DestReg, IncrReg,
505 MaskReg, Scratch1Reg);
510 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW32(Ordering)), Scratch1Reg)
519 MI.eraseFromParent();
554 if (
MBBI ==
E ||
MBBI->getOpcode() != RISCV::AND)
558 if (!(ANDOp1 == DestReg && ANDOp2 == MaskReg) &&
559 !(ANDOp1 == MaskReg && ANDOp2 == DestReg))
562 DestReg =
MBBI->getOperand(0).getReg();
568 if (
MBBI ==
E ||
MBBI->getOpcode() != RISCV::BNE)
572 if (!(BNEOp0 == DestReg && BNEOp1 == CmpValReg) &&
573 !(BNEOp0 == CmpValReg && BNEOp1 == DestReg))
576 LoopHeadBNETarget =
MBBI->getOperand(2).getMBB();
582 for (
auto *
MI : ToErase)
583 MI->eraseFromParent();
587bool RISCVExpandAtomicPseudo::expandAtomicCmpXchg(
598 Register ScratchReg =
MI.getOperand(1).getReg();
600 Register CmpValReg =
MI.getOperand(3).getReg();
601 Register NewValReg =
MI.getOperand(4).getReg();
605 tryToFoldBNEOnCmpXchgResult(
MBB, std::next(
MBBI), DestReg, CmpValReg, MaskReg,
610 MF->
insert(++LoopHeadMBB->getIterator(), LoopTailMBB);
611 MF->
insert(++LoopTailMBB->getIterator(), DoneMBB);
614 LoopHeadMBB->addSuccessor(LoopTailMBB);
615 LoopHeadMBB->addSuccessor(LoopHeadBNETarget);
616 LoopTailMBB->addSuccessor(DoneMBB);
617 LoopTailMBB->addSuccessor(LoopHeadMBB);
629 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
634 .
addMBB(LoopHeadBNETarget);
638 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
651 BuildMI(LoopHeadMBB,
DL,
TII->get(getLRForRMW(Ordering, Width)), DestReg)
653 BuildMI(LoopHeadMBB,
DL,
TII->get(RISCV::AND), ScratchReg)
659 .
addMBB(LoopHeadBNETarget);
667 insertMaskedMerge(
TII,
DL, LoopTailMBB, ScratchReg, DestReg, NewValReg,
668 MaskReg, ScratchReg);
669 BuildMI(LoopTailMBB,
DL,
TII->get(getSCForRMW(Ordering, Width)), ScratchReg)
679 MI.eraseFromParent();
697 return new RISCVExpandAtomicPseudo();
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
const HexagonInstrInfo * TII
This file implements the LivePhysRegs utility for tracking liveness of physical registers.
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
#define RISCV_EXPAND_ATOMIC_PSEUDO_NAME
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static unsigned getInstSizeInBytes(const MachineInstr &MI, const SystemZInstrInfo *TII)
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Min
*p = old <signed v ? old : v
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
FunctionPass class - This class is used to implement most global optimizations.
A set of physical registers with utility functions to track liveness when walking backward/forward th...
void transferSuccessors(MachineBasicBlock *FromMBB)
Transfers all the successors from MBB to this machine basic block (i.e., copies all the successors Fr...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
void removeSuccessor(MachineBasicBlock *Succ, bool NormalizeSuccProbs=false)
Remove successor from the successors list of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Wrapper class representing virtual and physical registers.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
virtual const TargetInstrInfo * getInstrInfo() const
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
FunctionPass * createRISCVExpandAtomicPseudoPass()
IterT skipDebugInstructionsForward(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It until it points to a non-debug instruction or to End and return the resulting iterator.
void initializeRISCVExpandAtomicPseudoPass(PassRegistry &)
AtomicOrdering
Atomic ordering for LLVM's memory model.
void computeAndAddLiveIns(LivePhysRegs &LiveRegs, MachineBasicBlock &MBB)
Convenience function combining computeLiveIns() and addLiveIns().