| File: | llvm/lib/CodeGen/PeepholeOptimizer.cpp |
| Warning: | line 546, column 30 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | //===- PeepholeOptimizer.cpp - Peephole Optimizations ---------------------===// | ||||||||
| 2 | // | ||||||||
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||
| 4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||
| 6 | // | ||||||||
| 7 | //===----------------------------------------------------------------------===// | ||||||||
| 8 | // | ||||||||
| 9 | // Perform peephole optimizations on the machine code: | ||||||||
| 10 | // | ||||||||
| 11 | // - Optimize Extensions | ||||||||
| 12 | // | ||||||||
| 13 | // Optimization of sign / zero extension instructions. It may be extended to | ||||||||
| 14 | // handle other instructions with similar properties. | ||||||||
| 15 | // | ||||||||
| 16 | // On some targets, some instructions, e.g. X86 sign / zero extension, may | ||||||||
| 17 | // leave the source value in the lower part of the result. This optimization | ||||||||
| 18 | // will replace some uses of the pre-extension value with uses of the | ||||||||
| 19 | // sub-register of the results. | ||||||||
| 20 | // | ||||||||
| 21 | // - Optimize Comparisons | ||||||||
| 22 | // | ||||||||
| 23 | // Optimization of comparison instructions. For instance, in this code: | ||||||||
| 24 | // | ||||||||
| 25 | // sub r1, 1 | ||||||||
| 26 | // cmp r1, 0 | ||||||||
| 27 | // bz L1 | ||||||||
| 28 | // | ||||||||
| 29 | // If the "sub" instruction all ready sets (or could be modified to set) the | ||||||||
| 30 | // same flag that the "cmp" instruction sets and that "bz" uses, then we can | ||||||||
| 31 | // eliminate the "cmp" instruction. | ||||||||
| 32 | // | ||||||||
| 33 | // Another instance, in this code: | ||||||||
| 34 | // | ||||||||
| 35 | // sub r1, r3 | sub r1, imm | ||||||||
| 36 | // cmp r3, r1 or cmp r1, r3 | cmp r1, imm | ||||||||
| 37 | // bge L1 | ||||||||
| 38 | // | ||||||||
| 39 | // If the branch instruction can use flag from "sub", then we can replace | ||||||||
| 40 | // "sub" with "subs" and eliminate the "cmp" instruction. | ||||||||
| 41 | // | ||||||||
| 42 | // - Optimize Loads: | ||||||||
| 43 | // | ||||||||
| 44 | // Loads that can be folded into a later instruction. A load is foldable | ||||||||
| 45 | // if it loads to virtual registers and the virtual register defined has | ||||||||
| 46 | // a single use. | ||||||||
| 47 | // | ||||||||
| 48 | // - Optimize Copies and Bitcast (more generally, target specific copies): | ||||||||
| 49 | // | ||||||||
| 50 | // Rewrite copies and bitcasts to avoid cross register bank copies | ||||||||
| 51 | // when possible. | ||||||||
| 52 | // E.g., Consider the following example, where capital and lower | ||||||||
| 53 | // letters denote different register file: | ||||||||
| 54 | // b = copy A <-- cross-bank copy | ||||||||
| 55 | // C = copy b <-- cross-bank copy | ||||||||
| 56 | // => | ||||||||
| 57 | // b = copy A <-- cross-bank copy | ||||||||
| 58 | // C = copy A <-- same-bank copy | ||||||||
| 59 | // | ||||||||
| 60 | // E.g., for bitcast: | ||||||||
| 61 | // b = bitcast A <-- cross-bank copy | ||||||||
| 62 | // C = bitcast b <-- cross-bank copy | ||||||||
| 63 | // => | ||||||||
| 64 | // b = bitcast A <-- cross-bank copy | ||||||||
| 65 | // C = copy A <-- same-bank copy | ||||||||
| 66 | //===----------------------------------------------------------------------===// | ||||||||
| 67 | |||||||||
| 68 | #include "llvm/ADT/DenseMap.h" | ||||||||
| 69 | #include "llvm/ADT/Optional.h" | ||||||||
| 70 | #include "llvm/ADT/SmallPtrSet.h" | ||||||||
| 71 | #include "llvm/ADT/SmallSet.h" | ||||||||
| 72 | #include "llvm/ADT/SmallVector.h" | ||||||||
| 73 | #include "llvm/ADT/Statistic.h" | ||||||||
| 74 | #include "llvm/CodeGen/MachineBasicBlock.h" | ||||||||
| 75 | #include "llvm/CodeGen/MachineDominators.h" | ||||||||
| 76 | #include "llvm/CodeGen/MachineFunction.h" | ||||||||
| 77 | #include "llvm/CodeGen/MachineFunctionPass.h" | ||||||||
| 78 | #include "llvm/CodeGen/MachineInstr.h" | ||||||||
| 79 | #include "llvm/CodeGen/MachineInstrBuilder.h" | ||||||||
| 80 | #include "llvm/CodeGen/MachineLoopInfo.h" | ||||||||
| 81 | #include "llvm/CodeGen/MachineOperand.h" | ||||||||
| 82 | #include "llvm/CodeGen/MachineRegisterInfo.h" | ||||||||
| 83 | #include "llvm/CodeGen/TargetInstrInfo.h" | ||||||||
| 84 | #include "llvm/CodeGen/TargetOpcodes.h" | ||||||||
| 85 | #include "llvm/CodeGen/TargetRegisterInfo.h" | ||||||||
| 86 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | ||||||||
| 87 | #include "llvm/InitializePasses.h" | ||||||||
| 88 | #include "llvm/MC/LaneBitmask.h" | ||||||||
| 89 | #include "llvm/MC/MCInstrDesc.h" | ||||||||
| 90 | #include "llvm/Pass.h" | ||||||||
| 91 | #include "llvm/Support/CommandLine.h" | ||||||||
| 92 | #include "llvm/Support/Debug.h" | ||||||||
| 93 | #include "llvm/Support/ErrorHandling.h" | ||||||||
| 94 | #include "llvm/Support/raw_ostream.h" | ||||||||
| 95 | #include <cassert> | ||||||||
| 96 | #include <cstdint> | ||||||||
| 97 | #include <memory> | ||||||||
| 98 | #include <utility> | ||||||||
| 99 | |||||||||
| 100 | using namespace llvm; | ||||||||
| 101 | using RegSubRegPair = TargetInstrInfo::RegSubRegPair; | ||||||||
| 102 | using RegSubRegPairAndIdx = TargetInstrInfo::RegSubRegPairAndIdx; | ||||||||
| 103 | |||||||||
| 104 | #define DEBUG_TYPE"peephole-opt" "peephole-opt" | ||||||||
| 105 | |||||||||
| 106 | // Optimize Extensions | ||||||||
| 107 | static cl::opt<bool> | ||||||||
| 108 | Aggressive("aggressive-ext-opt", cl::Hidden, | ||||||||
| 109 | cl::desc("Aggressive extension optimization")); | ||||||||
| 110 | |||||||||
| 111 | static cl::opt<bool> | ||||||||
| 112 | DisablePeephole("disable-peephole", cl::Hidden, cl::init(false), | ||||||||
| 113 | cl::desc("Disable the peephole optimizer")); | ||||||||
| 114 | |||||||||
| 115 | /// Specifiy whether or not the value tracking looks through | ||||||||
| 116 | /// complex instructions. When this is true, the value tracker | ||||||||
| 117 | /// bails on everything that is not a copy or a bitcast. | ||||||||
| 118 | static cl::opt<bool> | ||||||||
| 119 | DisableAdvCopyOpt("disable-adv-copy-opt", cl::Hidden, cl::init(false), | ||||||||
| 120 | cl::desc("Disable advanced copy optimization")); | ||||||||
| 121 | |||||||||
| 122 | static cl::opt<bool> DisableNAPhysCopyOpt( | ||||||||
| 123 | "disable-non-allocatable-phys-copy-opt", cl::Hidden, cl::init(false), | ||||||||
| 124 | cl::desc("Disable non-allocatable physical register copy optimization")); | ||||||||
| 125 | |||||||||
| 126 | // Limit the number of PHI instructions to process | ||||||||
| 127 | // in PeepholeOptimizer::getNextSource. | ||||||||
| 128 | static cl::opt<unsigned> RewritePHILimit( | ||||||||
| 129 | "rewrite-phi-limit", cl::Hidden, cl::init(10), | ||||||||
| 130 | cl::desc("Limit the length of PHI chains to lookup")); | ||||||||
| 131 | |||||||||
| 132 | // Limit the length of recurrence chain when evaluating the benefit of | ||||||||
| 133 | // commuting operands. | ||||||||
| 134 | static cl::opt<unsigned> MaxRecurrenceChain( | ||||||||
| 135 | "recurrence-chain-limit", cl::Hidden, cl::init(3), | ||||||||
| 136 | cl::desc("Maximum length of recurrence chain when evaluating the benefit " | ||||||||
| 137 | "of commuting operands")); | ||||||||
| 138 | |||||||||
| 139 | |||||||||
| 140 | STATISTIC(NumReuse, "Number of extension results reused")static llvm::Statistic NumReuse = {"peephole-opt", "NumReuse" , "Number of extension results reused"}; | ||||||||
| 141 | STATISTIC(NumCmps, "Number of compares eliminated")static llvm::Statistic NumCmps = {"peephole-opt", "NumCmps", "Number of compares eliminated" }; | ||||||||
| 142 | STATISTIC(NumImmFold, "Number of move immediate folded")static llvm::Statistic NumImmFold = {"peephole-opt", "NumImmFold" , "Number of move immediate folded"}; | ||||||||
| 143 | STATISTIC(NumLoadFold, "Number of loads folded")static llvm::Statistic NumLoadFold = {"peephole-opt", "NumLoadFold" , "Number of loads folded"}; | ||||||||
| 144 | STATISTIC(NumSelects, "Number of selects optimized")static llvm::Statistic NumSelects = {"peephole-opt", "NumSelects" , "Number of selects optimized"}; | ||||||||
| 145 | STATISTIC(NumUncoalescableCopies, "Number of uncoalescable copies optimized")static llvm::Statistic NumUncoalescableCopies = {"peephole-opt" , "NumUncoalescableCopies", "Number of uncoalescable copies optimized" }; | ||||||||
| 146 | STATISTIC(NumRewrittenCopies, "Number of copies rewritten")static llvm::Statistic NumRewrittenCopies = {"peephole-opt", "NumRewrittenCopies" , "Number of copies rewritten"}; | ||||||||
| 147 | STATISTIC(NumNAPhysCopies, "Number of non-allocatable physical copies removed")static llvm::Statistic NumNAPhysCopies = {"peephole-opt", "NumNAPhysCopies" , "Number of non-allocatable physical copies removed"}; | ||||||||
| 148 | |||||||||
| 149 | namespace { | ||||||||
| 150 | |||||||||
| 151 | class ValueTrackerResult; | ||||||||
| 152 | class RecurrenceInstr; | ||||||||
| 153 | |||||||||
| 154 | class PeepholeOptimizer : public MachineFunctionPass { | ||||||||
| 155 | const TargetInstrInfo *TII; | ||||||||
| 156 | const TargetRegisterInfo *TRI; | ||||||||
| 157 | MachineRegisterInfo *MRI; | ||||||||
| 158 | MachineDominatorTree *DT; // Machine dominator tree | ||||||||
| 159 | MachineLoopInfo *MLI; | ||||||||
| 160 | |||||||||
| 161 | public: | ||||||||
| 162 | static char ID; // Pass identification | ||||||||
| 163 | |||||||||
| 164 | PeepholeOptimizer() : MachineFunctionPass(ID) { | ||||||||
| 165 | initializePeepholeOptimizerPass(*PassRegistry::getPassRegistry()); | ||||||||
| 166 | } | ||||||||
| 167 | |||||||||
| 168 | bool runOnMachineFunction(MachineFunction &MF) override; | ||||||||
| 169 | |||||||||
| 170 | void getAnalysisUsage(AnalysisUsage &AU) const override { | ||||||||
| 171 | AU.setPreservesCFG(); | ||||||||
| 172 | MachineFunctionPass::getAnalysisUsage(AU); | ||||||||
| 173 | AU.addRequired<MachineLoopInfo>(); | ||||||||
| 174 | AU.addPreserved<MachineLoopInfo>(); | ||||||||
| 175 | if (Aggressive) { | ||||||||
| 176 | AU.addRequired<MachineDominatorTree>(); | ||||||||
| 177 | AU.addPreserved<MachineDominatorTree>(); | ||||||||
| 178 | } | ||||||||
| 179 | } | ||||||||
| 180 | |||||||||
| 181 | MachineFunctionProperties getRequiredProperties() const override { | ||||||||
| 182 | return MachineFunctionProperties() | ||||||||
| 183 | .set(MachineFunctionProperties::Property::IsSSA); | ||||||||
| 184 | } | ||||||||
| 185 | |||||||||
| 186 | /// Track Def -> Use info used for rewriting copies. | ||||||||
| 187 | using RewriteMapTy = SmallDenseMap<RegSubRegPair, ValueTrackerResult>; | ||||||||
| 188 | |||||||||
| 189 | /// Sequence of instructions that formulate recurrence cycle. | ||||||||
| 190 | using RecurrenceCycle = SmallVector<RecurrenceInstr, 4>; | ||||||||
| 191 | |||||||||
| 192 | private: | ||||||||
| 193 | bool optimizeCmpInstr(MachineInstr &MI); | ||||||||
| 194 | bool optimizeExtInstr(MachineInstr &MI, MachineBasicBlock &MBB, | ||||||||
| 195 | SmallPtrSetImpl<MachineInstr*> &LocalMIs); | ||||||||
| 196 | bool optimizeSelect(MachineInstr &MI, | ||||||||
| 197 | SmallPtrSetImpl<MachineInstr *> &LocalMIs); | ||||||||
| 198 | bool optimizeCondBranch(MachineInstr &MI); | ||||||||
| 199 | bool optimizeCoalescableCopy(MachineInstr &MI); | ||||||||
| 200 | bool optimizeUncoalescableCopy(MachineInstr &MI, | ||||||||
| 201 | SmallPtrSetImpl<MachineInstr *> &LocalMIs); | ||||||||
| 202 | bool optimizeRecurrence(MachineInstr &PHI); | ||||||||
| 203 | bool findNextSource(RegSubRegPair RegSubReg, RewriteMapTy &RewriteMap); | ||||||||
| 204 | bool isMoveImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs, | ||||||||
| 205 | DenseMap<Register, MachineInstr *> &ImmDefMIs); | ||||||||
| 206 | bool foldImmediate(MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs, | ||||||||
| 207 | DenseMap<Register, MachineInstr *> &ImmDefMIs); | ||||||||
| 208 | |||||||||
| 209 | /// Finds recurrence cycles, but only ones that formulated around | ||||||||
| 210 | /// a def operand and a use operand that are tied. If there is a use | ||||||||
| 211 | /// operand commutable with the tied use operand, find recurrence cycle | ||||||||
| 212 | /// along that operand as well. | ||||||||
| 213 | bool findTargetRecurrence(Register Reg, | ||||||||
| 214 | const SmallSet<Register, 2> &TargetReg, | ||||||||
| 215 | RecurrenceCycle &RC); | ||||||||
| 216 | |||||||||
| 217 | /// If copy instruction \p MI is a virtual register copy, track it in | ||||||||
| 218 | /// the set \p CopyMIs. If this virtual register was previously seen as a | ||||||||
| 219 | /// copy, replace the uses of this copy with the previously seen copy's | ||||||||
| 220 | /// destination register. | ||||||||
| 221 | bool foldRedundantCopy(MachineInstr &MI, | ||||||||
| 222 | DenseMap<RegSubRegPair, MachineInstr *> &CopyMIs); | ||||||||
| 223 | |||||||||
| 224 | /// Is the register \p Reg a non-allocatable physical register? | ||||||||
| 225 | bool isNAPhysCopy(Register Reg); | ||||||||
| 226 | |||||||||
| 227 | /// If copy instruction \p MI is a non-allocatable virtual<->physical | ||||||||
| 228 | /// register copy, track it in the \p NAPhysToVirtMIs map. If this | ||||||||
| 229 | /// non-allocatable physical register was previously copied to a virtual | ||||||||
| 230 | /// registered and hasn't been clobbered, the virt->phys copy can be | ||||||||
| 231 | /// deleted. | ||||||||
| 232 | bool foldRedundantNAPhysCopy( | ||||||||
| 233 | MachineInstr &MI, DenseMap<Register, MachineInstr *> &NAPhysToVirtMIs); | ||||||||
| 234 | |||||||||
| 235 | bool isLoadFoldable(MachineInstr &MI, | ||||||||
| 236 | SmallSet<Register, 16> &FoldAsLoadDefCandidates); | ||||||||
| 237 | |||||||||
| 238 | /// Check whether \p MI is understood by the register coalescer | ||||||||
| 239 | /// but may require some rewriting. | ||||||||
| 240 | bool isCoalescableCopy(const MachineInstr &MI) { | ||||||||
| 241 | // SubregToRegs are not interesting, because they are already register | ||||||||
| 242 | // coalescer friendly. | ||||||||
| 243 | return MI.isCopy() || (!DisableAdvCopyOpt && | ||||||||
| 244 | (MI.isRegSequence() || MI.isInsertSubreg() || | ||||||||
| 245 | MI.isExtractSubreg())); | ||||||||
| 246 | } | ||||||||
| 247 | |||||||||
| 248 | /// Check whether \p MI is a copy like instruction that is | ||||||||
| 249 | /// not recognized by the register coalescer. | ||||||||
| 250 | bool isUncoalescableCopy(const MachineInstr &MI) { | ||||||||
| 251 | return MI.isBitcast() || | ||||||||
| 252 | (!DisableAdvCopyOpt && | ||||||||
| 253 | (MI.isRegSequenceLike() || MI.isInsertSubregLike() || | ||||||||
| 254 | MI.isExtractSubregLike())); | ||||||||
| 255 | } | ||||||||
| 256 | |||||||||
| 257 | MachineInstr &rewriteSource(MachineInstr &CopyLike, | ||||||||
| 258 | RegSubRegPair Def, RewriteMapTy &RewriteMap); | ||||||||
| 259 | }; | ||||||||
| 260 | |||||||||
| 261 | /// Helper class to hold instructions that are inside recurrence cycles. | ||||||||
| 262 | /// The recurrence cycle is formulated around 1) a def operand and its | ||||||||
| 263 | /// tied use operand, or 2) a def operand and a use operand that is commutable | ||||||||
| 264 | /// with another use operand which is tied to the def operand. In the latter | ||||||||
| 265 | /// case, index of the tied use operand and the commutable use operand are | ||||||||
| 266 | /// maintained with CommutePair. | ||||||||
| 267 | class RecurrenceInstr { | ||||||||
| 268 | public: | ||||||||
| 269 | using IndexPair = std::pair<unsigned, unsigned>; | ||||||||
| 270 | |||||||||
| 271 | RecurrenceInstr(MachineInstr *MI) : MI(MI) {} | ||||||||
| 272 | RecurrenceInstr(MachineInstr *MI, unsigned Idx1, unsigned Idx2) | ||||||||
| 273 | : MI(MI), CommutePair(std::make_pair(Idx1, Idx2)) {} | ||||||||
| 274 | |||||||||
| 275 | MachineInstr *getMI() const { return MI; } | ||||||||
| 276 | Optional<IndexPair> getCommutePair() const { return CommutePair; } | ||||||||
| 277 | |||||||||
| 278 | private: | ||||||||
| 279 | MachineInstr *MI; | ||||||||
| 280 | Optional<IndexPair> CommutePair; | ||||||||
| 281 | }; | ||||||||
| 282 | |||||||||
| 283 | /// Helper class to hold a reply for ValueTracker queries. | ||||||||
| 284 | /// Contains the returned sources for a given search and the instructions | ||||||||
| 285 | /// where the sources were tracked from. | ||||||||
| 286 | class ValueTrackerResult { | ||||||||
| 287 | private: | ||||||||
| 288 | /// Track all sources found by one ValueTracker query. | ||||||||
| 289 | SmallVector<RegSubRegPair, 2> RegSrcs; | ||||||||
| 290 | |||||||||
| 291 | /// Instruction using the sources in 'RegSrcs'. | ||||||||
| 292 | const MachineInstr *Inst = nullptr; | ||||||||
| 293 | |||||||||
| 294 | public: | ||||||||
| 295 | ValueTrackerResult() = default; | ||||||||
| 296 | |||||||||
| 297 | ValueTrackerResult(Register Reg, unsigned SubReg) { | ||||||||
| 298 | addSource(Reg, SubReg); | ||||||||
| 299 | } | ||||||||
| 300 | |||||||||
| 301 | bool isValid() const { return getNumSources() > 0; } | ||||||||
| 302 | |||||||||
| 303 | void setInst(const MachineInstr *I) { Inst = I; } | ||||||||
| 304 | const MachineInstr *getInst() const { return Inst; } | ||||||||
| 305 | |||||||||
| 306 | void clear() { | ||||||||
| 307 | RegSrcs.clear(); | ||||||||
| 308 | Inst = nullptr; | ||||||||
| 309 | } | ||||||||
| 310 | |||||||||
| 311 | void addSource(Register SrcReg, unsigned SrcSubReg) { | ||||||||
| 312 | RegSrcs.push_back(RegSubRegPair(SrcReg, SrcSubReg)); | ||||||||
| 313 | } | ||||||||
| 314 | |||||||||
| 315 | void setSource(int Idx, Register SrcReg, unsigned SrcSubReg) { | ||||||||
| 316 | assert(Idx < getNumSources() && "Reg pair source out of index")(static_cast <bool> (Idx < getNumSources() && "Reg pair source out of index") ? void (0) : __assert_fail ( "Idx < getNumSources() && \"Reg pair source out of index\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 316, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 317 | RegSrcs[Idx] = RegSubRegPair(SrcReg, SrcSubReg); | ||||||||
| 318 | } | ||||||||
| 319 | |||||||||
| 320 | int getNumSources() const { return RegSrcs.size(); } | ||||||||
| 321 | |||||||||
| 322 | RegSubRegPair getSrc(int Idx) const { | ||||||||
| 323 | return RegSrcs[Idx]; | ||||||||
| 324 | } | ||||||||
| 325 | |||||||||
| 326 | Register getSrcReg(int Idx) const { | ||||||||
| 327 | assert(Idx < getNumSources() && "Reg source out of index")(static_cast <bool> (Idx < getNumSources() && "Reg source out of index") ? void (0) : __assert_fail ("Idx < getNumSources() && \"Reg source out of index\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 327, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 328 | return RegSrcs[Idx].Reg; | ||||||||
| 329 | } | ||||||||
| 330 | |||||||||
| 331 | unsigned getSrcSubReg(int Idx) const { | ||||||||
| 332 | assert(Idx < getNumSources() && "SubReg source out of index")(static_cast <bool> (Idx < getNumSources() && "SubReg source out of index") ? void (0) : __assert_fail ("Idx < getNumSources() && \"SubReg source out of index\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 332, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 333 | return RegSrcs[Idx].SubReg; | ||||||||
| 334 | } | ||||||||
| 335 | |||||||||
| 336 | bool operator==(const ValueTrackerResult &Other) const { | ||||||||
| 337 | if (Other.getInst() != getInst()) | ||||||||
| 338 | return false; | ||||||||
| 339 | |||||||||
| 340 | if (Other.getNumSources() != getNumSources()) | ||||||||
| 341 | return false; | ||||||||
| 342 | |||||||||
| 343 | for (int i = 0, e = Other.getNumSources(); i != e; ++i) | ||||||||
| 344 | if (Other.getSrcReg(i) != getSrcReg(i) || | ||||||||
| 345 | Other.getSrcSubReg(i) != getSrcSubReg(i)) | ||||||||
| 346 | return false; | ||||||||
| 347 | return true; | ||||||||
| 348 | } | ||||||||
| 349 | }; | ||||||||
| 350 | |||||||||
| 351 | /// Helper class to track the possible sources of a value defined by | ||||||||
| 352 | /// a (chain of) copy related instructions. | ||||||||
| 353 | /// Given a definition (instruction and definition index), this class | ||||||||
| 354 | /// follows the use-def chain to find successive suitable sources. | ||||||||
| 355 | /// The given source can be used to rewrite the definition into | ||||||||
| 356 | /// def = COPY src. | ||||||||
| 357 | /// | ||||||||
| 358 | /// For instance, let us consider the following snippet: | ||||||||
| 359 | /// v0 = | ||||||||
| 360 | /// v2 = INSERT_SUBREG v1, v0, sub0 | ||||||||
| 361 | /// def = COPY v2.sub0 | ||||||||
| 362 | /// | ||||||||
| 363 | /// Using a ValueTracker for def = COPY v2.sub0 will give the following | ||||||||
| 364 | /// suitable sources: | ||||||||
| 365 | /// v2.sub0 and v0. | ||||||||
| 366 | /// Then, def can be rewritten into def = COPY v0. | ||||||||
| 367 | class ValueTracker { | ||||||||
| 368 | private: | ||||||||
| 369 | /// The current point into the use-def chain. | ||||||||
| 370 | const MachineInstr *Def = nullptr; | ||||||||
| 371 | |||||||||
| 372 | /// The index of the definition in Def. | ||||||||
| 373 | unsigned DefIdx = 0; | ||||||||
| 374 | |||||||||
| 375 | /// The sub register index of the definition. | ||||||||
| 376 | unsigned DefSubReg; | ||||||||
| 377 | |||||||||
| 378 | /// The register where the value can be found. | ||||||||
| 379 | Register Reg; | ||||||||
| 380 | |||||||||
| 381 | /// MachineRegisterInfo used to perform tracking. | ||||||||
| 382 | const MachineRegisterInfo &MRI; | ||||||||
| 383 | |||||||||
| 384 | /// Optional TargetInstrInfo used to perform some complex tracking. | ||||||||
| 385 | const TargetInstrInfo *TII; | ||||||||
| 386 | |||||||||
| 387 | /// Dispatcher to the right underlying implementation of getNextSource. | ||||||||
| 388 | ValueTrackerResult getNextSourceImpl(); | ||||||||
| 389 | |||||||||
| 390 | /// Specialized version of getNextSource for Copy instructions. | ||||||||
| 391 | ValueTrackerResult getNextSourceFromCopy(); | ||||||||
| 392 | |||||||||
| 393 | /// Specialized version of getNextSource for Bitcast instructions. | ||||||||
| 394 | ValueTrackerResult getNextSourceFromBitcast(); | ||||||||
| 395 | |||||||||
| 396 | /// Specialized version of getNextSource for RegSequence instructions. | ||||||||
| 397 | ValueTrackerResult getNextSourceFromRegSequence(); | ||||||||
| 398 | |||||||||
| 399 | /// Specialized version of getNextSource for InsertSubreg instructions. | ||||||||
| 400 | ValueTrackerResult getNextSourceFromInsertSubreg(); | ||||||||
| 401 | |||||||||
| 402 | /// Specialized version of getNextSource for ExtractSubreg instructions. | ||||||||
| 403 | ValueTrackerResult getNextSourceFromExtractSubreg(); | ||||||||
| 404 | |||||||||
| 405 | /// Specialized version of getNextSource for SubregToReg instructions. | ||||||||
| 406 | ValueTrackerResult getNextSourceFromSubregToReg(); | ||||||||
| 407 | |||||||||
| 408 | /// Specialized version of getNextSource for PHI instructions. | ||||||||
| 409 | ValueTrackerResult getNextSourceFromPHI(); | ||||||||
| 410 | |||||||||
| 411 | public: | ||||||||
| 412 | /// Create a ValueTracker instance for the value defined by \p Reg. | ||||||||
| 413 | /// \p DefSubReg represents the sub register index the value tracker will | ||||||||
| 414 | /// track. It does not need to match the sub register index used in the | ||||||||
| 415 | /// definition of \p Reg. | ||||||||
| 416 | /// If \p Reg is a physical register, a value tracker constructed with | ||||||||
| 417 | /// this constructor will not find any alternative source. | ||||||||
| 418 | /// Indeed, when \p Reg is a physical register that constructor does not | ||||||||
| 419 | /// know which definition of \p Reg it should track. | ||||||||
| 420 | /// Use the next constructor to track a physical register. | ||||||||
| 421 | ValueTracker(Register Reg, unsigned DefSubReg, | ||||||||
| 422 | const MachineRegisterInfo &MRI, | ||||||||
| 423 | const TargetInstrInfo *TII = nullptr) | ||||||||
| 424 | : DefSubReg(DefSubReg), Reg(Reg), MRI(MRI), TII(TII) { | ||||||||
| 425 | if (!Reg.isPhysical()) { | ||||||||
| 426 | Def = MRI.getVRegDef(Reg); | ||||||||
| 427 | DefIdx = MRI.def_begin(Reg).getOperandNo(); | ||||||||
| 428 | } | ||||||||
| 429 | } | ||||||||
| 430 | |||||||||
| 431 | /// Following the use-def chain, get the next available source | ||||||||
| 432 | /// for the tracked value. | ||||||||
| 433 | /// \return A ValueTrackerResult containing a set of registers | ||||||||
| 434 | /// and sub registers with tracked values. A ValueTrackerResult with | ||||||||
| 435 | /// an empty set of registers means no source was found. | ||||||||
| 436 | ValueTrackerResult getNextSource(); | ||||||||
| 437 | }; | ||||||||
| 438 | |||||||||
| 439 | } // end anonymous namespace | ||||||||
| 440 | |||||||||
| 441 | char PeepholeOptimizer::ID = 0; | ||||||||
| 442 | |||||||||
| 443 | char &llvm::PeepholeOptimizerID = PeepholeOptimizer::ID; | ||||||||
| 444 | |||||||||
| 445 | INITIALIZE_PASS_BEGIN(PeepholeOptimizer, DEBUG_TYPE,static void *initializePeepholeOptimizerPassOnce(PassRegistry &Registry) { | ||||||||
| 446 | "Peephole Optimizations", false, false)static void *initializePeepholeOptimizerPassOnce(PassRegistry &Registry) { | ||||||||
| 447 | INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)initializeMachineDominatorTreePass(Registry); | ||||||||
| 448 | INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo)initializeMachineLoopInfoPass(Registry); | ||||||||
| 449 | INITIALIZE_PASS_END(PeepholeOptimizer, DEBUG_TYPE,PassInfo *PI = new PassInfo( "Peephole Optimizations", "peephole-opt" , &PeepholeOptimizer::ID, PassInfo::NormalCtor_t(callDefaultCtor <PeepholeOptimizer>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializePeepholeOptimizerPassFlag ; void llvm::initializePeepholeOptimizerPass(PassRegistry & Registry) { llvm::call_once(InitializePeepholeOptimizerPassFlag , initializePeepholeOptimizerPassOnce, std::ref(Registry)); } | ||||||||
| 450 | "Peephole Optimizations", false, false)PassInfo *PI = new PassInfo( "Peephole Optimizations", "peephole-opt" , &PeepholeOptimizer::ID, PassInfo::NormalCtor_t(callDefaultCtor <PeepholeOptimizer>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializePeepholeOptimizerPassFlag ; void llvm::initializePeepholeOptimizerPass(PassRegistry & Registry) { llvm::call_once(InitializePeepholeOptimizerPassFlag , initializePeepholeOptimizerPassOnce, std::ref(Registry)); } | ||||||||
| 451 | |||||||||
| 452 | /// If instruction is a copy-like instruction, i.e. it reads a single register | ||||||||
| 453 | /// and writes a single register and it does not modify the source, and if the | ||||||||
| 454 | /// source value is preserved as a sub-register of the result, then replace all | ||||||||
| 455 | /// reachable uses of the source with the subreg of the result. | ||||||||
| 456 | /// | ||||||||
| 457 | /// Do not generate an EXTRACT that is used only in a debug use, as this changes | ||||||||
| 458 | /// the code. Since this code does not currently share EXTRACTs, just ignore all | ||||||||
| 459 | /// debug uses. | ||||||||
| 460 | bool PeepholeOptimizer:: | ||||||||
| 461 | optimizeExtInstr(MachineInstr &MI, MachineBasicBlock &MBB, | ||||||||
| 462 | SmallPtrSetImpl<MachineInstr*> &LocalMIs) { | ||||||||
| 463 | Register SrcReg, DstReg; | ||||||||
| 464 | unsigned SubIdx; | ||||||||
| 465 | if (!TII->isCoalescableExtInstr(MI, SrcReg, DstReg, SubIdx)) | ||||||||
| 466 | return false; | ||||||||
| 467 | |||||||||
| 468 | if (DstReg.isPhysical() || SrcReg.isPhysical()) | ||||||||
| 469 | return false; | ||||||||
| 470 | |||||||||
| 471 | if (MRI->hasOneNonDBGUse(SrcReg)) | ||||||||
| 472 | // No other uses. | ||||||||
| 473 | return false; | ||||||||
| 474 | |||||||||
| 475 | // Ensure DstReg can get a register class that actually supports | ||||||||
| 476 | // sub-registers. Don't change the class until we commit. | ||||||||
| 477 | const TargetRegisterClass *DstRC = MRI->getRegClass(DstReg); | ||||||||
| 478 | DstRC = TRI->getSubClassWithSubReg(DstRC, SubIdx); | ||||||||
| 479 | if (!DstRC) | ||||||||
| 480 | return false; | ||||||||
| 481 | |||||||||
| 482 | // The ext instr may be operating on a sub-register of SrcReg as well. | ||||||||
| 483 | // PPC::EXTSW is a 32 -> 64-bit sign extension, but it reads a 64-bit | ||||||||
| 484 | // register. | ||||||||
| 485 | // If UseSrcSubIdx is Set, SubIdx also applies to SrcReg, and only uses of | ||||||||
| 486 | // SrcReg:SubIdx should be replaced. | ||||||||
| 487 | bool UseSrcSubIdx = | ||||||||
| 488 | TRI->getSubClassWithSubReg(MRI->getRegClass(SrcReg), SubIdx) != nullptr; | ||||||||
| 489 | |||||||||
| 490 | // The source has other uses. See if we can replace the other uses with use of | ||||||||
| 491 | // the result of the extension. | ||||||||
| 492 | SmallPtrSet<MachineBasicBlock*, 4> ReachedBBs; | ||||||||
| 493 | for (MachineInstr &UI : MRI->use_nodbg_instructions(DstReg)) | ||||||||
| 494 | ReachedBBs.insert(UI.getParent()); | ||||||||
| 495 | |||||||||
| 496 | // Uses that are in the same BB of uses of the result of the instruction. | ||||||||
| 497 | SmallVector<MachineOperand*, 8> Uses; | ||||||||
| 498 | |||||||||
| 499 | // Uses that the result of the instruction can reach. | ||||||||
| 500 | SmallVector<MachineOperand*, 8> ExtendedUses; | ||||||||
| 501 | |||||||||
| 502 | bool ExtendLife = true; | ||||||||
| 503 | for (MachineOperand &UseMO : MRI->use_nodbg_operands(SrcReg)) { | ||||||||
| 504 | MachineInstr *UseMI = UseMO.getParent(); | ||||||||
| 505 | if (UseMI == &MI) | ||||||||
| 506 | continue; | ||||||||
| 507 | |||||||||
| 508 | if (UseMI->isPHI()) { | ||||||||
| 509 | ExtendLife = false; | ||||||||
| 510 | continue; | ||||||||
| 511 | } | ||||||||
| 512 | |||||||||
| 513 | // Only accept uses of SrcReg:SubIdx. | ||||||||
| 514 | if (UseSrcSubIdx
| ||||||||
| 515 | continue; | ||||||||
| 516 | |||||||||
| 517 | // It's an error to translate this: | ||||||||
| 518 | // | ||||||||
| 519 | // %reg1025 = <sext> %reg1024 | ||||||||
| 520 | // ... | ||||||||
| 521 | // %reg1026 = SUBREG_TO_REG 0, %reg1024, 4 | ||||||||
| 522 | // | ||||||||
| 523 | // into this: | ||||||||
| 524 | // | ||||||||
| 525 | // %reg1025 = <sext> %reg1024 | ||||||||
| 526 | // ... | ||||||||
| 527 | // %reg1027 = COPY %reg1025:4 | ||||||||
| 528 | // %reg1026 = SUBREG_TO_REG 0, %reg1027, 4 | ||||||||
| 529 | // | ||||||||
| 530 | // The problem here is that SUBREG_TO_REG is there to assert that an | ||||||||
| 531 | // implicit zext occurs. It doesn't insert a zext instruction. If we allow | ||||||||
| 532 | // the COPY here, it will give us the value after the <sext>, not the | ||||||||
| 533 | // original value of %reg1024 before <sext>. | ||||||||
| 534 | if (UseMI->getOpcode() == TargetOpcode::SUBREG_TO_REG) | ||||||||
| 535 | continue; | ||||||||
| 536 | |||||||||
| 537 | MachineBasicBlock *UseMBB = UseMI->getParent(); | ||||||||
| 538 | if (UseMBB == &MBB) { | ||||||||
| 539 | // Local uses that come after the extension. | ||||||||
| 540 | if (!LocalMIs.count(UseMI)) | ||||||||
| 541 | Uses.push_back(&UseMO); | ||||||||
| 542 | } else if (ReachedBBs.count(UseMBB)) { | ||||||||
| 543 | // Non-local uses where the result of the extension is used. Always | ||||||||
| 544 | // replace these unless it's a PHI. | ||||||||
| 545 | Uses.push_back(&UseMO); | ||||||||
| 546 | } else if (Aggressive && DT->dominates(&MBB, UseMBB)) { | ||||||||
| |||||||||
| 547 | // We may want to extend the live range of the extension result in order | ||||||||
| 548 | // to replace these uses. | ||||||||
| 549 | ExtendedUses.push_back(&UseMO); | ||||||||
| 550 | } else { | ||||||||
| 551 | // Both will be live out of the def MBB anyway. Don't extend live range of | ||||||||
| 552 | // the extension result. | ||||||||
| 553 | ExtendLife = false; | ||||||||
| 554 | break; | ||||||||
| 555 | } | ||||||||
| 556 | } | ||||||||
| 557 | |||||||||
| 558 | if (ExtendLife && !ExtendedUses.empty()) | ||||||||
| 559 | // Extend the liveness of the extension result. | ||||||||
| 560 | Uses.append(ExtendedUses.begin(), ExtendedUses.end()); | ||||||||
| 561 | |||||||||
| 562 | // Now replace all uses. | ||||||||
| 563 | bool Changed = false; | ||||||||
| 564 | if (!Uses.empty()) { | ||||||||
| 565 | SmallPtrSet<MachineBasicBlock*, 4> PHIBBs; | ||||||||
| 566 | |||||||||
| 567 | // Look for PHI uses of the extended result, we don't want to extend the | ||||||||
| 568 | // liveness of a PHI input. It breaks all kinds of assumptions down | ||||||||
| 569 | // stream. A PHI use is expected to be the kill of its source values. | ||||||||
| 570 | for (MachineInstr &UI : MRI->use_nodbg_instructions(DstReg)) | ||||||||
| 571 | if (UI.isPHI()) | ||||||||
| 572 | PHIBBs.insert(UI.getParent()); | ||||||||
| 573 | |||||||||
| 574 | const TargetRegisterClass *RC = MRI->getRegClass(SrcReg); | ||||||||
| 575 | for (unsigned i = 0, e = Uses.size(); i != e; ++i) { | ||||||||
| 576 | MachineOperand *UseMO = Uses[i]; | ||||||||
| 577 | MachineInstr *UseMI = UseMO->getParent(); | ||||||||
| 578 | MachineBasicBlock *UseMBB = UseMI->getParent(); | ||||||||
| 579 | if (PHIBBs.count(UseMBB)) | ||||||||
| 580 | continue; | ||||||||
| 581 | |||||||||
| 582 | // About to add uses of DstReg, clear DstReg's kill flags. | ||||||||
| 583 | if (!Changed) { | ||||||||
| 584 | MRI->clearKillFlags(DstReg); | ||||||||
| 585 | MRI->constrainRegClass(DstReg, DstRC); | ||||||||
| 586 | } | ||||||||
| 587 | |||||||||
| 588 | // SubReg defs are illegal in machine SSA phase, | ||||||||
| 589 | // we should not generate SubReg defs. | ||||||||
| 590 | // | ||||||||
| 591 | // For example, for the instructions: | ||||||||
| 592 | // | ||||||||
| 593 | // %1:g8rc_and_g8rc_nox0 = EXTSW %0:g8rc | ||||||||
| 594 | // %3:gprc_and_gprc_nor0 = COPY %0.sub_32:g8rc | ||||||||
| 595 | // | ||||||||
| 596 | // We should generate: | ||||||||
| 597 | // | ||||||||
| 598 | // %1:g8rc_and_g8rc_nox0 = EXTSW %0:g8rc | ||||||||
| 599 | // %6:gprc_and_gprc_nor0 = COPY %1.sub_32:g8rc_and_g8rc_nox0 | ||||||||
| 600 | // %3:gprc_and_gprc_nor0 = COPY %6:gprc_and_gprc_nor0 | ||||||||
| 601 | // | ||||||||
| 602 | if (UseSrcSubIdx) | ||||||||
| 603 | RC = MRI->getRegClass(UseMI->getOperand(0).getReg()); | ||||||||
| 604 | |||||||||
| 605 | Register NewVR = MRI->createVirtualRegister(RC); | ||||||||
| 606 | BuildMI(*UseMBB, UseMI, UseMI->getDebugLoc(), | ||||||||
| 607 | TII->get(TargetOpcode::COPY), NewVR) | ||||||||
| 608 | .addReg(DstReg, 0, SubIdx); | ||||||||
| 609 | if (UseSrcSubIdx) | ||||||||
| 610 | UseMO->setSubReg(0); | ||||||||
| 611 | |||||||||
| 612 | UseMO->setReg(NewVR); | ||||||||
| 613 | ++NumReuse; | ||||||||
| 614 | Changed = true; | ||||||||
| 615 | } | ||||||||
| 616 | } | ||||||||
| 617 | |||||||||
| 618 | return Changed; | ||||||||
| 619 | } | ||||||||
| 620 | |||||||||
| 621 | /// If the instruction is a compare and the previous instruction it's comparing | ||||||||
| 622 | /// against already sets (or could be modified to set) the same flag as the | ||||||||
| 623 | /// compare, then we can remove the comparison and use the flag from the | ||||||||
| 624 | /// previous instruction. | ||||||||
| 625 | bool PeepholeOptimizer::optimizeCmpInstr(MachineInstr &MI) { | ||||||||
| 626 | // If this instruction is a comparison against zero and isn't comparing a | ||||||||
| 627 | // physical register, we can try to optimize it. | ||||||||
| 628 | Register SrcReg, SrcReg2; | ||||||||
| 629 | int CmpMask, CmpValue; | ||||||||
| 630 | if (!TII->analyzeCompare(MI, SrcReg, SrcReg2, CmpMask, CmpValue) || | ||||||||
| 631 | SrcReg.isPhysical() || SrcReg2.isPhysical()) | ||||||||
| 632 | return false; | ||||||||
| 633 | |||||||||
| 634 | // Attempt to optimize the comparison instruction. | ||||||||
| 635 | LLVM_DEBUG(dbgs() << "Attempting to optimize compare: " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Attempting to optimize compare: " << MI; } } while (false); | ||||||||
| 636 | if (TII->optimizeCompareInstr(MI, SrcReg, SrcReg2, CmpMask, CmpValue, MRI)) { | ||||||||
| 637 | LLVM_DEBUG(dbgs() << " -> Successfully optimized compare!\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << " -> Successfully optimized compare!\n" ; } } while (false); | ||||||||
| 638 | ++NumCmps; | ||||||||
| 639 | return true; | ||||||||
| 640 | } | ||||||||
| 641 | |||||||||
| 642 | return false; | ||||||||
| 643 | } | ||||||||
| 644 | |||||||||
| 645 | /// Optimize a select instruction. | ||||||||
| 646 | bool PeepholeOptimizer::optimizeSelect(MachineInstr &MI, | ||||||||
| 647 | SmallPtrSetImpl<MachineInstr *> &LocalMIs) { | ||||||||
| 648 | unsigned TrueOp = 0; | ||||||||
| 649 | unsigned FalseOp = 0; | ||||||||
| 650 | bool Optimizable = false; | ||||||||
| 651 | SmallVector<MachineOperand, 4> Cond; | ||||||||
| 652 | if (TII->analyzeSelect(MI, Cond, TrueOp, FalseOp, Optimizable)) | ||||||||
| 653 | return false; | ||||||||
| 654 | if (!Optimizable) | ||||||||
| 655 | return false; | ||||||||
| 656 | if (!TII->optimizeSelect(MI, LocalMIs)) | ||||||||
| 657 | return false; | ||||||||
| 658 | LLVM_DEBUG(dbgs() << "Deleting select: " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Deleting select: " << MI; } } while (false); | ||||||||
| 659 | MI.eraseFromParent(); | ||||||||
| 660 | ++NumSelects; | ||||||||
| 661 | return true; | ||||||||
| 662 | } | ||||||||
| 663 | |||||||||
| 664 | /// Check if a simpler conditional branch can be generated. | ||||||||
| 665 | bool PeepholeOptimizer::optimizeCondBranch(MachineInstr &MI) { | ||||||||
| 666 | return TII->optimizeCondBranch(MI); | ||||||||
| 667 | } | ||||||||
| 668 | |||||||||
| 669 | /// Try to find the next source that share the same register file | ||||||||
| 670 | /// for the value defined by \p Reg and \p SubReg. | ||||||||
| 671 | /// When true is returned, the \p RewriteMap can be used by the client to | ||||||||
| 672 | /// retrieve all Def -> Use along the way up to the next source. Any found | ||||||||
| 673 | /// Use that is not itself a key for another entry, is the next source to | ||||||||
| 674 | /// use. During the search for the next source, multiple sources can be found | ||||||||
| 675 | /// given multiple incoming sources of a PHI instruction. In this case, we | ||||||||
| 676 | /// look in each PHI source for the next source; all found next sources must | ||||||||
| 677 | /// share the same register file as \p Reg and \p SubReg. The client should | ||||||||
| 678 | /// then be capable to rewrite all intermediate PHIs to get the next source. | ||||||||
| 679 | /// \return False if no alternative sources are available. True otherwise. | ||||||||
| 680 | bool PeepholeOptimizer::findNextSource(RegSubRegPair RegSubReg, | ||||||||
| 681 | RewriteMapTy &RewriteMap) { | ||||||||
| 682 | // Do not try to find a new source for a physical register. | ||||||||
| 683 | // So far we do not have any motivating example for doing that. | ||||||||
| 684 | // Thus, instead of maintaining untested code, we will revisit that if | ||||||||
| 685 | // that changes at some point. | ||||||||
| 686 | Register Reg = RegSubReg.Reg; | ||||||||
| 687 | if (Reg.isPhysical()) | ||||||||
| 688 | return false; | ||||||||
| 689 | const TargetRegisterClass *DefRC = MRI->getRegClass(Reg); | ||||||||
| 690 | |||||||||
| 691 | SmallVector<RegSubRegPair, 4> SrcToLook; | ||||||||
| 692 | RegSubRegPair CurSrcPair = RegSubReg; | ||||||||
| 693 | SrcToLook.push_back(CurSrcPair); | ||||||||
| 694 | |||||||||
| 695 | unsigned PHICount = 0; | ||||||||
| 696 | do { | ||||||||
| 697 | CurSrcPair = SrcToLook.pop_back_val(); | ||||||||
| 698 | // As explained above, do not handle physical registers | ||||||||
| 699 | if (Register::isPhysicalRegister(CurSrcPair.Reg)) | ||||||||
| 700 | return false; | ||||||||
| 701 | |||||||||
| 702 | ValueTracker ValTracker(CurSrcPair.Reg, CurSrcPair.SubReg, *MRI, TII); | ||||||||
| 703 | |||||||||
| 704 | // Follow the chain of copies until we find a more suitable source, a phi | ||||||||
| 705 | // or have to abort. | ||||||||
| 706 | while (true) { | ||||||||
| 707 | ValueTrackerResult Res = ValTracker.getNextSource(); | ||||||||
| 708 | // Abort at the end of a chain (without finding a suitable source). | ||||||||
| 709 | if (!Res.isValid()) | ||||||||
| 710 | return false; | ||||||||
| 711 | |||||||||
| 712 | // Insert the Def -> Use entry for the recently found source. | ||||||||
| 713 | ValueTrackerResult CurSrcRes = RewriteMap.lookup(CurSrcPair); | ||||||||
| 714 | if (CurSrcRes.isValid()) { | ||||||||
| 715 | assert(CurSrcRes == Res && "ValueTrackerResult found must match")(static_cast <bool> (CurSrcRes == Res && "ValueTrackerResult found must match" ) ? void (0) : __assert_fail ("CurSrcRes == Res && \"ValueTrackerResult found must match\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 715, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 716 | // An existent entry with multiple sources is a PHI cycle we must avoid. | ||||||||
| 717 | // Otherwise it's an entry with a valid next source we already found. | ||||||||
| 718 | if (CurSrcRes.getNumSources() > 1) { | ||||||||
| 719 | LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "findNextSource: found PHI cycle, aborting...\n" ; } } while (false) | ||||||||
| 720 | << "findNextSource: found PHI cycle, aborting...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "findNextSource: found PHI cycle, aborting...\n" ; } } while (false); | ||||||||
| 721 | return false; | ||||||||
| 722 | } | ||||||||
| 723 | break; | ||||||||
| 724 | } | ||||||||
| 725 | RewriteMap.insert(std::make_pair(CurSrcPair, Res)); | ||||||||
| 726 | |||||||||
| 727 | // ValueTrackerResult usually have one source unless it's the result from | ||||||||
| 728 | // a PHI instruction. Add the found PHI edges to be looked up further. | ||||||||
| 729 | unsigned NumSrcs = Res.getNumSources(); | ||||||||
| 730 | if (NumSrcs > 1) { | ||||||||
| 731 | PHICount++; | ||||||||
| 732 | if (PHICount >= RewritePHILimit) { | ||||||||
| 733 | LLVM_DEBUG(dbgs() << "findNextSource: PHI limit reached\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "findNextSource: PHI limit reached\n" ; } } while (false); | ||||||||
| 734 | return false; | ||||||||
| 735 | } | ||||||||
| 736 | |||||||||
| 737 | for (unsigned i = 0; i < NumSrcs; ++i) | ||||||||
| 738 | SrcToLook.push_back(Res.getSrc(i)); | ||||||||
| 739 | break; | ||||||||
| 740 | } | ||||||||
| 741 | |||||||||
| 742 | CurSrcPair = Res.getSrc(0); | ||||||||
| 743 | // Do not extend the live-ranges of physical registers as they add | ||||||||
| 744 | // constraints to the register allocator. Moreover, if we want to extend | ||||||||
| 745 | // the live-range of a physical register, unlike SSA virtual register, | ||||||||
| 746 | // we will have to check that they aren't redefine before the related use. | ||||||||
| 747 | if (Register::isPhysicalRegister(CurSrcPair.Reg)) | ||||||||
| 748 | return false; | ||||||||
| 749 | |||||||||
| 750 | // Keep following the chain if the value isn't any better yet. | ||||||||
| 751 | const TargetRegisterClass *SrcRC = MRI->getRegClass(CurSrcPair.Reg); | ||||||||
| 752 | if (!TRI->shouldRewriteCopySrc(DefRC, RegSubReg.SubReg, SrcRC, | ||||||||
| 753 | CurSrcPair.SubReg)) | ||||||||
| 754 | continue; | ||||||||
| 755 | |||||||||
| 756 | // We currently cannot deal with subreg operands on PHI instructions | ||||||||
| 757 | // (see insertPHI()). | ||||||||
| 758 | if (PHICount > 0 && CurSrcPair.SubReg != 0) | ||||||||
| 759 | continue; | ||||||||
| 760 | |||||||||
| 761 | // We found a suitable source, and are done with this chain. | ||||||||
| 762 | break; | ||||||||
| 763 | } | ||||||||
| 764 | } while (!SrcToLook.empty()); | ||||||||
| 765 | |||||||||
| 766 | // If we did not find a more suitable source, there is nothing to optimize. | ||||||||
| 767 | return CurSrcPair.Reg != Reg; | ||||||||
| 768 | } | ||||||||
| 769 | |||||||||
| 770 | /// Insert a PHI instruction with incoming edges \p SrcRegs that are | ||||||||
| 771 | /// guaranteed to have the same register class. This is necessary whenever we | ||||||||
| 772 | /// successfully traverse a PHI instruction and find suitable sources coming | ||||||||
| 773 | /// from its edges. By inserting a new PHI, we provide a rewritten PHI def | ||||||||
| 774 | /// suitable to be used in a new COPY instruction. | ||||||||
| 775 | static MachineInstr & | ||||||||
| 776 | insertPHI(MachineRegisterInfo &MRI, const TargetInstrInfo &TII, | ||||||||
| 777 | const SmallVectorImpl<RegSubRegPair> &SrcRegs, | ||||||||
| 778 | MachineInstr &OrigPHI) { | ||||||||
| 779 | assert(!SrcRegs.empty() && "No sources to create a PHI instruction?")(static_cast <bool> (!SrcRegs.empty() && "No sources to create a PHI instruction?" ) ? void (0) : __assert_fail ("!SrcRegs.empty() && \"No sources to create a PHI instruction?\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 779, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 780 | |||||||||
| 781 | const TargetRegisterClass *NewRC = MRI.getRegClass(SrcRegs[0].Reg); | ||||||||
| 782 | // NewRC is only correct if no subregisters are involved. findNextSource() | ||||||||
| 783 | // should have rejected those cases already. | ||||||||
| 784 | assert(SrcRegs[0].SubReg == 0 && "should not have subreg operand")(static_cast <bool> (SrcRegs[0].SubReg == 0 && "should not have subreg operand" ) ? void (0) : __assert_fail ("SrcRegs[0].SubReg == 0 && \"should not have subreg operand\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 784, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 785 | Register NewVR = MRI.createVirtualRegister(NewRC); | ||||||||
| 786 | MachineBasicBlock *MBB = OrigPHI.getParent(); | ||||||||
| 787 | MachineInstrBuilder MIB = BuildMI(*MBB, &OrigPHI, OrigPHI.getDebugLoc(), | ||||||||
| 788 | TII.get(TargetOpcode::PHI), NewVR); | ||||||||
| 789 | |||||||||
| 790 | unsigned MBBOpIdx = 2; | ||||||||
| 791 | for (const RegSubRegPair &RegPair : SrcRegs) { | ||||||||
| 792 | MIB.addReg(RegPair.Reg, 0, RegPair.SubReg); | ||||||||
| 793 | MIB.addMBB(OrigPHI.getOperand(MBBOpIdx).getMBB()); | ||||||||
| 794 | // Since we're extended the lifetime of RegPair.Reg, clear the | ||||||||
| 795 | // kill flags to account for that and make RegPair.Reg reaches | ||||||||
| 796 | // the new PHI. | ||||||||
| 797 | MRI.clearKillFlags(RegPair.Reg); | ||||||||
| 798 | MBBOpIdx += 2; | ||||||||
| 799 | } | ||||||||
| 800 | |||||||||
| 801 | return *MIB; | ||||||||
| 802 | } | ||||||||
| 803 | |||||||||
| 804 | namespace { | ||||||||
| 805 | |||||||||
| 806 | /// Interface to query instructions amenable to copy rewriting. | ||||||||
| 807 | class Rewriter { | ||||||||
| 808 | protected: | ||||||||
| 809 | MachineInstr &CopyLike; | ||||||||
| 810 | unsigned CurrentSrcIdx = 0; ///< The index of the source being rewritten. | ||||||||
| 811 | public: | ||||||||
| 812 | Rewriter(MachineInstr &CopyLike) : CopyLike(CopyLike) {} | ||||||||
| 813 | virtual ~Rewriter() {} | ||||||||
| 814 | |||||||||
| 815 | /// Get the next rewritable source (SrcReg, SrcSubReg) and | ||||||||
| 816 | /// the related value that it affects (DstReg, DstSubReg). | ||||||||
| 817 | /// A source is considered rewritable if its register class and the | ||||||||
| 818 | /// register class of the related DstReg may not be register | ||||||||
| 819 | /// coalescer friendly. In other words, given a copy-like instruction | ||||||||
| 820 | /// not all the arguments may be returned at rewritable source, since | ||||||||
| 821 | /// some arguments are none to be register coalescer friendly. | ||||||||
| 822 | /// | ||||||||
| 823 | /// Each call of this method moves the current source to the next | ||||||||
| 824 | /// rewritable source. | ||||||||
| 825 | /// For instance, let CopyLike be the instruction to rewrite. | ||||||||
| 826 | /// CopyLike has one definition and one source: | ||||||||
| 827 | /// dst.dstSubIdx = CopyLike src.srcSubIdx. | ||||||||
| 828 | /// | ||||||||
| 829 | /// The first call will give the first rewritable source, i.e., | ||||||||
| 830 | /// the only source this instruction has: | ||||||||
| 831 | /// (SrcReg, SrcSubReg) = (src, srcSubIdx). | ||||||||
| 832 | /// This source defines the whole definition, i.e., | ||||||||
| 833 | /// (DstReg, DstSubReg) = (dst, dstSubIdx). | ||||||||
| 834 | /// | ||||||||
| 835 | /// The second and subsequent calls will return false, as there is only one | ||||||||
| 836 | /// rewritable source. | ||||||||
| 837 | /// | ||||||||
| 838 | /// \return True if a rewritable source has been found, false otherwise. | ||||||||
| 839 | /// The output arguments are valid if and only if true is returned. | ||||||||
| 840 | virtual bool getNextRewritableSource(RegSubRegPair &Src, | ||||||||
| 841 | RegSubRegPair &Dst) = 0; | ||||||||
| 842 | |||||||||
| 843 | /// Rewrite the current source with \p NewReg and \p NewSubReg if possible. | ||||||||
| 844 | /// \return True if the rewriting was possible, false otherwise. | ||||||||
| 845 | virtual bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) = 0; | ||||||||
| 846 | }; | ||||||||
| 847 | |||||||||
| 848 | /// Rewriter for COPY instructions. | ||||||||
| 849 | class CopyRewriter : public Rewriter { | ||||||||
| 850 | public: | ||||||||
| 851 | CopyRewriter(MachineInstr &MI) : Rewriter(MI) { | ||||||||
| 852 | assert(MI.isCopy() && "Expected copy instruction")(static_cast <bool> (MI.isCopy() && "Expected copy instruction" ) ? void (0) : __assert_fail ("MI.isCopy() && \"Expected copy instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 852, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 853 | } | ||||||||
| 854 | virtual ~CopyRewriter() = default; | ||||||||
| 855 | |||||||||
| 856 | bool getNextRewritableSource(RegSubRegPair &Src, | ||||||||
| 857 | RegSubRegPair &Dst) override { | ||||||||
| 858 | // CurrentSrcIdx > 0 means this function has already been called. | ||||||||
| 859 | if (CurrentSrcIdx > 0) | ||||||||
| 860 | return false; | ||||||||
| 861 | // This is the first call to getNextRewritableSource. | ||||||||
| 862 | // Move the CurrentSrcIdx to remember that we made that call. | ||||||||
| 863 | CurrentSrcIdx = 1; | ||||||||
| 864 | // The rewritable source is the argument. | ||||||||
| 865 | const MachineOperand &MOSrc = CopyLike.getOperand(1); | ||||||||
| 866 | Src = RegSubRegPair(MOSrc.getReg(), MOSrc.getSubReg()); | ||||||||
| 867 | // What we track are the alternative sources of the definition. | ||||||||
| 868 | const MachineOperand &MODef = CopyLike.getOperand(0); | ||||||||
| 869 | Dst = RegSubRegPair(MODef.getReg(), MODef.getSubReg()); | ||||||||
| 870 | return true; | ||||||||
| 871 | } | ||||||||
| 872 | |||||||||
| 873 | bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override { | ||||||||
| 874 | if (CurrentSrcIdx != 1) | ||||||||
| 875 | return false; | ||||||||
| 876 | MachineOperand &MOSrc = CopyLike.getOperand(CurrentSrcIdx); | ||||||||
| 877 | MOSrc.setReg(NewReg); | ||||||||
| 878 | MOSrc.setSubReg(NewSubReg); | ||||||||
| 879 | return true; | ||||||||
| 880 | } | ||||||||
| 881 | }; | ||||||||
| 882 | |||||||||
| 883 | /// Helper class to rewrite uncoalescable copy like instructions | ||||||||
| 884 | /// into new COPY (coalescable friendly) instructions. | ||||||||
| 885 | class UncoalescableRewriter : public Rewriter { | ||||||||
| 886 | unsigned NumDefs; ///< Number of defs in the bitcast. | ||||||||
| 887 | |||||||||
| 888 | public: | ||||||||
| 889 | UncoalescableRewriter(MachineInstr &MI) : Rewriter(MI) { | ||||||||
| 890 | NumDefs = MI.getDesc().getNumDefs(); | ||||||||
| 891 | } | ||||||||
| 892 | |||||||||
| 893 | /// \see See Rewriter::getNextRewritableSource() | ||||||||
| 894 | /// All such sources need to be considered rewritable in order to | ||||||||
| 895 | /// rewrite a uncoalescable copy-like instruction. This method return | ||||||||
| 896 | /// each definition that must be checked if rewritable. | ||||||||
| 897 | bool getNextRewritableSource(RegSubRegPair &Src, | ||||||||
| 898 | RegSubRegPair &Dst) override { | ||||||||
| 899 | // Find the next non-dead definition and continue from there. | ||||||||
| 900 | if (CurrentSrcIdx == NumDefs) | ||||||||
| 901 | return false; | ||||||||
| 902 | |||||||||
| 903 | while (CopyLike.getOperand(CurrentSrcIdx).isDead()) { | ||||||||
| 904 | ++CurrentSrcIdx; | ||||||||
| 905 | if (CurrentSrcIdx == NumDefs) | ||||||||
| 906 | return false; | ||||||||
| 907 | } | ||||||||
| 908 | |||||||||
| 909 | // What we track are the alternative sources of the definition. | ||||||||
| 910 | Src = RegSubRegPair(0, 0); | ||||||||
| 911 | const MachineOperand &MODef = CopyLike.getOperand(CurrentSrcIdx); | ||||||||
| 912 | Dst = RegSubRegPair(MODef.getReg(), MODef.getSubReg()); | ||||||||
| 913 | |||||||||
| 914 | CurrentSrcIdx++; | ||||||||
| 915 | return true; | ||||||||
| 916 | } | ||||||||
| 917 | |||||||||
| 918 | bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override { | ||||||||
| 919 | return false; | ||||||||
| 920 | } | ||||||||
| 921 | }; | ||||||||
| 922 | |||||||||
| 923 | /// Specialized rewriter for INSERT_SUBREG instruction. | ||||||||
| 924 | class InsertSubregRewriter : public Rewriter { | ||||||||
| 925 | public: | ||||||||
| 926 | InsertSubregRewriter(MachineInstr &MI) : Rewriter(MI) { | ||||||||
| 927 | assert(MI.isInsertSubreg() && "Invalid instruction")(static_cast <bool> (MI.isInsertSubreg() && "Invalid instruction" ) ? void (0) : __assert_fail ("MI.isInsertSubreg() && \"Invalid instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 927, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 928 | } | ||||||||
| 929 | |||||||||
| 930 | /// \see See Rewriter::getNextRewritableSource() | ||||||||
| 931 | /// Here CopyLike has the following form: | ||||||||
| 932 | /// dst = INSERT_SUBREG Src1, Src2.src2SubIdx, subIdx. | ||||||||
| 933 | /// Src1 has the same register class has dst, hence, there is | ||||||||
| 934 | /// nothing to rewrite. | ||||||||
| 935 | /// Src2.src2SubIdx, may not be register coalescer friendly. | ||||||||
| 936 | /// Therefore, the first call to this method returns: | ||||||||
| 937 | /// (SrcReg, SrcSubReg) = (Src2, src2SubIdx). | ||||||||
| 938 | /// (DstReg, DstSubReg) = (dst, subIdx). | ||||||||
| 939 | /// | ||||||||
| 940 | /// Subsequence calls will return false. | ||||||||
| 941 | bool getNextRewritableSource(RegSubRegPair &Src, | ||||||||
| 942 | RegSubRegPair &Dst) override { | ||||||||
| 943 | // If we already get the only source we can rewrite, return false. | ||||||||
| 944 | if (CurrentSrcIdx == 2) | ||||||||
| 945 | return false; | ||||||||
| 946 | // We are looking at v2 = INSERT_SUBREG v0, v1, sub0. | ||||||||
| 947 | CurrentSrcIdx = 2; | ||||||||
| 948 | const MachineOperand &MOInsertedReg = CopyLike.getOperand(2); | ||||||||
| 949 | Src = RegSubRegPair(MOInsertedReg.getReg(), MOInsertedReg.getSubReg()); | ||||||||
| 950 | const MachineOperand &MODef = CopyLike.getOperand(0); | ||||||||
| 951 | |||||||||
| 952 | // We want to track something that is compatible with the | ||||||||
| 953 | // partial definition. | ||||||||
| 954 | if (MODef.getSubReg()) | ||||||||
| 955 | // Bail if we have to compose sub-register indices. | ||||||||
| 956 | return false; | ||||||||
| 957 | Dst = RegSubRegPair(MODef.getReg(), | ||||||||
| 958 | (unsigned)CopyLike.getOperand(3).getImm()); | ||||||||
| 959 | return true; | ||||||||
| 960 | } | ||||||||
| 961 | |||||||||
| 962 | bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override { | ||||||||
| 963 | if (CurrentSrcIdx != 2) | ||||||||
| 964 | return false; | ||||||||
| 965 | // We are rewriting the inserted reg. | ||||||||
| 966 | MachineOperand &MO = CopyLike.getOperand(CurrentSrcIdx); | ||||||||
| 967 | MO.setReg(NewReg); | ||||||||
| 968 | MO.setSubReg(NewSubReg); | ||||||||
| 969 | return true; | ||||||||
| 970 | } | ||||||||
| 971 | }; | ||||||||
| 972 | |||||||||
| 973 | /// Specialized rewriter for EXTRACT_SUBREG instruction. | ||||||||
| 974 | class ExtractSubregRewriter : public Rewriter { | ||||||||
| 975 | const TargetInstrInfo &TII; | ||||||||
| 976 | |||||||||
| 977 | public: | ||||||||
| 978 | ExtractSubregRewriter(MachineInstr &MI, const TargetInstrInfo &TII) | ||||||||
| 979 | : Rewriter(MI), TII(TII) { | ||||||||
| 980 | assert(MI.isExtractSubreg() && "Invalid instruction")(static_cast <bool> (MI.isExtractSubreg() && "Invalid instruction" ) ? void (0) : __assert_fail ("MI.isExtractSubreg() && \"Invalid instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 980, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 981 | } | ||||||||
| 982 | |||||||||
| 983 | /// \see Rewriter::getNextRewritableSource() | ||||||||
| 984 | /// Here CopyLike has the following form: | ||||||||
| 985 | /// dst.dstSubIdx = EXTRACT_SUBREG Src, subIdx. | ||||||||
| 986 | /// There is only one rewritable source: Src.subIdx, | ||||||||
| 987 | /// which defines dst.dstSubIdx. | ||||||||
| 988 | bool getNextRewritableSource(RegSubRegPair &Src, | ||||||||
| 989 | RegSubRegPair &Dst) override { | ||||||||
| 990 | // If we already get the only source we can rewrite, return false. | ||||||||
| 991 | if (CurrentSrcIdx == 1) | ||||||||
| 992 | return false; | ||||||||
| 993 | // We are looking at v1 = EXTRACT_SUBREG v0, sub0. | ||||||||
| 994 | CurrentSrcIdx = 1; | ||||||||
| 995 | const MachineOperand &MOExtractedReg = CopyLike.getOperand(1); | ||||||||
| 996 | // If we have to compose sub-register indices, bail out. | ||||||||
| 997 | if (MOExtractedReg.getSubReg()) | ||||||||
| 998 | return false; | ||||||||
| 999 | |||||||||
| 1000 | Src = RegSubRegPair(MOExtractedReg.getReg(), | ||||||||
| 1001 | CopyLike.getOperand(2).getImm()); | ||||||||
| 1002 | |||||||||
| 1003 | // We want to track something that is compatible with the definition. | ||||||||
| 1004 | const MachineOperand &MODef = CopyLike.getOperand(0); | ||||||||
| 1005 | Dst = RegSubRegPair(MODef.getReg(), MODef.getSubReg()); | ||||||||
| 1006 | return true; | ||||||||
| 1007 | } | ||||||||
| 1008 | |||||||||
| 1009 | bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override { | ||||||||
| 1010 | // The only source we can rewrite is the input register. | ||||||||
| 1011 | if (CurrentSrcIdx != 1) | ||||||||
| 1012 | return false; | ||||||||
| 1013 | |||||||||
| 1014 | CopyLike.getOperand(CurrentSrcIdx).setReg(NewReg); | ||||||||
| 1015 | |||||||||
| 1016 | // If we find a source that does not require to extract something, | ||||||||
| 1017 | // rewrite the operation with a copy. | ||||||||
| 1018 | if (!NewSubReg) { | ||||||||
| 1019 | // Move the current index to an invalid position. | ||||||||
| 1020 | // We do not want another call to this method to be able | ||||||||
| 1021 | // to do any change. | ||||||||
| 1022 | CurrentSrcIdx = -1; | ||||||||
| 1023 | // Rewrite the operation as a COPY. | ||||||||
| 1024 | // Get rid of the sub-register index. | ||||||||
| 1025 | CopyLike.RemoveOperand(2); | ||||||||
| 1026 | // Morph the operation into a COPY. | ||||||||
| 1027 | CopyLike.setDesc(TII.get(TargetOpcode::COPY)); | ||||||||
| 1028 | return true; | ||||||||
| 1029 | } | ||||||||
| 1030 | CopyLike.getOperand(CurrentSrcIdx + 1).setImm(NewSubReg); | ||||||||
| 1031 | return true; | ||||||||
| 1032 | } | ||||||||
| 1033 | }; | ||||||||
| 1034 | |||||||||
| 1035 | /// Specialized rewriter for REG_SEQUENCE instruction. | ||||||||
| 1036 | class RegSequenceRewriter : public Rewriter { | ||||||||
| 1037 | public: | ||||||||
| 1038 | RegSequenceRewriter(MachineInstr &MI) : Rewriter(MI) { | ||||||||
| 1039 | assert(MI.isRegSequence() && "Invalid instruction")(static_cast <bool> (MI.isRegSequence() && "Invalid instruction" ) ? void (0) : __assert_fail ("MI.isRegSequence() && \"Invalid instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1039, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1040 | } | ||||||||
| 1041 | |||||||||
| 1042 | /// \see Rewriter::getNextRewritableSource() | ||||||||
| 1043 | /// Here CopyLike has the following form: | ||||||||
| 1044 | /// dst = REG_SEQUENCE Src1.src1SubIdx, subIdx1, Src2.src2SubIdx, subIdx2. | ||||||||
| 1045 | /// Each call will return a different source, walking all the available | ||||||||
| 1046 | /// source. | ||||||||
| 1047 | /// | ||||||||
| 1048 | /// The first call returns: | ||||||||
| 1049 | /// (SrcReg, SrcSubReg) = (Src1, src1SubIdx). | ||||||||
| 1050 | /// (DstReg, DstSubReg) = (dst, subIdx1). | ||||||||
| 1051 | /// | ||||||||
| 1052 | /// The second call returns: | ||||||||
| 1053 | /// (SrcReg, SrcSubReg) = (Src2, src2SubIdx). | ||||||||
| 1054 | /// (DstReg, DstSubReg) = (dst, subIdx2). | ||||||||
| 1055 | /// | ||||||||
| 1056 | /// And so on, until all the sources have been traversed, then | ||||||||
| 1057 | /// it returns false. | ||||||||
| 1058 | bool getNextRewritableSource(RegSubRegPair &Src, | ||||||||
| 1059 | RegSubRegPair &Dst) override { | ||||||||
| 1060 | // We are looking at v0 = REG_SEQUENCE v1, sub1, v2, sub2, etc. | ||||||||
| 1061 | |||||||||
| 1062 | // If this is the first call, move to the first argument. | ||||||||
| 1063 | if (CurrentSrcIdx == 0) { | ||||||||
| 1064 | CurrentSrcIdx = 1; | ||||||||
| 1065 | } else { | ||||||||
| 1066 | // Otherwise, move to the next argument and check that it is valid. | ||||||||
| 1067 | CurrentSrcIdx += 2; | ||||||||
| 1068 | if (CurrentSrcIdx >= CopyLike.getNumOperands()) | ||||||||
| 1069 | return false; | ||||||||
| 1070 | } | ||||||||
| 1071 | const MachineOperand &MOInsertedReg = CopyLike.getOperand(CurrentSrcIdx); | ||||||||
| 1072 | Src.Reg = MOInsertedReg.getReg(); | ||||||||
| 1073 | // If we have to compose sub-register indices, bail out. | ||||||||
| 1074 | if ((Src.SubReg = MOInsertedReg.getSubReg())) | ||||||||
| 1075 | return false; | ||||||||
| 1076 | |||||||||
| 1077 | // We want to track something that is compatible with the related | ||||||||
| 1078 | // partial definition. | ||||||||
| 1079 | Dst.SubReg = CopyLike.getOperand(CurrentSrcIdx + 1).getImm(); | ||||||||
| 1080 | |||||||||
| 1081 | const MachineOperand &MODef = CopyLike.getOperand(0); | ||||||||
| 1082 | Dst.Reg = MODef.getReg(); | ||||||||
| 1083 | // If we have to compose sub-registers, bail. | ||||||||
| 1084 | return MODef.getSubReg() == 0; | ||||||||
| 1085 | } | ||||||||
| 1086 | |||||||||
| 1087 | bool RewriteCurrentSource(Register NewReg, unsigned NewSubReg) override { | ||||||||
| 1088 | // We cannot rewrite out of bound operands. | ||||||||
| 1089 | // Moreover, rewritable sources are at odd positions. | ||||||||
| 1090 | if ((CurrentSrcIdx & 1) != 1 || CurrentSrcIdx > CopyLike.getNumOperands()) | ||||||||
| 1091 | return false; | ||||||||
| 1092 | |||||||||
| 1093 | MachineOperand &MO = CopyLike.getOperand(CurrentSrcIdx); | ||||||||
| 1094 | MO.setReg(NewReg); | ||||||||
| 1095 | MO.setSubReg(NewSubReg); | ||||||||
| 1096 | return true; | ||||||||
| 1097 | } | ||||||||
| 1098 | }; | ||||||||
| 1099 | |||||||||
| 1100 | } // end anonymous namespace | ||||||||
| 1101 | |||||||||
| 1102 | /// Get the appropriated Rewriter for \p MI. | ||||||||
| 1103 | /// \return A pointer to a dynamically allocated Rewriter or nullptr if no | ||||||||
| 1104 | /// rewriter works for \p MI. | ||||||||
| 1105 | static Rewriter *getCopyRewriter(MachineInstr &MI, const TargetInstrInfo &TII) { | ||||||||
| 1106 | // Handle uncoalescable copy-like instructions. | ||||||||
| 1107 | if (MI.isBitcast() || MI.isRegSequenceLike() || MI.isInsertSubregLike() || | ||||||||
| 1108 | MI.isExtractSubregLike()) | ||||||||
| 1109 | return new UncoalescableRewriter(MI); | ||||||||
| 1110 | |||||||||
| 1111 | switch (MI.getOpcode()) { | ||||||||
| 1112 | default: | ||||||||
| 1113 | return nullptr; | ||||||||
| 1114 | case TargetOpcode::COPY: | ||||||||
| 1115 | return new CopyRewriter(MI); | ||||||||
| 1116 | case TargetOpcode::INSERT_SUBREG: | ||||||||
| 1117 | return new InsertSubregRewriter(MI); | ||||||||
| 1118 | case TargetOpcode::EXTRACT_SUBREG: | ||||||||
| 1119 | return new ExtractSubregRewriter(MI, TII); | ||||||||
| 1120 | case TargetOpcode::REG_SEQUENCE: | ||||||||
| 1121 | return new RegSequenceRewriter(MI); | ||||||||
| 1122 | } | ||||||||
| 1123 | } | ||||||||
| 1124 | |||||||||
| 1125 | /// Given a \p Def.Reg and Def.SubReg pair, use \p RewriteMap to find | ||||||||
| 1126 | /// the new source to use for rewrite. If \p HandleMultipleSources is true and | ||||||||
| 1127 | /// multiple sources for a given \p Def are found along the way, we found a | ||||||||
| 1128 | /// PHI instructions that needs to be rewritten. | ||||||||
| 1129 | /// TODO: HandleMultipleSources should be removed once we test PHI handling | ||||||||
| 1130 | /// with coalescable copies. | ||||||||
| 1131 | static RegSubRegPair | ||||||||
| 1132 | getNewSource(MachineRegisterInfo *MRI, const TargetInstrInfo *TII, | ||||||||
| 1133 | RegSubRegPair Def, | ||||||||
| 1134 | const PeepholeOptimizer::RewriteMapTy &RewriteMap, | ||||||||
| 1135 | bool HandleMultipleSources = true) { | ||||||||
| 1136 | RegSubRegPair LookupSrc(Def.Reg, Def.SubReg); | ||||||||
| 1137 | while (true) { | ||||||||
| 1138 | ValueTrackerResult Res = RewriteMap.lookup(LookupSrc); | ||||||||
| 1139 | // If there are no entries on the map, LookupSrc is the new source. | ||||||||
| 1140 | if (!Res.isValid()) | ||||||||
| 1141 | return LookupSrc; | ||||||||
| 1142 | |||||||||
| 1143 | // There's only one source for this definition, keep searching... | ||||||||
| 1144 | unsigned NumSrcs = Res.getNumSources(); | ||||||||
| 1145 | if (NumSrcs == 1) { | ||||||||
| 1146 | LookupSrc.Reg = Res.getSrcReg(0); | ||||||||
| 1147 | LookupSrc.SubReg = Res.getSrcSubReg(0); | ||||||||
| 1148 | continue; | ||||||||
| 1149 | } | ||||||||
| 1150 | |||||||||
| 1151 | // TODO: Remove once multiple srcs w/ coalescable copies are supported. | ||||||||
| 1152 | if (!HandleMultipleSources) | ||||||||
| 1153 | break; | ||||||||
| 1154 | |||||||||
| 1155 | // Multiple sources, recurse into each source to find a new source | ||||||||
| 1156 | // for it. Then, rewrite the PHI accordingly to its new edges. | ||||||||
| 1157 | SmallVector<RegSubRegPair, 4> NewPHISrcs; | ||||||||
| 1158 | for (unsigned i = 0; i < NumSrcs; ++i) { | ||||||||
| 1159 | RegSubRegPair PHISrc(Res.getSrcReg(i), Res.getSrcSubReg(i)); | ||||||||
| 1160 | NewPHISrcs.push_back( | ||||||||
| 1161 | getNewSource(MRI, TII, PHISrc, RewriteMap, HandleMultipleSources)); | ||||||||
| 1162 | } | ||||||||
| 1163 | |||||||||
| 1164 | // Build the new PHI node and return its def register as the new source. | ||||||||
| 1165 | MachineInstr &OrigPHI = const_cast<MachineInstr &>(*Res.getInst()); | ||||||||
| 1166 | MachineInstr &NewPHI = insertPHI(*MRI, *TII, NewPHISrcs, OrigPHI); | ||||||||
| 1167 | LLVM_DEBUG(dbgs() << "-- getNewSource\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "-- getNewSource\n"; } } while (false); | ||||||||
| 1168 | LLVM_DEBUG(dbgs() << " Replacing: " << OrigPHI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << " Replacing: " << OrigPHI; } } while (false); | ||||||||
| 1169 | LLVM_DEBUG(dbgs() << " With: " << NewPHI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << " With: " << NewPHI; } } while (false); | ||||||||
| 1170 | const MachineOperand &MODef = NewPHI.getOperand(0); | ||||||||
| 1171 | return RegSubRegPair(MODef.getReg(), MODef.getSubReg()); | ||||||||
| 1172 | } | ||||||||
| 1173 | |||||||||
| 1174 | return RegSubRegPair(0, 0); | ||||||||
| 1175 | } | ||||||||
| 1176 | |||||||||
| 1177 | /// Optimize generic copy instructions to avoid cross register bank copy. | ||||||||
| 1178 | /// The optimization looks through a chain of copies and tries to find a source | ||||||||
| 1179 | /// that has a compatible register class. | ||||||||
| 1180 | /// Two register classes are considered to be compatible if they share the same | ||||||||
| 1181 | /// register bank. | ||||||||
| 1182 | /// New copies issued by this optimization are register allocator | ||||||||
| 1183 | /// friendly. This optimization does not remove any copy as it may | ||||||||
| 1184 | /// overconstrain the register allocator, but replaces some operands | ||||||||
| 1185 | /// when possible. | ||||||||
| 1186 | /// \pre isCoalescableCopy(*MI) is true. | ||||||||
| 1187 | /// \return True, when \p MI has been rewritten. False otherwise. | ||||||||
| 1188 | bool PeepholeOptimizer::optimizeCoalescableCopy(MachineInstr &MI) { | ||||||||
| 1189 | assert(isCoalescableCopy(MI) && "Invalid argument")(static_cast <bool> (isCoalescableCopy(MI) && "Invalid argument" ) ? void (0) : __assert_fail ("isCoalescableCopy(MI) && \"Invalid argument\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1189, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1190 | assert(MI.getDesc().getNumDefs() == 1 &&(static_cast <bool> (MI.getDesc().getNumDefs() == 1 && "Coalescer can understand multiple defs?!") ? void (0) : __assert_fail ("MI.getDesc().getNumDefs() == 1 && \"Coalescer can understand multiple defs?!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1191, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1191 | "Coalescer can understand multiple defs?!")(static_cast <bool> (MI.getDesc().getNumDefs() == 1 && "Coalescer can understand multiple defs?!") ? void (0) : __assert_fail ("MI.getDesc().getNumDefs() == 1 && \"Coalescer can understand multiple defs?!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1191, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1192 | const MachineOperand &MODef = MI.getOperand(0); | ||||||||
| 1193 | // Do not rewrite physical definitions. | ||||||||
| 1194 | if (Register::isPhysicalRegister(MODef.getReg())) | ||||||||
| 1195 | return false; | ||||||||
| 1196 | |||||||||
| 1197 | bool Changed = false; | ||||||||
| 1198 | // Get the right rewriter for the current copy. | ||||||||
| 1199 | std::unique_ptr<Rewriter> CpyRewriter(getCopyRewriter(MI, *TII)); | ||||||||
| 1200 | // If none exists, bail out. | ||||||||
| 1201 | if (!CpyRewriter) | ||||||||
| 1202 | return false; | ||||||||
| 1203 | // Rewrite each rewritable source. | ||||||||
| 1204 | RegSubRegPair Src; | ||||||||
| 1205 | RegSubRegPair TrackPair; | ||||||||
| 1206 | while (CpyRewriter->getNextRewritableSource(Src, TrackPair)) { | ||||||||
| 1207 | // Keep track of PHI nodes and its incoming edges when looking for sources. | ||||||||
| 1208 | RewriteMapTy RewriteMap; | ||||||||
| 1209 | // Try to find a more suitable source. If we failed to do so, or get the | ||||||||
| 1210 | // actual source, move to the next source. | ||||||||
| 1211 | if (!findNextSource(TrackPair, RewriteMap)) | ||||||||
| 1212 | continue; | ||||||||
| 1213 | |||||||||
| 1214 | // Get the new source to rewrite. TODO: Only enable handling of multiple | ||||||||
| 1215 | // sources (PHIs) once we have a motivating example and testcases for it. | ||||||||
| 1216 | RegSubRegPair NewSrc = getNewSource(MRI, TII, TrackPair, RewriteMap, | ||||||||
| 1217 | /*HandleMultipleSources=*/false); | ||||||||
| 1218 | if (Src.Reg == NewSrc.Reg || NewSrc.Reg == 0) | ||||||||
| 1219 | continue; | ||||||||
| 1220 | |||||||||
| 1221 | // Rewrite source. | ||||||||
| 1222 | if (CpyRewriter->RewriteCurrentSource(NewSrc.Reg, NewSrc.SubReg)) { | ||||||||
| 1223 | // We may have extended the live-range of NewSrc, account for that. | ||||||||
| 1224 | MRI->clearKillFlags(NewSrc.Reg); | ||||||||
| 1225 | Changed = true; | ||||||||
| 1226 | } | ||||||||
| 1227 | } | ||||||||
| 1228 | // TODO: We could have a clean-up method to tidy the instruction. | ||||||||
| 1229 | // E.g., v0 = INSERT_SUBREG v1, v1.sub0, sub0 | ||||||||
| 1230 | // => v0 = COPY v1 | ||||||||
| 1231 | // Currently we haven't seen motivating example for that and we | ||||||||
| 1232 | // want to avoid untested code. | ||||||||
| 1233 | NumRewrittenCopies += Changed; | ||||||||
| 1234 | return Changed; | ||||||||
| 1235 | } | ||||||||
| 1236 | |||||||||
| 1237 | /// Rewrite the source found through \p Def, by using the \p RewriteMap | ||||||||
| 1238 | /// and create a new COPY instruction. More info about RewriteMap in | ||||||||
| 1239 | /// PeepholeOptimizer::findNextSource. Right now this is only used to handle | ||||||||
| 1240 | /// Uncoalescable copies, since they are copy like instructions that aren't | ||||||||
| 1241 | /// recognized by the register allocator. | ||||||||
| 1242 | MachineInstr & | ||||||||
| 1243 | PeepholeOptimizer::rewriteSource(MachineInstr &CopyLike, | ||||||||
| 1244 | RegSubRegPair Def, RewriteMapTy &RewriteMap) { | ||||||||
| 1245 | assert(!Register::isPhysicalRegister(Def.Reg) &&(static_cast <bool> (!Register::isPhysicalRegister(Def. Reg) && "We do not rewrite physical registers") ? void (0) : __assert_fail ("!Register::isPhysicalRegister(Def.Reg) && \"We do not rewrite physical registers\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1246, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1246 | "We do not rewrite physical registers")(static_cast <bool> (!Register::isPhysicalRegister(Def. Reg) && "We do not rewrite physical registers") ? void (0) : __assert_fail ("!Register::isPhysicalRegister(Def.Reg) && \"We do not rewrite physical registers\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1246, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1247 | |||||||||
| 1248 | // Find the new source to use in the COPY rewrite. | ||||||||
| 1249 | RegSubRegPair NewSrc = getNewSource(MRI, TII, Def, RewriteMap); | ||||||||
| 1250 | |||||||||
| 1251 | // Insert the COPY. | ||||||||
| 1252 | const TargetRegisterClass *DefRC = MRI->getRegClass(Def.Reg); | ||||||||
| 1253 | Register NewVReg = MRI->createVirtualRegister(DefRC); | ||||||||
| 1254 | |||||||||
| 1255 | MachineInstr *NewCopy = | ||||||||
| 1256 | BuildMI(*CopyLike.getParent(), &CopyLike, CopyLike.getDebugLoc(), | ||||||||
| 1257 | TII->get(TargetOpcode::COPY), NewVReg) | ||||||||
| 1258 | .addReg(NewSrc.Reg, 0, NewSrc.SubReg); | ||||||||
| 1259 | |||||||||
| 1260 | if (Def.SubReg) { | ||||||||
| 1261 | NewCopy->getOperand(0).setSubReg(Def.SubReg); | ||||||||
| 1262 | NewCopy->getOperand(0).setIsUndef(); | ||||||||
| 1263 | } | ||||||||
| 1264 | |||||||||
| 1265 | LLVM_DEBUG(dbgs() << "-- RewriteSource\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "-- RewriteSource\n"; } } while (false); | ||||||||
| 1266 | LLVM_DEBUG(dbgs() << " Replacing: " << CopyLike)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << " Replacing: " << CopyLike; } } while (false); | ||||||||
| 1267 | LLVM_DEBUG(dbgs() << " With: " << *NewCopy)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << " With: " << *NewCopy; } } while (false); | ||||||||
| 1268 | MRI->replaceRegWith(Def.Reg, NewVReg); | ||||||||
| 1269 | MRI->clearKillFlags(NewVReg); | ||||||||
| 1270 | |||||||||
| 1271 | // We extended the lifetime of NewSrc.Reg, clear the kill flags to | ||||||||
| 1272 | // account for that. | ||||||||
| 1273 | MRI->clearKillFlags(NewSrc.Reg); | ||||||||
| 1274 | |||||||||
| 1275 | return *NewCopy; | ||||||||
| 1276 | } | ||||||||
| 1277 | |||||||||
| 1278 | /// Optimize copy-like instructions to create | ||||||||
| 1279 | /// register coalescer friendly instruction. | ||||||||
| 1280 | /// The optimization tries to kill-off the \p MI by looking | ||||||||
| 1281 | /// through a chain of copies to find a source that has a compatible | ||||||||
| 1282 | /// register class. | ||||||||
| 1283 | /// If such a source is found, it replace \p MI by a generic COPY | ||||||||
| 1284 | /// operation. | ||||||||
| 1285 | /// \pre isUncoalescableCopy(*MI) is true. | ||||||||
| 1286 | /// \return True, when \p MI has been optimized. In that case, \p MI has | ||||||||
| 1287 | /// been removed from its parent. | ||||||||
| 1288 | /// All COPY instructions created, are inserted in \p LocalMIs. | ||||||||
| 1289 | bool PeepholeOptimizer::optimizeUncoalescableCopy( | ||||||||
| 1290 | MachineInstr &MI, SmallPtrSetImpl<MachineInstr *> &LocalMIs) { | ||||||||
| 1291 | assert(isUncoalescableCopy(MI) && "Invalid argument")(static_cast <bool> (isUncoalescableCopy(MI) && "Invalid argument") ? void (0) : __assert_fail ("isUncoalescableCopy(MI) && \"Invalid argument\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1291, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1292 | UncoalescableRewriter CpyRewriter(MI); | ||||||||
| 1293 | |||||||||
| 1294 | // Rewrite each rewritable source by generating new COPYs. This works | ||||||||
| 1295 | // differently from optimizeCoalescableCopy since it first makes sure that all | ||||||||
| 1296 | // definitions can be rewritten. | ||||||||
| 1297 | RewriteMapTy RewriteMap; | ||||||||
| 1298 | RegSubRegPair Src; | ||||||||
| 1299 | RegSubRegPair Def; | ||||||||
| 1300 | SmallVector<RegSubRegPair, 4> RewritePairs; | ||||||||
| 1301 | while (CpyRewriter.getNextRewritableSource(Src, Def)) { | ||||||||
| 1302 | // If a physical register is here, this is probably for a good reason. | ||||||||
| 1303 | // Do not rewrite that. | ||||||||
| 1304 | if (Register::isPhysicalRegister(Def.Reg)) | ||||||||
| 1305 | return false; | ||||||||
| 1306 | |||||||||
| 1307 | // If we do not know how to rewrite this definition, there is no point | ||||||||
| 1308 | // in trying to kill this instruction. | ||||||||
| 1309 | if (!findNextSource(Def, RewriteMap)) | ||||||||
| 1310 | return false; | ||||||||
| 1311 | |||||||||
| 1312 | RewritePairs.push_back(Def); | ||||||||
| 1313 | } | ||||||||
| 1314 | |||||||||
| 1315 | // The change is possible for all defs, do it. | ||||||||
| 1316 | for (const RegSubRegPair &Def : RewritePairs) { | ||||||||
| 1317 | // Rewrite the "copy" in a way the register coalescer understands. | ||||||||
| 1318 | MachineInstr &NewCopy = rewriteSource(MI, Def, RewriteMap); | ||||||||
| 1319 | LocalMIs.insert(&NewCopy); | ||||||||
| 1320 | } | ||||||||
| 1321 | |||||||||
| 1322 | // MI is now dead. | ||||||||
| 1323 | LLVM_DEBUG(dbgs() << "Deleting uncoalescable copy: " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Deleting uncoalescable copy: " << MI; } } while (false); | ||||||||
| 1324 | MI.eraseFromParent(); | ||||||||
| 1325 | ++NumUncoalescableCopies; | ||||||||
| 1326 | return true; | ||||||||
| 1327 | } | ||||||||
| 1328 | |||||||||
| 1329 | /// Check whether MI is a candidate for folding into a later instruction. | ||||||||
| 1330 | /// We only fold loads to virtual registers and the virtual register defined | ||||||||
| 1331 | /// has a single user. | ||||||||
| 1332 | bool PeepholeOptimizer::isLoadFoldable( | ||||||||
| 1333 | MachineInstr &MI, SmallSet<Register, 16> &FoldAsLoadDefCandidates) { | ||||||||
| 1334 | if (!MI.canFoldAsLoad() || !MI.mayLoad()) | ||||||||
| 1335 | return false; | ||||||||
| 1336 | const MCInstrDesc &MCID = MI.getDesc(); | ||||||||
| 1337 | if (MCID.getNumDefs() != 1) | ||||||||
| 1338 | return false; | ||||||||
| 1339 | |||||||||
| 1340 | Register Reg = MI.getOperand(0).getReg(); | ||||||||
| 1341 | // To reduce compilation time, we check MRI->hasOneNonDBGUser when inserting | ||||||||
| 1342 | // loads. It should be checked when processing uses of the load, since | ||||||||
| 1343 | // uses can be removed during peephole. | ||||||||
| 1344 | if (Reg.isVirtual() && !MI.getOperand(0).getSubReg() && | ||||||||
| 1345 | MRI->hasOneNonDBGUser(Reg)) { | ||||||||
| 1346 | FoldAsLoadDefCandidates.insert(Reg); | ||||||||
| 1347 | return true; | ||||||||
| 1348 | } | ||||||||
| 1349 | return false; | ||||||||
| 1350 | } | ||||||||
| 1351 | |||||||||
| 1352 | bool PeepholeOptimizer::isMoveImmediate( | ||||||||
| 1353 | MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs, | ||||||||
| 1354 | DenseMap<Register, MachineInstr *> &ImmDefMIs) { | ||||||||
| 1355 | const MCInstrDesc &MCID = MI.getDesc(); | ||||||||
| 1356 | if (!MI.isMoveImmediate()) | ||||||||
| 1357 | return false; | ||||||||
| 1358 | if (MCID.getNumDefs() != 1) | ||||||||
| 1359 | return false; | ||||||||
| 1360 | Register Reg = MI.getOperand(0).getReg(); | ||||||||
| 1361 | if (Reg.isVirtual()) { | ||||||||
| 1362 | ImmDefMIs.insert(std::make_pair(Reg, &MI)); | ||||||||
| 1363 | ImmDefRegs.insert(Reg); | ||||||||
| 1364 | return true; | ||||||||
| 1365 | } | ||||||||
| 1366 | |||||||||
| 1367 | return false; | ||||||||
| 1368 | } | ||||||||
| 1369 | |||||||||
| 1370 | /// Try folding register operands that are defined by move immediate | ||||||||
| 1371 | /// instructions, i.e. a trivial constant folding optimization, if | ||||||||
| 1372 | /// and only if the def and use are in the same BB. | ||||||||
| 1373 | bool PeepholeOptimizer::foldImmediate( | ||||||||
| 1374 | MachineInstr &MI, SmallSet<Register, 4> &ImmDefRegs, | ||||||||
| 1375 | DenseMap<Register, MachineInstr *> &ImmDefMIs) { | ||||||||
| 1376 | for (unsigned i = 0, e = MI.getDesc().getNumOperands(); i != e; ++i) { | ||||||||
| 1377 | MachineOperand &MO = MI.getOperand(i); | ||||||||
| 1378 | if (!MO.isReg() || MO.isDef()) | ||||||||
| 1379 | continue; | ||||||||
| 1380 | Register Reg = MO.getReg(); | ||||||||
| 1381 | if (!Reg.isVirtual()) | ||||||||
| 1382 | continue; | ||||||||
| 1383 | if (ImmDefRegs.count(Reg) == 0) | ||||||||
| 1384 | continue; | ||||||||
| 1385 | DenseMap<Register, MachineInstr *>::iterator II = ImmDefMIs.find(Reg); | ||||||||
| 1386 | assert(II != ImmDefMIs.end() && "couldn't find immediate definition")(static_cast <bool> (II != ImmDefMIs.end() && "couldn't find immediate definition" ) ? void (0) : __assert_fail ("II != ImmDefMIs.end() && \"couldn't find immediate definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1386, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1387 | if (TII->FoldImmediate(MI, *II->second, Reg, MRI)) { | ||||||||
| 1388 | ++NumImmFold; | ||||||||
| 1389 | return true; | ||||||||
| 1390 | } | ||||||||
| 1391 | } | ||||||||
| 1392 | return false; | ||||||||
| 1393 | } | ||||||||
| 1394 | |||||||||
| 1395 | // FIXME: This is very simple and misses some cases which should be handled when | ||||||||
| 1396 | // motivating examples are found. | ||||||||
| 1397 | // | ||||||||
| 1398 | // The copy rewriting logic should look at uses as well as defs and be able to | ||||||||
| 1399 | // eliminate copies across blocks. | ||||||||
| 1400 | // | ||||||||
| 1401 | // Later copies that are subregister extracts will also not be eliminated since | ||||||||
| 1402 | // only the first copy is considered. | ||||||||
| 1403 | // | ||||||||
| 1404 | // e.g. | ||||||||
| 1405 | // %1 = COPY %0 | ||||||||
| 1406 | // %2 = COPY %0:sub1 | ||||||||
| 1407 | // | ||||||||
| 1408 | // Should replace %2 uses with %1:sub1 | ||||||||
| 1409 | bool PeepholeOptimizer::foldRedundantCopy( | ||||||||
| 1410 | MachineInstr &MI, DenseMap<RegSubRegPair, MachineInstr *> &CopyMIs) { | ||||||||
| 1411 | assert(MI.isCopy() && "expected a COPY machine instruction")(static_cast <bool> (MI.isCopy() && "expected a COPY machine instruction" ) ? void (0) : __assert_fail ("MI.isCopy() && \"expected a COPY machine instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1411, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1412 | |||||||||
| 1413 | Register SrcReg = MI.getOperand(1).getReg(); | ||||||||
| 1414 | unsigned SrcSubReg = MI.getOperand(1).getSubReg(); | ||||||||
| 1415 | if (!SrcReg.isVirtual()) | ||||||||
| 1416 | return false; | ||||||||
| 1417 | |||||||||
| 1418 | Register DstReg = MI.getOperand(0).getReg(); | ||||||||
| 1419 | if (!DstReg.isVirtual()) | ||||||||
| 1420 | return false; | ||||||||
| 1421 | |||||||||
| 1422 | RegSubRegPair SrcPair(SrcReg, SrcSubReg); | ||||||||
| 1423 | |||||||||
| 1424 | if (CopyMIs.insert(std::make_pair(SrcPair, &MI)).second) { | ||||||||
| 1425 | // First copy of this reg seen. | ||||||||
| 1426 | return false; | ||||||||
| 1427 | } | ||||||||
| 1428 | |||||||||
| 1429 | MachineInstr *PrevCopy = CopyMIs.find(SrcPair)->second; | ||||||||
| 1430 | |||||||||
| 1431 | assert(SrcSubReg == PrevCopy->getOperand(1).getSubReg() &&(static_cast <bool> (SrcSubReg == PrevCopy->getOperand (1).getSubReg() && "Unexpected mismatching subreg!") ? void (0) : __assert_fail ("SrcSubReg == PrevCopy->getOperand(1).getSubReg() && \"Unexpected mismatching subreg!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1432, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1432 | "Unexpected mismatching subreg!")(static_cast <bool> (SrcSubReg == PrevCopy->getOperand (1).getSubReg() && "Unexpected mismatching subreg!") ? void (0) : __assert_fail ("SrcSubReg == PrevCopy->getOperand(1).getSubReg() && \"Unexpected mismatching subreg!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1432, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1433 | |||||||||
| 1434 | Register PrevDstReg = PrevCopy->getOperand(0).getReg(); | ||||||||
| 1435 | |||||||||
| 1436 | // Only replace if the copy register class is the same. | ||||||||
| 1437 | // | ||||||||
| 1438 | // TODO: If we have multiple copies to different register classes, we may want | ||||||||
| 1439 | // to track multiple copies of the same source register. | ||||||||
| 1440 | if (MRI->getRegClass(DstReg) != MRI->getRegClass(PrevDstReg)) | ||||||||
| 1441 | return false; | ||||||||
| 1442 | |||||||||
| 1443 | MRI->replaceRegWith(DstReg, PrevDstReg); | ||||||||
| 1444 | |||||||||
| 1445 | // Lifetime of the previous copy has been extended. | ||||||||
| 1446 | MRI->clearKillFlags(PrevDstReg); | ||||||||
| 1447 | return true; | ||||||||
| 1448 | } | ||||||||
| 1449 | |||||||||
| 1450 | bool PeepholeOptimizer::isNAPhysCopy(Register Reg) { | ||||||||
| 1451 | return Reg.isPhysical() && !MRI->isAllocatable(Reg); | ||||||||
| 1452 | } | ||||||||
| 1453 | |||||||||
| 1454 | bool PeepholeOptimizer::foldRedundantNAPhysCopy( | ||||||||
| 1455 | MachineInstr &MI, DenseMap<Register, MachineInstr *> &NAPhysToVirtMIs) { | ||||||||
| 1456 | assert(MI.isCopy() && "expected a COPY machine instruction")(static_cast <bool> (MI.isCopy() && "expected a COPY machine instruction" ) ? void (0) : __assert_fail ("MI.isCopy() && \"expected a COPY machine instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1456, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1457 | |||||||||
| 1458 | if (DisableNAPhysCopyOpt) | ||||||||
| 1459 | return false; | ||||||||
| 1460 | |||||||||
| 1461 | Register DstReg = MI.getOperand(0).getReg(); | ||||||||
| 1462 | Register SrcReg = MI.getOperand(1).getReg(); | ||||||||
| 1463 | if (isNAPhysCopy(SrcReg) && Register::isVirtualRegister(DstReg)) { | ||||||||
| 1464 | // %vreg = COPY $physreg | ||||||||
| 1465 | // Avoid using a datastructure which can track multiple live non-allocatable | ||||||||
| 1466 | // phys->virt copies since LLVM doesn't seem to do this. | ||||||||
| 1467 | NAPhysToVirtMIs.insert({SrcReg, &MI}); | ||||||||
| 1468 | return false; | ||||||||
| 1469 | } | ||||||||
| 1470 | |||||||||
| 1471 | if (!(SrcReg.isVirtual() && isNAPhysCopy(DstReg))) | ||||||||
| 1472 | return false; | ||||||||
| 1473 | |||||||||
| 1474 | // $physreg = COPY %vreg | ||||||||
| 1475 | auto PrevCopy = NAPhysToVirtMIs.find(DstReg); | ||||||||
| 1476 | if (PrevCopy == NAPhysToVirtMIs.end()) { | ||||||||
| 1477 | // We can't remove the copy: there was an intervening clobber of the | ||||||||
| 1478 | // non-allocatable physical register after the copy to virtual. | ||||||||
| 1479 | LLVM_DEBUG(dbgs() << "NAPhysCopy: intervening clobber forbids erasing "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: intervening clobber forbids erasing " << MI; } } while (false) | ||||||||
| 1480 | << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: intervening clobber forbids erasing " << MI; } } while (false); | ||||||||
| 1481 | return false; | ||||||||
| 1482 | } | ||||||||
| 1483 | |||||||||
| 1484 | Register PrevDstReg = PrevCopy->second->getOperand(0).getReg(); | ||||||||
| 1485 | if (PrevDstReg == SrcReg) { | ||||||||
| 1486 | // Remove the virt->phys copy: we saw the virtual register definition, and | ||||||||
| 1487 | // the non-allocatable physical register's state hasn't changed since then. | ||||||||
| 1488 | LLVM_DEBUG(dbgs() << "NAPhysCopy: erasing " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: erasing " << MI; } } while (false); | ||||||||
| 1489 | ++NumNAPhysCopies; | ||||||||
| 1490 | return true; | ||||||||
| 1491 | } | ||||||||
| 1492 | |||||||||
| 1493 | // Potential missed optimization opportunity: we saw a different virtual | ||||||||
| 1494 | // register get a copy of the non-allocatable physical register, and we only | ||||||||
| 1495 | // track one such copy. Avoid getting confused by this new non-allocatable | ||||||||
| 1496 | // physical register definition, and remove it from the tracked copies. | ||||||||
| 1497 | LLVM_DEBUG(dbgs() << "NAPhysCopy: missed opportunity " << MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: missed opportunity " << MI; } } while (false); | ||||||||
| 1498 | NAPhysToVirtMIs.erase(PrevCopy); | ||||||||
| 1499 | return false; | ||||||||
| 1500 | } | ||||||||
| 1501 | |||||||||
| 1502 | /// \bried Returns true if \p MO is a virtual register operand. | ||||||||
| 1503 | static bool isVirtualRegisterOperand(MachineOperand &MO) { | ||||||||
| 1504 | return MO.isReg() && MO.getReg().isVirtual(); | ||||||||
| 1505 | } | ||||||||
| 1506 | |||||||||
| 1507 | bool PeepholeOptimizer::findTargetRecurrence( | ||||||||
| 1508 | Register Reg, const SmallSet<Register, 2> &TargetRegs, | ||||||||
| 1509 | RecurrenceCycle &RC) { | ||||||||
| 1510 | // Recurrence found if Reg is in TargetRegs. | ||||||||
| 1511 | if (TargetRegs.count(Reg)) | ||||||||
| 1512 | return true; | ||||||||
| 1513 | |||||||||
| 1514 | // TODO: Curerntly, we only allow the last instruction of the recurrence | ||||||||
| 1515 | // cycle (the instruction that feeds the PHI instruction) to have more than | ||||||||
| 1516 | // one uses to guarantee that commuting operands does not tie registers | ||||||||
| 1517 | // with overlapping live range. Once we have actual live range info of | ||||||||
| 1518 | // each register, this constraint can be relaxed. | ||||||||
| 1519 | if (!MRI->hasOneNonDBGUse(Reg)) | ||||||||
| 1520 | return false; | ||||||||
| 1521 | |||||||||
| 1522 | // Give up if the reccurrence chain length is longer than the limit. | ||||||||
| 1523 | if (RC.size() >= MaxRecurrenceChain) | ||||||||
| 1524 | return false; | ||||||||
| 1525 | |||||||||
| 1526 | MachineInstr &MI = *(MRI->use_instr_nodbg_begin(Reg)); | ||||||||
| 1527 | unsigned Idx = MI.findRegisterUseOperandIdx(Reg); | ||||||||
| 1528 | |||||||||
| 1529 | // Only interested in recurrences whose instructions have only one def, which | ||||||||
| 1530 | // is a virtual register. | ||||||||
| 1531 | if (MI.getDesc().getNumDefs() != 1) | ||||||||
| 1532 | return false; | ||||||||
| 1533 | |||||||||
| 1534 | MachineOperand &DefOp = MI.getOperand(0); | ||||||||
| 1535 | if (!isVirtualRegisterOperand(DefOp)) | ||||||||
| 1536 | return false; | ||||||||
| 1537 | |||||||||
| 1538 | // Check if def operand of MI is tied to any use operand. We are only | ||||||||
| 1539 | // interested in the case that all the instructions in the recurrence chain | ||||||||
| 1540 | // have there def operand tied with one of the use operand. | ||||||||
| 1541 | unsigned TiedUseIdx; | ||||||||
| 1542 | if (!MI.isRegTiedToUseOperand(0, &TiedUseIdx)) | ||||||||
| 1543 | return false; | ||||||||
| 1544 | |||||||||
| 1545 | if (Idx == TiedUseIdx) { | ||||||||
| 1546 | RC.push_back(RecurrenceInstr(&MI)); | ||||||||
| 1547 | return findTargetRecurrence(DefOp.getReg(), TargetRegs, RC); | ||||||||
| 1548 | } else { | ||||||||
| 1549 | // If Idx is not TiedUseIdx, check if Idx is commutable with TiedUseIdx. | ||||||||
| 1550 | unsigned CommIdx = TargetInstrInfo::CommuteAnyOperandIndex; | ||||||||
| 1551 | if (TII->findCommutedOpIndices(MI, Idx, CommIdx) && CommIdx == TiedUseIdx) { | ||||||||
| 1552 | RC.push_back(RecurrenceInstr(&MI, Idx, CommIdx)); | ||||||||
| 1553 | return findTargetRecurrence(DefOp.getReg(), TargetRegs, RC); | ||||||||
| 1554 | } | ||||||||
| 1555 | } | ||||||||
| 1556 | |||||||||
| 1557 | return false; | ||||||||
| 1558 | } | ||||||||
| 1559 | |||||||||
| 1560 | /// Phi instructions will eventually be lowered to copy instructions. | ||||||||
| 1561 | /// If phi is in a loop header, a recurrence may formulated around the source | ||||||||
| 1562 | /// and destination of the phi. For such case commuting operands of the | ||||||||
| 1563 | /// instructions in the recurrence may enable coalescing of the copy instruction | ||||||||
| 1564 | /// generated from the phi. For example, if there is a recurrence of | ||||||||
| 1565 | /// | ||||||||
| 1566 | /// LoopHeader: | ||||||||
| 1567 | /// %1 = phi(%0, %100) | ||||||||
| 1568 | /// LoopLatch: | ||||||||
| 1569 | /// %0<def, tied1> = ADD %2<def, tied0>, %1 | ||||||||
| 1570 | /// | ||||||||
| 1571 | /// , the fact that %0 and %2 are in the same tied operands set makes | ||||||||
| 1572 | /// the coalescing of copy instruction generated from the phi in | ||||||||
| 1573 | /// LoopHeader(i.e. %1 = COPY %0) impossible, because %1 and | ||||||||
| 1574 | /// %2 have overlapping live range. This introduces additional move | ||||||||
| 1575 | /// instruction to the final assembly. However, if we commute %2 and | ||||||||
| 1576 | /// %1 of ADD instruction, the redundant move instruction can be | ||||||||
| 1577 | /// avoided. | ||||||||
| 1578 | bool PeepholeOptimizer::optimizeRecurrence(MachineInstr &PHI) { | ||||||||
| 1579 | SmallSet<Register, 2> TargetRegs; | ||||||||
| 1580 | for (unsigned Idx = 1; Idx < PHI.getNumOperands(); Idx += 2) { | ||||||||
| 1581 | MachineOperand &MO = PHI.getOperand(Idx); | ||||||||
| 1582 | assert(isVirtualRegisterOperand(MO) && "Invalid PHI instruction")(static_cast <bool> (isVirtualRegisterOperand(MO) && "Invalid PHI instruction") ? void (0) : __assert_fail ("isVirtualRegisterOperand(MO) && \"Invalid PHI instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1582, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1583 | TargetRegs.insert(MO.getReg()); | ||||||||
| 1584 | } | ||||||||
| 1585 | |||||||||
| 1586 | bool Changed = false; | ||||||||
| 1587 | RecurrenceCycle RC; | ||||||||
| 1588 | if (findTargetRecurrence(PHI.getOperand(0).getReg(), TargetRegs, RC)) { | ||||||||
| 1589 | // Commutes operands of instructions in RC if necessary so that the copy to | ||||||||
| 1590 | // be generated from PHI can be coalesced. | ||||||||
| 1591 | LLVM_DEBUG(dbgs() << "Optimize recurrence chain from " << PHI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Optimize recurrence chain from " << PHI; } } while (false); | ||||||||
| 1592 | for (auto &RI : RC) { | ||||||||
| 1593 | LLVM_DEBUG(dbgs() << "\tInst: " << *(RI.getMI()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "\tInst: " << *(RI. getMI()); } } while (false); | ||||||||
| 1594 | auto CP = RI.getCommutePair(); | ||||||||
| 1595 | if (CP) { | ||||||||
| 1596 | Changed = true; | ||||||||
| 1597 | TII->commuteInstruction(*(RI.getMI()), false, (*CP).first, | ||||||||
| 1598 | (*CP).second); | ||||||||
| 1599 | LLVM_DEBUG(dbgs() << "\t\tCommuted: " << *(RI.getMI()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "\t\tCommuted: " << *(RI.getMI()); } } while (false); | ||||||||
| 1600 | } | ||||||||
| 1601 | } | ||||||||
| 1602 | } | ||||||||
| 1603 | |||||||||
| 1604 | return Changed; | ||||||||
| 1605 | } | ||||||||
| 1606 | |||||||||
| 1607 | bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { | ||||||||
| 1608 | if (skipFunction(MF.getFunction())) | ||||||||
| |||||||||
| 1609 | return false; | ||||||||
| 1610 | |||||||||
| 1611 | LLVM_DEBUG(dbgs() << "********** PEEPHOLE OPTIMIZER **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "********** PEEPHOLE OPTIMIZER **********\n" ; } } while (false); | ||||||||
| 1612 | LLVM_DEBUG(dbgs() << "********** Function: " << MF.getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "********** Function: " << MF.getName() << '\n'; } } while (false); | ||||||||
| 1613 | |||||||||
| 1614 | if (DisablePeephole) | ||||||||
| 1615 | return false; | ||||||||
| 1616 | |||||||||
| 1617 | TII = MF.getSubtarget().getInstrInfo(); | ||||||||
| 1618 | TRI = MF.getSubtarget().getRegisterInfo(); | ||||||||
| 1619 | MRI = &MF.getRegInfo(); | ||||||||
| 1620 | DT = Aggressive ? &getAnalysis<MachineDominatorTree>() : nullptr; | ||||||||
| 1621 | MLI = &getAnalysis<MachineLoopInfo>(); | ||||||||
| 1622 | |||||||||
| 1623 | bool Changed = false; | ||||||||
| 1624 | |||||||||
| 1625 | for (MachineBasicBlock &MBB : MF) { | ||||||||
| 1626 | bool SeenMoveImm = false; | ||||||||
| 1627 | |||||||||
| 1628 | // During this forward scan, at some point it needs to answer the question | ||||||||
| 1629 | // "given a pointer to an MI in the current BB, is it located before or | ||||||||
| 1630 | // after the current instruction". | ||||||||
| 1631 | // To perform this, the following set keeps track of the MIs already seen | ||||||||
| 1632 | // during the scan, if a MI is not in the set, it is assumed to be located | ||||||||
| 1633 | // after. Newly created MIs have to be inserted in the set as well. | ||||||||
| 1634 | SmallPtrSet<MachineInstr*, 16> LocalMIs; | ||||||||
| 1635 | SmallSet<Register, 4> ImmDefRegs; | ||||||||
| 1636 | DenseMap<Register, MachineInstr *> ImmDefMIs; | ||||||||
| 1637 | SmallSet<Register, 16> FoldAsLoadDefCandidates; | ||||||||
| 1638 | |||||||||
| 1639 | // Track when a non-allocatable physical register is copied to a virtual | ||||||||
| 1640 | // register so that useless moves can be removed. | ||||||||
| 1641 | // | ||||||||
| 1642 | // $physreg is the map index; MI is the last valid `%vreg = COPY $physreg` | ||||||||
| 1643 | // without any intervening re-definition of $physreg. | ||||||||
| 1644 | DenseMap<Register, MachineInstr *> NAPhysToVirtMIs; | ||||||||
| 1645 | |||||||||
| 1646 | // Set of pairs of virtual registers and their subregs that are copied | ||||||||
| 1647 | // from. | ||||||||
| 1648 | DenseMap<RegSubRegPair, MachineInstr *> CopySrcMIs; | ||||||||
| 1649 | |||||||||
| 1650 | bool IsLoopHeader = MLI->isLoopHeader(&MBB); | ||||||||
| 1651 | |||||||||
| 1652 | for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end(); | ||||||||
| 1653 | MII != MIE; ) { | ||||||||
| 1654 | MachineInstr *MI = &*MII; | ||||||||
| 1655 | // We may be erasing MI below, increment MII now. | ||||||||
| 1656 | ++MII; | ||||||||
| 1657 | LocalMIs.insert(MI); | ||||||||
| 1658 | |||||||||
| 1659 | // Skip debug instructions. They should not affect this peephole | ||||||||
| 1660 | // optimization. | ||||||||
| 1661 | if (MI->isDebugInstr()) | ||||||||
| 1662 | continue; | ||||||||
| 1663 | |||||||||
| 1664 | if (MI->isPosition()) | ||||||||
| 1665 | continue; | ||||||||
| 1666 | |||||||||
| 1667 | if (IsLoopHeader && MI->isPHI()) { | ||||||||
| 1668 | if (optimizeRecurrence(*MI)) { | ||||||||
| 1669 | Changed = true; | ||||||||
| 1670 | continue; | ||||||||
| 1671 | } | ||||||||
| 1672 | } | ||||||||
| 1673 | |||||||||
| 1674 | if (!MI->isCopy()) { | ||||||||
| 1675 | for (const MachineOperand &MO : MI->operands()) { | ||||||||
| 1676 | // Visit all operands: definitions can be implicit or explicit. | ||||||||
| 1677 | if (MO.isReg()) { | ||||||||
| 1678 | Register Reg = MO.getReg(); | ||||||||
| 1679 | if (MO.isDef() && isNAPhysCopy(Reg)) { | ||||||||
| 1680 | const auto &Def = NAPhysToVirtMIs.find(Reg); | ||||||||
| 1681 | if (Def != NAPhysToVirtMIs.end()) { | ||||||||
| 1682 | // A new definition of the non-allocatable physical register | ||||||||
| 1683 | // invalidates previous copies. | ||||||||
| 1684 | LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: invalidating because of " << *MI; } } while (false) | ||||||||
| 1685 | << "NAPhysCopy: invalidating because of " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: invalidating because of " << *MI; } } while (false); | ||||||||
| 1686 | NAPhysToVirtMIs.erase(Def); | ||||||||
| 1687 | } | ||||||||
| 1688 | } | ||||||||
| 1689 | } else if (MO.isRegMask()) { | ||||||||
| 1690 | const uint32_t *RegMask = MO.getRegMask(); | ||||||||
| 1691 | for (auto &RegMI : NAPhysToVirtMIs) { | ||||||||
| 1692 | Register Def = RegMI.first; | ||||||||
| 1693 | if (MachineOperand::clobbersPhysReg(RegMask, Def)) { | ||||||||
| 1694 | LLVM_DEBUG(dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: invalidating because of " << *MI; } } while (false) | ||||||||
| 1695 | << "NAPhysCopy: invalidating because of " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: invalidating because of " << *MI; } } while (false); | ||||||||
| 1696 | NAPhysToVirtMIs.erase(Def); | ||||||||
| 1697 | } | ||||||||
| 1698 | } | ||||||||
| 1699 | } | ||||||||
| 1700 | } | ||||||||
| 1701 | } | ||||||||
| 1702 | |||||||||
| 1703 | if (MI->isImplicitDef() || MI->isKill()) | ||||||||
| 1704 | continue; | ||||||||
| 1705 | |||||||||
| 1706 | if (MI->isInlineAsm() || MI->hasUnmodeledSideEffects()) { | ||||||||
| 1707 | // Blow away all non-allocatable physical registers knowledge since we | ||||||||
| 1708 | // don't know what's correct anymore. | ||||||||
| 1709 | // | ||||||||
| 1710 | // FIXME: handle explicit asm clobbers. | ||||||||
| 1711 | LLVM_DEBUG(dbgs() << "NAPhysCopy: blowing away all info due to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: blowing away all info due to " << *MI; } } while (false) | ||||||||
| 1712 | << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "NAPhysCopy: blowing away all info due to " << *MI; } } while (false); | ||||||||
| 1713 | NAPhysToVirtMIs.clear(); | ||||||||
| 1714 | } | ||||||||
| 1715 | |||||||||
| 1716 | if ((isUncoalescableCopy(*MI) && | ||||||||
| 1717 | optimizeUncoalescableCopy(*MI, LocalMIs)) || | ||||||||
| 1718 | (MI->isCompare() && optimizeCmpInstr(*MI)) || | ||||||||
| 1719 | (MI->isSelect() && optimizeSelect(*MI, LocalMIs))) { | ||||||||
| 1720 | // MI is deleted. | ||||||||
| 1721 | LocalMIs.erase(MI); | ||||||||
| 1722 | Changed = true; | ||||||||
| 1723 | continue; | ||||||||
| 1724 | } | ||||||||
| 1725 | |||||||||
| 1726 | if (MI->isConditionalBranch() && optimizeCondBranch(*MI)) { | ||||||||
| 1727 | Changed = true; | ||||||||
| 1728 | continue; | ||||||||
| 1729 | } | ||||||||
| 1730 | |||||||||
| 1731 | if (isCoalescableCopy(*MI) && optimizeCoalescableCopy(*MI)) { | ||||||||
| 1732 | // MI is just rewritten. | ||||||||
| 1733 | Changed = true; | ||||||||
| 1734 | continue; | ||||||||
| 1735 | } | ||||||||
| 1736 | |||||||||
| 1737 | if (MI->isCopy() && (foldRedundantCopy(*MI, CopySrcMIs) || | ||||||||
| 1738 | foldRedundantNAPhysCopy(*MI, NAPhysToVirtMIs))) { | ||||||||
| 1739 | LocalMIs.erase(MI); | ||||||||
| 1740 | LLVM_DEBUG(dbgs() << "Deleting redundant copy: " << *MI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Deleting redundant copy: " << *MI << "\n"; } } while (false); | ||||||||
| 1741 | MI->eraseFromParent(); | ||||||||
| 1742 | Changed = true; | ||||||||
| 1743 | continue; | ||||||||
| 1744 | } | ||||||||
| 1745 | |||||||||
| 1746 | if (isMoveImmediate(*MI, ImmDefRegs, ImmDefMIs)) { | ||||||||
| 1747 | SeenMoveImm = true; | ||||||||
| 1748 | } else { | ||||||||
| 1749 | Changed |= optimizeExtInstr(*MI, MBB, LocalMIs); | ||||||||
| 1750 | // optimizeExtInstr might have created new instructions after MI | ||||||||
| 1751 | // and before the already incremented MII. Adjust MII so that the | ||||||||
| 1752 | // next iteration sees the new instructions. | ||||||||
| 1753 | MII = MI; | ||||||||
| 1754 | ++MII; | ||||||||
| 1755 | if (SeenMoveImm) | ||||||||
| 1756 | Changed |= foldImmediate(*MI, ImmDefRegs, ImmDefMIs); | ||||||||
| 1757 | } | ||||||||
| 1758 | |||||||||
| 1759 | // Check whether MI is a load candidate for folding into a later | ||||||||
| 1760 | // instruction. If MI is not a candidate, check whether we can fold an | ||||||||
| 1761 | // earlier load into MI. | ||||||||
| 1762 | if (!isLoadFoldable(*MI, FoldAsLoadDefCandidates) && | ||||||||
| 1763 | !FoldAsLoadDefCandidates.empty()) { | ||||||||
| 1764 | |||||||||
| 1765 | // We visit each operand even after successfully folding a previous | ||||||||
| 1766 | // one. This allows us to fold multiple loads into a single | ||||||||
| 1767 | // instruction. We do assume that optimizeLoadInstr doesn't insert | ||||||||
| 1768 | // foldable uses earlier in the argument list. Since we don't restart | ||||||||
| 1769 | // iteration, we'd miss such cases. | ||||||||
| 1770 | const MCInstrDesc &MIDesc = MI->getDesc(); | ||||||||
| 1771 | for (unsigned i = MIDesc.getNumDefs(); i != MI->getNumOperands(); | ||||||||
| 1772 | ++i) { | ||||||||
| 1773 | const MachineOperand &MOp = MI->getOperand(i); | ||||||||
| 1774 | if (!MOp.isReg()) | ||||||||
| 1775 | continue; | ||||||||
| 1776 | Register FoldAsLoadDefReg = MOp.getReg(); | ||||||||
| 1777 | if (FoldAsLoadDefCandidates.count(FoldAsLoadDefReg)) { | ||||||||
| 1778 | // We need to fold load after optimizeCmpInstr, since | ||||||||
| 1779 | // optimizeCmpInstr can enable folding by converting SUB to CMP. | ||||||||
| 1780 | // Save FoldAsLoadDefReg because optimizeLoadInstr() resets it and | ||||||||
| 1781 | // we need it for markUsesInDebugValueAsUndef(). | ||||||||
| 1782 | Register FoldedReg = FoldAsLoadDefReg; | ||||||||
| 1783 | MachineInstr *DefMI = nullptr; | ||||||||
| 1784 | if (MachineInstr *FoldMI = | ||||||||
| 1785 | TII->optimizeLoadInstr(*MI, MRI, FoldAsLoadDefReg, DefMI)) { | ||||||||
| 1786 | // Update LocalMIs since we replaced MI with FoldMI and deleted | ||||||||
| 1787 | // DefMI. | ||||||||
| 1788 | LLVM_DEBUG(dbgs() << "Replacing: " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Replacing: " << *MI ; } } while (false); | ||||||||
| 1789 | LLVM_DEBUG(dbgs() << " With: " << *FoldMI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << " With: " << *FoldMI ; } } while (false); | ||||||||
| 1790 | LocalMIs.erase(MI); | ||||||||
| 1791 | LocalMIs.erase(DefMI); | ||||||||
| 1792 | LocalMIs.insert(FoldMI); | ||||||||
| 1793 | // Update the call site info. | ||||||||
| 1794 | if (MI->shouldUpdateCallSiteInfo()) | ||||||||
| 1795 | MI->getMF()->moveCallSiteInfo(MI, FoldMI); | ||||||||
| 1796 | MI->eraseFromParent(); | ||||||||
| 1797 | DefMI->eraseFromParent(); | ||||||||
| 1798 | MRI->markUsesInDebugValueAsUndef(FoldedReg); | ||||||||
| 1799 | FoldAsLoadDefCandidates.erase(FoldedReg); | ||||||||
| 1800 | ++NumLoadFold; | ||||||||
| 1801 | |||||||||
| 1802 | // MI is replaced with FoldMI so we can continue trying to fold | ||||||||
| 1803 | Changed = true; | ||||||||
| 1804 | MI = FoldMI; | ||||||||
| 1805 | } | ||||||||
| 1806 | } | ||||||||
| 1807 | } | ||||||||
| 1808 | } | ||||||||
| 1809 | |||||||||
| 1810 | // If we run into an instruction we can't fold across, discard | ||||||||
| 1811 | // the load candidates. Note: We might be able to fold *into* this | ||||||||
| 1812 | // instruction, so this needs to be after the folding logic. | ||||||||
| 1813 | if (MI->isLoadFoldBarrier()) { | ||||||||
| 1814 | LLVM_DEBUG(dbgs() << "Encountered load fold barrier on " << *MI)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("peephole-opt")) { dbgs() << "Encountered load fold barrier on " << *MI; } } while (false); | ||||||||
| 1815 | FoldAsLoadDefCandidates.clear(); | ||||||||
| 1816 | } | ||||||||
| 1817 | } | ||||||||
| 1818 | } | ||||||||
| 1819 | |||||||||
| 1820 | return Changed; | ||||||||
| 1821 | } | ||||||||
| 1822 | |||||||||
| 1823 | ValueTrackerResult ValueTracker::getNextSourceFromCopy() { | ||||||||
| 1824 | assert(Def->isCopy() && "Invalid definition")(static_cast <bool> (Def->isCopy() && "Invalid definition" ) ? void (0) : __assert_fail ("Def->isCopy() && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1824, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1825 | // Copy instruction are supposed to be: Def = Src. | ||||||||
| 1826 | // If someone breaks this assumption, bad things will happen everywhere. | ||||||||
| 1827 | // There may be implicit uses preventing the copy to be moved across | ||||||||
| 1828 | // some target specific register definitions | ||||||||
| 1829 | assert(Def->getNumOperands() - Def->getNumImplicitOperands() == 2 &&(static_cast <bool> (Def->getNumOperands() - Def-> getNumImplicitOperands() == 2 && "Invalid number of operands" ) ? void (0) : __assert_fail ("Def->getNumOperands() - Def->getNumImplicitOperands() == 2 && \"Invalid number of operands\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1830, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1830 | "Invalid number of operands")(static_cast <bool> (Def->getNumOperands() - Def-> getNumImplicitOperands() == 2 && "Invalid number of operands" ) ? void (0) : __assert_fail ("Def->getNumOperands() - Def->getNumImplicitOperands() == 2 && \"Invalid number of operands\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1830, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1831 | assert(!Def->hasImplicitDef() && "Only implicit uses are allowed")(static_cast <bool> (!Def->hasImplicitDef() && "Only implicit uses are allowed") ? void (0) : __assert_fail ("!Def->hasImplicitDef() && \"Only implicit uses are allowed\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1831, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1832 | |||||||||
| 1833 | if (Def->getOperand(DefIdx).getSubReg() != DefSubReg) | ||||||||
| 1834 | // If we look for a different subreg, it means we want a subreg of src. | ||||||||
| 1835 | // Bails as we do not support composing subregs yet. | ||||||||
| 1836 | return ValueTrackerResult(); | ||||||||
| 1837 | // Otherwise, we want the whole source. | ||||||||
| 1838 | const MachineOperand &Src = Def->getOperand(1); | ||||||||
| 1839 | if (Src.isUndef()) | ||||||||
| 1840 | return ValueTrackerResult(); | ||||||||
| 1841 | return ValueTrackerResult(Src.getReg(), Src.getSubReg()); | ||||||||
| 1842 | } | ||||||||
| 1843 | |||||||||
| 1844 | ValueTrackerResult ValueTracker::getNextSourceFromBitcast() { | ||||||||
| 1845 | assert(Def->isBitcast() && "Invalid definition")(static_cast <bool> (Def->isBitcast() && "Invalid definition" ) ? void (0) : __assert_fail ("Def->isBitcast() && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1845, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1846 | |||||||||
| 1847 | // Bail if there are effects that a plain copy will not expose. | ||||||||
| 1848 | if (Def->mayRaiseFPException() || Def->hasUnmodeledSideEffects()) | ||||||||
| 1849 | return ValueTrackerResult(); | ||||||||
| 1850 | |||||||||
| 1851 | // Bitcasts with more than one def are not supported. | ||||||||
| 1852 | if (Def->getDesc().getNumDefs() != 1) | ||||||||
| 1853 | return ValueTrackerResult(); | ||||||||
| 1854 | const MachineOperand DefOp = Def->getOperand(DefIdx); | ||||||||
| 1855 | if (DefOp.getSubReg() != DefSubReg) | ||||||||
| 1856 | // If we look for a different subreg, it means we want a subreg of the src. | ||||||||
| 1857 | // Bails as we do not support composing subregs yet. | ||||||||
| 1858 | return ValueTrackerResult(); | ||||||||
| 1859 | |||||||||
| 1860 | unsigned SrcIdx = Def->getNumOperands(); | ||||||||
| 1861 | for (unsigned OpIdx = DefIdx + 1, EndOpIdx = SrcIdx; OpIdx != EndOpIdx; | ||||||||
| 1862 | ++OpIdx) { | ||||||||
| 1863 | const MachineOperand &MO = Def->getOperand(OpIdx); | ||||||||
| 1864 | if (!MO.isReg() || !MO.getReg()) | ||||||||
| 1865 | continue; | ||||||||
| 1866 | // Ignore dead implicit defs. | ||||||||
| 1867 | if (MO.isImplicit() && MO.isDead()) | ||||||||
| 1868 | continue; | ||||||||
| 1869 | assert(!MO.isDef() && "We should have skipped all the definitions by now")(static_cast <bool> (!MO.isDef() && "We should have skipped all the definitions by now" ) ? void (0) : __assert_fail ("!MO.isDef() && \"We should have skipped all the definitions by now\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1869, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1870 | if (SrcIdx != EndOpIdx) | ||||||||
| 1871 | // Multiple sources? | ||||||||
| 1872 | return ValueTrackerResult(); | ||||||||
| 1873 | SrcIdx = OpIdx; | ||||||||
| 1874 | } | ||||||||
| 1875 | |||||||||
| 1876 | // In some rare case, Def has no input, SrcIdx is out of bound, | ||||||||
| 1877 | // getOperand(SrcIdx) will fail below. | ||||||||
| 1878 | if (SrcIdx >= Def->getNumOperands()) | ||||||||
| 1879 | return ValueTrackerResult(); | ||||||||
| 1880 | |||||||||
| 1881 | // Stop when any user of the bitcast is a SUBREG_TO_REG, replacing with a COPY | ||||||||
| 1882 | // will break the assumed guarantees for the upper bits. | ||||||||
| 1883 | for (const MachineInstr &UseMI : MRI.use_nodbg_instructions(DefOp.getReg())) { | ||||||||
| 1884 | if (UseMI.isSubregToReg()) | ||||||||
| 1885 | return ValueTrackerResult(); | ||||||||
| 1886 | } | ||||||||
| 1887 | |||||||||
| 1888 | const MachineOperand &Src = Def->getOperand(SrcIdx); | ||||||||
| 1889 | if (Src.isUndef()) | ||||||||
| 1890 | return ValueTrackerResult(); | ||||||||
| 1891 | return ValueTrackerResult(Src.getReg(), Src.getSubReg()); | ||||||||
| 1892 | } | ||||||||
| 1893 | |||||||||
| 1894 | ValueTrackerResult ValueTracker::getNextSourceFromRegSequence() { | ||||||||
| 1895 | assert((Def->isRegSequence() || Def->isRegSequenceLike()) &&(static_cast <bool> ((Def->isRegSequence() || Def-> isRegSequenceLike()) && "Invalid definition") ? void ( 0) : __assert_fail ("(Def->isRegSequence() || Def->isRegSequenceLike()) && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1896, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1896 | "Invalid definition")(static_cast <bool> ((Def->isRegSequence() || Def-> isRegSequenceLike()) && "Invalid definition") ? void ( 0) : __assert_fail ("(Def->isRegSequence() || Def->isRegSequenceLike()) && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1896, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1897 | |||||||||
| 1898 | if (Def->getOperand(DefIdx).getSubReg()) | ||||||||
| 1899 | // If we are composing subregs, bail out. | ||||||||
| 1900 | // The case we are checking is Def.<subreg> = REG_SEQUENCE. | ||||||||
| 1901 | // This should almost never happen as the SSA property is tracked at | ||||||||
| 1902 | // the register level (as opposed to the subreg level). | ||||||||
| 1903 | // I.e., | ||||||||
| 1904 | // Def.sub0 = | ||||||||
| 1905 | // Def.sub1 = | ||||||||
| 1906 | // is a valid SSA representation for Def.sub0 and Def.sub1, but not for | ||||||||
| 1907 | // Def. Thus, it must not be generated. | ||||||||
| 1908 | // However, some code could theoretically generates a single | ||||||||
| 1909 | // Def.sub0 (i.e, not defining the other subregs) and we would | ||||||||
| 1910 | // have this case. | ||||||||
| 1911 | // If we can ascertain (or force) that this never happens, we could | ||||||||
| 1912 | // turn that into an assertion. | ||||||||
| 1913 | return ValueTrackerResult(); | ||||||||
| 1914 | |||||||||
| 1915 | if (!TII) | ||||||||
| 1916 | // We could handle the REG_SEQUENCE here, but we do not want to | ||||||||
| 1917 | // duplicate the code from the generic TII. | ||||||||
| 1918 | return ValueTrackerResult(); | ||||||||
| 1919 | |||||||||
| 1920 | SmallVector<RegSubRegPairAndIdx, 8> RegSeqInputRegs; | ||||||||
| 1921 | if (!TII->getRegSequenceInputs(*Def, DefIdx, RegSeqInputRegs)) | ||||||||
| 1922 | return ValueTrackerResult(); | ||||||||
| 1923 | |||||||||
| 1924 | // We are looking at: | ||||||||
| 1925 | // Def = REG_SEQUENCE v0, sub0, v1, sub1, ... | ||||||||
| 1926 | // Check if one of the operand defines the subreg we are interested in. | ||||||||
| 1927 | for (const RegSubRegPairAndIdx &RegSeqInput : RegSeqInputRegs) { | ||||||||
| 1928 | if (RegSeqInput.SubIdx == DefSubReg) | ||||||||
| 1929 | return ValueTrackerResult(RegSeqInput.Reg, RegSeqInput.SubReg); | ||||||||
| 1930 | } | ||||||||
| 1931 | |||||||||
| 1932 | // If the subreg we are tracking is super-defined by another subreg, | ||||||||
| 1933 | // we could follow this value. However, this would require to compose | ||||||||
| 1934 | // the subreg and we do not do that for now. | ||||||||
| 1935 | return ValueTrackerResult(); | ||||||||
| 1936 | } | ||||||||
| 1937 | |||||||||
| 1938 | ValueTrackerResult ValueTracker::getNextSourceFromInsertSubreg() { | ||||||||
| 1939 | assert((Def->isInsertSubreg() || Def->isInsertSubregLike()) &&(static_cast <bool> ((Def->isInsertSubreg() || Def-> isInsertSubregLike()) && "Invalid definition") ? void (0) : __assert_fail ("(Def->isInsertSubreg() || Def->isInsertSubregLike()) && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1940, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1940 | "Invalid definition")(static_cast <bool> ((Def->isInsertSubreg() || Def-> isInsertSubregLike()) && "Invalid definition") ? void (0) : __assert_fail ("(Def->isInsertSubreg() || Def->isInsertSubregLike()) && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1940, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1941 | |||||||||
| 1942 | if (Def->getOperand(DefIdx).getSubReg()) | ||||||||
| 1943 | // If we are composing subreg, bail out. | ||||||||
| 1944 | // Same remark as getNextSourceFromRegSequence. | ||||||||
| 1945 | // I.e., this may be turned into an assert. | ||||||||
| 1946 | return ValueTrackerResult(); | ||||||||
| 1947 | |||||||||
| 1948 | if (!TII) | ||||||||
| 1949 | // We could handle the REG_SEQUENCE here, but we do not want to | ||||||||
| 1950 | // duplicate the code from the generic TII. | ||||||||
| 1951 | return ValueTrackerResult(); | ||||||||
| 1952 | |||||||||
| 1953 | RegSubRegPair BaseReg; | ||||||||
| 1954 | RegSubRegPairAndIdx InsertedReg; | ||||||||
| 1955 | if (!TII->getInsertSubregInputs(*Def, DefIdx, BaseReg, InsertedReg)) | ||||||||
| 1956 | return ValueTrackerResult(); | ||||||||
| 1957 | |||||||||
| 1958 | // We are looking at: | ||||||||
| 1959 | // Def = INSERT_SUBREG v0, v1, sub1 | ||||||||
| 1960 | // There are two cases: | ||||||||
| 1961 | // 1. DefSubReg == sub1, get v1. | ||||||||
| 1962 | // 2. DefSubReg != sub1, the value may be available through v0. | ||||||||
| 1963 | |||||||||
| 1964 | // #1 Check if the inserted register matches the required sub index. | ||||||||
| 1965 | if (InsertedReg.SubIdx == DefSubReg) { | ||||||||
| 1966 | return ValueTrackerResult(InsertedReg.Reg, InsertedReg.SubReg); | ||||||||
| 1967 | } | ||||||||
| 1968 | // #2 Otherwise, if the sub register we are looking for is not partial | ||||||||
| 1969 | // defined by the inserted element, we can look through the main | ||||||||
| 1970 | // register (v0). | ||||||||
| 1971 | const MachineOperand &MODef = Def->getOperand(DefIdx); | ||||||||
| 1972 | // If the result register (Def) and the base register (v0) do not | ||||||||
| 1973 | // have the same register class or if we have to compose | ||||||||
| 1974 | // subregisters, bail out. | ||||||||
| 1975 | if (MRI.getRegClass(MODef.getReg()) != MRI.getRegClass(BaseReg.Reg) || | ||||||||
| 1976 | BaseReg.SubReg) | ||||||||
| 1977 | return ValueTrackerResult(); | ||||||||
| 1978 | |||||||||
| 1979 | // Get the TRI and check if the inserted sub-register overlaps with the | ||||||||
| 1980 | // sub-register we are tracking. | ||||||||
| 1981 | const TargetRegisterInfo *TRI = MRI.getTargetRegisterInfo(); | ||||||||
| 1982 | if (!TRI || | ||||||||
| 1983 | !(TRI->getSubRegIndexLaneMask(DefSubReg) & | ||||||||
| 1984 | TRI->getSubRegIndexLaneMask(InsertedReg.SubIdx)).none()) | ||||||||
| 1985 | return ValueTrackerResult(); | ||||||||
| 1986 | // At this point, the value is available in v0 via the same subreg | ||||||||
| 1987 | // we used for Def. | ||||||||
| 1988 | return ValueTrackerResult(BaseReg.Reg, DefSubReg); | ||||||||
| 1989 | } | ||||||||
| 1990 | |||||||||
| 1991 | ValueTrackerResult ValueTracker::getNextSourceFromExtractSubreg() { | ||||||||
| 1992 | assert((Def->isExtractSubreg() ||(static_cast <bool> ((Def->isExtractSubreg() || Def-> isExtractSubregLike()) && "Invalid definition") ? void (0) : __assert_fail ("(Def->isExtractSubreg() || Def->isExtractSubregLike()) && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1993, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 1993 | Def->isExtractSubregLike()) && "Invalid definition")(static_cast <bool> ((Def->isExtractSubreg() || Def-> isExtractSubregLike()) && "Invalid definition") ? void (0) : __assert_fail ("(Def->isExtractSubreg() || Def->isExtractSubregLike()) && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 1993, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 1994 | // We are looking at: | ||||||||
| 1995 | // Def = EXTRACT_SUBREG v0, sub0 | ||||||||
| 1996 | |||||||||
| 1997 | // Bail if we have to compose sub registers. | ||||||||
| 1998 | // Indeed, if DefSubReg != 0, we would have to compose it with sub0. | ||||||||
| 1999 | if (DefSubReg) | ||||||||
| 2000 | return ValueTrackerResult(); | ||||||||
| 2001 | |||||||||
| 2002 | if (!TII) | ||||||||
| 2003 | // We could handle the EXTRACT_SUBREG here, but we do not want to | ||||||||
| 2004 | // duplicate the code from the generic TII. | ||||||||
| 2005 | return ValueTrackerResult(); | ||||||||
| 2006 | |||||||||
| 2007 | RegSubRegPairAndIdx ExtractSubregInputReg; | ||||||||
| 2008 | if (!TII->getExtractSubregInputs(*Def, DefIdx, ExtractSubregInputReg)) | ||||||||
| 2009 | return ValueTrackerResult(); | ||||||||
| 2010 | |||||||||
| 2011 | // Bail if we have to compose sub registers. | ||||||||
| 2012 | // Likewise, if v0.subreg != 0, we would have to compose v0.subreg with sub0. | ||||||||
| 2013 | if (ExtractSubregInputReg.SubReg) | ||||||||
| 2014 | return ValueTrackerResult(); | ||||||||
| 2015 | // Otherwise, the value is available in the v0.sub0. | ||||||||
| 2016 | return ValueTrackerResult(ExtractSubregInputReg.Reg, | ||||||||
| 2017 | ExtractSubregInputReg.SubIdx); | ||||||||
| 2018 | } | ||||||||
| 2019 | |||||||||
| 2020 | ValueTrackerResult ValueTracker::getNextSourceFromSubregToReg() { | ||||||||
| 2021 | assert(Def->isSubregToReg() && "Invalid definition")(static_cast <bool> (Def->isSubregToReg() && "Invalid definition") ? void (0) : __assert_fail ("Def->isSubregToReg() && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2021, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2022 | // We are looking at: | ||||||||
| 2023 | // Def = SUBREG_TO_REG Imm, v0, sub0 | ||||||||
| 2024 | |||||||||
| 2025 | // Bail if we have to compose sub registers. | ||||||||
| 2026 | // If DefSubReg != sub0, we would have to check that all the bits | ||||||||
| 2027 | // we track are included in sub0 and if yes, we would have to | ||||||||
| 2028 | // determine the right subreg in v0. | ||||||||
| 2029 | if (DefSubReg != Def->getOperand(3).getImm()) | ||||||||
| 2030 | return ValueTrackerResult(); | ||||||||
| 2031 | // Bail if we have to compose sub registers. | ||||||||
| 2032 | // Likewise, if v0.subreg != 0, we would have to compose it with sub0. | ||||||||
| 2033 | if (Def->getOperand(2).getSubReg()) | ||||||||
| 2034 | return ValueTrackerResult(); | ||||||||
| 2035 | |||||||||
| 2036 | return ValueTrackerResult(Def->getOperand(2).getReg(), | ||||||||
| 2037 | Def->getOperand(3).getImm()); | ||||||||
| 2038 | } | ||||||||
| 2039 | |||||||||
| 2040 | /// Explore each PHI incoming operand and return its sources. | ||||||||
| 2041 | ValueTrackerResult ValueTracker::getNextSourceFromPHI() { | ||||||||
| 2042 | assert(Def->isPHI() && "Invalid definition")(static_cast <bool> (Def->isPHI() && "Invalid definition" ) ? void (0) : __assert_fail ("Def->isPHI() && \"Invalid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2042, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2043 | ValueTrackerResult Res; | ||||||||
| 2044 | |||||||||
| 2045 | // If we look for a different subreg, bail as we do not support composing | ||||||||
| 2046 | // subregs yet. | ||||||||
| 2047 | if (Def->getOperand(0).getSubReg() != DefSubReg) | ||||||||
| 2048 | return ValueTrackerResult(); | ||||||||
| 2049 | |||||||||
| 2050 | // Return all register sources for PHI instructions. | ||||||||
| 2051 | for (unsigned i = 1, e = Def->getNumOperands(); i < e; i += 2) { | ||||||||
| 2052 | const MachineOperand &MO = Def->getOperand(i); | ||||||||
| 2053 | assert(MO.isReg() && "Invalid PHI instruction")(static_cast <bool> (MO.isReg() && "Invalid PHI instruction" ) ? void (0) : __assert_fail ("MO.isReg() && \"Invalid PHI instruction\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2053, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2054 | // We have no code to deal with undef operands. They shouldn't happen in | ||||||||
| 2055 | // normal programs anyway. | ||||||||
| 2056 | if (MO.isUndef()) | ||||||||
| 2057 | return ValueTrackerResult(); | ||||||||
| 2058 | Res.addSource(MO.getReg(), MO.getSubReg()); | ||||||||
| 2059 | } | ||||||||
| 2060 | |||||||||
| 2061 | return Res; | ||||||||
| 2062 | } | ||||||||
| 2063 | |||||||||
| 2064 | ValueTrackerResult ValueTracker::getNextSourceImpl() { | ||||||||
| 2065 | assert(Def && "This method needs a valid definition")(static_cast <bool> (Def && "This method needs a valid definition" ) ? void (0) : __assert_fail ("Def && \"This method needs a valid definition\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2065, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2066 | |||||||||
| 2067 | assert(((Def->getOperand(DefIdx).isDef() &&(static_cast <bool> (((Def->getOperand(DefIdx).isDef () && (DefIdx < Def->getDesc().getNumDefs() || Def ->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit ()) && "Invalid DefIdx") ? void (0) : __assert_fail ( "((Def->getOperand(DefIdx).isDef() && (DefIdx < Def->getDesc().getNumDefs() || Def->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit()) && \"Invalid DefIdx\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2071, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2068 | (DefIdx < Def->getDesc().getNumDefs() ||(static_cast <bool> (((Def->getOperand(DefIdx).isDef () && (DefIdx < Def->getDesc().getNumDefs() || Def ->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit ()) && "Invalid DefIdx") ? void (0) : __assert_fail ( "((Def->getOperand(DefIdx).isDef() && (DefIdx < Def->getDesc().getNumDefs() || Def->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit()) && \"Invalid DefIdx\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2071, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2069 | Def->getDesc().isVariadic())) ||(static_cast <bool> (((Def->getOperand(DefIdx).isDef () && (DefIdx < Def->getDesc().getNumDefs() || Def ->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit ()) && "Invalid DefIdx") ? void (0) : __assert_fail ( "((Def->getOperand(DefIdx).isDef() && (DefIdx < Def->getDesc().getNumDefs() || Def->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit()) && \"Invalid DefIdx\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2071, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2070 | Def->getOperand(DefIdx).isImplicit()) &&(static_cast <bool> (((Def->getOperand(DefIdx).isDef () && (DefIdx < Def->getDesc().getNumDefs() || Def ->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit ()) && "Invalid DefIdx") ? void (0) : __assert_fail ( "((Def->getOperand(DefIdx).isDef() && (DefIdx < Def->getDesc().getNumDefs() || Def->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit()) && \"Invalid DefIdx\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2071, __extension__ __PRETTY_FUNCTION__)) | ||||||||
| 2071 | "Invalid DefIdx")(static_cast <bool> (((Def->getOperand(DefIdx).isDef () && (DefIdx < Def->getDesc().getNumDefs() || Def ->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit ()) && "Invalid DefIdx") ? void (0) : __assert_fail ( "((Def->getOperand(DefIdx).isDef() && (DefIdx < Def->getDesc().getNumDefs() || Def->getDesc().isVariadic())) || Def->getOperand(DefIdx).isImplicit()) && \"Invalid DefIdx\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/lib/CodeGen/PeepholeOptimizer.cpp" , 2071, __extension__ __PRETTY_FUNCTION__)); | ||||||||
| 2072 | if (Def->isCopy()) | ||||||||
| 2073 | return getNextSourceFromCopy(); | ||||||||
| 2074 | if (Def->isBitcast()) | ||||||||
| 2075 | return getNextSourceFromBitcast(); | ||||||||
| 2076 | // All the remaining cases involve "complex" instructions. | ||||||||
| 2077 | // Bail if we did not ask for the advanced tracking. | ||||||||
| 2078 | if (DisableAdvCopyOpt) | ||||||||
| 2079 | return ValueTrackerResult(); | ||||||||
| 2080 | if (Def->isRegSequence() || Def->isRegSequenceLike()) | ||||||||
| 2081 | return getNextSourceFromRegSequence(); | ||||||||
| 2082 | if (Def->isInsertSubreg() || Def->isInsertSubregLike()) | ||||||||
| 2083 | return getNextSourceFromInsertSubreg(); | ||||||||
| 2084 | if (Def->isExtractSubreg() || Def->isExtractSubregLike()) | ||||||||
| 2085 | return getNextSourceFromExtractSubreg(); | ||||||||
| 2086 | if (Def->isSubregToReg()) | ||||||||
| 2087 | return getNextSourceFromSubregToReg(); | ||||||||
| 2088 | if (Def->isPHI()) | ||||||||
| 2089 | return getNextSourceFromPHI(); | ||||||||
| 2090 | return ValueTrackerResult(); | ||||||||
| 2091 | } | ||||||||
| 2092 | |||||||||
| 2093 | ValueTrackerResult ValueTracker::getNextSource() { | ||||||||
| 2094 | // If we reach a point where we cannot move up in the use-def chain, | ||||||||
| 2095 | // there is nothing we can get. | ||||||||
| 2096 | if (!Def) | ||||||||
| 2097 | return ValueTrackerResult(); | ||||||||
| 2098 | |||||||||
| 2099 | ValueTrackerResult Res = getNextSourceImpl(); | ||||||||
| 2100 | if (Res.isValid()) { | ||||||||
| 2101 | // Update definition, definition index, and subregister for the | ||||||||
| 2102 | // next call of getNextSource. | ||||||||
| 2103 | // Update the current register. | ||||||||
| 2104 | bool OneRegSrc = Res.getNumSources() == 1; | ||||||||
| 2105 | if (OneRegSrc) | ||||||||
| 2106 | Reg = Res.getSrcReg(0); | ||||||||
| 2107 | // Update the result before moving up in the use-def chain | ||||||||
| 2108 | // with the instruction containing the last found sources. | ||||||||
| 2109 | Res.setInst(Def); | ||||||||
| 2110 | |||||||||
| 2111 | // If we can still move up in the use-def chain, move to the next | ||||||||
| 2112 | // definition. | ||||||||
| 2113 | if (!Register::isPhysicalRegister(Reg) && OneRegSrc) { | ||||||||
| 2114 | MachineRegisterInfo::def_iterator DI = MRI.def_begin(Reg); | ||||||||
| 2115 | if (DI != MRI.def_end()) { | ||||||||
| 2116 | Def = DI->getParent(); | ||||||||
| 2117 | DefIdx = DI.getOperandNo(); | ||||||||
| 2118 | DefSubReg = Res.getSrcSubReg(0); | ||||||||
| 2119 | } else { | ||||||||
| 2120 | Def = nullptr; | ||||||||
| 2121 | } | ||||||||
| 2122 | return Res; | ||||||||
| 2123 | } | ||||||||
| 2124 | } | ||||||||
| 2125 | // If we end up here, this means we will not be able to find another source | ||||||||
| 2126 | // for the next iteration. Make sure any new call to getNextSource bails out | ||||||||
| 2127 | // early by cutting the use-def chain. | ||||||||
| 2128 | Def = nullptr; | ||||||||
| 2129 | return Res; | ||||||||
| 2130 | } |
| 1 | //===-- llvm/CodeGen/Register.h ---------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef LLVM_CODEGEN_REGISTER_H |
| 10 | #define LLVM_CODEGEN_REGISTER_H |
| 11 | |
| 12 | #include "llvm/MC/MCRegister.h" |
| 13 | #include <cassert> |
| 14 | |
| 15 | namespace llvm { |
| 16 | |
| 17 | /// Wrapper class representing virtual and physical registers. Should be passed |
| 18 | /// by value. |
| 19 | class Register { |
| 20 | unsigned Reg; |
| 21 | |
| 22 | public: |
| 23 | constexpr Register(unsigned Val = 0): Reg(Val) {} |
| 24 | constexpr Register(MCRegister Val): Reg(Val) {} |
| 25 | |
| 26 | // Register numbers can represent physical registers, virtual registers, and |
| 27 | // sometimes stack slots. The unsigned values are divided into these ranges: |
| 28 | // |
| 29 | // 0 Not a register, can be used as a sentinel. |
| 30 | // [1;2^30) Physical registers assigned by TableGen. |
| 31 | // [2^30;2^31) Stack slots. (Rarely used.) |
| 32 | // [2^31;2^32) Virtual registers assigned by MachineRegisterInfo. |
| 33 | // |
| 34 | // Further sentinels can be allocated from the small negative integers. |
| 35 | // DenseMapInfo<unsigned> uses -1u and -2u. |
| 36 | static_assert(std::numeric_limits<decltype(Reg)>::max() >= 0xFFFFFFFF, |
| 37 | "Reg isn't large enough to hold full range."); |
| 38 | |
| 39 | /// isStackSlot - Sometimes it is useful the be able to store a non-negative |
| 40 | /// frame index in a variable that normally holds a register. isStackSlot() |
| 41 | /// returns true if Reg is in the range used for stack slots. |
| 42 | /// |
| 43 | /// FIXME: remove in favor of member. |
| 44 | static bool isStackSlot(unsigned Reg) { |
| 45 | return MCRegister::isStackSlot(Reg); |
| 46 | } |
| 47 | |
| 48 | /// Return true if this is a stack slot. |
| 49 | bool isStack() const { return MCRegister::isStackSlot(Reg); } |
| 50 | |
| 51 | /// Compute the frame index from a register value representing a stack slot. |
| 52 | static int stackSlot2Index(Register Reg) { |
| 53 | assert(Reg.isStack() && "Not a stack slot")(static_cast <bool> (Reg.isStack() && "Not a stack slot" ) ? void (0) : __assert_fail ("Reg.isStack() && \"Not a stack slot\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/Register.h" , 53, __extension__ __PRETTY_FUNCTION__)); |
| 54 | return int(Reg - MCRegister::FirstStackSlot); |
| 55 | } |
| 56 | |
| 57 | /// Convert a non-negative frame index to a stack slot register value. |
| 58 | static Register index2StackSlot(int FI) { |
| 59 | assert(FI >= 0 && "Cannot hold a negative frame index.")(static_cast <bool> (FI >= 0 && "Cannot hold a negative frame index." ) ? void (0) : __assert_fail ("FI >= 0 && \"Cannot hold a negative frame index.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/Register.h" , 59, __extension__ __PRETTY_FUNCTION__)); |
| 60 | return Register(FI + MCRegister::FirstStackSlot); |
| 61 | } |
| 62 | |
| 63 | /// Return true if the specified register number is in |
| 64 | /// the physical register namespace. |
| 65 | static bool isPhysicalRegister(unsigned Reg) { |
| 66 | return MCRegister::isPhysicalRegister(Reg); |
| 67 | } |
| 68 | |
| 69 | /// Return true if the specified register number is in |
| 70 | /// the virtual register namespace. |
| 71 | static bool isVirtualRegister(unsigned Reg) { |
| 72 | return Reg & MCRegister::VirtualRegFlag && !isStackSlot(Reg); |
| 73 | } |
| 74 | |
| 75 | /// Convert a virtual register number to a 0-based index. |
| 76 | /// The first virtual register in a function will get the index 0. |
| 77 | static unsigned virtReg2Index(Register Reg) { |
| 78 | assert(isVirtualRegister(Reg) && "Not a virtual register")(static_cast <bool> (isVirtualRegister(Reg) && "Not a virtual register" ) ? void (0) : __assert_fail ("isVirtualRegister(Reg) && \"Not a virtual register\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/Register.h" , 78, __extension__ __PRETTY_FUNCTION__)); |
| 79 | return Reg & ~MCRegister::VirtualRegFlag; |
| 80 | } |
| 81 | |
| 82 | /// Convert a 0-based index to a virtual register number. |
| 83 | /// This is the inverse operation of VirtReg2IndexFunctor below. |
| 84 | static Register index2VirtReg(unsigned Index) { |
| 85 | assert(Index < (1u << 31) && "Index too large for virtual register range.")(static_cast <bool> (Index < (1u << 31) && "Index too large for virtual register range.") ? void (0) : __assert_fail ("Index < (1u << 31) && \"Index too large for virtual register range.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/Register.h" , 85, __extension__ __PRETTY_FUNCTION__)); |
| 86 | return Index | MCRegister::VirtualRegFlag; |
| 87 | } |
| 88 | |
| 89 | /// Return true if the specified register number is in the virtual register |
| 90 | /// namespace. |
| 91 | bool isVirtual() const { |
| 92 | return isVirtualRegister(Reg); |
| 93 | } |
| 94 | |
| 95 | /// Return true if the specified register number is in the physical register |
| 96 | /// namespace. |
| 97 | bool isPhysical() const { |
| 98 | return isPhysicalRegister(Reg); |
| 99 | } |
| 100 | |
| 101 | /// Convert a virtual register number to a 0-based index. The first virtual |
| 102 | /// register in a function will get the index 0. |
| 103 | unsigned virtRegIndex() const { |
| 104 | return virtReg2Index(Reg); |
| 105 | } |
| 106 | |
| 107 | constexpr operator unsigned() const { |
| 108 | return Reg; |
| 109 | } |
| 110 | |
| 111 | unsigned id() const { return Reg; } |
| 112 | |
| 113 | operator MCRegister() const { |
| 114 | return MCRegister(Reg); |
| 115 | } |
| 116 | |
| 117 | /// Utility to check-convert this value to a MCRegister. The caller is |
| 118 | /// expected to have already validated that this Register is, indeed, |
| 119 | /// physical. |
| 120 | MCRegister asMCReg() const { |
| 121 | assert(Reg == MCRegister::NoRegister ||(static_cast <bool> (Reg == MCRegister::NoRegister || MCRegister ::isPhysicalRegister(Reg)) ? void (0) : __assert_fail ("Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister(Reg)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/Register.h" , 122, __extension__ __PRETTY_FUNCTION__)) |
| 122 | MCRegister::isPhysicalRegister(Reg))(static_cast <bool> (Reg == MCRegister::NoRegister || MCRegister ::isPhysicalRegister(Reg)) ? void (0) : __assert_fail ("Reg == MCRegister::NoRegister || MCRegister::isPhysicalRegister(Reg)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/Register.h" , 122, __extension__ __PRETTY_FUNCTION__)); |
| 123 | return MCRegister(Reg); |
| 124 | } |
| 125 | |
| 126 | bool isValid() const { return Reg != MCRegister::NoRegister; } |
| 127 | |
| 128 | /// Comparisons between register objects |
| 129 | bool operator==(const Register &Other) const { return Reg == Other.Reg; } |
| 130 | bool operator!=(const Register &Other) const { return Reg != Other.Reg; } |
| 131 | bool operator==(const MCRegister &Other) const { return Reg == Other.id(); } |
| 132 | bool operator!=(const MCRegister &Other) const { return Reg != Other.id(); } |
| 133 | |
| 134 | /// Comparisons against register constants. E.g. |
| 135 | /// * R == AArch64::WZR |
| 136 | /// * R == 0 |
| 137 | /// * R == VirtRegMap::NO_PHYS_REG |
| 138 | bool operator==(unsigned Other) const { return Reg == Other; } |
| 139 | bool operator!=(unsigned Other) const { return Reg != Other; } |
| 140 | bool operator==(int Other) const { return Reg == unsigned(Other); } |
| 141 | bool operator!=(int Other) const { return Reg != unsigned(Other); } |
| 142 | // MSVC requires that we explicitly declare these two as well. |
| 143 | bool operator==(MCPhysReg Other) const { return Reg == unsigned(Other); } |
| 144 | bool operator!=(MCPhysReg Other) const { return Reg != unsigned(Other); } |
| 145 | }; |
| 146 | |
| 147 | // Provide DenseMapInfo for Register |
| 148 | template<> struct DenseMapInfo<Register> { |
| 149 | static inline unsigned getEmptyKey() { |
| 150 | return DenseMapInfo<unsigned>::getEmptyKey(); |
| 151 | } |
| 152 | static inline unsigned getTombstoneKey() { |
| 153 | return DenseMapInfo<unsigned>::getTombstoneKey(); |
| 154 | } |
| 155 | static unsigned getHashValue(const Register &Val) { |
| 156 | return DenseMapInfo<unsigned>::getHashValue(Val.id()); |
| 157 | } |
| 158 | static bool isEqual(const Register &LHS, const Register &RHS) { |
| 159 | return DenseMapInfo<unsigned>::isEqual(LHS.id(), RHS.id()); |
| 160 | } |
| 161 | }; |
| 162 | |
| 163 | } |
| 164 | |
| 165 | #endif // LLVM_CODEGEN_REGISTER_H |
| 1 | //===-- llvm/MC/Register.h --------------------------------------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | |
| 9 | #ifndef LLVM_MC_MCREGISTER_H |
| 10 | #define LLVM_MC_MCREGISTER_H |
| 11 | |
| 12 | #include "llvm/ADT/DenseMapInfo.h" |
| 13 | #include <cassert> |
| 14 | #include <limits> |
| 15 | |
| 16 | namespace llvm { |
| 17 | |
| 18 | /// An unsigned integer type large enough to represent all physical registers, |
| 19 | /// but not necessarily virtual registers. |
| 20 | using MCPhysReg = uint16_t; |
| 21 | |
| 22 | /// Wrapper class representing physical registers. Should be passed by value. |
| 23 | class MCRegister { |
| 24 | friend hash_code hash_value(const MCRegister &); |
| 25 | unsigned Reg; |
| 26 | |
| 27 | public: |
| 28 | constexpr MCRegister(unsigned Val = 0): Reg(Val) {} |
| 29 | |
| 30 | // Register numbers can represent physical registers, virtual registers, and |
| 31 | // sometimes stack slots. The unsigned values are divided into these ranges: |
| 32 | // |
| 33 | // 0 Not a register, can be used as a sentinel. |
| 34 | // [1;2^30) Physical registers assigned by TableGen. |
| 35 | // [2^30;2^31) Stack slots. (Rarely used.) |
| 36 | // [2^31;2^32) Virtual registers assigned by MachineRegisterInfo. |
| 37 | // |
| 38 | // Further sentinels can be allocated from the small negative integers. |
| 39 | // DenseMapInfo<unsigned> uses -1u and -2u. |
| 40 | static_assert(std::numeric_limits<decltype(Reg)>::max() >= 0xFFFFFFFF, |
| 41 | "Reg isn't large enough to hold full range."); |
| 42 | static constexpr unsigned NoRegister = 0u; |
| 43 | static constexpr unsigned FirstPhysicalReg = 1u; |
| 44 | static constexpr unsigned FirstStackSlot = 1u << 30; |
| 45 | static constexpr unsigned VirtualRegFlag = 1u << 31; |
| 46 | |
| 47 | /// This is the portion of the positive number space that is not a physical |
| 48 | /// register. StackSlot values do not exist in the MC layer, see |
| 49 | /// Register::isStackSlot() for the more information on them. |
| 50 | /// |
| 51 | static bool isStackSlot(unsigned Reg) { |
| 52 | return FirstStackSlot <= Reg && Reg < VirtualRegFlag; |
| 53 | } |
| 54 | |
| 55 | /// Return true if the specified register number is in |
| 56 | /// the physical register namespace. |
| 57 | static bool isPhysicalRegister(unsigned Reg) { |
| 58 | return FirstPhysicalReg <= Reg && Reg < FirstStackSlot; |
| 59 | } |
| 60 | |
| 61 | constexpr operator unsigned() const { |
| 62 | return Reg; |
| 63 | } |
| 64 | |
| 65 | /// Check the provided unsigned value is a valid MCRegister. |
| 66 | static MCRegister from(unsigned Val) { |
| 67 | assert(Val == NoRegister || isPhysicalRegister(Val))(static_cast <bool> (Val == NoRegister || isPhysicalRegister (Val)) ? void (0) : __assert_fail ("Val == NoRegister || isPhysicalRegister(Val)" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/MC/MCRegister.h" , 67, __extension__ __PRETTY_FUNCTION__)); |
| 68 | return MCRegister(Val); |
| 69 | } |
| 70 | |
| 71 | unsigned id() const { |
| 72 | return Reg; |
| 73 | } |
| 74 | |
| 75 | bool isValid() const { return Reg != NoRegister; } |
| 76 | |
| 77 | /// Comparisons between register objects |
| 78 | bool operator==(const MCRegister &Other) const { return Reg == Other.Reg; } |
| 79 | bool operator!=(const MCRegister &Other) const { return Reg != Other.Reg; } |
| 80 | |
| 81 | /// Comparisons against register constants. E.g. |
| 82 | /// * R == AArch64::WZR |
| 83 | /// * R == 0 |
| 84 | /// * R == VirtRegMap::NO_PHYS_REG |
| 85 | bool operator==(unsigned Other) const { return Reg == Other; } |
| 86 | bool operator!=(unsigned Other) const { return Reg != Other; } |
| 87 | bool operator==(int Other) const { return Reg == unsigned(Other); } |
| 88 | bool operator!=(int Other) const { return Reg != unsigned(Other); } |
| 89 | // MSVC requires that we explicitly declare these two as well. |
| 90 | bool operator==(MCPhysReg Other) const { return Reg == unsigned(Other); } |
| 91 | bool operator!=(MCPhysReg Other) const { return Reg != unsigned(Other); } |
| 92 | }; |
| 93 | |
| 94 | // Provide DenseMapInfo for MCRegister |
| 95 | template<> struct DenseMapInfo<MCRegister> { |
| 96 | static inline unsigned getEmptyKey() { |
| 97 | return DenseMapInfo<unsigned>::getEmptyKey(); |
| 98 | } |
| 99 | static inline unsigned getTombstoneKey() { |
| 100 | return DenseMapInfo<unsigned>::getTombstoneKey(); |
| 101 | } |
| 102 | static unsigned getHashValue(const MCRegister &Val) { |
| 103 | return DenseMapInfo<unsigned>::getHashValue(Val.id()); |
| 104 | } |
| 105 | static bool isEqual(const MCRegister &LHS, const MCRegister &RHS) { |
| 106 | return DenseMapInfo<unsigned>::isEqual(LHS.id(), RHS.id()); |
| 107 | } |
| 108 | }; |
| 109 | |
| 110 | inline hash_code hash_value(const MCRegister &Reg) { |
| 111 | return hash_value(Reg.id()); |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | #endif // LLVM_MC_MCREGISTER_H |
| 1 | //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===// |
| 2 | // |
| 3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| 4 | // See https://llvm.org/LICENSE.txt for license information. |
| 5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| 6 | // |
| 7 | //===----------------------------------------------------------------------===// |
| 8 | // |
| 9 | // This file contains the declaration of the MachineInstr class, which is the |
| 10 | // basic representation for all target dependent machine instructions used by |
| 11 | // the back end. |
| 12 | // |
| 13 | //===----------------------------------------------------------------------===// |
| 14 | |
| 15 | #ifndef LLVM_CODEGEN_MACHINEINSTR_H |
| 16 | #define LLVM_CODEGEN_MACHINEINSTR_H |
| 17 | |
| 18 | #include "llvm/ADT/DenseMapInfo.h" |
| 19 | #include "llvm/ADT/PointerSumType.h" |
| 20 | #include "llvm/ADT/SmallSet.h" |
| 21 | #include "llvm/ADT/ilist.h" |
| 22 | #include "llvm/ADT/ilist_node.h" |
| 23 | #include "llvm/ADT/iterator_range.h" |
| 24 | #include "llvm/CodeGen/MachineMemOperand.h" |
| 25 | #include "llvm/CodeGen/MachineOperand.h" |
| 26 | #include "llvm/CodeGen/TargetOpcodes.h" |
| 27 | #include "llvm/IR/DebugLoc.h" |
| 28 | #include "llvm/IR/InlineAsm.h" |
| 29 | #include "llvm/IR/PseudoProbe.h" |
| 30 | #include "llvm/MC/MCInstrDesc.h" |
| 31 | #include "llvm/MC/MCSymbol.h" |
| 32 | #include "llvm/Support/ArrayRecycler.h" |
| 33 | #include "llvm/Support/TrailingObjects.h" |
| 34 | #include <algorithm> |
| 35 | #include <cassert> |
| 36 | #include <cstdint> |
| 37 | #include <utility> |
| 38 | |
| 39 | namespace llvm { |
| 40 | |
| 41 | class AAResults; |
| 42 | template <typename T> class ArrayRef; |
| 43 | class DIExpression; |
| 44 | class DILocalVariable; |
| 45 | class MachineBasicBlock; |
| 46 | class MachineFunction; |
| 47 | class MachineRegisterInfo; |
| 48 | class ModuleSlotTracker; |
| 49 | class raw_ostream; |
| 50 | template <typename T> class SmallVectorImpl; |
| 51 | class SmallBitVector; |
| 52 | class StringRef; |
| 53 | class TargetInstrInfo; |
| 54 | class TargetRegisterClass; |
| 55 | class TargetRegisterInfo; |
| 56 | |
| 57 | //===----------------------------------------------------------------------===// |
| 58 | /// Representation of each machine instruction. |
| 59 | /// |
| 60 | /// This class isn't a POD type, but it must have a trivial destructor. When a |
| 61 | /// MachineFunction is deleted, all the contained MachineInstrs are deallocated |
| 62 | /// without having their destructor called. |
| 63 | /// |
| 64 | class MachineInstr |
| 65 | : public ilist_node_with_parent<MachineInstr, MachineBasicBlock, |
| 66 | ilist_sentinel_tracking<true>> { |
| 67 | public: |
| 68 | using mmo_iterator = ArrayRef<MachineMemOperand *>::iterator; |
| 69 | |
| 70 | /// Flags to specify different kinds of comments to output in |
| 71 | /// assembly code. These flags carry semantic information not |
| 72 | /// otherwise easily derivable from the IR text. |
| 73 | /// |
| 74 | enum CommentFlag { |
| 75 | ReloadReuse = 0x1, // higher bits are reserved for target dep comments. |
| 76 | NoSchedComment = 0x2, |
| 77 | TAsmComments = 0x4 // Target Asm comments should start from this value. |
| 78 | }; |
| 79 | |
| 80 | enum MIFlag { |
| 81 | NoFlags = 0, |
| 82 | FrameSetup = 1 << 0, // Instruction is used as a part of |
| 83 | // function frame setup code. |
| 84 | FrameDestroy = 1 << 1, // Instruction is used as a part of |
| 85 | // function frame destruction code. |
| 86 | BundledPred = 1 << 2, // Instruction has bundled predecessors. |
| 87 | BundledSucc = 1 << 3, // Instruction has bundled successors. |
| 88 | FmNoNans = 1 << 4, // Instruction does not support Fast |
| 89 | // math nan values. |
| 90 | FmNoInfs = 1 << 5, // Instruction does not support Fast |
| 91 | // math infinity values. |
| 92 | FmNsz = 1 << 6, // Instruction is not required to retain |
| 93 | // signed zero values. |
| 94 | FmArcp = 1 << 7, // Instruction supports Fast math |
| 95 | // reciprocal approximations. |
| 96 | FmContract = 1 << 8, // Instruction supports Fast math |
| 97 | // contraction operations like fma. |
| 98 | FmAfn = 1 << 9, // Instruction may map to Fast math |
| 99 | // instrinsic approximation. |
| 100 | FmReassoc = 1 << 10, // Instruction supports Fast math |
| 101 | // reassociation of operand order. |
| 102 | NoUWrap = 1 << 11, // Instruction supports binary operator |
| 103 | // no unsigned wrap. |
| 104 | NoSWrap = 1 << 12, // Instruction supports binary operator |
| 105 | // no signed wrap. |
| 106 | IsExact = 1 << 13, // Instruction supports division is |
| 107 | // known to be exact. |
| 108 | NoFPExcept = 1 << 14, // Instruction does not raise |
| 109 | // floatint-point exceptions. |
| 110 | NoMerge = 1 << 15, // Passes that drop source location info |
| 111 | // (e.g. branch folding) should skip |
| 112 | // this instruction. |
| 113 | }; |
| 114 | |
| 115 | private: |
| 116 | const MCInstrDesc *MCID; // Instruction descriptor. |
| 117 | MachineBasicBlock *Parent = nullptr; // Pointer to the owning basic block. |
| 118 | |
| 119 | // Operands are allocated by an ArrayRecycler. |
| 120 | MachineOperand *Operands = nullptr; // Pointer to the first operand. |
| 121 | unsigned NumOperands = 0; // Number of operands on instruction. |
| 122 | |
| 123 | uint16_t Flags = 0; // Various bits of additional |
| 124 | // information about machine |
| 125 | // instruction. |
| 126 | |
| 127 | uint8_t AsmPrinterFlags = 0; // Various bits of information used by |
| 128 | // the AsmPrinter to emit helpful |
| 129 | // comments. This is *not* semantic |
| 130 | // information. Do not use this for |
| 131 | // anything other than to convey comment |
| 132 | // information to AsmPrinter. |
| 133 | |
| 134 | // OperandCapacity has uint8_t size, so it should be next to AsmPrinterFlags |
| 135 | // to properly pack. |
| 136 | using OperandCapacity = ArrayRecycler<MachineOperand>::Capacity; |
| 137 | OperandCapacity CapOperands; // Capacity of the Operands array. |
| 138 | |
| 139 | /// Internal implementation detail class that provides out-of-line storage for |
| 140 | /// extra info used by the machine instruction when this info cannot be stored |
| 141 | /// in-line within the instruction itself. |
| 142 | /// |
| 143 | /// This has to be defined eagerly due to the implementation constraints of |
| 144 | /// `PointerSumType` where it is used. |
| 145 | class ExtraInfo final |
| 146 | : TrailingObjects<ExtraInfo, MachineMemOperand *, MCSymbol *, MDNode *> { |
| 147 | public: |
| 148 | static ExtraInfo *create(BumpPtrAllocator &Allocator, |
| 149 | ArrayRef<MachineMemOperand *> MMOs, |
| 150 | MCSymbol *PreInstrSymbol = nullptr, |
| 151 | MCSymbol *PostInstrSymbol = nullptr, |
| 152 | MDNode *HeapAllocMarker = nullptr) { |
| 153 | bool HasPreInstrSymbol = PreInstrSymbol != nullptr; |
| 154 | bool HasPostInstrSymbol = PostInstrSymbol != nullptr; |
| 155 | bool HasHeapAllocMarker = HeapAllocMarker != nullptr; |
| 156 | auto *Result = new (Allocator.Allocate( |
| 157 | totalSizeToAlloc<MachineMemOperand *, MCSymbol *, MDNode *>( |
| 158 | MMOs.size(), HasPreInstrSymbol + HasPostInstrSymbol, |
| 159 | HasHeapAllocMarker), |
| 160 | alignof(ExtraInfo))) |
| 161 | ExtraInfo(MMOs.size(), HasPreInstrSymbol, HasPostInstrSymbol, |
| 162 | HasHeapAllocMarker); |
| 163 | |
| 164 | // Copy the actual data into the trailing objects. |
| 165 | std::copy(MMOs.begin(), MMOs.end(), |
| 166 | Result->getTrailingObjects<MachineMemOperand *>()); |
| 167 | |
| 168 | if (HasPreInstrSymbol) |
| 169 | Result->getTrailingObjects<MCSymbol *>()[0] = PreInstrSymbol; |
| 170 | if (HasPostInstrSymbol) |
| 171 | Result->getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] = |
| 172 | PostInstrSymbol; |
| 173 | if (HasHeapAllocMarker) |
| 174 | Result->getTrailingObjects<MDNode *>()[0] = HeapAllocMarker; |
| 175 | |
| 176 | return Result; |
| 177 | } |
| 178 | |
| 179 | ArrayRef<MachineMemOperand *> getMMOs() const { |
| 180 | return makeArrayRef(getTrailingObjects<MachineMemOperand *>(), NumMMOs); |
| 181 | } |
| 182 | |
| 183 | MCSymbol *getPreInstrSymbol() const { |
| 184 | return HasPreInstrSymbol ? getTrailingObjects<MCSymbol *>()[0] : nullptr; |
| 185 | } |
| 186 | |
| 187 | MCSymbol *getPostInstrSymbol() const { |
| 188 | return HasPostInstrSymbol |
| 189 | ? getTrailingObjects<MCSymbol *>()[HasPreInstrSymbol] |
| 190 | : nullptr; |
| 191 | } |
| 192 | |
| 193 | MDNode *getHeapAllocMarker() const { |
| 194 | return HasHeapAllocMarker ? getTrailingObjects<MDNode *>()[0] : nullptr; |
| 195 | } |
| 196 | |
| 197 | private: |
| 198 | friend TrailingObjects; |
| 199 | |
| 200 | // Description of the extra info, used to interpret the actual optional |
| 201 | // data appended. |
| 202 | // |
| 203 | // Note that this is not terribly space optimized. This leaves a great deal |
| 204 | // of flexibility to fit more in here later. |
| 205 | const int NumMMOs; |
| 206 | const bool HasPreInstrSymbol; |
| 207 | const bool HasPostInstrSymbol; |
| 208 | const bool HasHeapAllocMarker; |
| 209 | |
| 210 | // Implement the `TrailingObjects` internal API. |
| 211 | size_t numTrailingObjects(OverloadToken<MachineMemOperand *>) const { |
| 212 | return NumMMOs; |
| 213 | } |
| 214 | size_t numTrailingObjects(OverloadToken<MCSymbol *>) const { |
| 215 | return HasPreInstrSymbol + HasPostInstrSymbol; |
| 216 | } |
| 217 | size_t numTrailingObjects(OverloadToken<MDNode *>) const { |
| 218 | return HasHeapAllocMarker; |
| 219 | } |
| 220 | |
| 221 | // Just a boring constructor to allow us to initialize the sizes. Always use |
| 222 | // the `create` routine above. |
| 223 | ExtraInfo(int NumMMOs, bool HasPreInstrSymbol, bool HasPostInstrSymbol, |
| 224 | bool HasHeapAllocMarker) |
| 225 | : NumMMOs(NumMMOs), HasPreInstrSymbol(HasPreInstrSymbol), |
| 226 | HasPostInstrSymbol(HasPostInstrSymbol), |
| 227 | HasHeapAllocMarker(HasHeapAllocMarker) {} |
| 228 | }; |
| 229 | |
| 230 | /// Enumeration of the kinds of inline extra info available. It is important |
| 231 | /// that the `MachineMemOperand` inline kind has a tag value of zero to make |
| 232 | /// it accessible as an `ArrayRef`. |
| 233 | enum ExtraInfoInlineKinds { |
| 234 | EIIK_MMO = 0, |
| 235 | EIIK_PreInstrSymbol, |
| 236 | EIIK_PostInstrSymbol, |
| 237 | EIIK_OutOfLine |
| 238 | }; |
| 239 | |
| 240 | // We store extra information about the instruction here. The common case is |
| 241 | // expected to be nothing or a single pointer (typically a MMO or a symbol). |
| 242 | // We work to optimize this common case by storing it inline here rather than |
| 243 | // requiring a separate allocation, but we fall back to an allocation when |
| 244 | // multiple pointers are needed. |
| 245 | PointerSumType<ExtraInfoInlineKinds, |
| 246 | PointerSumTypeMember<EIIK_MMO, MachineMemOperand *>, |
| 247 | PointerSumTypeMember<EIIK_PreInstrSymbol, MCSymbol *>, |
| 248 | PointerSumTypeMember<EIIK_PostInstrSymbol, MCSymbol *>, |
| 249 | PointerSumTypeMember<EIIK_OutOfLine, ExtraInfo *>> |
| 250 | Info; |
| 251 | |
| 252 | DebugLoc debugLoc; // Source line information. |
| 253 | |
| 254 | /// Unique instruction number. Used by DBG_INSTR_REFs to refer to the values |
| 255 | /// defined by this instruction. |
| 256 | unsigned DebugInstrNum; |
| 257 | |
| 258 | // Intrusive list support |
| 259 | friend struct ilist_traits<MachineInstr>; |
| 260 | friend struct ilist_callback_traits<MachineBasicBlock>; |
| 261 | void setParent(MachineBasicBlock *P) { Parent = P; } |
| 262 | |
| 263 | /// This constructor creates a copy of the given |
| 264 | /// MachineInstr in the given MachineFunction. |
| 265 | MachineInstr(MachineFunction &, const MachineInstr &); |
| 266 | |
| 267 | /// This constructor create a MachineInstr and add the implicit operands. |
| 268 | /// It reserves space for number of operands specified by |
| 269 | /// MCInstrDesc. An explicit DebugLoc is supplied. |
| 270 | MachineInstr(MachineFunction &, const MCInstrDesc &tid, DebugLoc dl, |
| 271 | bool NoImp = false); |
| 272 | |
| 273 | // MachineInstrs are pool-allocated and owned by MachineFunction. |
| 274 | friend class MachineFunction; |
| 275 | |
| 276 | void |
| 277 | dumprImpl(const MachineRegisterInfo &MRI, unsigned Depth, unsigned MaxDepth, |
| 278 | SmallPtrSetImpl<const MachineInstr *> &AlreadySeenInstrs) const; |
| 279 | |
| 280 | public: |
| 281 | MachineInstr(const MachineInstr &) = delete; |
| 282 | MachineInstr &operator=(const MachineInstr &) = delete; |
| 283 | // Use MachineFunction::DeleteMachineInstr() instead. |
| 284 | ~MachineInstr() = delete; |
| 285 | |
| 286 | const MachineBasicBlock* getParent() const { return Parent; } |
| 287 | MachineBasicBlock* getParent() { return Parent; } |
| 288 | |
| 289 | /// Move the instruction before \p MovePos. |
| 290 | void moveBefore(MachineInstr *MovePos); |
| 291 | |
| 292 | /// Return the function that contains the basic block that this instruction |
| 293 | /// belongs to. |
| 294 | /// |
| 295 | /// Note: this is undefined behaviour if the instruction does not have a |
| 296 | /// parent. |
| 297 | const MachineFunction *getMF() const; |
| 298 | MachineFunction *getMF() { |
| 299 | return const_cast<MachineFunction *>( |
| 300 | static_cast<const MachineInstr *>(this)->getMF()); |
| 301 | } |
| 302 | |
| 303 | /// Return the asm printer flags bitvector. |
| 304 | uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; } |
| 305 | |
| 306 | /// Clear the AsmPrinter bitvector. |
| 307 | void clearAsmPrinterFlags() { AsmPrinterFlags = 0; } |
| 308 | |
| 309 | /// Return whether an AsmPrinter flag is set. |
| 310 | bool getAsmPrinterFlag(CommentFlag Flag) const { |
| 311 | return AsmPrinterFlags & Flag; |
| 312 | } |
| 313 | |
| 314 | /// Set a flag for the AsmPrinter. |
| 315 | void setAsmPrinterFlag(uint8_t Flag) { |
| 316 | AsmPrinterFlags |= Flag; |
| 317 | } |
| 318 | |
| 319 | /// Clear specific AsmPrinter flags. |
| 320 | void clearAsmPrinterFlag(CommentFlag Flag) { |
| 321 | AsmPrinterFlags &= ~Flag; |
| 322 | } |
| 323 | |
| 324 | /// Return the MI flags bitvector. |
| 325 | uint16_t getFlags() const { |
| 326 | return Flags; |
| 327 | } |
| 328 | |
| 329 | /// Return whether an MI flag is set. |
| 330 | bool getFlag(MIFlag Flag) const { |
| 331 | return Flags & Flag; |
| 332 | } |
| 333 | |
| 334 | /// Set a MI flag. |
| 335 | void setFlag(MIFlag Flag) { |
| 336 | Flags |= (uint16_t)Flag; |
| 337 | } |
| 338 | |
| 339 | void setFlags(unsigned flags) { |
| 340 | // Filter out the automatically maintained flags. |
| 341 | unsigned Mask = BundledPred | BundledSucc; |
| 342 | Flags = (Flags & Mask) | (flags & ~Mask); |
| 343 | } |
| 344 | |
| 345 | /// clearFlag - Clear a MI flag. |
| 346 | void clearFlag(MIFlag Flag) { |
| 347 | Flags &= ~((uint16_t)Flag); |
| 348 | } |
| 349 | |
| 350 | /// Return true if MI is in a bundle (but not the first MI in a bundle). |
| 351 | /// |
| 352 | /// A bundle looks like this before it's finalized: |
| 353 | /// ---------------- |
| 354 | /// | MI | |
| 355 | /// ---------------- |
| 356 | /// | |
| 357 | /// ---------------- |
| 358 | /// | MI * | |
| 359 | /// ---------------- |
| 360 | /// | |
| 361 | /// ---------------- |
| 362 | /// | MI * | |
| 363 | /// ---------------- |
| 364 | /// In this case, the first MI starts a bundle but is not inside a bundle, the |
| 365 | /// next 2 MIs are considered "inside" the bundle. |
| 366 | /// |
| 367 | /// After a bundle is finalized, it looks like this: |
| 368 | /// ---------------- |
| 369 | /// | Bundle | |
| 370 | /// ---------------- |
| 371 | /// | |
| 372 | /// ---------------- |
| 373 | /// | MI * | |
| 374 | /// ---------------- |
| 375 | /// | |
| 376 | /// ---------------- |
| 377 | /// | MI * | |
| 378 | /// ---------------- |
| 379 | /// | |
| 380 | /// ---------------- |
| 381 | /// | MI * | |
| 382 | /// ---------------- |
| 383 | /// The first instruction has the special opcode "BUNDLE". It's not "inside" |
| 384 | /// a bundle, but the next three MIs are. |
| 385 | bool isInsideBundle() const { |
| 386 | return getFlag(BundledPred); |
| 387 | } |
| 388 | |
| 389 | /// Return true if this instruction part of a bundle. This is true |
| 390 | /// if either itself or its following instruction is marked "InsideBundle". |
| 391 | bool isBundled() const { |
| 392 | return isBundledWithPred() || isBundledWithSucc(); |
| 393 | } |
| 394 | |
| 395 | /// Return true if this instruction is part of a bundle, and it is not the |
| 396 | /// first instruction in the bundle. |
| 397 | bool isBundledWithPred() const { return getFlag(BundledPred); } |
| 398 | |
| 399 | /// Return true if this instruction is part of a bundle, and it is not the |
| 400 | /// last instruction in the bundle. |
| 401 | bool isBundledWithSucc() const { return getFlag(BundledSucc); } |
| 402 | |
| 403 | /// Bundle this instruction with its predecessor. This can be an unbundled |
| 404 | /// instruction, or it can be the first instruction in a bundle. |
| 405 | void bundleWithPred(); |
| 406 | |
| 407 | /// Bundle this instruction with its successor. This can be an unbundled |
| 408 | /// instruction, or it can be the last instruction in a bundle. |
| 409 | void bundleWithSucc(); |
| 410 | |
| 411 | /// Break bundle above this instruction. |
| 412 | void unbundleFromPred(); |
| 413 | |
| 414 | /// Break bundle below this instruction. |
| 415 | void unbundleFromSucc(); |
| 416 | |
| 417 | /// Returns the debug location id of this MachineInstr. |
| 418 | const DebugLoc &getDebugLoc() const { return debugLoc; } |
| 419 | |
| 420 | /// Return the operand containing the offset to be used if this DBG_VALUE |
| 421 | /// instruction is indirect; will be an invalid register if this value is |
| 422 | /// not indirect, and an immediate with value 0 otherwise. |
| 423 | const MachineOperand &getDebugOffset() const { |
| 424 | assert(isNonListDebugValue() && "not a DBG_VALUE")(static_cast <bool> (isNonListDebugValue() && "not a DBG_VALUE" ) ? void (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 424, __extension__ __PRETTY_FUNCTION__)); |
| 425 | return getOperand(1); |
| 426 | } |
| 427 | MachineOperand &getDebugOffset() { |
| 428 | assert(isNonListDebugValue() && "not a DBG_VALUE")(static_cast <bool> (isNonListDebugValue() && "not a DBG_VALUE" ) ? void (0) : __assert_fail ("isNonListDebugValue() && \"not a DBG_VALUE\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 428, __extension__ __PRETTY_FUNCTION__)); |
| 429 | return getOperand(1); |
| 430 | } |
| 431 | |
| 432 | /// Return the operand for the debug variable referenced by |
| 433 | /// this DBG_VALUE instruction. |
| 434 | const MachineOperand &getDebugVariableOp() const; |
| 435 | MachineOperand &getDebugVariableOp(); |
| 436 | |
| 437 | /// Return the debug variable referenced by |
| 438 | /// this DBG_VALUE instruction. |
| 439 | const DILocalVariable *getDebugVariable() const; |
| 440 | |
| 441 | /// Return the operand for the complex address expression referenced by |
| 442 | /// this DBG_VALUE instruction. |
| 443 | const MachineOperand &getDebugExpressionOp() const; |
| 444 | MachineOperand &getDebugExpressionOp(); |
| 445 | |
| 446 | /// Return the complex address expression referenced by |
| 447 | /// this DBG_VALUE instruction. |
| 448 | const DIExpression *getDebugExpression() const; |
| 449 | |
| 450 | /// Return the debug label referenced by |
| 451 | /// this DBG_LABEL instruction. |
| 452 | const DILabel *getDebugLabel() const; |
| 453 | |
| 454 | /// Fetch the instruction number of this MachineInstr. If it does not have |
| 455 | /// one already, a new and unique number will be assigned. |
| 456 | unsigned getDebugInstrNum(); |
| 457 | |
| 458 | /// Fetch instruction number of this MachineInstr -- but before it's inserted |
| 459 | /// into \p MF. Needed for transformations that create an instruction but |
| 460 | /// don't immediately insert them. |
| 461 | unsigned getDebugInstrNum(MachineFunction &MF); |
| 462 | |
| 463 | /// Examine the instruction number of this MachineInstr. May be zero if |
| 464 | /// it hasn't been assigned a number yet. |
| 465 | unsigned peekDebugInstrNum() const { return DebugInstrNum; } |
| 466 | |
| 467 | /// Set instruction number of this MachineInstr. Avoid using unless you're |
| 468 | /// deserializing this information. |
| 469 | void setDebugInstrNum(unsigned Num) { DebugInstrNum = Num; } |
| 470 | |
| 471 | /// Drop any variable location debugging information associated with this |
| 472 | /// instruction. Use when an instruction is modified in such a way that it no |
| 473 | /// longer defines the value it used to. Variable locations using that value |
| 474 | /// will be dropped. |
| 475 | void dropDebugNumber() { DebugInstrNum = 0; } |
| 476 | |
| 477 | /// Emit an error referring to the source location of this instruction. |
| 478 | /// This should only be used for inline assembly that is somehow |
| 479 | /// impossible to compile. Other errors should have been handled much |
| 480 | /// earlier. |
| 481 | /// |
| 482 | /// If this method returns, the caller should try to recover from the error. |
| 483 | void emitError(StringRef Msg) const; |
| 484 | |
| 485 | /// Returns the target instruction descriptor of this MachineInstr. |
| 486 | const MCInstrDesc &getDesc() const { return *MCID; } |
| 487 | |
| 488 | /// Returns the opcode of this MachineInstr. |
| 489 | unsigned getOpcode() const { return MCID->Opcode; } |
| 490 | |
| 491 | /// Retuns the total number of operands. |
| 492 | unsigned getNumOperands() const { return NumOperands; } |
| 493 | |
| 494 | /// Returns the total number of operands which are debug locations. |
| 495 | unsigned getNumDebugOperands() const { |
| 496 | return std::distance(debug_operands().begin(), debug_operands().end()); |
| 497 | } |
| 498 | |
| 499 | const MachineOperand& getOperand(unsigned i) const { |
| 500 | assert(i < getNumOperands() && "getOperand() out of range!")(static_cast <bool> (i < getNumOperands() && "getOperand() out of range!") ? void (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 500, __extension__ __PRETTY_FUNCTION__)); |
| 501 | return Operands[i]; |
| 502 | } |
| 503 | MachineOperand& getOperand(unsigned i) { |
| 504 | assert(i < getNumOperands() && "getOperand() out of range!")(static_cast <bool> (i < getNumOperands() && "getOperand() out of range!") ? void (0) : __assert_fail ("i < getNumOperands() && \"getOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 504, __extension__ __PRETTY_FUNCTION__)); |
| 505 | return Operands[i]; |
| 506 | } |
| 507 | |
| 508 | MachineOperand &getDebugOperand(unsigned Index) { |
| 509 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")(static_cast <bool> (Index < getNumDebugOperands() && "getDebugOperand() out of range!") ? void (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 509, __extension__ __PRETTY_FUNCTION__)); |
| 510 | return *(debug_operands().begin() + Index); |
| 511 | } |
| 512 | const MachineOperand &getDebugOperand(unsigned Index) const { |
| 513 | assert(Index < getNumDebugOperands() && "getDebugOperand() out of range!")(static_cast <bool> (Index < getNumDebugOperands() && "getDebugOperand() out of range!") ? void (0) : __assert_fail ("Index < getNumDebugOperands() && \"getDebugOperand() out of range!\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 513, __extension__ __PRETTY_FUNCTION__)); |
| 514 | return *(debug_operands().begin() + Index); |
| 515 | } |
| 516 | |
| 517 | SmallSet<Register, 4> getUsedDebugRegs() const { |
| 518 | assert(isDebugValue() && "not a DBG_VALUE*")(static_cast <bool> (isDebugValue() && "not a DBG_VALUE*" ) ? void (0) : __assert_fail ("isDebugValue() && \"not a DBG_VALUE*\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 518, __extension__ __PRETTY_FUNCTION__)); |
| 519 | SmallSet<Register, 4> UsedRegs; |
| 520 | for (auto MO : debug_operands()) |
| 521 | if (MO.isReg() && MO.getReg()) |
| 522 | UsedRegs.insert(MO.getReg()); |
| 523 | return UsedRegs; |
| 524 | } |
| 525 | |
| 526 | /// Returns whether this debug value has at least one debug operand with the |
| 527 | /// register \p Reg. |
| 528 | bool hasDebugOperandForReg(Register Reg) const { |
| 529 | return any_of(debug_operands(), [Reg](const MachineOperand &Op) { |
| 530 | return Op.isReg() && Op.getReg() == Reg; |
| 531 | }); |
| 532 | } |
| 533 | |
| 534 | /// Returns a range of all of the operands that correspond to a debug use of |
| 535 | /// \p Reg. |
| 536 | template <typename Operand, typename Instruction> |
| 537 | static iterator_range< |
| 538 | filter_iterator<Operand *, std::function<bool(Operand &Op)>>> |
| 539 | getDebugOperandsForReg(Instruction *MI, Register Reg) { |
| 540 | std::function<bool(Operand & Op)> OpUsesReg( |
| 541 | [Reg](Operand &Op) { return Op.isReg() && Op.getReg() == Reg; }); |
| 542 | return make_filter_range(MI->debug_operands(), OpUsesReg); |
| 543 | } |
| 544 | iterator_range<filter_iterator<const MachineOperand *, |
| 545 | std::function<bool(const MachineOperand &Op)>>> |
| 546 | getDebugOperandsForReg(Register Reg) const { |
| 547 | return MachineInstr::getDebugOperandsForReg<const MachineOperand, |
| 548 | const MachineInstr>(this, Reg); |
| 549 | } |
| 550 | iterator_range<filter_iterator<MachineOperand *, |
| 551 | std::function<bool(MachineOperand &Op)>>> |
| 552 | getDebugOperandsForReg(Register Reg) { |
| 553 | return MachineInstr::getDebugOperandsForReg<MachineOperand, MachineInstr>( |
| 554 | this, Reg); |
| 555 | } |
| 556 | |
| 557 | bool isDebugOperand(const MachineOperand *Op) const { |
| 558 | return Op >= adl_begin(debug_operands()) && Op <= adl_end(debug_operands()); |
| 559 | } |
| 560 | |
| 561 | unsigned getDebugOperandIndex(const MachineOperand *Op) const { |
| 562 | assert(isDebugOperand(Op) && "Expected a debug operand.")(static_cast <bool> (isDebugOperand(Op) && "Expected a debug operand." ) ? void (0) : __assert_fail ("isDebugOperand(Op) && \"Expected a debug operand.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 562, __extension__ __PRETTY_FUNCTION__)); |
| 563 | return std::distance(adl_begin(debug_operands()), Op); |
| 564 | } |
| 565 | |
| 566 | /// Returns the total number of definitions. |
| 567 | unsigned getNumDefs() const { |
| 568 | return getNumExplicitDefs() + MCID->getNumImplicitDefs(); |
| 569 | } |
| 570 | |
| 571 | /// Returns true if the instruction has implicit definition. |
| 572 | bool hasImplicitDef() const { |
| 573 | for (unsigned I = getNumExplicitOperands(), E = getNumOperands(); |
| 574 | I != E; ++I) { |
| 575 | const MachineOperand &MO = getOperand(I); |
| 576 | if (MO.isDef() && MO.isImplicit()) |
| 577 | return true; |
| 578 | } |
| 579 | return false; |
| 580 | } |
| 581 | |
| 582 | /// Returns the implicit operands number. |
| 583 | unsigned getNumImplicitOperands() const { |
| 584 | return getNumOperands() - getNumExplicitOperands(); |
| 585 | } |
| 586 | |
| 587 | /// Return true if operand \p OpIdx is a subregister index. |
| 588 | bool isOperandSubregIdx(unsigned OpIdx) const { |
| 589 | assert(getOperand(OpIdx).getType() == MachineOperand::MO_Immediate &&(static_cast <bool> (getOperand(OpIdx).getType() == MachineOperand ::MO_Immediate && "Expected MO_Immediate operand type." ) ? void (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 590, __extension__ __PRETTY_FUNCTION__)) |
| 590 | "Expected MO_Immediate operand type.")(static_cast <bool> (getOperand(OpIdx).getType() == MachineOperand ::MO_Immediate && "Expected MO_Immediate operand type." ) ? void (0) : __assert_fail ("getOperand(OpIdx).getType() == MachineOperand::MO_Immediate && \"Expected MO_Immediate operand type.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 590, __extension__ __PRETTY_FUNCTION__)); |
| 591 | if (isExtractSubreg() && OpIdx == 2) |
| 592 | return true; |
| 593 | if (isInsertSubreg() && OpIdx == 3) |
| 594 | return true; |
| 595 | if (isRegSequence() && OpIdx > 1 && (OpIdx % 2) == 0) |
| 596 | return true; |
| 597 | if (isSubregToReg() && OpIdx == 3) |
| 598 | return true; |
| 599 | return false; |
| 600 | } |
| 601 | |
| 602 | /// Returns the number of non-implicit operands. |
| 603 | unsigned getNumExplicitOperands() const; |
| 604 | |
| 605 | /// Returns the number of non-implicit definitions. |
| 606 | unsigned getNumExplicitDefs() const; |
| 607 | |
| 608 | /// iterator/begin/end - Iterate over all operands of a machine instruction. |
| 609 | using mop_iterator = MachineOperand *; |
| 610 | using const_mop_iterator = const MachineOperand *; |
| 611 | |
| 612 | mop_iterator operands_begin() { return Operands; } |
| 613 | mop_iterator operands_end() { return Operands + NumOperands; } |
| 614 | |
| 615 | const_mop_iterator operands_begin() const { return Operands; } |
| 616 | const_mop_iterator operands_end() const { return Operands + NumOperands; } |
| 617 | |
| 618 | iterator_range<mop_iterator> operands() { |
| 619 | return make_range(operands_begin(), operands_end()); |
| 620 | } |
| 621 | iterator_range<const_mop_iterator> operands() const { |
| 622 | return make_range(operands_begin(), operands_end()); |
| 623 | } |
| 624 | iterator_range<mop_iterator> explicit_operands() { |
| 625 | return make_range(operands_begin(), |
| 626 | operands_begin() + getNumExplicitOperands()); |
| 627 | } |
| 628 | iterator_range<const_mop_iterator> explicit_operands() const { |
| 629 | return make_range(operands_begin(), |
| 630 | operands_begin() + getNumExplicitOperands()); |
| 631 | } |
| 632 | iterator_range<mop_iterator> implicit_operands() { |
| 633 | return make_range(explicit_operands().end(), operands_end()); |
| 634 | } |
| 635 | iterator_range<const_mop_iterator> implicit_operands() const { |
| 636 | return make_range(explicit_operands().end(), operands_end()); |
| 637 | } |
| 638 | /// Returns a range over all operands that are used to determine the variable |
| 639 | /// location for this DBG_VALUE instruction. |
| 640 | iterator_range<mop_iterator> debug_operands() { |
| 641 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 641, __extension__ __PRETTY_FUNCTION__)); |
| 642 | return isDebugValueList() |
| 643 | ? make_range(operands_begin() + 2, operands_end()) |
| 644 | : make_range(operands_begin(), operands_begin() + 1); |
| 645 | } |
| 646 | /// \copydoc debug_operands() |
| 647 | iterator_range<const_mop_iterator> debug_operands() const { |
| 648 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 648, __extension__ __PRETTY_FUNCTION__)); |
| 649 | return isDebugValueList() |
| 650 | ? make_range(operands_begin() + 2, operands_end()) |
| 651 | : make_range(operands_begin(), operands_begin() + 1); |
| 652 | } |
| 653 | /// Returns a range over all explicit operands that are register definitions. |
| 654 | /// Implicit definition are not included! |
| 655 | iterator_range<mop_iterator> defs() { |
| 656 | return make_range(operands_begin(), |
| 657 | operands_begin() + getNumExplicitDefs()); |
| 658 | } |
| 659 | /// \copydoc defs() |
| 660 | iterator_range<const_mop_iterator> defs() const { |
| 661 | return make_range(operands_begin(), |
| 662 | operands_begin() + getNumExplicitDefs()); |
| 663 | } |
| 664 | /// Returns a range that includes all operands that are register uses. |
| 665 | /// This may include unrelated operands which are not register uses. |
| 666 | iterator_range<mop_iterator> uses() { |
| 667 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
| 668 | } |
| 669 | /// \copydoc uses() |
| 670 | iterator_range<const_mop_iterator> uses() const { |
| 671 | return make_range(operands_begin() + getNumExplicitDefs(), operands_end()); |
| 672 | } |
| 673 | iterator_range<mop_iterator> explicit_uses() { |
| 674 | return make_range(operands_begin() + getNumExplicitDefs(), |
| 675 | operands_begin() + getNumExplicitOperands()); |
| 676 | } |
| 677 | iterator_range<const_mop_iterator> explicit_uses() const { |
| 678 | return make_range(operands_begin() + getNumExplicitDefs(), |
| 679 | operands_begin() + getNumExplicitOperands()); |
| 680 | } |
| 681 | |
| 682 | /// Returns the number of the operand iterator \p I points to. |
| 683 | unsigned getOperandNo(const_mop_iterator I) const { |
| 684 | return I - operands_begin(); |
| 685 | } |
| 686 | |
| 687 | /// Access to memory operands of the instruction. If there are none, that does |
| 688 | /// not imply anything about whether the function accesses memory. Instead, |
| 689 | /// the caller must behave conservatively. |
| 690 | ArrayRef<MachineMemOperand *> memoperands() const { |
| 691 | if (!Info) |
| 692 | return {}; |
| 693 | |
| 694 | if (Info.is<EIIK_MMO>()) |
| 695 | return makeArrayRef(Info.getAddrOfZeroTagPointer(), 1); |
| 696 | |
| 697 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 698 | return EI->getMMOs(); |
| 699 | |
| 700 | return {}; |
| 701 | } |
| 702 | |
| 703 | /// Access to memory operands of the instruction. |
| 704 | /// |
| 705 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
| 706 | /// anything about whether the function accesses memory. Instead, the caller |
| 707 | /// must behave conservatively. |
| 708 | mmo_iterator memoperands_begin() const { return memoperands().begin(); } |
| 709 | |
| 710 | /// Access to memory operands of the instruction. |
| 711 | /// |
| 712 | /// If `memoperands_begin() == memoperands_end()`, that does not imply |
| 713 | /// anything about whether the function accesses memory. Instead, the caller |
| 714 | /// must behave conservatively. |
| 715 | mmo_iterator memoperands_end() const { return memoperands().end(); } |
| 716 | |
| 717 | /// Return true if we don't have any memory operands which described the |
| 718 | /// memory access done by this instruction. If this is true, calling code |
| 719 | /// must be conservative. |
| 720 | bool memoperands_empty() const { return memoperands().empty(); } |
| 721 | |
| 722 | /// Return true if this instruction has exactly one MachineMemOperand. |
| 723 | bool hasOneMemOperand() const { return memoperands().size() == 1; } |
| 724 | |
| 725 | /// Return the number of memory operands. |
| 726 | unsigned getNumMemOperands() const { return memoperands().size(); } |
| 727 | |
| 728 | /// Helper to extract a pre-instruction symbol if one has been added. |
| 729 | MCSymbol *getPreInstrSymbol() const { |
| 730 | if (!Info) |
| 731 | return nullptr; |
| 732 | if (MCSymbol *S = Info.get<EIIK_PreInstrSymbol>()) |
| 733 | return S; |
| 734 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 735 | return EI->getPreInstrSymbol(); |
| 736 | |
| 737 | return nullptr; |
| 738 | } |
| 739 | |
| 740 | /// Helper to extract a post-instruction symbol if one has been added. |
| 741 | MCSymbol *getPostInstrSymbol() const { |
| 742 | if (!Info) |
| 743 | return nullptr; |
| 744 | if (MCSymbol *S = Info.get<EIIK_PostInstrSymbol>()) |
| 745 | return S; |
| 746 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 747 | return EI->getPostInstrSymbol(); |
| 748 | |
| 749 | return nullptr; |
| 750 | } |
| 751 | |
| 752 | /// Helper to extract a heap alloc marker if one has been added. |
| 753 | MDNode *getHeapAllocMarker() const { |
| 754 | if (!Info) |
| 755 | return nullptr; |
| 756 | if (ExtraInfo *EI = Info.get<EIIK_OutOfLine>()) |
| 757 | return EI->getHeapAllocMarker(); |
| 758 | |
| 759 | return nullptr; |
| 760 | } |
| 761 | |
| 762 | /// API for querying MachineInstr properties. They are the same as MCInstrDesc |
| 763 | /// queries but they are bundle aware. |
| 764 | |
| 765 | enum QueryType { |
| 766 | IgnoreBundle, // Ignore bundles |
| 767 | AnyInBundle, // Return true if any instruction in bundle has property |
| 768 | AllInBundle // Return true if all instructions in bundle have property |
| 769 | }; |
| 770 | |
| 771 | /// Return true if the instruction (or in the case of a bundle, |
| 772 | /// the instructions inside the bundle) has the specified property. |
| 773 | /// The first argument is the property being queried. |
| 774 | /// The second argument indicates whether the query should look inside |
| 775 | /// instruction bundles. |
| 776 | bool hasProperty(unsigned MCFlag, QueryType Type = AnyInBundle) const { |
| 777 | assert(MCFlag < 64 &&(static_cast <bool> (MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? void (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 778, __extension__ __PRETTY_FUNCTION__)) |
| 778 | "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.")(static_cast <bool> (MCFlag < 64 && "MCFlag out of range for bit mask in getFlags/hasPropertyInBundle." ) ? void (0) : __assert_fail ("MCFlag < 64 && \"MCFlag out of range for bit mask in getFlags/hasPropertyInBundle.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 778, __extension__ __PRETTY_FUNCTION__)); |
| 779 | // Inline the fast path for unbundled or bundle-internal instructions. |
| 780 | if (Type == IgnoreBundle || !isBundled() || isBundledWithPred()) |
| 781 | return getDesc().getFlags() & (1ULL << MCFlag); |
| 782 | |
| 783 | // If this is the first instruction in a bundle, take the slow path. |
| 784 | return hasPropertyInBundle(1ULL << MCFlag, Type); |
| 785 | } |
| 786 | |
| 787 | /// Return true if this is an instruction that should go through the usual |
| 788 | /// legalization steps. |
| 789 | bool isPreISelOpcode(QueryType Type = IgnoreBundle) const { |
| 790 | return hasProperty(MCID::PreISelOpcode, Type); |
| 791 | } |
| 792 | |
| 793 | /// Return true if this instruction can have a variable number of operands. |
| 794 | /// In this case, the variable operands will be after the normal |
| 795 | /// operands but before the implicit definitions and uses (if any are |
| 796 | /// present). |
| 797 | bool isVariadic(QueryType Type = IgnoreBundle) const { |
| 798 | return hasProperty(MCID::Variadic, Type); |
| 799 | } |
| 800 | |
| 801 | /// Set if this instruction has an optional definition, e.g. |
| 802 | /// ARM instructions which can set condition code if 's' bit is set. |
| 803 | bool hasOptionalDef(QueryType Type = IgnoreBundle) const { |
| 804 | return hasProperty(MCID::HasOptionalDef, Type); |
| 805 | } |
| 806 | |
| 807 | /// Return true if this is a pseudo instruction that doesn't |
| 808 | /// correspond to a real machine instruction. |
| 809 | bool isPseudo(QueryType Type = IgnoreBundle) const { |
| 810 | return hasProperty(MCID::Pseudo, Type); |
| 811 | } |
| 812 | |
| 813 | bool isReturn(QueryType Type = AnyInBundle) const { |
| 814 | return hasProperty(MCID::Return, Type); |
| 815 | } |
| 816 | |
| 817 | /// Return true if this is an instruction that marks the end of an EH scope, |
| 818 | /// i.e., a catchpad or a cleanuppad instruction. |
| 819 | bool isEHScopeReturn(QueryType Type = AnyInBundle) const { |
| 820 | return hasProperty(MCID::EHScopeReturn, Type); |
| 821 | } |
| 822 | |
| 823 | bool isCall(QueryType Type = AnyInBundle) const { |
| 824 | return hasProperty(MCID::Call, Type); |
| 825 | } |
| 826 | |
| 827 | /// Return true if this is a call instruction that may have an associated |
| 828 | /// call site entry in the debug info. |
| 829 | bool isCandidateForCallSiteEntry(QueryType Type = IgnoreBundle) const; |
| 830 | /// Return true if copying, moving, or erasing this instruction requires |
| 831 | /// updating Call Site Info (see \ref copyCallSiteInfo, \ref moveCallSiteInfo, |
| 832 | /// \ref eraseCallSiteInfo). |
| 833 | bool shouldUpdateCallSiteInfo() const; |
| 834 | |
| 835 | /// Returns true if the specified instruction stops control flow |
| 836 | /// from executing the instruction immediately following it. Examples include |
| 837 | /// unconditional branches and return instructions. |
| 838 | bool isBarrier(QueryType Type = AnyInBundle) const { |
| 839 | return hasProperty(MCID::Barrier, Type); |
| 840 | } |
| 841 | |
| 842 | /// Returns true if this instruction part of the terminator for a basic block. |
| 843 | /// Typically this is things like return and branch instructions. |
| 844 | /// |
| 845 | /// Various passes use this to insert code into the bottom of a basic block, |
| 846 | /// but before control flow occurs. |
| 847 | bool isTerminator(QueryType Type = AnyInBundle) const { |
| 848 | return hasProperty(MCID::Terminator, Type); |
| 849 | } |
| 850 | |
| 851 | /// Returns true if this is a conditional, unconditional, or indirect branch. |
| 852 | /// Predicates below can be used to discriminate between |
| 853 | /// these cases, and the TargetInstrInfo::analyzeBranch method can be used to |
| 854 | /// get more information. |
| 855 | bool isBranch(QueryType Type = AnyInBundle) const { |
| 856 | return hasProperty(MCID::Branch, Type); |
| 857 | } |
| 858 | |
| 859 | /// Return true if this is an indirect branch, such as a |
| 860 | /// branch through a register. |
| 861 | bool isIndirectBranch(QueryType Type = AnyInBundle) const { |
| 862 | return hasProperty(MCID::IndirectBranch, Type); |
| 863 | } |
| 864 | |
| 865 | /// Return true if this is a branch which may fall |
| 866 | /// through to the next instruction or may transfer control flow to some other |
| 867 | /// block. The TargetInstrInfo::analyzeBranch method can be used to get more |
| 868 | /// information about this branch. |
| 869 | bool isConditionalBranch(QueryType Type = AnyInBundle) const { |
| 870 | return isBranch(Type) && !isBarrier(Type) && !isIndirectBranch(Type); |
| 871 | } |
| 872 | |
| 873 | /// Return true if this is a branch which always |
| 874 | /// transfers control flow to some other block. The |
| 875 | /// TargetInstrInfo::analyzeBranch method can be used to get more information |
| 876 | /// about this branch. |
| 877 | bool isUnconditionalBranch(QueryType Type = AnyInBundle) const { |
| 878 | return isBranch(Type) && isBarrier(Type) && !isIndirectBranch(Type); |
| 879 | } |
| 880 | |
| 881 | /// Return true if this instruction has a predicate operand that |
| 882 | /// controls execution. It may be set to 'always', or may be set to other |
| 883 | /// values. There are various methods in TargetInstrInfo that can be used to |
| 884 | /// control and modify the predicate in this instruction. |
| 885 | bool isPredicable(QueryType Type = AllInBundle) const { |
| 886 | // If it's a bundle than all bundled instructions must be predicable for this |
| 887 | // to return true. |
| 888 | return hasProperty(MCID::Predicable, Type); |
| 889 | } |
| 890 | |
| 891 | /// Return true if this instruction is a comparison. |
| 892 | bool isCompare(QueryType Type = IgnoreBundle) const { |
| 893 | return hasProperty(MCID::Compare, Type); |
| 894 | } |
| 895 | |
| 896 | /// Return true if this instruction is a move immediate |
| 897 | /// (including conditional moves) instruction. |
| 898 | bool isMoveImmediate(QueryType Type = IgnoreBundle) const { |
| 899 | return hasProperty(MCID::MoveImm, Type); |
| 900 | } |
| 901 | |
| 902 | /// Return true if this instruction is a register move. |
| 903 | /// (including moving values from subreg to reg) |
| 904 | bool isMoveReg(QueryType Type = IgnoreBundle) const { |
| 905 | return hasProperty(MCID::MoveReg, Type); |
| 906 | } |
| 907 | |
| 908 | /// Return true if this instruction is a bitcast instruction. |
| 909 | bool isBitcast(QueryType Type = IgnoreBundle) const { |
| 910 | return hasProperty(MCID::Bitcast, Type); |
| 911 | } |
| 912 | |
| 913 | /// Return true if this instruction is a select instruction. |
| 914 | bool isSelect(QueryType Type = IgnoreBundle) const { |
| 915 | return hasProperty(MCID::Select, Type); |
| 916 | } |
| 917 | |
| 918 | /// Return true if this instruction cannot be safely duplicated. |
| 919 | /// For example, if the instruction has a unique labels attached |
| 920 | /// to it, duplicating it would cause multiple definition errors. |
| 921 | bool isNotDuplicable(QueryType Type = AnyInBundle) const { |
| 922 | return hasProperty(MCID::NotDuplicable, Type); |
| 923 | } |
| 924 | |
| 925 | /// Return true if this instruction is convergent. |
| 926 | /// Convergent instructions can not be made control-dependent on any |
| 927 | /// additional values. |
| 928 | bool isConvergent(QueryType Type = AnyInBundle) const { |
| 929 | if (isInlineAsm()) { |
| 930 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
| 931 | if (ExtraInfo & InlineAsm::Extra_IsConvergent) |
| 932 | return true; |
| 933 | } |
| 934 | return hasProperty(MCID::Convergent, Type); |
| 935 | } |
| 936 | |
| 937 | /// Returns true if the specified instruction has a delay slot |
| 938 | /// which must be filled by the code generator. |
| 939 | bool hasDelaySlot(QueryType Type = AnyInBundle) const { |
| 940 | return hasProperty(MCID::DelaySlot, Type); |
| 941 | } |
| 942 | |
| 943 | /// Return true for instructions that can be folded as |
| 944 | /// memory operands in other instructions. The most common use for this |
| 945 | /// is instructions that are simple loads from memory that don't modify |
| 946 | /// the loaded value in any way, but it can also be used for instructions |
| 947 | /// that can be expressed as constant-pool loads, such as V_SETALLONES |
| 948 | /// on x86, to allow them to be folded when it is beneficial. |
| 949 | /// This should only be set on instructions that return a value in their |
| 950 | /// only virtual register definition. |
| 951 | bool canFoldAsLoad(QueryType Type = IgnoreBundle) const { |
| 952 | return hasProperty(MCID::FoldableAsLoad, Type); |
| 953 | } |
| 954 | |
| 955 | /// Return true if this instruction behaves |
| 956 | /// the same way as the generic REG_SEQUENCE instructions. |
| 957 | /// E.g., on ARM, |
| 958 | /// dX VMOVDRR rY, rZ |
| 959 | /// is equivalent to |
| 960 | /// dX = REG_SEQUENCE rY, ssub_0, rZ, ssub_1. |
| 961 | /// |
| 962 | /// Note that for the optimizers to be able to take advantage of |
| 963 | /// this property, TargetInstrInfo::getRegSequenceLikeInputs has to be |
| 964 | /// override accordingly. |
| 965 | bool isRegSequenceLike(QueryType Type = IgnoreBundle) const { |
| 966 | return hasProperty(MCID::RegSequence, Type); |
| 967 | } |
| 968 | |
| 969 | /// Return true if this instruction behaves |
| 970 | /// the same way as the generic EXTRACT_SUBREG instructions. |
| 971 | /// E.g., on ARM, |
| 972 | /// rX, rY VMOVRRD dZ |
| 973 | /// is equivalent to two EXTRACT_SUBREG: |
| 974 | /// rX = EXTRACT_SUBREG dZ, ssub_0 |
| 975 | /// rY = EXTRACT_SUBREG dZ, ssub_1 |
| 976 | /// |
| 977 | /// Note that for the optimizers to be able to take advantage of |
| 978 | /// this property, TargetInstrInfo::getExtractSubregLikeInputs has to be |
| 979 | /// override accordingly. |
| 980 | bool isExtractSubregLike(QueryType Type = IgnoreBundle) const { |
| 981 | return hasProperty(MCID::ExtractSubreg, Type); |
| 982 | } |
| 983 | |
| 984 | /// Return true if this instruction behaves |
| 985 | /// the same way as the generic INSERT_SUBREG instructions. |
| 986 | /// E.g., on ARM, |
| 987 | /// dX = VSETLNi32 dY, rZ, Imm |
| 988 | /// is equivalent to a INSERT_SUBREG: |
| 989 | /// dX = INSERT_SUBREG dY, rZ, translateImmToSubIdx(Imm) |
| 990 | /// |
| 991 | /// Note that for the optimizers to be able to take advantage of |
| 992 | /// this property, TargetInstrInfo::getInsertSubregLikeInputs has to be |
| 993 | /// override accordingly. |
| 994 | bool isInsertSubregLike(QueryType Type = IgnoreBundle) const { |
| 995 | return hasProperty(MCID::InsertSubreg, Type); |
| 996 | } |
| 997 | |
| 998 | //===--------------------------------------------------------------------===// |
| 999 | // Side Effect Analysis |
| 1000 | //===--------------------------------------------------------------------===// |
| 1001 | |
| 1002 | /// Return true if this instruction could possibly read memory. |
| 1003 | /// Instructions with this flag set are not necessarily simple load |
| 1004 | /// instructions, they may load a value and modify it, for example. |
| 1005 | bool mayLoad(QueryType Type = AnyInBundle) const { |
| 1006 | if (isInlineAsm()) { |
| 1007 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
| 1008 | if (ExtraInfo & InlineAsm::Extra_MayLoad) |
| 1009 | return true; |
| 1010 | } |
| 1011 | return hasProperty(MCID::MayLoad, Type); |
| 1012 | } |
| 1013 | |
| 1014 | /// Return true if this instruction could possibly modify memory. |
| 1015 | /// Instructions with this flag set are not necessarily simple store |
| 1016 | /// instructions, they may store a modified value based on their operands, or |
| 1017 | /// may not actually modify anything, for example. |
| 1018 | bool mayStore(QueryType Type = AnyInBundle) const { |
| 1019 | if (isInlineAsm()) { |
| 1020 | unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm(); |
| 1021 | if (ExtraInfo & InlineAsm::Extra_MayStore) |
| 1022 | return true; |
| 1023 | } |
| 1024 | return hasProperty(MCID::MayStore, Type); |
| 1025 | } |
| 1026 | |
| 1027 | /// Return true if this instruction could possibly read or modify memory. |
| 1028 | bool mayLoadOrStore(QueryType Type = AnyInBundle) const { |
| 1029 | return mayLoad(Type) || mayStore(Type); |
| 1030 | } |
| 1031 | |
| 1032 | /// Return true if this instruction could possibly raise a floating-point |
| 1033 | /// exception. This is the case if the instruction is a floating-point |
| 1034 | /// instruction that can in principle raise an exception, as indicated |
| 1035 | /// by the MCID::MayRaiseFPException property, *and* at the same time, |
| 1036 | /// the instruction is used in a context where we expect floating-point |
| 1037 | /// exceptions are not disabled, as indicated by the NoFPExcept MI flag. |
| 1038 | bool mayRaiseFPException() const { |
| 1039 | return hasProperty(MCID::MayRaiseFPException) && |
| 1040 | !getFlag(MachineInstr::MIFlag::NoFPExcept); |
| 1041 | } |
| 1042 | |
| 1043 | //===--------------------------------------------------------------------===// |
| 1044 | // Flags that indicate whether an instruction can be modified by a method. |
| 1045 | //===--------------------------------------------------------------------===// |
| 1046 | |
| 1047 | /// Return true if this may be a 2- or 3-address |
| 1048 | /// instruction (of the form "X = op Y, Z, ..."), which produces the same |
| 1049 | /// result if Y and Z are exchanged. If this flag is set, then the |
| 1050 | /// TargetInstrInfo::commuteInstruction method may be used to hack on the |
| 1051 | /// instruction. |
| 1052 | /// |
| 1053 | /// Note that this flag may be set on instructions that are only commutable |
| 1054 | /// sometimes. In these cases, the call to commuteInstruction will fail. |
| 1055 | /// Also note that some instructions require non-trivial modification to |
| 1056 | /// commute them. |
| 1057 | bool isCommutable(QueryType Type = IgnoreBundle) const { |
| 1058 | return hasProperty(MCID::Commutable, Type); |
| 1059 | } |
| 1060 | |
| 1061 | /// Return true if this is a 2-address instruction |
| 1062 | /// which can be changed into a 3-address instruction if needed. Doing this |
| 1063 | /// transformation can be profitable in the register allocator, because it |
| 1064 | /// means that the instruction can use a 2-address form if possible, but |
| 1065 | /// degrade into a less efficient form if the source and dest register cannot |
| 1066 | /// be assigned to the same register. For example, this allows the x86 |
| 1067 | /// backend to turn a "shl reg, 3" instruction into an LEA instruction, which |
| 1068 | /// is the same speed as the shift but has bigger code size. |
| 1069 | /// |
| 1070 | /// If this returns true, then the target must implement the |
| 1071 | /// TargetInstrInfo::convertToThreeAddress method for this instruction, which |
| 1072 | /// is allowed to fail if the transformation isn't valid for this specific |
| 1073 | /// instruction (e.g. shl reg, 4 on x86). |
| 1074 | /// |
| 1075 | bool isConvertibleTo3Addr(QueryType Type = IgnoreBundle) const { |
| 1076 | return hasProperty(MCID::ConvertibleTo3Addr, Type); |
| 1077 | } |
| 1078 | |
| 1079 | /// Return true if this instruction requires |
| 1080 | /// custom insertion support when the DAG scheduler is inserting it into a |
| 1081 | /// machine basic block. If this is true for the instruction, it basically |
| 1082 | /// means that it is a pseudo instruction used at SelectionDAG time that is |
| 1083 | /// expanded out into magic code by the target when MachineInstrs are formed. |
| 1084 | /// |
| 1085 | /// If this is true, the TargetLoweringInfo::InsertAtEndOfBasicBlock method |
| 1086 | /// is used to insert this into the MachineBasicBlock. |
| 1087 | bool usesCustomInsertionHook(QueryType Type = IgnoreBundle) const { |
| 1088 | return hasProperty(MCID::UsesCustomInserter, Type); |
| 1089 | } |
| 1090 | |
| 1091 | /// Return true if this instruction requires *adjustment* |
| 1092 | /// after instruction selection by calling a target hook. For example, this |
| 1093 | /// can be used to fill in ARM 's' optional operand depending on whether |
| 1094 | /// the conditional flag register is used. |
| 1095 | bool hasPostISelHook(QueryType Type = IgnoreBundle) const { |
| 1096 | return hasProperty(MCID::HasPostISelHook, Type); |
| 1097 | } |
| 1098 | |
| 1099 | /// Returns true if this instruction is a candidate for remat. |
| 1100 | /// This flag is deprecated, please don't use it anymore. If this |
| 1101 | /// flag is set, the isReallyTriviallyReMaterializable() method is called to |
| 1102 | /// verify the instruction is really rematable. |
| 1103 | bool isRematerializable(QueryType Type = AllInBundle) const { |
| 1104 | // It's only possible to re-mat a bundle if all bundled instructions are |
| 1105 | // re-materializable. |
| 1106 | return hasProperty(MCID::Rematerializable, Type); |
| 1107 | } |
| 1108 | |
| 1109 | /// Returns true if this instruction has the same cost (or less) than a move |
| 1110 | /// instruction. This is useful during certain types of optimizations |
| 1111 | /// (e.g., remat during two-address conversion or machine licm) |
| 1112 | /// where we would like to remat or hoist the instruction, but not if it costs |
| 1113 | /// more than moving the instruction into the appropriate register. Note, we |
| 1114 | /// are not marking copies from and to the same register class with this flag. |
| 1115 | bool isAsCheapAsAMove(QueryType Type = AllInBundle) const { |
| 1116 | // Only returns true for a bundle if all bundled instructions are cheap. |
| 1117 | return hasProperty(MCID::CheapAsAMove, Type); |
| 1118 | } |
| 1119 | |
| 1120 | /// Returns true if this instruction source operands |
| 1121 | /// have special register allocation requirements that are not captured by the |
| 1122 | /// operand register classes. e.g. ARM::STRD's two source registers must be an |
| 1123 | /// even / odd pair, ARM::STM registers have to be in ascending order. |
| 1124 | /// Post-register allocation passes should not attempt to change allocations |
| 1125 | /// for sources of instructions with this flag. |
| 1126 | bool hasExtraSrcRegAllocReq(QueryType Type = AnyInBundle) const { |
| 1127 | return hasProperty(MCID::ExtraSrcRegAllocReq, Type); |
| 1128 | } |
| 1129 | |
| 1130 | /// Returns true if this instruction def operands |
| 1131 | /// have special register allocation requirements that are not captured by the |
| 1132 | /// operand register classes. e.g. ARM::LDRD's two def registers must be an |
| 1133 | /// even / odd pair, ARM::LDM registers have to be in ascending order. |
| 1134 | /// Post-register allocation passes should not attempt to change allocations |
| 1135 | /// for definitions of instructions with this flag. |
| 1136 | bool hasExtraDefRegAllocReq(QueryType Type = AnyInBundle) const { |
| 1137 | return hasProperty(MCID::ExtraDefRegAllocReq, Type); |
| 1138 | } |
| 1139 | |
| 1140 | enum MICheckType { |
| 1141 | CheckDefs, // Check all operands for equality |
| 1142 | CheckKillDead, // Check all operands including kill / dead markers |
| 1143 | IgnoreDefs, // Ignore all definitions |
| 1144 | IgnoreVRegDefs // Ignore virtual register definitions |
| 1145 | }; |
| 1146 | |
| 1147 | /// Return true if this instruction is identical to \p Other. |
| 1148 | /// Two instructions are identical if they have the same opcode and all their |
| 1149 | /// operands are identical (with respect to MachineOperand::isIdenticalTo()). |
| 1150 | /// Note that this means liveness related flags (dead, undef, kill) do not |
| 1151 | /// affect the notion of identical. |
| 1152 | bool isIdenticalTo(const MachineInstr &Other, |
| 1153 | MICheckType Check = CheckDefs) const; |
| 1154 | |
| 1155 | /// Unlink 'this' from the containing basic block, and return it without |
| 1156 | /// deleting it. |
| 1157 | /// |
| 1158 | /// This function can not be used on bundled instructions, use |
| 1159 | /// removeFromBundle() to remove individual instructions from a bundle. |
| 1160 | MachineInstr *removeFromParent(); |
| 1161 | |
| 1162 | /// Unlink this instruction from its basic block and return it without |
| 1163 | /// deleting it. |
| 1164 | /// |
| 1165 | /// If the instruction is part of a bundle, the other instructions in the |
| 1166 | /// bundle remain bundled. |
| 1167 | MachineInstr *removeFromBundle(); |
| 1168 | |
| 1169 | /// Unlink 'this' from the containing basic block and delete it. |
| 1170 | /// |
| 1171 | /// If this instruction is the header of a bundle, the whole bundle is erased. |
| 1172 | /// This function can not be used for instructions inside a bundle, use |
| 1173 | /// eraseFromBundle() to erase individual bundled instructions. |
| 1174 | void eraseFromParent(); |
| 1175 | |
| 1176 | /// Unlink 'this' from the containing basic block and delete it. |
| 1177 | /// |
| 1178 | /// For all definitions mark their uses in DBG_VALUE nodes |
| 1179 | /// as undefined. Otherwise like eraseFromParent(). |
| 1180 | void eraseFromParentAndMarkDBGValuesForRemoval(); |
| 1181 | |
| 1182 | /// Unlink 'this' form its basic block and delete it. |
| 1183 | /// |
| 1184 | /// If the instruction is part of a bundle, the other instructions in the |
| 1185 | /// bundle remain bundled. |
| 1186 | void eraseFromBundle(); |
| 1187 | |
| 1188 | bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; } |
| 1189 | bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; } |
| 1190 | bool isAnnotationLabel() const { |
| 1191 | return getOpcode() == TargetOpcode::ANNOTATION_LABEL; |
| 1192 | } |
| 1193 | |
| 1194 | /// Returns true if the MachineInstr represents a label. |
| 1195 | bool isLabel() const { |
| 1196 | return isEHLabel() || isGCLabel() || isAnnotationLabel(); |
| 1197 | } |
| 1198 | |
| 1199 | bool isCFIInstruction() const { |
| 1200 | return getOpcode() == TargetOpcode::CFI_INSTRUCTION; |
| 1201 | } |
| 1202 | |
| 1203 | bool isPseudoProbe() const { |
| 1204 | return getOpcode() == TargetOpcode::PSEUDO_PROBE; |
| 1205 | } |
| 1206 | |
| 1207 | // True if the instruction represents a position in the function. |
| 1208 | bool isPosition() const { return isLabel() || isCFIInstruction(); } |
| 1209 | |
| 1210 | bool isNonListDebugValue() const { |
| 1211 | return getOpcode() == TargetOpcode::DBG_VALUE; |
| 1212 | } |
| 1213 | bool isDebugValueList() const { |
| 1214 | return getOpcode() == TargetOpcode::DBG_VALUE_LIST; |
| 1215 | } |
| 1216 | bool isDebugValue() const { |
| 1217 | return isNonListDebugValue() || isDebugValueList(); |
| 1218 | } |
| 1219 | bool isDebugLabel() const { return getOpcode() == TargetOpcode::DBG_LABEL; } |
| 1220 | bool isDebugRef() const { return getOpcode() == TargetOpcode::DBG_INSTR_REF; } |
| 1221 | bool isDebugPHI() const { return getOpcode() == TargetOpcode::DBG_PHI; } |
| 1222 | bool isDebugInstr() const { |
| 1223 | return isDebugValue() || isDebugLabel() || isDebugRef() || isDebugPHI(); |
| 1224 | } |
| 1225 | bool isDebugOrPseudoInstr() const { |
| 1226 | return isDebugInstr() || isPseudoProbe(); |
| 1227 | } |
| 1228 | |
| 1229 | bool isDebugOffsetImm() const { |
| 1230 | return isNonListDebugValue() && getDebugOffset().isImm(); |
| 1231 | } |
| 1232 | |
| 1233 | /// A DBG_VALUE is indirect iff the location operand is a register and |
| 1234 | /// the offset operand is an immediate. |
| 1235 | bool isIndirectDebugValue() const { |
| 1236 | return isDebugOffsetImm() && getDebugOperand(0).isReg(); |
| 1237 | } |
| 1238 | |
| 1239 | /// A DBG_VALUE is an entry value iff its debug expression contains the |
| 1240 | /// DW_OP_LLVM_entry_value operation. |
| 1241 | bool isDebugEntryValue() const; |
| 1242 | |
| 1243 | /// Return true if the instruction is a debug value which describes a part of |
| 1244 | /// a variable as unavailable. |
| 1245 | bool isUndefDebugValue() const { |
| 1246 | if (!isDebugValue()) |
| 1247 | return false; |
| 1248 | // If any $noreg locations are given, this DV is undef. |
| 1249 | for (const MachineOperand &Op : debug_operands()) |
| 1250 | if (Op.isReg() && !Op.getReg().isValid()) |
| 1251 | return true; |
| 1252 | return false; |
| 1253 | } |
| 1254 | |
| 1255 | bool isPHI() const { |
| 1256 | return getOpcode() == TargetOpcode::PHI || |
| 1257 | getOpcode() == TargetOpcode::G_PHI; |
| 1258 | } |
| 1259 | bool isKill() const { return getOpcode() == TargetOpcode::KILL; } |
| 1260 | bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } |
| 1261 | bool isInlineAsm() const { |
| 1262 | return getOpcode() == TargetOpcode::INLINEASM || |
| 1263 | getOpcode() == TargetOpcode::INLINEASM_BR; |
| 1264 | } |
| 1265 | |
| 1266 | /// FIXME: Seems like a layering violation that the AsmDialect, which is X86 |
| 1267 | /// specific, be attached to a generic MachineInstr. |
| 1268 | bool isMSInlineAsm() const { |
| 1269 | return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; |
| 1270 | } |
| 1271 | |
| 1272 | bool isStackAligningInlineAsm() const; |
| 1273 | InlineAsm::AsmDialect getInlineAsmDialect() const; |
| 1274 | |
| 1275 | bool isInsertSubreg() const { |
| 1276 | return getOpcode() == TargetOpcode::INSERT_SUBREG; |
| 1277 | } |
| 1278 | |
| 1279 | bool isSubregToReg() const { |
| 1280 | return getOpcode() == TargetOpcode::SUBREG_TO_REG; |
| 1281 | } |
| 1282 | |
| 1283 | bool isRegSequence() const { |
| 1284 | return getOpcode() == TargetOpcode::REG_SEQUENCE; |
| 1285 | } |
| 1286 | |
| 1287 | bool isBundle() const { |
| 1288 | return getOpcode() == TargetOpcode::BUNDLE; |
| 1289 | } |
| 1290 | |
| 1291 | bool isCopy() const { |
| 1292 | return getOpcode() == TargetOpcode::COPY; |
| 1293 | } |
| 1294 | |
| 1295 | bool isFullCopy() const { |
| 1296 | return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg(); |
| 1297 | } |
| 1298 | |
| 1299 | bool isExtractSubreg() const { |
| 1300 | return getOpcode() == TargetOpcode::EXTRACT_SUBREG; |
| 1301 | } |
| 1302 | |
| 1303 | /// Return true if the instruction behaves like a copy. |
| 1304 | /// This does not include native copy instructions. |
| 1305 | bool isCopyLike() const { |
| 1306 | return isCopy() || isSubregToReg(); |
| 1307 | } |
| 1308 | |
| 1309 | /// Return true is the instruction is an identity copy. |
| 1310 | bool isIdentityCopy() const { |
| 1311 | return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() && |
| 1312 | getOperand(0).getSubReg() == getOperand(1).getSubReg(); |
| 1313 | } |
| 1314 | |
| 1315 | /// Return true if this instruction doesn't produce any output in the form of |
| 1316 | /// executable instructions. |
| 1317 | bool isMetaInstruction() const { |
| 1318 | switch (getOpcode()) { |
| 1319 | default: |
| 1320 | return false; |
| 1321 | case TargetOpcode::IMPLICIT_DEF: |
| 1322 | case TargetOpcode::KILL: |
| 1323 | case TargetOpcode::CFI_INSTRUCTION: |
| 1324 | case TargetOpcode::EH_LABEL: |
| 1325 | case TargetOpcode::GC_LABEL: |
| 1326 | case TargetOpcode::DBG_VALUE: |
| 1327 | case TargetOpcode::DBG_VALUE_LIST: |
| 1328 | case TargetOpcode::DBG_INSTR_REF: |
| 1329 | case TargetOpcode::DBG_PHI: |
| 1330 | case TargetOpcode::DBG_LABEL: |
| 1331 | case TargetOpcode::LIFETIME_START: |
| 1332 | case TargetOpcode::LIFETIME_END: |
| 1333 | case TargetOpcode::PSEUDO_PROBE: |
| 1334 | return true; |
| 1335 | } |
| 1336 | } |
| 1337 | |
| 1338 | /// Return true if this is a transient instruction that is either very likely |
| 1339 | /// to be eliminated during register allocation (such as copy-like |
| 1340 | /// instructions), or if this instruction doesn't have an execution-time cost. |
| 1341 | bool isTransient() const { |
| 1342 | switch (getOpcode()) { |
| 1343 | default: |
| 1344 | return isMetaInstruction(); |
| 1345 | // Copy-like instructions are usually eliminated during register allocation. |
| 1346 | case TargetOpcode::PHI: |
| 1347 | case TargetOpcode::G_PHI: |
| 1348 | case TargetOpcode::COPY: |
| 1349 | case TargetOpcode::INSERT_SUBREG: |
| 1350 | case TargetOpcode::SUBREG_TO_REG: |
| 1351 | case TargetOpcode::REG_SEQUENCE: |
| 1352 | return true; |
| 1353 | } |
| 1354 | } |
| 1355 | |
| 1356 | /// Return the number of instructions inside the MI bundle, excluding the |
| 1357 | /// bundle header. |
| 1358 | /// |
| 1359 | /// This is the number of instructions that MachineBasicBlock::iterator |
| 1360 | /// skips, 0 for unbundled instructions. |
| 1361 | unsigned getBundleSize() const; |
| 1362 | |
| 1363 | /// Return true if the MachineInstr reads the specified register. |
| 1364 | /// If TargetRegisterInfo is passed, then it also checks if there |
| 1365 | /// is a read of a super-register. |
| 1366 | /// This does not count partial redefines of virtual registers as reads: |
| 1367 | /// %reg1024:6 = OP. |
| 1368 | bool readsRegister(Register Reg, |
| 1369 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1370 | return findRegisterUseOperandIdx(Reg, false, TRI) != -1; |
| 1371 | } |
| 1372 | |
| 1373 | /// Return true if the MachineInstr reads the specified virtual register. |
| 1374 | /// Take into account that a partial define is a |
| 1375 | /// read-modify-write operation. |
| 1376 | bool readsVirtualRegister(Register Reg) const { |
| 1377 | return readsWritesVirtualRegister(Reg).first; |
| 1378 | } |
| 1379 | |
| 1380 | /// Return a pair of bools (reads, writes) indicating if this instruction |
| 1381 | /// reads or writes Reg. This also considers partial defines. |
| 1382 | /// If Ops is not null, all operand indices for Reg are added. |
| 1383 | std::pair<bool,bool> readsWritesVirtualRegister(Register Reg, |
| 1384 | SmallVectorImpl<unsigned> *Ops = nullptr) const; |
| 1385 | |
| 1386 | /// Return true if the MachineInstr kills the specified register. |
| 1387 | /// If TargetRegisterInfo is passed, then it also checks if there is |
| 1388 | /// a kill of a super-register. |
| 1389 | bool killsRegister(Register Reg, |
| 1390 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1391 | return findRegisterUseOperandIdx(Reg, true, TRI) != -1; |
| 1392 | } |
| 1393 | |
| 1394 | /// Return true if the MachineInstr fully defines the specified register. |
| 1395 | /// If TargetRegisterInfo is passed, then it also checks |
| 1396 | /// if there is a def of a super-register. |
| 1397 | /// NOTE: It's ignoring subreg indices on virtual registers. |
| 1398 | bool definesRegister(Register Reg, |
| 1399 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1400 | return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1; |
| 1401 | } |
| 1402 | |
| 1403 | /// Return true if the MachineInstr modifies (fully define or partially |
| 1404 | /// define) the specified register. |
| 1405 | /// NOTE: It's ignoring subreg indices on virtual registers. |
| 1406 | bool modifiesRegister(Register Reg, |
| 1407 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1408 | return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1; |
| 1409 | } |
| 1410 | |
| 1411 | /// Returns true if the register is dead in this machine instruction. |
| 1412 | /// If TargetRegisterInfo is passed, then it also checks |
| 1413 | /// if there is a dead def of a super-register. |
| 1414 | bool registerDefIsDead(Register Reg, |
| 1415 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1416 | return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1; |
| 1417 | } |
| 1418 | |
| 1419 | /// Returns true if the MachineInstr has an implicit-use operand of exactly |
| 1420 | /// the given register (not considering sub/super-registers). |
| 1421 | bool hasRegisterImplicitUseOperand(Register Reg) const; |
| 1422 | |
| 1423 | /// Returns the operand index that is a use of the specific register or -1 |
| 1424 | /// if it is not found. It further tightens the search criteria to a use |
| 1425 | /// that kills the register if isKill is true. |
| 1426 | int findRegisterUseOperandIdx(Register Reg, bool isKill = false, |
| 1427 | const TargetRegisterInfo *TRI = nullptr) const; |
| 1428 | |
| 1429 | /// Wrapper for findRegisterUseOperandIdx, it returns |
| 1430 | /// a pointer to the MachineOperand rather than an index. |
| 1431 | MachineOperand *findRegisterUseOperand(Register Reg, bool isKill = false, |
| 1432 | const TargetRegisterInfo *TRI = nullptr) { |
| 1433 | int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI); |
| 1434 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
| 1435 | } |
| 1436 | |
| 1437 | const MachineOperand *findRegisterUseOperand( |
| 1438 | Register Reg, bool isKill = false, |
| 1439 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1440 | return const_cast<MachineInstr *>(this)-> |
| 1441 | findRegisterUseOperand(Reg, isKill, TRI); |
| 1442 | } |
| 1443 | |
| 1444 | /// Returns the operand index that is a def of the specified register or |
| 1445 | /// -1 if it is not found. If isDead is true, defs that are not dead are |
| 1446 | /// skipped. If Overlap is true, then it also looks for defs that merely |
| 1447 | /// overlap the specified register. If TargetRegisterInfo is non-null, |
| 1448 | /// then it also checks if there is a def of a super-register. |
| 1449 | /// This may also return a register mask operand when Overlap is true. |
| 1450 | int findRegisterDefOperandIdx(Register Reg, |
| 1451 | bool isDead = false, bool Overlap = false, |
| 1452 | const TargetRegisterInfo *TRI = nullptr) const; |
| 1453 | |
| 1454 | /// Wrapper for findRegisterDefOperandIdx, it returns |
| 1455 | /// a pointer to the MachineOperand rather than an index. |
| 1456 | MachineOperand * |
| 1457 | findRegisterDefOperand(Register Reg, bool isDead = false, |
| 1458 | bool Overlap = false, |
| 1459 | const TargetRegisterInfo *TRI = nullptr) { |
| 1460 | int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); |
| 1461 | return (Idx == -1) ? nullptr : &getOperand(Idx); |
| 1462 | } |
| 1463 | |
| 1464 | const MachineOperand * |
| 1465 | findRegisterDefOperand(Register Reg, bool isDead = false, |
| 1466 | bool Overlap = false, |
| 1467 | const TargetRegisterInfo *TRI = nullptr) const { |
| 1468 | return const_cast<MachineInstr *>(this)->findRegisterDefOperand( |
| 1469 | Reg, isDead, Overlap, TRI); |
| 1470 | } |
| 1471 | |
| 1472 | /// Find the index of the first operand in the |
| 1473 | /// operand list that is used to represent the predicate. It returns -1 if |
| 1474 | /// none is found. |
| 1475 | int findFirstPredOperandIdx() const; |
| 1476 | |
| 1477 | /// Find the index of the flag word operand that |
| 1478 | /// corresponds to operand OpIdx on an inline asm instruction. Returns -1 if |
| 1479 | /// getOperand(OpIdx) does not belong to an inline asm operand group. |
| 1480 | /// |
| 1481 | /// If GroupNo is not NULL, it will receive the number of the operand group |
| 1482 | /// containing OpIdx. |
| 1483 | int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const; |
| 1484 | |
| 1485 | /// Compute the static register class constraint for operand OpIdx. |
| 1486 | /// For normal instructions, this is derived from the MCInstrDesc. |
| 1487 | /// For inline assembly it is derived from the flag words. |
| 1488 | /// |
| 1489 | /// Returns NULL if the static register class constraint cannot be |
| 1490 | /// determined. |
| 1491 | const TargetRegisterClass* |
| 1492 | getRegClassConstraint(unsigned OpIdx, |
| 1493 | const TargetInstrInfo *TII, |
| 1494 | const TargetRegisterInfo *TRI) const; |
| 1495 | |
| 1496 | /// Applies the constraints (def/use) implied by this MI on \p Reg to |
| 1497 | /// the given \p CurRC. |
| 1498 | /// If \p ExploreBundle is set and MI is part of a bundle, all the |
| 1499 | /// instructions inside the bundle will be taken into account. In other words, |
| 1500 | /// this method accumulates all the constraints of the operand of this MI and |
| 1501 | /// the related bundle if MI is a bundle or inside a bundle. |
| 1502 | /// |
| 1503 | /// Returns the register class that satisfies both \p CurRC and the |
| 1504 | /// constraints set by MI. Returns NULL if such a register class does not |
| 1505 | /// exist. |
| 1506 | /// |
| 1507 | /// \pre CurRC must not be NULL. |
| 1508 | const TargetRegisterClass *getRegClassConstraintEffectForVReg( |
| 1509 | Register Reg, const TargetRegisterClass *CurRC, |
| 1510 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI, |
| 1511 | bool ExploreBundle = false) const; |
| 1512 | |
| 1513 | /// Applies the constraints (def/use) implied by the \p OpIdx operand |
| 1514 | /// to the given \p CurRC. |
| 1515 | /// |
| 1516 | /// Returns the register class that satisfies both \p CurRC and the |
| 1517 | /// constraints set by \p OpIdx MI. Returns NULL if such a register class |
| 1518 | /// does not exist. |
| 1519 | /// |
| 1520 | /// \pre CurRC must not be NULL. |
| 1521 | /// \pre The operand at \p OpIdx must be a register. |
| 1522 | const TargetRegisterClass * |
| 1523 | getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC, |
| 1524 | const TargetInstrInfo *TII, |
| 1525 | const TargetRegisterInfo *TRI) const; |
| 1526 | |
| 1527 | /// Add a tie between the register operands at DefIdx and UseIdx. |
| 1528 | /// The tie will cause the register allocator to ensure that the two |
| 1529 | /// operands are assigned the same physical register. |
| 1530 | /// |
| 1531 | /// Tied operands are managed automatically for explicit operands in the |
| 1532 | /// MCInstrDesc. This method is for exceptional cases like inline asm. |
| 1533 | void tieOperands(unsigned DefIdx, unsigned UseIdx); |
| 1534 | |
| 1535 | /// Given the index of a tied register operand, find the |
| 1536 | /// operand it is tied to. Defs are tied to uses and vice versa. Returns the |
| 1537 | /// index of the tied operand which must exist. |
| 1538 | unsigned findTiedOperandIdx(unsigned OpIdx) const; |
| 1539 | |
| 1540 | /// Given the index of a register def operand, |
| 1541 | /// check if the register def is tied to a source operand, due to either |
| 1542 | /// two-address elimination or inline assembly constraints. Returns the |
| 1543 | /// first tied use operand index by reference if UseOpIdx is not null. |
| 1544 | bool isRegTiedToUseOperand(unsigned DefOpIdx, |
| 1545 | unsigned *UseOpIdx = nullptr) const { |
| 1546 | const MachineOperand &MO = getOperand(DefOpIdx); |
| 1547 | if (!MO.isReg() || !MO.isDef() || !MO.isTied()) |
| 1548 | return false; |
| 1549 | if (UseOpIdx) |
| 1550 | *UseOpIdx = findTiedOperandIdx(DefOpIdx); |
| 1551 | return true; |
| 1552 | } |
| 1553 | |
| 1554 | /// Return true if the use operand of the specified index is tied to a def |
| 1555 | /// operand. It also returns the def operand index by reference if DefOpIdx |
| 1556 | /// is not null. |
| 1557 | bool isRegTiedToDefOperand(unsigned UseOpIdx, |
| 1558 | unsigned *DefOpIdx = nullptr) const { |
| 1559 | const MachineOperand &MO = getOperand(UseOpIdx); |
| 1560 | if (!MO.isReg() || !MO.isUse() || !MO.isTied()) |
| 1561 | return false; |
| 1562 | if (DefOpIdx) |
| 1563 | *DefOpIdx = findTiedOperandIdx(UseOpIdx); |
| 1564 | return true; |
| 1565 | } |
| 1566 | |
| 1567 | /// Clears kill flags on all operands. |
| 1568 | void clearKillInfo(); |
| 1569 | |
| 1570 | /// Replace all occurrences of FromReg with ToReg:SubIdx, |
| 1571 | /// properly composing subreg indices where necessary. |
| 1572 | void substituteRegister(Register FromReg, Register ToReg, unsigned SubIdx, |
| 1573 | const TargetRegisterInfo &RegInfo); |
| 1574 | |
| 1575 | /// We have determined MI kills a register. Look for the |
| 1576 | /// operand that uses it and mark it as IsKill. If AddIfNotFound is true, |
| 1577 | /// add a implicit operand if it's not found. Returns true if the operand |
| 1578 | /// exists / is added. |
| 1579 | bool addRegisterKilled(Register IncomingReg, |
| 1580 | const TargetRegisterInfo *RegInfo, |
| 1581 | bool AddIfNotFound = false); |
| 1582 | |
| 1583 | /// Clear all kill flags affecting Reg. If RegInfo is provided, this includes |
| 1584 | /// all aliasing registers. |
| 1585 | void clearRegisterKills(Register Reg, const TargetRegisterInfo *RegInfo); |
| 1586 | |
| 1587 | /// We have determined MI defined a register without a use. |
| 1588 | /// Look for the operand that defines it and mark it as IsDead. If |
| 1589 | /// AddIfNotFound is true, add a implicit operand if it's not found. Returns |
| 1590 | /// true if the operand exists / is added. |
| 1591 | bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, |
| 1592 | bool AddIfNotFound = false); |
| 1593 | |
| 1594 | /// Clear all dead flags on operands defining register @p Reg. |
| 1595 | void clearRegisterDeads(Register Reg); |
| 1596 | |
| 1597 | /// Mark all subregister defs of register @p Reg with the undef flag. |
| 1598 | /// This function is used when we determined to have a subregister def in an |
| 1599 | /// otherwise undefined super register. |
| 1600 | void setRegisterDefReadUndef(Register Reg, bool IsUndef = true); |
| 1601 | |
| 1602 | /// We have determined MI defines a register. Make sure there is an operand |
| 1603 | /// defining Reg. |
| 1604 | void addRegisterDefined(Register Reg, |
| 1605 | const TargetRegisterInfo *RegInfo = nullptr); |
| 1606 | |
| 1607 | /// Mark every physreg used by this instruction as |
| 1608 | /// dead except those in the UsedRegs list. |
| 1609 | /// |
| 1610 | /// On instructions with register mask operands, also add implicit-def |
| 1611 | /// operands for all registers in UsedRegs. |
| 1612 | void setPhysRegsDeadExcept(ArrayRef<Register> UsedRegs, |
| 1613 | const TargetRegisterInfo &TRI); |
| 1614 | |
| 1615 | /// Return true if it is safe to move this instruction. If |
| 1616 | /// SawStore is set to true, it means that there is a store (or call) between |
| 1617 | /// the instruction's location and its intended destination. |
| 1618 | bool isSafeToMove(AAResults *AA, bool &SawStore) const; |
| 1619 | |
| 1620 | /// Returns true if this instruction's memory access aliases the memory |
| 1621 | /// access of Other. |
| 1622 | // |
| 1623 | /// Assumes any physical registers used to compute addresses |
| 1624 | /// have the same value for both instructions. Returns false if neither |
| 1625 | /// instruction writes to memory. |
| 1626 | /// |
| 1627 | /// @param AA Optional alias analysis, used to compare memory operands. |
| 1628 | /// @param Other MachineInstr to check aliasing against. |
| 1629 | /// @param UseTBAA Whether to pass TBAA information to alias analysis. |
| 1630 | bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; |
| 1631 | |
| 1632 | /// Return true if this instruction may have an ordered |
| 1633 | /// or volatile memory reference, or if the information describing the memory |
| 1634 | /// reference is not available. Return false if it is known to have no |
| 1635 | /// ordered or volatile memory references. |
| 1636 | bool hasOrderedMemoryRef() const; |
| 1637 | |
| 1638 | /// Return true if this load instruction never traps and points to a memory |
| 1639 | /// location whose value doesn't change during the execution of this function. |
| 1640 | /// |
| 1641 | /// Examples include loading a value from the constant pool or from the |
| 1642 | /// argument area of a function (if it does not change). If the instruction |
| 1643 | /// does multiple loads, this returns true only if all of the loads are |
| 1644 | /// dereferenceable and invariant. |
| 1645 | bool isDereferenceableInvariantLoad(AAResults *AA) const; |
| 1646 | |
| 1647 | /// If the specified instruction is a PHI that always merges together the |
| 1648 | /// same virtual register, return the register, otherwise return 0. |
| 1649 | unsigned isConstantValuePHI() const; |
| 1650 | |
| 1651 | /// Return true if this instruction has side effects that are not modeled |
| 1652 | /// by mayLoad / mayStore, etc. |
| 1653 | /// For all instructions, the property is encoded in MCInstrDesc::Flags |
| 1654 | /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is |
| 1655 | /// INLINEASM instruction, in which case the side effect property is encoded |
| 1656 | /// in one of its operands (see InlineAsm::Extra_HasSideEffect). |
| 1657 | /// |
| 1658 | bool hasUnmodeledSideEffects() const; |
| 1659 | |
| 1660 | /// Returns true if it is illegal to fold a load across this instruction. |
| 1661 | bool isLoadFoldBarrier() const; |
| 1662 | |
| 1663 | /// Return true if all the defs of this instruction are dead. |
| 1664 | bool allDefsAreDead() const; |
| 1665 | |
| 1666 | /// Return a valid size if the instruction is a spill instruction. |
| 1667 | Optional<unsigned> getSpillSize(const TargetInstrInfo *TII) const; |
| 1668 | |
| 1669 | /// Return a valid size if the instruction is a folded spill instruction. |
| 1670 | Optional<unsigned> getFoldedSpillSize(const TargetInstrInfo *TII) const; |
| 1671 | |
| 1672 | /// Return a valid size if the instruction is a restore instruction. |
| 1673 | Optional<unsigned> getRestoreSize(const TargetInstrInfo *TII) const; |
| 1674 | |
| 1675 | /// Return a valid size if the instruction is a folded restore instruction. |
| 1676 | Optional<unsigned> |
| 1677 | getFoldedRestoreSize(const TargetInstrInfo *TII) const; |
| 1678 | |
| 1679 | /// Copy implicit register operands from specified |
| 1680 | /// instruction to this instruction. |
| 1681 | void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI); |
| 1682 | |
| 1683 | /// Debugging support |
| 1684 | /// @{ |
| 1685 | /// Determine the generic type to be printed (if needed) on uses and defs. |
| 1686 | LLT getTypeToPrint(unsigned OpIdx, SmallBitVector &PrintedTypes, |
| 1687 | const MachineRegisterInfo &MRI) const; |
| 1688 | |
| 1689 | /// Return true when an instruction has tied register that can't be determined |
| 1690 | /// by the instruction's descriptor. This is useful for MIR printing, to |
| 1691 | /// determine whether we need to print the ties or not. |
| 1692 | bool hasComplexRegisterTies() const; |
| 1693 | |
| 1694 | /// Print this MI to \p OS. |
| 1695 | /// Don't print information that can be inferred from other instructions if |
| 1696 | /// \p IsStandalone is false. It is usually true when only a fragment of the |
| 1697 | /// function is printed. |
| 1698 | /// Only print the defs and the opcode if \p SkipOpers is true. |
| 1699 | /// Otherwise, also print operands if \p SkipDebugLoc is true. |
| 1700 | /// Otherwise, also print the debug loc, with a terminating newline. |
| 1701 | /// \p TII is used to print the opcode name. If it's not present, but the |
| 1702 | /// MI is in a function, the opcode will be printed using the function's TII. |
| 1703 | void print(raw_ostream &OS, bool IsStandalone = true, bool SkipOpers = false, |
| 1704 | bool SkipDebugLoc = false, bool AddNewLine = true, |
| 1705 | const TargetInstrInfo *TII = nullptr) const; |
| 1706 | void print(raw_ostream &OS, ModuleSlotTracker &MST, bool IsStandalone = true, |
| 1707 | bool SkipOpers = false, bool SkipDebugLoc = false, |
| 1708 | bool AddNewLine = true, |
| 1709 | const TargetInstrInfo *TII = nullptr) const; |
| 1710 | void dump() const; |
| 1711 | /// Print on dbgs() the current instruction and the instructions defining its |
| 1712 | /// operands and so on until we reach \p MaxDepth. |
| 1713 | void dumpr(const MachineRegisterInfo &MRI, |
| 1714 | unsigned MaxDepth = UINT_MAX(2147483647 *2U +1U)) const; |
| 1715 | /// @} |
| 1716 | |
| 1717 | //===--------------------------------------------------------------------===// |
| 1718 | // Accessors used to build up machine instructions. |
| 1719 | |
| 1720 | /// Add the specified operand to the instruction. If it is an implicit |
| 1721 | /// operand, it is added to the end of the operand list. If it is an |
| 1722 | /// explicit operand it is added at the end of the explicit operand list |
| 1723 | /// (before the first implicit operand). |
| 1724 | /// |
| 1725 | /// MF must be the machine function that was used to allocate this |
| 1726 | /// instruction. |
| 1727 | /// |
| 1728 | /// MachineInstrBuilder provides a more convenient interface for creating |
| 1729 | /// instructions and adding operands. |
| 1730 | void addOperand(MachineFunction &MF, const MachineOperand &Op); |
| 1731 | |
| 1732 | /// Add an operand without providing an MF reference. This only works for |
| 1733 | /// instructions that are inserted in a basic block. |
| 1734 | /// |
| 1735 | /// MachineInstrBuilder and the two-argument addOperand(MF, MO) should be |
| 1736 | /// preferred. |
| 1737 | void addOperand(const MachineOperand &Op); |
| 1738 | |
| 1739 | /// Replace the instruction descriptor (thus opcode) of |
| 1740 | /// the current instruction with a new one. |
| 1741 | void setDesc(const MCInstrDesc &tid) { MCID = &tid; } |
| 1742 | |
| 1743 | /// Replace current source information with new such. |
| 1744 | /// Avoid using this, the constructor argument is preferable. |
| 1745 | void setDebugLoc(DebugLoc dl) { |
| 1746 | debugLoc = std::move(dl); |
| 1747 | assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() && "Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 1747, __extension__ __PRETTY_FUNCTION__)); |
| 1748 | } |
| 1749 | |
| 1750 | /// Erase an operand from an instruction, leaving it with one |
| 1751 | /// fewer operand than it started with. |
| 1752 | void RemoveOperand(unsigned OpNo); |
| 1753 | |
| 1754 | /// Clear this MachineInstr's memory reference descriptor list. This resets |
| 1755 | /// the memrefs to their most conservative state. This should be used only |
| 1756 | /// as a last resort since it greatly pessimizes our knowledge of the memory |
| 1757 | /// access performed by the instruction. |
| 1758 | void dropMemRefs(MachineFunction &MF); |
| 1759 | |
| 1760 | /// Assign this MachineInstr's memory reference descriptor list. |
| 1761 | /// |
| 1762 | /// Unlike other methods, this *will* allocate them into a new array |
| 1763 | /// associated with the provided `MachineFunction`. |
| 1764 | void setMemRefs(MachineFunction &MF, ArrayRef<MachineMemOperand *> MemRefs); |
| 1765 | |
| 1766 | /// Add a MachineMemOperand to the machine instruction. |
| 1767 | /// This function should be used only occasionally. The setMemRefs function |
| 1768 | /// is the primary method for setting up a MachineInstr's MemRefs list. |
| 1769 | void addMemOperand(MachineFunction &MF, MachineMemOperand *MO); |
| 1770 | |
| 1771 | /// Clone another MachineInstr's memory reference descriptor list and replace |
| 1772 | /// ours with it. |
| 1773 | /// |
| 1774 | /// Note that `*this` may be the incoming MI! |
| 1775 | /// |
| 1776 | /// Prefer this API whenever possible as it can avoid allocations in common |
| 1777 | /// cases. |
| 1778 | void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI); |
| 1779 | |
| 1780 | /// Clone the merge of multiple MachineInstrs' memory reference descriptors |
| 1781 | /// list and replace ours with it. |
| 1782 | /// |
| 1783 | /// Note that `*this` may be one of the incoming MIs! |
| 1784 | /// |
| 1785 | /// Prefer this API whenever possible as it can avoid allocations in common |
| 1786 | /// cases. |
| 1787 | void cloneMergedMemRefs(MachineFunction &MF, |
| 1788 | ArrayRef<const MachineInstr *> MIs); |
| 1789 | |
| 1790 | /// Set a symbol that will be emitted just prior to the instruction itself. |
| 1791 | /// |
| 1792 | /// Setting this to a null pointer will remove any such symbol. |
| 1793 | /// |
| 1794 | /// FIXME: This is not fully implemented yet. |
| 1795 | void setPreInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
| 1796 | |
| 1797 | /// Set a symbol that will be emitted just after the instruction itself. |
| 1798 | /// |
| 1799 | /// Setting this to a null pointer will remove any such symbol. |
| 1800 | /// |
| 1801 | /// FIXME: This is not fully implemented yet. |
| 1802 | void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); |
| 1803 | |
| 1804 | /// Clone another MachineInstr's pre- and post- instruction symbols and |
| 1805 | /// replace ours with it. |
| 1806 | void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); |
| 1807 | |
| 1808 | /// Set a marker on instructions that denotes where we should create and emit |
| 1809 | /// heap alloc site labels. This waits until after instruction selection and |
| 1810 | /// optimizations to create the label, so it should still work if the |
| 1811 | /// instruction is removed or duplicated. |
| 1812 | void setHeapAllocMarker(MachineFunction &MF, MDNode *MD); |
| 1813 | |
| 1814 | /// Return the MIFlags which represent both MachineInstrs. This |
| 1815 | /// should be used when merging two MachineInstrs into one. This routine does |
| 1816 | /// not modify the MIFlags of this MachineInstr. |
| 1817 | uint16_t mergeFlagsWith(const MachineInstr& Other) const; |
| 1818 | |
| 1819 | static uint16_t copyFlagsFromInstruction(const Instruction &I); |
| 1820 | |
| 1821 | /// Copy all flags to MachineInst MIFlags |
| 1822 | void copyIRFlags(const Instruction &I); |
| 1823 | |
| 1824 | /// Break any tie involving OpIdx. |
| 1825 | void untieRegOperand(unsigned OpIdx) { |
| 1826 | MachineOperand &MO = getOperand(OpIdx); |
| 1827 | if (MO.isReg() && MO.isTied()) { |
| 1828 | getOperand(findTiedOperandIdx(OpIdx)).TiedTo = 0; |
| 1829 | MO.TiedTo = 0; |
| 1830 | } |
| 1831 | } |
| 1832 | |
| 1833 | /// Add all implicit def and use operands to this instruction. |
| 1834 | void addImplicitDefUseOperands(MachineFunction &MF); |
| 1835 | |
| 1836 | /// Scan instructions immediately following MI and collect any matching |
| 1837 | /// DBG_VALUEs. |
| 1838 | void collectDebugValues(SmallVectorImpl<MachineInstr *> &DbgValues); |
| 1839 | |
| 1840 | /// Find all DBG_VALUEs that point to the register def in this instruction |
| 1841 | /// and point them to \p Reg instead. |
| 1842 | void changeDebugValuesDefReg(Register Reg); |
| 1843 | |
| 1844 | /// Returns the Intrinsic::ID for this instruction. |
| 1845 | /// \pre Must have an intrinsic ID operand. |
| 1846 | unsigned getIntrinsicID() const { |
| 1847 | return getOperand(getNumExplicitDefs()).getIntrinsicID(); |
| 1848 | } |
| 1849 | |
| 1850 | /// Sets all register debug operands in this debug value instruction to be |
| 1851 | /// undef. |
| 1852 | void setDebugValueUndef() { |
| 1853 | assert(isDebugValue() && "Must be a debug value instruction.")(static_cast <bool> (isDebugValue() && "Must be a debug value instruction." ) ? void (0) : __assert_fail ("isDebugValue() && \"Must be a debug value instruction.\"" , "/build/llvm-toolchain-snapshot-14~++20210828111110+16086d47c0d0/llvm/include/llvm/CodeGen/MachineInstr.h" , 1853, __extension__ __PRETTY_FUNCTION__)); |
| 1854 | for (MachineOperand &MO : debug_operands()) { |
| 1855 | if (MO.isReg()) { |
| 1856 | MO.setReg(0); |
| 1857 | MO.setSubReg(0); |
| 1858 | } |
| 1859 | } |
| 1860 | } |
| 1861 | |
| 1862 | private: |
| 1863 | /// If this instruction is embedded into a MachineFunction, return the |
| 1864 | /// MachineRegisterInfo object for the current function, otherwise |
| 1865 | /// return null. |
| 1866 | MachineRegisterInfo *getRegInfo(); |
| 1867 | |
| 1868 | /// Unlink all of the register operands in this instruction from their |
| 1869 | /// respective use lists. This requires that the operands already be on their |
| 1870 | /// use lists. |
| 1871 | void RemoveRegOperandsFromUseLists(MachineRegisterInfo&); |
| 1872 | |
| 1873 | /// Add all of the register operands in this instruction from their |
| 1874 | /// respective use lists. This requires that the operands not be on their |
| 1875 | /// use lists yet. |
| 1876 | void AddRegOperandsToUseLists(MachineRegisterInfo&); |
| 1877 | |
| 1878 | /// Slow path for hasProperty when we're dealing with a bundle. |
| 1879 | bool hasPropertyInBundle(uint64_t Mask, QueryType Type) const; |
| 1880 | |
| 1881 | /// Implements the logic of getRegClassConstraintEffectForVReg for the |
| 1882 | /// this MI and the given operand index \p OpIdx. |
| 1883 | /// If the related operand does not constrained Reg, this returns CurRC. |
| 1884 | const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl( |
| 1885 | unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC, |
| 1886 | const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const; |
| 1887 | |
| 1888 | /// Stores extra instruction information inline or allocates as ExtraInfo |
| 1889 | /// based on the number of pointers. |
| 1890 | void setExtraInfo(MachineFunction &MF, ArrayRef<MachineMemOperand *> MMOs, |
| 1891 | MCSymbol *PreInstrSymbol, MCSymbol *PostInstrSymbol, |
| 1892 | MDNode *HeapAllocMarker); |
| 1893 | }; |
| 1894 | |
| 1895 | /// Special DenseMapInfo traits to compare MachineInstr* by *value* of the |
| 1896 | /// instruction rather than by pointer value. |
| 1897 | /// The hashing and equality testing functions ignore definitions so this is |
| 1898 | /// useful for CSE, etc. |
| 1899 | struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> { |
| 1900 | static inline MachineInstr *getEmptyKey() { |
| 1901 | return nullptr; |
| 1902 | } |
| 1903 | |
| 1904 | static inline MachineInstr *getTombstoneKey() { |
| 1905 | return reinterpret_cast<MachineInstr*>(-1); |
| 1906 | } |
| 1907 | |
| 1908 | static unsigned getHashValue(const MachineInstr* const &MI); |
| 1909 | |
| 1910 | static bool isEqual(const MachineInstr* const &LHS, |
| 1911 | const MachineInstr* const &RHS) { |
| 1912 | if (RHS == getEmptyKey() || RHS == getTombstoneKey() || |
| 1913 | LHS == getEmptyKey() || LHS == getTombstoneKey()) |
| 1914 | return LHS == RHS; |
| 1915 | return LHS->isIdenticalTo(*RHS, MachineInstr::IgnoreVRegDefs); |
| 1916 | } |
| 1917 | }; |
| 1918 | |
| 1919 | //===----------------------------------------------------------------------===// |
| 1920 | // Debugging Support |
| 1921 | |
| 1922 | inline raw_ostream& operator<<(raw_ostream &OS, const MachineInstr &MI) { |
| 1923 | MI.print(OS); |
| 1924 | return OS; |
| 1925 | } |
| 1926 | |
| 1927 | } // end namespace llvm |
| 1928 | |
| 1929 | #endif // LLVM_CODEGEN_MACHINEINSTR_H |