File: | lib/Transforms/Vectorize/VPlan.cpp |
Warning: | line 376, column 15 Value stored to 'VectorLatchBB' during its initialization is never read |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- VPlan.cpp - Vectorizer Plan ----------------------------------------===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | /// |
9 | /// \file |
10 | /// This is the LLVM vectorization plan. It represents a candidate for |
11 | /// vectorization, allowing to plan and optimize how to vectorize a given loop |
12 | /// before generating LLVM-IR. |
13 | /// The vectorizer uses vectorization plans to estimate the costs of potential |
14 | /// candidates and if profitable to execute the desired plan, generating vector |
15 | /// LLVM-IR code. |
16 | /// |
17 | //===----------------------------------------------------------------------===// |
18 | |
19 | #include "VPlan.h" |
20 | #include "VPlanDominatorTree.h" |
21 | #include "llvm/ADT/DepthFirstIterator.h" |
22 | #include "llvm/ADT/PostOrderIterator.h" |
23 | #include "llvm/ADT/SmallVector.h" |
24 | #include "llvm/ADT/Twine.h" |
25 | #include "llvm/Analysis/LoopInfo.h" |
26 | #include "llvm/IR/BasicBlock.h" |
27 | #include "llvm/IR/CFG.h" |
28 | #include "llvm/IR/InstrTypes.h" |
29 | #include "llvm/IR/Instruction.h" |
30 | #include "llvm/IR/Instructions.h" |
31 | #include "llvm/IR/Type.h" |
32 | #include "llvm/IR/Value.h" |
33 | #include "llvm/Support/Casting.h" |
34 | #include "llvm/Support/Debug.h" |
35 | #include "llvm/Support/ErrorHandling.h" |
36 | #include "llvm/Support/GenericDomTreeConstruction.h" |
37 | #include "llvm/Support/GraphWriter.h" |
38 | #include "llvm/Support/raw_ostream.h" |
39 | #include "llvm/Transforms/Utils/BasicBlockUtils.h" |
40 | #include <cassert> |
41 | #include <iterator> |
42 | #include <string> |
43 | #include <vector> |
44 | |
45 | using namespace llvm; |
46 | extern cl::opt<bool> EnableVPlanNativePath; |
47 | |
48 | #define DEBUG_TYPE"vplan" "vplan" |
49 | |
50 | raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) { |
51 | if (const VPInstruction *Instr = dyn_cast<VPInstruction>(&V)) |
52 | Instr->print(OS); |
53 | else |
54 | V.printAsOperand(OS); |
55 | return OS; |
56 | } |
57 | |
58 | /// \return the VPBasicBlock that is the entry of Block, possibly indirectly. |
59 | const VPBasicBlock *VPBlockBase::getEntryBasicBlock() const { |
60 | const VPBlockBase *Block = this; |
61 | while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) |
62 | Block = Region->getEntry(); |
63 | return cast<VPBasicBlock>(Block); |
64 | } |
65 | |
66 | VPBasicBlock *VPBlockBase::getEntryBasicBlock() { |
67 | VPBlockBase *Block = this; |
68 | while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) |
69 | Block = Region->getEntry(); |
70 | return cast<VPBasicBlock>(Block); |
71 | } |
72 | |
73 | /// \return the VPBasicBlock that is the exit of Block, possibly indirectly. |
74 | const VPBasicBlock *VPBlockBase::getExitBasicBlock() const { |
75 | const VPBlockBase *Block = this; |
76 | while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) |
77 | Block = Region->getExit(); |
78 | return cast<VPBasicBlock>(Block); |
79 | } |
80 | |
81 | VPBasicBlock *VPBlockBase::getExitBasicBlock() { |
82 | VPBlockBase *Block = this; |
83 | while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) |
84 | Block = Region->getExit(); |
85 | return cast<VPBasicBlock>(Block); |
86 | } |
87 | |
88 | VPBlockBase *VPBlockBase::getEnclosingBlockWithSuccessors() { |
89 | if (!Successors.empty() || !Parent) |
90 | return this; |
91 | assert(Parent->getExit() == this &&((Parent->getExit() == this && "Block w/o successors not the exit of its parent." ) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 92, __PRETTY_FUNCTION__)) |
92 | "Block w/o successors not the exit of its parent.")((Parent->getExit() == this && "Block w/o successors not the exit of its parent." ) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 92, __PRETTY_FUNCTION__)); |
93 | return Parent->getEnclosingBlockWithSuccessors(); |
94 | } |
95 | |
96 | VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() { |
97 | if (!Predecessors.empty() || !Parent) |
98 | return this; |
99 | assert(Parent->getEntry() == this &&((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent." ) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 100, __PRETTY_FUNCTION__)) |
100 | "Block w/o predecessors not the entry of its parent.")((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent." ) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 100, __PRETTY_FUNCTION__)); |
101 | return Parent->getEnclosingBlockWithPredecessors(); |
102 | } |
103 | |
104 | void VPBlockBase::deleteCFG(VPBlockBase *Entry) { |
105 | SmallVector<VPBlockBase *, 8> Blocks; |
106 | for (VPBlockBase *Block : depth_first(Entry)) |
107 | Blocks.push_back(Block); |
108 | |
109 | for (VPBlockBase *Block : Blocks) |
110 | delete Block; |
111 | } |
112 | |
113 | BasicBlock * |
114 | VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) { |
115 | // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks. |
116 | // Pred stands for Predessor. Prev stands for Previous - last visited/created. |
117 | BasicBlock *PrevBB = CFG.PrevBB; |
118 | BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(), |
119 | PrevBB->getParent(), CFG.LastBB); |
120 | LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: created " << NewBB-> getName() << '\n'; } } while (false); |
121 | |
122 | // Hook up the new basic block to its predecessors. |
123 | for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) { |
124 | VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock(); |
125 | auto &PredVPSuccessors = PredVPBB->getSuccessors(); |
126 | BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB]; |
127 | |
128 | // In outer loop vectorization scenario, the predecessor BBlock may not yet |
129 | // be visited(backedge). Mark the VPBasicBlock for fixup at the end of |
130 | // vectorization. We do not encounter this case in inner loop vectorization |
131 | // as we start out by building a loop skeleton with the vector loop header |
132 | // and latch blocks. As a result, we never enter this function for the |
133 | // header block in the non VPlan-native path. |
134 | if (!PredBB) { |
135 | assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path" ) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 136, __PRETTY_FUNCTION__)) |
136 | "Unexpected null predecessor in non VPlan-native path")((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path" ) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 136, __PRETTY_FUNCTION__)); |
137 | CFG.VPBBsToFix.push_back(PredVPBB); |
138 | continue; |
139 | } |
140 | |
141 | assert(PredBB && "Predecessor basic-block not found building successor.")((PredBB && "Predecessor basic-block not found building successor." ) ? static_cast<void> (0) : __assert_fail ("PredBB && \"Predecessor basic-block not found building successor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 141, __PRETTY_FUNCTION__)); |
142 | auto *PredBBTerminator = PredBB->getTerminator(); |
143 | LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: draw edge from" << PredBB ->getName() << '\n'; } } while (false); |
144 | if (isa<UnreachableInst>(PredBBTerminator)) { |
145 | assert(PredVPSuccessors.size() == 1 &&((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor." ) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 146, __PRETTY_FUNCTION__)) |
146 | "Predecessor ending w/o branch must have single successor.")((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor." ) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 146, __PRETTY_FUNCTION__)); |
147 | PredBBTerminator->eraseFromParent(); |
148 | BranchInst::Create(NewBB, PredBB); |
149 | } else { |
150 | assert(PredVPSuccessors.size() == 2 &&((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors." ) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 151, __PRETTY_FUNCTION__)) |
151 | "Predecessor ending with branch must have two successors.")((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors." ) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 151, __PRETTY_FUNCTION__)); |
152 | unsigned idx = PredVPSuccessors.front() == this ? 0 : 1; |
153 | assert(!PredBBTerminator->getSuccessor(idx) &&((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block." ) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 154, __PRETTY_FUNCTION__)) |
154 | "Trying to reset an existing successor block.")((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block." ) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 154, __PRETTY_FUNCTION__)); |
155 | PredBBTerminator->setSuccessor(idx, NewBB); |
156 | } |
157 | } |
158 | return NewBB; |
159 | } |
160 | |
161 | void VPBasicBlock::execute(VPTransformState *State) { |
162 | bool Replica = State->Instance && |
163 | !(State->Instance->Part == 0 && State->Instance->Lane == 0); |
164 | VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB; |
165 | VPBlockBase *SingleHPred = nullptr; |
166 | BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible. |
167 | |
168 | // 1. Create an IR basic block, or reuse the last one if possible. |
169 | // The last IR basic block is reused, as an optimization, in three cases: |
170 | // A. the first VPBB reuses the loop header BB - when PrevVPBB is null; |
171 | // B. when the current VPBB has a single (hierarchical) predecessor which |
172 | // is PrevVPBB and the latter has a single (hierarchical) successor; and |
173 | // C. when the current VPBB is an entry of a region replica - where PrevVPBB |
174 | // is the exit of this region from a previous instance, or the predecessor |
175 | // of this region. |
176 | if (PrevVPBB && /* A */ |
177 | !((SingleHPred = getSingleHierarchicalPredecessor()) && |
178 | SingleHPred->getExitBasicBlock() == PrevVPBB && |
179 | PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */ |
180 | !(Replica && getPredecessors().empty())) { /* C */ |
181 | NewBB = createEmptyBasicBlock(State->CFG); |
182 | State->Builder.SetInsertPoint(NewBB); |
183 | // Temporarily terminate with unreachable until CFG is rewired. |
184 | UnreachableInst *Terminator = State->Builder.CreateUnreachable(); |
185 | State->Builder.SetInsertPoint(Terminator); |
186 | // Register NewBB in its loop. In innermost loops its the same for all BB's. |
187 | Loop *L = State->LI->getLoopFor(State->CFG.LastBB); |
188 | L->addBasicBlockToLoop(NewBB, *State->LI); |
189 | State->CFG.PrevBB = NewBB; |
190 | } |
191 | |
192 | // 2. Fill the IR basic block with IR instructions. |
193 | LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: vectorizing VPBB:" << getName() << " in BB:" << NewBB->getName() << '\n'; } } while (false) |
194 | << " in BB:" << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: vectorizing VPBB:" << getName() << " in BB:" << NewBB->getName() << '\n'; } } while (false); |
195 | |
196 | State->CFG.VPBB2IRBB[this] = NewBB; |
197 | State->CFG.PrevVPBB = this; |
198 | |
199 | for (VPRecipeBase &Recipe : Recipes) |
200 | Recipe.execute(*State); |
201 | |
202 | VPValue *CBV; |
203 | if (EnableVPlanNativePath && (CBV = getCondBit())) { |
204 | Value *IRCBV = CBV->getUnderlyingValue(); |
205 | assert(IRCBV && "Unexpected null underlying value for condition bit")((IRCBV && "Unexpected null underlying value for condition bit" ) ? static_cast<void> (0) : __assert_fail ("IRCBV && \"Unexpected null underlying value for condition bit\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 205, __PRETTY_FUNCTION__)); |
206 | |
207 | // Condition bit value in a VPBasicBlock is used as the branch selector. In |
208 | // the VPlan-native path case, since all branches are uniform we generate a |
209 | // branch instruction using the condition value from vector lane 0 and dummy |
210 | // successors. The successors are fixed later when the successor blocks are |
211 | // visited. |
212 | Value *NewCond = State->Callback.getOrCreateVectorValues(IRCBV, 0); |
213 | NewCond = State->Builder.CreateExtractElement(NewCond, |
214 | State->Builder.getInt32(0)); |
215 | |
216 | // Replace the temporary unreachable terminator with the new conditional |
217 | // branch. |
218 | auto *CurrentTerminator = NewBB->getTerminator(); |
219 | assert(isa<UnreachableInst>(CurrentTerminator) &&((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional " "branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 221, __PRETTY_FUNCTION__)) |
220 | "Expected to replace unreachable terminator with conditional "((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional " "branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 221, __PRETTY_FUNCTION__)) |
221 | "branch.")((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional " "branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 221, __PRETTY_FUNCTION__)); |
222 | auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond); |
223 | CondBr->setSuccessor(0, nullptr); |
224 | ReplaceInstWithInst(CurrentTerminator, CondBr); |
225 | } |
226 | |
227 | LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: filled BB:" << *NewBB ; } } while (false); |
228 | } |
229 | |
230 | void VPRegionBlock::execute(VPTransformState *State) { |
231 | ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry); |
232 | |
233 | if (!isReplicator()) { |
234 | // Visit the VPBlocks connected to "this", starting from it. |
235 | for (VPBlockBase *Block : RPOT) { |
236 | if (EnableVPlanNativePath) { |
237 | // The inner loop vectorization path does not represent loop preheader |
238 | // and exit blocks as part of the VPlan. In the VPlan-native path, skip |
239 | // vectorizing loop preheader block. In future, we may replace this |
240 | // check with the check for loop preheader. |
241 | if (Block->getNumPredecessors() == 0) |
242 | continue; |
243 | |
244 | // Skip vectorizing loop exit block. In future, we may replace this |
245 | // check with the check for loop exit. |
246 | if (Block->getNumSuccessors() == 0) |
247 | continue; |
248 | } |
249 | |
250 | LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block ->getName() << '\n'; } } while (false); |
251 | Block->execute(State); |
252 | } |
253 | return; |
254 | } |
255 | |
256 | assert(!State->Instance && "Replicating a Region with non-null instance.")((!State->Instance && "Replicating a Region with non-null instance." ) ? static_cast<void> (0) : __assert_fail ("!State->Instance && \"Replicating a Region with non-null instance.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 256, __PRETTY_FUNCTION__)); |
257 | |
258 | // Enter replicating mode. |
259 | State->Instance = {0, 0}; |
260 | |
261 | for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) { |
262 | State->Instance->Part = Part; |
263 | for (unsigned Lane = 0, VF = State->VF; Lane < VF; ++Lane) { |
264 | State->Instance->Lane = Lane; |
265 | // Visit the VPBlocks connected to \p this, starting from it. |
266 | for (VPBlockBase *Block : RPOT) { |
267 | LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block ->getName() << '\n'; } } while (false); |
268 | Block->execute(State); |
269 | } |
270 | } |
271 | } |
272 | |
273 | // Exit replicating mode. |
274 | State->Instance.reset(); |
275 | } |
276 | |
277 | void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) { |
278 | Parent = InsertPos->getParent(); |
279 | Parent->getRecipeList().insert(InsertPos->getIterator(), this); |
280 | } |
281 | |
282 | iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() { |
283 | return getParent()->getRecipeList().erase(getIterator()); |
284 | } |
285 | |
286 | void VPInstruction::generateInstruction(VPTransformState &State, |
287 | unsigned Part) { |
288 | IRBuilder<> &Builder = State.Builder; |
289 | |
290 | if (Instruction::isBinaryOp(getOpcode())) { |
291 | Value *A = State.get(getOperand(0), Part); |
292 | Value *B = State.get(getOperand(1), Part); |
293 | Value *V = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B); |
294 | State.set(this, V, Part); |
295 | return; |
296 | } |
297 | |
298 | switch (getOpcode()) { |
299 | case VPInstruction::Not: { |
300 | Value *A = State.get(getOperand(0), Part); |
301 | Value *V = Builder.CreateNot(A); |
302 | State.set(this, V, Part); |
303 | break; |
304 | } |
305 | case VPInstruction::ICmpULE: { |
306 | Value *IV = State.get(getOperand(0), Part); |
307 | Value *TC = State.get(getOperand(1), Part); |
308 | Value *V = Builder.CreateICmpULE(IV, TC); |
309 | State.set(this, V, Part); |
310 | break; |
311 | } |
312 | default: |
313 | llvm_unreachable("Unsupported opcode for instruction")::llvm::llvm_unreachable_internal("Unsupported opcode for instruction" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 313); |
314 | } |
315 | } |
316 | |
317 | void VPInstruction::execute(VPTransformState &State) { |
318 | assert(!State.Instance && "VPInstruction executing an Instance")((!State.Instance && "VPInstruction executing an Instance" ) ? static_cast<void> (0) : __assert_fail ("!State.Instance && \"VPInstruction executing an Instance\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 318, __PRETTY_FUNCTION__)); |
319 | for (unsigned Part = 0; Part < State.UF; ++Part) |
320 | generateInstruction(State, Part); |
321 | } |
322 | |
323 | void VPInstruction::print(raw_ostream &O, const Twine &Indent) const { |
324 | O << " +\n" << Indent << "\"EMIT "; |
325 | print(O); |
326 | O << "\\l\""; |
327 | } |
328 | |
329 | void VPInstruction::print(raw_ostream &O) const { |
330 | printAsOperand(O); |
331 | O << " = "; |
332 | |
333 | switch (getOpcode()) { |
334 | case VPInstruction::Not: |
335 | O << "not"; |
336 | break; |
337 | case VPInstruction::ICmpULE: |
338 | O << "icmp ule"; |
339 | break; |
340 | case VPInstruction::SLPLoad: |
341 | O << "combined load"; |
342 | break; |
343 | case VPInstruction::SLPStore: |
344 | O << "combined store"; |
345 | break; |
346 | default: |
347 | O << Instruction::getOpcodeName(getOpcode()); |
348 | } |
349 | |
350 | for (const VPValue *Operand : operands()) { |
351 | O << " "; |
352 | Operand->printAsOperand(O); |
353 | } |
354 | } |
355 | |
356 | /// Generate the code inside the body of the vectorized loop. Assumes a single |
357 | /// LoopVectorBody basic-block was created for this. Introduce additional |
358 | /// basic-blocks as needed, and fill them all. |
359 | void VPlan::execute(VPTransformState *State) { |
360 | // -1. Check if the backedge taken count is needed, and if so build it. |
361 | if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) { |
362 | Value *TC = State->TripCount; |
363 | IRBuilder<> Builder(State->CFG.PrevBB->getTerminator()); |
364 | auto *TCMO = Builder.CreateSub(TC, ConstantInt::get(TC->getType(), 1), |
365 | "trip.count.minus.1"); |
366 | Value2VPValue[TCMO] = BackedgeTakenCount; |
367 | } |
368 | |
369 | // 0. Set the reverse mapping from VPValues to Values for code generation. |
370 | for (auto &Entry : Value2VPValue) |
371 | State->VPValue2Value[Entry.second] = Entry.first; |
372 | |
373 | BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB; |
374 | BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor(); |
375 | assert(VectorHeaderBB && "Loop preheader does not have a single successor.")((VectorHeaderBB && "Loop preheader does not have a single successor." ) ? static_cast<void> (0) : __assert_fail ("VectorHeaderBB && \"Loop preheader does not have a single successor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 375, __PRETTY_FUNCTION__)); |
376 | BasicBlock *VectorLatchBB = VectorHeaderBB; |
Value stored to 'VectorLatchBB' during its initialization is never read | |
377 | |
378 | // 1. Make room to generate basic-blocks inside loop body if needed. |
379 | VectorLatchBB = VectorHeaderBB->splitBasicBlock( |
380 | VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch"); |
381 | Loop *L = State->LI->getLoopFor(VectorHeaderBB); |
382 | L->addBasicBlockToLoop(VectorLatchBB, *State->LI); |
383 | // Remove the edge between Header and Latch to allow other connections. |
384 | // Temporarily terminate with unreachable until CFG is rewired. |
385 | // Note: this asserts the generated code's assumption that |
386 | // getFirstInsertionPt() can be dereferenced into an Instruction. |
387 | VectorHeaderBB->getTerminator()->eraseFromParent(); |
388 | State->Builder.SetInsertPoint(VectorHeaderBB); |
389 | UnreachableInst *Terminator = State->Builder.CreateUnreachable(); |
390 | State->Builder.SetInsertPoint(Terminator); |
391 | |
392 | // 2. Generate code in loop body. |
393 | State->CFG.PrevVPBB = nullptr; |
394 | State->CFG.PrevBB = VectorHeaderBB; |
395 | State->CFG.LastBB = VectorLatchBB; |
396 | |
397 | for (VPBlockBase *Block : depth_first(Entry)) |
398 | Block->execute(State); |
399 | |
400 | // Setup branch terminator successors for VPBBs in VPBBsToFix based on |
401 | // VPBB's successors. |
402 | for (auto VPBB : State->CFG.VPBBsToFix) { |
403 | assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path" ) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 404, __PRETTY_FUNCTION__)) |
404 | "Unexpected VPBBsToFix in non VPlan-native path")((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path" ) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 404, __PRETTY_FUNCTION__)); |
405 | BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB]; |
406 | assert(BB && "Unexpected null basic block for VPBB")((BB && "Unexpected null basic block for VPBB") ? static_cast <void> (0) : __assert_fail ("BB && \"Unexpected null basic block for VPBB\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 406, __PRETTY_FUNCTION__)); |
407 | |
408 | unsigned Idx = 0; |
409 | auto *BBTerminator = BB->getTerminator(); |
410 | |
411 | for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) { |
412 | VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock(); |
413 | BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]); |
414 | ++Idx; |
415 | } |
416 | } |
417 | |
418 | // 3. Merge the temporary latch created with the last basic-block filled. |
419 | BasicBlock *LastBB = State->CFG.PrevBB; |
420 | // Connect LastBB to VectorLatchBB to facilitate their merge. |
421 | assert((EnableVPlanNativePath ||(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB ->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable" ) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 423, __PRETTY_FUNCTION__)) |
422 | isa<UnreachableInst>(LastBB->getTerminator())) &&(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB ->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable" ) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 423, __PRETTY_FUNCTION__)) |
423 | "Expected InnerLoop VPlan CFG to terminate with unreachable")(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB ->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable" ) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 423, __PRETTY_FUNCTION__)); |
424 | assert((!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) &&(((!EnableVPlanNativePath || isa<BranchInst>(LastBB-> getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath" ) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 425, __PRETTY_FUNCTION__)) |
425 | "Expected VPlan CFG to terminate with branch in NativePath")(((!EnableVPlanNativePath || isa<BranchInst>(LastBB-> getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath" ) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 425, __PRETTY_FUNCTION__)); |
426 | LastBB->getTerminator()->eraseFromParent(); |
427 | BranchInst::Create(VectorLatchBB, LastBB); |
428 | |
429 | // Merge LastBB with Latch. |
430 | bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI); |
431 | (void)Merged; |
432 | assert(Merged && "Could not merge last basic block with latch.")((Merged && "Could not merge last basic block with latch." ) ? static_cast<void> (0) : __assert_fail ("Merged && \"Could not merge last basic block with latch.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 432, __PRETTY_FUNCTION__)); |
433 | VectorLatchBB = LastBB; |
434 | |
435 | // We do not attempt to preserve DT for outer loop vectorization currently. |
436 | if (!EnableVPlanNativePath) |
437 | updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB); |
438 | } |
439 | |
440 | void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB, |
441 | BasicBlock *LoopLatchBB) { |
442 | BasicBlock *LoopHeaderBB = LoopPreHeaderBB->getSingleSuccessor(); |
443 | assert(LoopHeaderBB && "Loop preheader does not have a single successor.")((LoopHeaderBB && "Loop preheader does not have a single successor." ) ? static_cast<void> (0) : __assert_fail ("LoopHeaderBB && \"Loop preheader does not have a single successor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 443, __PRETTY_FUNCTION__)); |
444 | DT->addNewBlock(LoopHeaderBB, LoopPreHeaderBB); |
445 | // The vector body may be more than a single basic-block by this point. |
446 | // Update the dominator tree information inside the vector body by propagating |
447 | // it from header to latch, expecting only triangular control-flow, if any. |
448 | BasicBlock *PostDomSucc = nullptr; |
449 | for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) { |
450 | // Get the list of successors of this block. |
451 | std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB)); |
452 | assert(Succs.size() <= 2 &&((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors." ) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 453, __PRETTY_FUNCTION__)) |
453 | "Basic block in vector loop has more than 2 successors.")((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors." ) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 453, __PRETTY_FUNCTION__)); |
454 | PostDomSucc = Succs[0]; |
455 | if (Succs.size() == 1) { |
456 | assert(PostDomSucc->getSinglePredecessor() &&((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor." ) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 457, __PRETTY_FUNCTION__)) |
457 | "PostDom successor has more than one predecessor.")((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor." ) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 457, __PRETTY_FUNCTION__)); |
458 | DT->addNewBlock(PostDomSucc, BB); |
459 | continue; |
460 | } |
461 | BasicBlock *InterimSucc = Succs[1]; |
462 | if (PostDomSucc->getSingleSuccessor() == InterimSucc) { |
463 | PostDomSucc = Succs[1]; |
464 | InterimSucc = Succs[0]; |
465 | } |
466 | assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&((InterimSucc->getSingleSuccessor() == PostDomSucc && "One successor of a basic block does not lead to the other." ) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 467, __PRETTY_FUNCTION__)) |
467 | "One successor of a basic block does not lead to the other.")((InterimSucc->getSingleSuccessor() == PostDomSucc && "One successor of a basic block does not lead to the other." ) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 467, __PRETTY_FUNCTION__)); |
468 | assert(InterimSucc->getSinglePredecessor() &&((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor." ) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 469, __PRETTY_FUNCTION__)) |
469 | "Interim successor has more than one predecessor.")((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor." ) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 469, __PRETTY_FUNCTION__)); |
470 | assert(PostDomSucc->hasNPredecessors(2) &&((PostDomSucc->hasNPredecessors(2) && "PostDom successor has more than two predecessors." ) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->hasNPredecessors(2) && \"PostDom successor has more than two predecessors.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 471, __PRETTY_FUNCTION__)) |
471 | "PostDom successor has more than two predecessors.")((PostDomSucc->hasNPredecessors(2) && "PostDom successor has more than two predecessors." ) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->hasNPredecessors(2) && \"PostDom successor has more than two predecessors.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 471, __PRETTY_FUNCTION__)); |
472 | DT->addNewBlock(InterimSucc, BB); |
473 | DT->addNewBlock(PostDomSucc, BB); |
474 | } |
475 | } |
476 | |
477 | const Twine VPlanPrinter::getUID(const VPBlockBase *Block) { |
478 | return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") + |
479 | Twine(getOrCreateBID(Block)); |
480 | } |
481 | |
482 | const Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) { |
483 | const std::string &Name = Block->getName(); |
484 | if (!Name.empty()) |
485 | return Name; |
486 | return "VPB" + Twine(getOrCreateBID(Block)); |
487 | } |
488 | |
489 | void VPlanPrinter::dump() { |
490 | Depth = 1; |
491 | bumpIndent(0); |
492 | OS << "digraph VPlan {\n"; |
493 | OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan"; |
494 | if (!Plan.getName().empty()) |
495 | OS << "\\n" << DOT::EscapeString(Plan.getName()); |
496 | if (!Plan.Value2VPValue.empty() || Plan.BackedgeTakenCount) { |
497 | OS << ", where:"; |
498 | if (Plan.BackedgeTakenCount) |
499 | OS << "\\n" |
500 | << *Plan.getOrCreateBackedgeTakenCount() << " := BackedgeTakenCount"; |
501 | for (auto Entry : Plan.Value2VPValue) { |
502 | OS << "\\n" << *Entry.second; |
503 | OS << DOT::EscapeString(" := "); |
504 | Entry.first->printAsOperand(OS, false); |
505 | } |
506 | } |
507 | OS << "\"]\n"; |
508 | OS << "node [shape=rect, fontname=Courier, fontsize=30]\n"; |
509 | OS << "edge [fontname=Courier, fontsize=30]\n"; |
510 | OS << "compound=true\n"; |
511 | |
512 | for (VPBlockBase *Block : depth_first(Plan.getEntry())) |
513 | dumpBlock(Block); |
514 | |
515 | OS << "}\n"; |
516 | } |
517 | |
518 | void VPlanPrinter::dumpBlock(const VPBlockBase *Block) { |
519 | if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block)) |
520 | dumpBasicBlock(BasicBlock); |
521 | else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) |
522 | dumpRegion(Region); |
523 | else |
524 | llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock." , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 524); |
525 | } |
526 | |
527 | void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To, |
528 | bool Hidden, const Twine &Label) { |
529 | // Due to "dot" we print an edge between two regions as an edge between the |
530 | // exit basic block and the entry basic of the respective regions. |
531 | const VPBlockBase *Tail = From->getExitBasicBlock(); |
532 | const VPBlockBase *Head = To->getEntryBasicBlock(); |
533 | OS << Indent << getUID(Tail) << " -> " << getUID(Head); |
534 | OS << " [ label=\"" << Label << '\"'; |
535 | if (Tail != From) |
536 | OS << " ltail=" << getUID(From); |
537 | if (Head != To) |
538 | OS << " lhead=" << getUID(To); |
539 | if (Hidden) |
540 | OS << "; splines=none"; |
541 | OS << "]\n"; |
542 | } |
543 | |
544 | void VPlanPrinter::dumpEdges(const VPBlockBase *Block) { |
545 | auto &Successors = Block->getSuccessors(); |
546 | if (Successors.size() == 1) |
547 | drawEdge(Block, Successors.front(), false, ""); |
548 | else if (Successors.size() == 2) { |
549 | drawEdge(Block, Successors.front(), false, "T"); |
550 | drawEdge(Block, Successors.back(), false, "F"); |
551 | } else { |
552 | unsigned SuccessorNumber = 0; |
553 | for (auto *Successor : Successors) |
554 | drawEdge(Block, Successor, false, Twine(SuccessorNumber++)); |
555 | } |
556 | } |
557 | |
558 | void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) { |
559 | OS << Indent << getUID(BasicBlock) << " [label =\n"; |
560 | bumpIndent(1); |
561 | OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\""; |
562 | bumpIndent(1); |
563 | |
564 | // Dump the block predicate. |
565 | const VPValue *Pred = BasicBlock->getPredicate(); |
566 | if (Pred) { |
567 | OS << " +\n" << Indent << " \"BlockPredicate: "; |
568 | if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) { |
569 | PredI->printAsOperand(OS); |
570 | OS << " (" << DOT::EscapeString(PredI->getParent()->getName()) |
571 | << ")\\l\""; |
572 | } else |
573 | Pred->printAsOperand(OS); |
574 | } |
575 | |
576 | for (const VPRecipeBase &Recipe : *BasicBlock) |
577 | Recipe.print(OS, Indent); |
578 | |
579 | // Dump the condition bit. |
580 | const VPValue *CBV = BasicBlock->getCondBit(); |
581 | if (CBV) { |
582 | OS << " +\n" << Indent << " \"CondBit: "; |
583 | if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) { |
584 | CBI->printAsOperand(OS); |
585 | OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\""; |
586 | } else { |
587 | CBV->printAsOperand(OS); |
588 | OS << "\""; |
589 | } |
590 | } |
591 | |
592 | bumpIndent(-2); |
593 | OS << "\n" << Indent << "]\n"; |
594 | dumpEdges(BasicBlock); |
595 | } |
596 | |
597 | void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) { |
598 | OS << Indent << "subgraph " << getUID(Region) << " {\n"; |
599 | bumpIndent(1); |
600 | OS << Indent << "fontname=Courier\n" |
601 | << Indent << "label=\"" |
602 | << DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ") |
603 | << DOT::EscapeString(Region->getName()) << "\"\n"; |
604 | // Dump the blocks of the region. |
605 | assert(Region->getEntry() && "Region contains no inner blocks.")((Region->getEntry() && "Region contains no inner blocks." ) ? static_cast<void> (0) : __assert_fail ("Region->getEntry() && \"Region contains no inner blocks.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 605, __PRETTY_FUNCTION__)); |
606 | for (const VPBlockBase *Block : depth_first(Region->getEntry())) |
607 | dumpBlock(Block); |
608 | bumpIndent(-1); |
609 | OS << Indent << "}\n"; |
610 | dumpEdges(Region); |
611 | } |
612 | |
613 | void VPlanPrinter::printAsIngredient(raw_ostream &O, Value *V) { |
614 | std::string IngredientString; |
615 | raw_string_ostream RSO(IngredientString); |
616 | if (auto *Inst = dyn_cast<Instruction>(V)) { |
617 | if (!Inst->getType()->isVoidTy()) { |
618 | Inst->printAsOperand(RSO, false); |
619 | RSO << " = "; |
620 | } |
621 | RSO << Inst->getOpcodeName() << " "; |
622 | unsigned E = Inst->getNumOperands(); |
623 | if (E > 0) { |
624 | Inst->getOperand(0)->printAsOperand(RSO, false); |
625 | for (unsigned I = 1; I < E; ++I) |
626 | Inst->getOperand(I)->printAsOperand(RSO << ", ", false); |
627 | } |
628 | } else // !Inst |
629 | V->printAsOperand(RSO, false); |
630 | RSO.flush(); |
631 | O << DOT::EscapeString(IngredientString); |
632 | } |
633 | |
634 | void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent) const { |
635 | O << " +\n" << Indent << "\"WIDEN\\l\""; |
636 | for (auto &Instr : make_range(Begin, End)) |
637 | O << " +\n" << Indent << "\" " << VPlanIngredient(&Instr) << "\\l\""; |
638 | } |
639 | |
640 | void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O, |
641 | const Twine &Indent) const { |
642 | O << " +\n" << Indent << "\"WIDEN-INDUCTION"; |
643 | if (Trunc) { |
644 | O << "\\l\""; |
645 | O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\""; |
646 | O << " +\n" << Indent << "\" " << VPlanIngredient(Trunc) << "\\l\""; |
647 | } else |
648 | O << " " << VPlanIngredient(IV) << "\\l\""; |
649 | } |
650 | |
651 | void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent) const { |
652 | O << " +\n" << Indent << "\"WIDEN-PHI " << VPlanIngredient(Phi) << "\\l\""; |
653 | } |
654 | |
655 | void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent) const { |
656 | O << " +\n" << Indent << "\"BLEND "; |
657 | Phi->printAsOperand(O, false); |
658 | O << " ="; |
659 | if (!User) { |
660 | // Not a User of any mask: not really blending, this is a |
661 | // single-predecessor phi. |
662 | O << " "; |
663 | Phi->getIncomingValue(0)->printAsOperand(O, false); |
664 | } else { |
665 | for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) { |
666 | O << " "; |
667 | Phi->getIncomingValue(I)->printAsOperand(O, false); |
668 | O << "/"; |
669 | User->getOperand(I)->printAsOperand(O); |
670 | } |
671 | } |
672 | O << "\\l\""; |
673 | } |
674 | |
675 | void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent) const { |
676 | O << " +\n" |
677 | << Indent << "\"" << (IsUniform ? "CLONE " : "REPLICATE ") |
678 | << VPlanIngredient(Ingredient); |
679 | if (AlsoPack) |
680 | O << " (S->V)"; |
681 | O << "\\l\""; |
682 | } |
683 | |
684 | void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent) const { |
685 | O << " +\n" |
686 | << Indent << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst) |
687 | << "\\l\""; |
688 | } |
689 | |
690 | void VPWidenMemoryInstructionRecipe::print(raw_ostream &O, |
691 | const Twine &Indent) const { |
692 | O << " +\n" << Indent << "\"WIDEN " << VPlanIngredient(&Instr); |
693 | if (User) { |
694 | O << ", "; |
695 | User->getOperand(0)->printAsOperand(O); |
696 | } |
697 | O << "\\l\""; |
698 | } |
699 | |
700 | template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT); |
701 | |
702 | void VPValue::replaceAllUsesWith(VPValue *New) { |
703 | for (VPUser *User : users()) |
704 | for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) |
705 | if (User->getOperand(I) == this) |
706 | User->setOperand(I, New); |
707 | } |
708 | |
709 | void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region, |
710 | Old2NewTy &Old2New, |
711 | InterleavedAccessInfo &IAI) { |
712 | ReversePostOrderTraversal<VPBlockBase *> RPOT(Region->getEntry()); |
713 | for (VPBlockBase *Base : RPOT) { |
714 | visitBlock(Base, Old2New, IAI); |
715 | } |
716 | } |
717 | |
718 | void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New, |
719 | InterleavedAccessInfo &IAI) { |
720 | if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) { |
721 | for (VPRecipeBase &VPI : *VPBB) { |
722 | assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions")((isa<VPInstruction>(&VPI) && "Can only handle VPInstructions" ) ? static_cast<void> (0) : __assert_fail ("isa<VPInstruction>(&VPI) && \"Can only handle VPInstructions\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 722, __PRETTY_FUNCTION__)); |
723 | auto *VPInst = cast<VPInstruction>(&VPI); |
724 | auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue()); |
725 | auto *IG = IAI.getInterleaveGroup(Inst); |
726 | if (!IG) |
727 | continue; |
728 | |
729 | auto NewIGIter = Old2New.find(IG); |
730 | if (NewIGIter == Old2New.end()) |
731 | Old2New[IG] = new InterleaveGroup<VPInstruction>( |
732 | IG->getFactor(), IG->isReverse(), IG->getAlignment()); |
733 | |
734 | if (Inst == IG->getInsertPos()) |
735 | Old2New[IG]->setInsertPos(VPInst); |
736 | |
737 | InterleaveGroupMap[VPInst] = Old2New[IG]; |
738 | InterleaveGroupMap[VPInst]->insertMember( |
739 | VPInst, IG->getIndex(Inst), |
740 | IG->isReverse() ? (-1) * int(IG->getFactor()) : IG->getFactor()); |
741 | } |
742 | } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block)) |
743 | visitRegion(Region, Old2New, IAI); |
744 | else |
745 | llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock." , "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp" , 745); |
746 | } |
747 | |
748 | VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan, |
749 | InterleavedAccessInfo &IAI) { |
750 | Old2NewTy Old2New; |
751 | visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI); |
752 | } |