Bug Summary

File:lib/Transforms/Vectorize/VPlan.cpp
Warning:line 376, column 15
Value stored to 'VectorLatchBB' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name VPlan.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn359999/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn359999=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-05-06-051613-19774-1 -x c++ /build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp -faddrsig
1//===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This is the LLVM vectorization plan. It represents a candidate for
11/// vectorization, allowing to plan and optimize how to vectorize a given loop
12/// before generating LLVM-IR.
13/// The vectorizer uses vectorization plans to estimate the costs of potential
14/// candidates and if profitable to execute the desired plan, generating vector
15/// LLVM-IR code.
16///
17//===----------------------------------------------------------------------===//
18
19#include "VPlan.h"
20#include "VPlanDominatorTree.h"
21#include "llvm/ADT/DepthFirstIterator.h"
22#include "llvm/ADT/PostOrderIterator.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Twine.h"
25#include "llvm/Analysis/LoopInfo.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/CFG.h"
28#include "llvm/IR/InstrTypes.h"
29#include "llvm/IR/Instruction.h"
30#include "llvm/IR/Instructions.h"
31#include "llvm/IR/Type.h"
32#include "llvm/IR/Value.h"
33#include "llvm/Support/Casting.h"
34#include "llvm/Support/Debug.h"
35#include "llvm/Support/ErrorHandling.h"
36#include "llvm/Support/GenericDomTreeConstruction.h"
37#include "llvm/Support/GraphWriter.h"
38#include "llvm/Support/raw_ostream.h"
39#include "llvm/Transforms/Utils/BasicBlockUtils.h"
40#include <cassert>
41#include <iterator>
42#include <string>
43#include <vector>
44
45using namespace llvm;
46extern cl::opt<bool> EnableVPlanNativePath;
47
48#define DEBUG_TYPE"vplan" "vplan"
49
50raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
51 if (const VPInstruction *Instr = dyn_cast<VPInstruction>(&V))
52 Instr->print(OS);
53 else
54 V.printAsOperand(OS);
55 return OS;
56}
57
58/// \return the VPBasicBlock that is the entry of Block, possibly indirectly.
59const VPBasicBlock *VPBlockBase::getEntryBasicBlock() const {
60 const VPBlockBase *Block = this;
61 while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
62 Block = Region->getEntry();
63 return cast<VPBasicBlock>(Block);
64}
65
66VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
67 VPBlockBase *Block = this;
68 while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
69 Block = Region->getEntry();
70 return cast<VPBasicBlock>(Block);
71}
72
73/// \return the VPBasicBlock that is the exit of Block, possibly indirectly.
74const VPBasicBlock *VPBlockBase::getExitBasicBlock() const {
75 const VPBlockBase *Block = this;
76 while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
77 Block = Region->getExit();
78 return cast<VPBasicBlock>(Block);
79}
80
81VPBasicBlock *VPBlockBase::getExitBasicBlock() {
82 VPBlockBase *Block = this;
83 while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
84 Block = Region->getExit();
85 return cast<VPBasicBlock>(Block);
86}
87
88VPBlockBase *VPBlockBase::getEnclosingBlockWithSuccessors() {
89 if (!Successors.empty() || !Parent)
90 return this;
91 assert(Parent->getExit() == this &&((Parent->getExit() == this && "Block w/o successors not the exit of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 92, __PRETTY_FUNCTION__))
92 "Block w/o successors not the exit of its parent.")((Parent->getExit() == this && "Block w/o successors not the exit of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 92, __PRETTY_FUNCTION__))
;
93 return Parent->getEnclosingBlockWithSuccessors();
94}
95
96VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() {
97 if (!Predecessors.empty() || !Parent)
98 return this;
99 assert(Parent->getEntry() == this &&((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 100, __PRETTY_FUNCTION__))
100 "Block w/o predecessors not the entry of its parent.")((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 100, __PRETTY_FUNCTION__))
;
101 return Parent->getEnclosingBlockWithPredecessors();
102}
103
104void VPBlockBase::deleteCFG(VPBlockBase *Entry) {
105 SmallVector<VPBlockBase *, 8> Blocks;
106 for (VPBlockBase *Block : depth_first(Entry))
107 Blocks.push_back(Block);
108
109 for (VPBlockBase *Block : Blocks)
110 delete Block;
111}
112
113BasicBlock *
114VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
115 // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
116 // Pred stands for Predessor. Prev stands for Previous - last visited/created.
117 BasicBlock *PrevBB = CFG.PrevBB;
118 BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
119 PrevBB->getParent(), CFG.LastBB);
120 LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: created " << NewBB->
getName() << '\n'; } } while (false)
;
121
122 // Hook up the new basic block to its predecessors.
123 for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
124 VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock();
125 auto &PredVPSuccessors = PredVPBB->getSuccessors();
126 BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
127
128 // In outer loop vectorization scenario, the predecessor BBlock may not yet
129 // be visited(backedge). Mark the VPBasicBlock for fixup at the end of
130 // vectorization. We do not encounter this case in inner loop vectorization
131 // as we start out by building a loop skeleton with the vector loop header
132 // and latch blocks. As a result, we never enter this function for the
133 // header block in the non VPlan-native path.
134 if (!PredBB) {
135 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 136, __PRETTY_FUNCTION__))
136 "Unexpected null predecessor in non VPlan-native path")((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 136, __PRETTY_FUNCTION__))
;
137 CFG.VPBBsToFix.push_back(PredVPBB);
138 continue;
139 }
140
141 assert(PredBB && "Predecessor basic-block not found building successor.")((PredBB && "Predecessor basic-block not found building successor."
) ? static_cast<void> (0) : __assert_fail ("PredBB && \"Predecessor basic-block not found building successor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 141, __PRETTY_FUNCTION__))
;
142 auto *PredBBTerminator = PredBB->getTerminator();
143 LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: draw edge from" << PredBB
->getName() << '\n'; } } while (false)
;
144 if (isa<UnreachableInst>(PredBBTerminator)) {
145 assert(PredVPSuccessors.size() == 1 &&((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 146, __PRETTY_FUNCTION__))
146 "Predecessor ending w/o branch must have single successor.")((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 146, __PRETTY_FUNCTION__))
;
147 PredBBTerminator->eraseFromParent();
148 BranchInst::Create(NewBB, PredBB);
149 } else {
150 assert(PredVPSuccessors.size() == 2 &&((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 151, __PRETTY_FUNCTION__))
151 "Predecessor ending with branch must have two successors.")((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 151, __PRETTY_FUNCTION__))
;
152 unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
153 assert(!PredBBTerminator->getSuccessor(idx) &&((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block."
) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 154, __PRETTY_FUNCTION__))
154 "Trying to reset an existing successor block.")((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block."
) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 154, __PRETTY_FUNCTION__))
;
155 PredBBTerminator->setSuccessor(idx, NewBB);
156 }
157 }
158 return NewBB;
159}
160
161void VPBasicBlock::execute(VPTransformState *State) {
162 bool Replica = State->Instance &&
163 !(State->Instance->Part == 0 && State->Instance->Lane == 0);
164 VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB;
165 VPBlockBase *SingleHPred = nullptr;
166 BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
167
168 // 1. Create an IR basic block, or reuse the last one if possible.
169 // The last IR basic block is reused, as an optimization, in three cases:
170 // A. the first VPBB reuses the loop header BB - when PrevVPBB is null;
171 // B. when the current VPBB has a single (hierarchical) predecessor which
172 // is PrevVPBB and the latter has a single (hierarchical) successor; and
173 // C. when the current VPBB is an entry of a region replica - where PrevVPBB
174 // is the exit of this region from a previous instance, or the predecessor
175 // of this region.
176 if (PrevVPBB && /* A */
177 !((SingleHPred = getSingleHierarchicalPredecessor()) &&
178 SingleHPred->getExitBasicBlock() == PrevVPBB &&
179 PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
180 !(Replica && getPredecessors().empty())) { /* C */
181 NewBB = createEmptyBasicBlock(State->CFG);
182 State->Builder.SetInsertPoint(NewBB);
183 // Temporarily terminate with unreachable until CFG is rewired.
184 UnreachableInst *Terminator = State->Builder.CreateUnreachable();
185 State->Builder.SetInsertPoint(Terminator);
186 // Register NewBB in its loop. In innermost loops its the same for all BB's.
187 Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
188 L->addBasicBlockToLoop(NewBB, *State->LI);
189 State->CFG.PrevBB = NewBB;
190 }
191
192 // 2. Fill the IR basic block with IR instructions.
193 LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: vectorizing VPBB:" <<
getName() << " in BB:" << NewBB->getName() <<
'\n'; } } while (false)
194 << " in BB:" << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: vectorizing VPBB:" <<
getName() << " in BB:" << NewBB->getName() <<
'\n'; } } while (false)
;
195
196 State->CFG.VPBB2IRBB[this] = NewBB;
197 State->CFG.PrevVPBB = this;
198
199 for (VPRecipeBase &Recipe : Recipes)
200 Recipe.execute(*State);
201
202 VPValue *CBV;
203 if (EnableVPlanNativePath && (CBV = getCondBit())) {
204 Value *IRCBV = CBV->getUnderlyingValue();
205 assert(IRCBV && "Unexpected null underlying value for condition bit")((IRCBV && "Unexpected null underlying value for condition bit"
) ? static_cast<void> (0) : __assert_fail ("IRCBV && \"Unexpected null underlying value for condition bit\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 205, __PRETTY_FUNCTION__))
;
206
207 // Condition bit value in a VPBasicBlock is used as the branch selector. In
208 // the VPlan-native path case, since all branches are uniform we generate a
209 // branch instruction using the condition value from vector lane 0 and dummy
210 // successors. The successors are fixed later when the successor blocks are
211 // visited.
212 Value *NewCond = State->Callback.getOrCreateVectorValues(IRCBV, 0);
213 NewCond = State->Builder.CreateExtractElement(NewCond,
214 State->Builder.getInt32(0));
215
216 // Replace the temporary unreachable terminator with the new conditional
217 // branch.
218 auto *CurrentTerminator = NewBB->getTerminator();
219 assert(isa<UnreachableInst>(CurrentTerminator) &&((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 221, __PRETTY_FUNCTION__))
220 "Expected to replace unreachable terminator with conditional "((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 221, __PRETTY_FUNCTION__))
221 "branch.")((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 221, __PRETTY_FUNCTION__))
;
222 auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond);
223 CondBr->setSuccessor(0, nullptr);
224 ReplaceInstWithInst(CurrentTerminator, CondBr);
225 }
226
227 LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: filled BB:" << *NewBB
; } } while (false)
;
228}
229
230void VPRegionBlock::execute(VPTransformState *State) {
231 ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry);
232
233 if (!isReplicator()) {
234 // Visit the VPBlocks connected to "this", starting from it.
235 for (VPBlockBase *Block : RPOT) {
236 if (EnableVPlanNativePath) {
237 // The inner loop vectorization path does not represent loop preheader
238 // and exit blocks as part of the VPlan. In the VPlan-native path, skip
239 // vectorizing loop preheader block. In future, we may replace this
240 // check with the check for loop preheader.
241 if (Block->getNumPredecessors() == 0)
242 continue;
243
244 // Skip vectorizing loop exit block. In future, we may replace this
245 // check with the check for loop exit.
246 if (Block->getNumSuccessors() == 0)
247 continue;
248 }
249
250 LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block
->getName() << '\n'; } } while (false)
;
251 Block->execute(State);
252 }
253 return;
254 }
255
256 assert(!State->Instance && "Replicating a Region with non-null instance.")((!State->Instance && "Replicating a Region with non-null instance."
) ? static_cast<void> (0) : __assert_fail ("!State->Instance && \"Replicating a Region with non-null instance.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 256, __PRETTY_FUNCTION__))
;
257
258 // Enter replicating mode.
259 State->Instance = {0, 0};
260
261 for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) {
262 State->Instance->Part = Part;
263 for (unsigned Lane = 0, VF = State->VF; Lane < VF; ++Lane) {
264 State->Instance->Lane = Lane;
265 // Visit the VPBlocks connected to \p this, starting from it.
266 for (VPBlockBase *Block : RPOT) {
267 LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block
->getName() << '\n'; } } while (false)
;
268 Block->execute(State);
269 }
270 }
271 }
272
273 // Exit replicating mode.
274 State->Instance.reset();
275}
276
277void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
278 Parent = InsertPos->getParent();
279 Parent->getRecipeList().insert(InsertPos->getIterator(), this);
280}
281
282iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() {
283 return getParent()->getRecipeList().erase(getIterator());
284}
285
286void VPInstruction::generateInstruction(VPTransformState &State,
287 unsigned Part) {
288 IRBuilder<> &Builder = State.Builder;
289
290 if (Instruction::isBinaryOp(getOpcode())) {
291 Value *A = State.get(getOperand(0), Part);
292 Value *B = State.get(getOperand(1), Part);
293 Value *V = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B);
294 State.set(this, V, Part);
295 return;
296 }
297
298 switch (getOpcode()) {
299 case VPInstruction::Not: {
300 Value *A = State.get(getOperand(0), Part);
301 Value *V = Builder.CreateNot(A);
302 State.set(this, V, Part);
303 break;
304 }
305 case VPInstruction::ICmpULE: {
306 Value *IV = State.get(getOperand(0), Part);
307 Value *TC = State.get(getOperand(1), Part);
308 Value *V = Builder.CreateICmpULE(IV, TC);
309 State.set(this, V, Part);
310 break;
311 }
312 default:
313 llvm_unreachable("Unsupported opcode for instruction")::llvm::llvm_unreachable_internal("Unsupported opcode for instruction"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 313)
;
314 }
315}
316
317void VPInstruction::execute(VPTransformState &State) {
318 assert(!State.Instance && "VPInstruction executing an Instance")((!State.Instance && "VPInstruction executing an Instance"
) ? static_cast<void> (0) : __assert_fail ("!State.Instance && \"VPInstruction executing an Instance\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 318, __PRETTY_FUNCTION__))
;
319 for (unsigned Part = 0; Part < State.UF; ++Part)
320 generateInstruction(State, Part);
321}
322
323void VPInstruction::print(raw_ostream &O, const Twine &Indent) const {
324 O << " +\n" << Indent << "\"EMIT ";
325 print(O);
326 O << "\\l\"";
327}
328
329void VPInstruction::print(raw_ostream &O) const {
330 printAsOperand(O);
331 O << " = ";
332
333 switch (getOpcode()) {
334 case VPInstruction::Not:
335 O << "not";
336 break;
337 case VPInstruction::ICmpULE:
338 O << "icmp ule";
339 break;
340 case VPInstruction::SLPLoad:
341 O << "combined load";
342 break;
343 case VPInstruction::SLPStore:
344 O << "combined store";
345 break;
346 default:
347 O << Instruction::getOpcodeName(getOpcode());
348 }
349
350 for (const VPValue *Operand : operands()) {
351 O << " ";
352 Operand->printAsOperand(O);
353 }
354}
355
356/// Generate the code inside the body of the vectorized loop. Assumes a single
357/// LoopVectorBody basic-block was created for this. Introduce additional
358/// basic-blocks as needed, and fill them all.
359void VPlan::execute(VPTransformState *State) {
360 // -1. Check if the backedge taken count is needed, and if so build it.
361 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
362 Value *TC = State->TripCount;
363 IRBuilder<> Builder(State->CFG.PrevBB->getTerminator());
364 auto *TCMO = Builder.CreateSub(TC, ConstantInt::get(TC->getType(), 1),
365 "trip.count.minus.1");
366 Value2VPValue[TCMO] = BackedgeTakenCount;
367 }
368
369 // 0. Set the reverse mapping from VPValues to Values for code generation.
370 for (auto &Entry : Value2VPValue)
371 State->VPValue2Value[Entry.second] = Entry.first;
372
373 BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
374 BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
375 assert(VectorHeaderBB && "Loop preheader does not have a single successor.")((VectorHeaderBB && "Loop preheader does not have a single successor."
) ? static_cast<void> (0) : __assert_fail ("VectorHeaderBB && \"Loop preheader does not have a single successor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 375, __PRETTY_FUNCTION__))
;
376 BasicBlock *VectorLatchBB = VectorHeaderBB;
Value stored to 'VectorLatchBB' during its initialization is never read
377
378 // 1. Make room to generate basic-blocks inside loop body if needed.
379 VectorLatchBB = VectorHeaderBB->splitBasicBlock(
380 VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
381 Loop *L = State->LI->getLoopFor(VectorHeaderBB);
382 L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
383 // Remove the edge between Header and Latch to allow other connections.
384 // Temporarily terminate with unreachable until CFG is rewired.
385 // Note: this asserts the generated code's assumption that
386 // getFirstInsertionPt() can be dereferenced into an Instruction.
387 VectorHeaderBB->getTerminator()->eraseFromParent();
388 State->Builder.SetInsertPoint(VectorHeaderBB);
389 UnreachableInst *Terminator = State->Builder.CreateUnreachable();
390 State->Builder.SetInsertPoint(Terminator);
391
392 // 2. Generate code in loop body.
393 State->CFG.PrevVPBB = nullptr;
394 State->CFG.PrevBB = VectorHeaderBB;
395 State->CFG.LastBB = VectorLatchBB;
396
397 for (VPBlockBase *Block : depth_first(Entry))
398 Block->execute(State);
399
400 // Setup branch terminator successors for VPBBs in VPBBsToFix based on
401 // VPBB's successors.
402 for (auto VPBB : State->CFG.VPBBsToFix) {
403 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 404, __PRETTY_FUNCTION__))
404 "Unexpected VPBBsToFix in non VPlan-native path")((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 404, __PRETTY_FUNCTION__))
;
405 BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
406 assert(BB && "Unexpected null basic block for VPBB")((BB && "Unexpected null basic block for VPBB") ? static_cast
<void> (0) : __assert_fail ("BB && \"Unexpected null basic block for VPBB\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 406, __PRETTY_FUNCTION__))
;
407
408 unsigned Idx = 0;
409 auto *BBTerminator = BB->getTerminator();
410
411 for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
412 VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
413 BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
414 ++Idx;
415 }
416 }
417
418 // 3. Merge the temporary latch created with the last basic-block filled.
419 BasicBlock *LastBB = State->CFG.PrevBB;
420 // Connect LastBB to VectorLatchBB to facilitate their merge.
421 assert((EnableVPlanNativePath ||(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 423, __PRETTY_FUNCTION__))
422 isa<UnreachableInst>(LastBB->getTerminator())) &&(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 423, __PRETTY_FUNCTION__))
423 "Expected InnerLoop VPlan CFG to terminate with unreachable")(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 423, __PRETTY_FUNCTION__))
;
424 assert((!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) &&(((!EnableVPlanNativePath || isa<BranchInst>(LastBB->
getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath"
) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 425, __PRETTY_FUNCTION__))
425 "Expected VPlan CFG to terminate with branch in NativePath")(((!EnableVPlanNativePath || isa<BranchInst>(LastBB->
getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath"
) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 425, __PRETTY_FUNCTION__))
;
426 LastBB->getTerminator()->eraseFromParent();
427 BranchInst::Create(VectorLatchBB, LastBB);
428
429 // Merge LastBB with Latch.
430 bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
431 (void)Merged;
432 assert(Merged && "Could not merge last basic block with latch.")((Merged && "Could not merge last basic block with latch."
) ? static_cast<void> (0) : __assert_fail ("Merged && \"Could not merge last basic block with latch.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 432, __PRETTY_FUNCTION__))
;
433 VectorLatchBB = LastBB;
434
435 // We do not attempt to preserve DT for outer loop vectorization currently.
436 if (!EnableVPlanNativePath)
437 updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB);
438}
439
440void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
441 BasicBlock *LoopLatchBB) {
442 BasicBlock *LoopHeaderBB = LoopPreHeaderBB->getSingleSuccessor();
443 assert(LoopHeaderBB && "Loop preheader does not have a single successor.")((LoopHeaderBB && "Loop preheader does not have a single successor."
) ? static_cast<void> (0) : __assert_fail ("LoopHeaderBB && \"Loop preheader does not have a single successor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 443, __PRETTY_FUNCTION__))
;
444 DT->addNewBlock(LoopHeaderBB, LoopPreHeaderBB);
445 // The vector body may be more than a single basic-block by this point.
446 // Update the dominator tree information inside the vector body by propagating
447 // it from header to latch, expecting only triangular control-flow, if any.
448 BasicBlock *PostDomSucc = nullptr;
449 for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
450 // Get the list of successors of this block.
451 std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
452 assert(Succs.size() <= 2 &&((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors."
) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 453, __PRETTY_FUNCTION__))
453 "Basic block in vector loop has more than 2 successors.")((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors."
) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 453, __PRETTY_FUNCTION__))
;
454 PostDomSucc = Succs[0];
455 if (Succs.size() == 1) {
456 assert(PostDomSucc->getSinglePredecessor() &&((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 457, __PRETTY_FUNCTION__))
457 "PostDom successor has more than one predecessor.")((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 457, __PRETTY_FUNCTION__))
;
458 DT->addNewBlock(PostDomSucc, BB);
459 continue;
460 }
461 BasicBlock *InterimSucc = Succs[1];
462 if (PostDomSucc->getSingleSuccessor() == InterimSucc) {
463 PostDomSucc = Succs[1];
464 InterimSucc = Succs[0];
465 }
466 assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&((InterimSucc->getSingleSuccessor() == PostDomSucc &&
"One successor of a basic block does not lead to the other."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 467, __PRETTY_FUNCTION__))
467 "One successor of a basic block does not lead to the other.")((InterimSucc->getSingleSuccessor() == PostDomSucc &&
"One successor of a basic block does not lead to the other."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 467, __PRETTY_FUNCTION__))
;
468 assert(InterimSucc->getSinglePredecessor() &&((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 469, __PRETTY_FUNCTION__))
469 "Interim successor has more than one predecessor.")((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 469, __PRETTY_FUNCTION__))
;
470 assert(PostDomSucc->hasNPredecessors(2) &&((PostDomSucc->hasNPredecessors(2) && "PostDom successor has more than two predecessors."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->hasNPredecessors(2) && \"PostDom successor has more than two predecessors.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 471, __PRETTY_FUNCTION__))
471 "PostDom successor has more than two predecessors.")((PostDomSucc->hasNPredecessors(2) && "PostDom successor has more than two predecessors."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->hasNPredecessors(2) && \"PostDom successor has more than two predecessors.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 471, __PRETTY_FUNCTION__))
;
472 DT->addNewBlock(InterimSucc, BB);
473 DT->addNewBlock(PostDomSucc, BB);
474 }
475}
476
477const Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
478 return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
479 Twine(getOrCreateBID(Block));
480}
481
482const Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) {
483 const std::string &Name = Block->getName();
484 if (!Name.empty())
485 return Name;
486 return "VPB" + Twine(getOrCreateBID(Block));
487}
488
489void VPlanPrinter::dump() {
490 Depth = 1;
491 bumpIndent(0);
492 OS << "digraph VPlan {\n";
493 OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
494 if (!Plan.getName().empty())
495 OS << "\\n" << DOT::EscapeString(Plan.getName());
496 if (!Plan.Value2VPValue.empty() || Plan.BackedgeTakenCount) {
497 OS << ", where:";
498 if (Plan.BackedgeTakenCount)
499 OS << "\\n"
500 << *Plan.getOrCreateBackedgeTakenCount() << " := BackedgeTakenCount";
501 for (auto Entry : Plan.Value2VPValue) {
502 OS << "\\n" << *Entry.second;
503 OS << DOT::EscapeString(" := ");
504 Entry.first->printAsOperand(OS, false);
505 }
506 }
507 OS << "\"]\n";
508 OS << "node [shape=rect, fontname=Courier, fontsize=30]\n";
509 OS << "edge [fontname=Courier, fontsize=30]\n";
510 OS << "compound=true\n";
511
512 for (VPBlockBase *Block : depth_first(Plan.getEntry()))
513 dumpBlock(Block);
514
515 OS << "}\n";
516}
517
518void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
519 if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block))
520 dumpBasicBlock(BasicBlock);
521 else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
522 dumpRegion(Region);
523 else
524 llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock."
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 524)
;
525}
526
527void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
528 bool Hidden, const Twine &Label) {
529 // Due to "dot" we print an edge between two regions as an edge between the
530 // exit basic block and the entry basic of the respective regions.
531 const VPBlockBase *Tail = From->getExitBasicBlock();
532 const VPBlockBase *Head = To->getEntryBasicBlock();
533 OS << Indent << getUID(Tail) << " -> " << getUID(Head);
534 OS << " [ label=\"" << Label << '\"';
535 if (Tail != From)
536 OS << " ltail=" << getUID(From);
537 if (Head != To)
538 OS << " lhead=" << getUID(To);
539 if (Hidden)
540 OS << "; splines=none";
541 OS << "]\n";
542}
543
544void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
545 auto &Successors = Block->getSuccessors();
546 if (Successors.size() == 1)
547 drawEdge(Block, Successors.front(), false, "");
548 else if (Successors.size() == 2) {
549 drawEdge(Block, Successors.front(), false, "T");
550 drawEdge(Block, Successors.back(), false, "F");
551 } else {
552 unsigned SuccessorNumber = 0;
553 for (auto *Successor : Successors)
554 drawEdge(Block, Successor, false, Twine(SuccessorNumber++));
555 }
556}
557
558void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
559 OS << Indent << getUID(BasicBlock) << " [label =\n";
560 bumpIndent(1);
561 OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
562 bumpIndent(1);
563
564 // Dump the block predicate.
565 const VPValue *Pred = BasicBlock->getPredicate();
566 if (Pred) {
567 OS << " +\n" << Indent << " \"BlockPredicate: ";
568 if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) {
569 PredI->printAsOperand(OS);
570 OS << " (" << DOT::EscapeString(PredI->getParent()->getName())
571 << ")\\l\"";
572 } else
573 Pred->printAsOperand(OS);
574 }
575
576 for (const VPRecipeBase &Recipe : *BasicBlock)
577 Recipe.print(OS, Indent);
578
579 // Dump the condition bit.
580 const VPValue *CBV = BasicBlock->getCondBit();
581 if (CBV) {
582 OS << " +\n" << Indent << " \"CondBit: ";
583 if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
584 CBI->printAsOperand(OS);
585 OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
586 } else {
587 CBV->printAsOperand(OS);
588 OS << "\"";
589 }
590 }
591
592 bumpIndent(-2);
593 OS << "\n" << Indent << "]\n";
594 dumpEdges(BasicBlock);
595}
596
597void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
598 OS << Indent << "subgraph " << getUID(Region) << " {\n";
599 bumpIndent(1);
600 OS << Indent << "fontname=Courier\n"
601 << Indent << "label=\""
602 << DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ")
603 << DOT::EscapeString(Region->getName()) << "\"\n";
604 // Dump the blocks of the region.
605 assert(Region->getEntry() && "Region contains no inner blocks.")((Region->getEntry() && "Region contains no inner blocks."
) ? static_cast<void> (0) : __assert_fail ("Region->getEntry() && \"Region contains no inner blocks.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 605, __PRETTY_FUNCTION__))
;
606 for (const VPBlockBase *Block : depth_first(Region->getEntry()))
607 dumpBlock(Block);
608 bumpIndent(-1);
609 OS << Indent << "}\n";
610 dumpEdges(Region);
611}
612
613void VPlanPrinter::printAsIngredient(raw_ostream &O, Value *V) {
614 std::string IngredientString;
615 raw_string_ostream RSO(IngredientString);
616 if (auto *Inst = dyn_cast<Instruction>(V)) {
617 if (!Inst->getType()->isVoidTy()) {
618 Inst->printAsOperand(RSO, false);
619 RSO << " = ";
620 }
621 RSO << Inst->getOpcodeName() << " ";
622 unsigned E = Inst->getNumOperands();
623 if (E > 0) {
624 Inst->getOperand(0)->printAsOperand(RSO, false);
625 for (unsigned I = 1; I < E; ++I)
626 Inst->getOperand(I)->printAsOperand(RSO << ", ", false);
627 }
628 } else // !Inst
629 V->printAsOperand(RSO, false);
630 RSO.flush();
631 O << DOT::EscapeString(IngredientString);
632}
633
634void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent) const {
635 O << " +\n" << Indent << "\"WIDEN\\l\"";
636 for (auto &Instr : make_range(Begin, End))
637 O << " +\n" << Indent << "\" " << VPlanIngredient(&Instr) << "\\l\"";
638}
639
640void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O,
641 const Twine &Indent) const {
642 O << " +\n" << Indent << "\"WIDEN-INDUCTION";
643 if (Trunc) {
644 O << "\\l\"";
645 O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
646 O << " +\n" << Indent << "\" " << VPlanIngredient(Trunc) << "\\l\"";
647 } else
648 O << " " << VPlanIngredient(IV) << "\\l\"";
649}
650
651void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
652 O << " +\n" << Indent << "\"WIDEN-PHI " << VPlanIngredient(Phi) << "\\l\"";
653}
654
655void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent) const {
656 O << " +\n" << Indent << "\"BLEND ";
657 Phi->printAsOperand(O, false);
658 O << " =";
659 if (!User) {
660 // Not a User of any mask: not really blending, this is a
661 // single-predecessor phi.
662 O << " ";
663 Phi->getIncomingValue(0)->printAsOperand(O, false);
664 } else {
665 for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) {
666 O << " ";
667 Phi->getIncomingValue(I)->printAsOperand(O, false);
668 O << "/";
669 User->getOperand(I)->printAsOperand(O);
670 }
671 }
672 O << "\\l\"";
673}
674
675void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent) const {
676 O << " +\n"
677 << Indent << "\"" << (IsUniform ? "CLONE " : "REPLICATE ")
678 << VPlanIngredient(Ingredient);
679 if (AlsoPack)
680 O << " (S->V)";
681 O << "\\l\"";
682}
683
684void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
685 O << " +\n"
686 << Indent << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst)
687 << "\\l\"";
688}
689
690void VPWidenMemoryInstructionRecipe::print(raw_ostream &O,
691 const Twine &Indent) const {
692 O << " +\n" << Indent << "\"WIDEN " << VPlanIngredient(&Instr);
693 if (User) {
694 O << ", ";
695 User->getOperand(0)->printAsOperand(O);
696 }
697 O << "\\l\"";
698}
699
700template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
701
702void VPValue::replaceAllUsesWith(VPValue *New) {
703 for (VPUser *User : users())
704 for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I)
705 if (User->getOperand(I) == this)
706 User->setOperand(I, New);
707}
708
709void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
710 Old2NewTy &Old2New,
711 InterleavedAccessInfo &IAI) {
712 ReversePostOrderTraversal<VPBlockBase *> RPOT(Region->getEntry());
713 for (VPBlockBase *Base : RPOT) {
714 visitBlock(Base, Old2New, IAI);
715 }
716}
717
718void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
719 InterleavedAccessInfo &IAI) {
720 if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
721 for (VPRecipeBase &VPI : *VPBB) {
722 assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions")((isa<VPInstruction>(&VPI) && "Can only handle VPInstructions"
) ? static_cast<void> (0) : __assert_fail ("isa<VPInstruction>(&VPI) && \"Can only handle VPInstructions\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 722, __PRETTY_FUNCTION__))
;
723 auto *VPInst = cast<VPInstruction>(&VPI);
724 auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
725 auto *IG = IAI.getInterleaveGroup(Inst);
726 if (!IG)
727 continue;
728
729 auto NewIGIter = Old2New.find(IG);
730 if (NewIGIter == Old2New.end())
731 Old2New[IG] = new InterleaveGroup<VPInstruction>(
732 IG->getFactor(), IG->isReverse(), IG->getAlignment());
733
734 if (Inst == IG->getInsertPos())
735 Old2New[IG]->setInsertPos(VPInst);
736
737 InterleaveGroupMap[VPInst] = Old2New[IG];
738 InterleaveGroupMap[VPInst]->insertMember(
739 VPInst, IG->getIndex(Inst),
740 IG->isReverse() ? (-1) * int(IG->getFactor()) : IG->getFactor());
741 }
742 } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
743 visitRegion(Region, Old2New, IAI);
744 else
745 llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock."
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/Transforms/Vectorize/VPlan.cpp"
, 745)
;
746}
747
748VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
749 InterleavedAccessInfo &IAI) {
750 Old2NewTy Old2New;
751 visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
752}