File: | llvm/lib/Transforms/Scalar/SROA.cpp |
Warning: | line 2144, column 3 Called C++ object pointer is null |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===// | ||||||||||||
2 | // | ||||||||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||||||||
6 | // | ||||||||||||
7 | //===----------------------------------------------------------------------===// | ||||||||||||
8 | /// \file | ||||||||||||
9 | /// This transformation implements the well known scalar replacement of | ||||||||||||
10 | /// aggregates transformation. It tries to identify promotable elements of an | ||||||||||||
11 | /// aggregate alloca, and promote them to registers. It will also try to | ||||||||||||
12 | /// convert uses of an element (or set of elements) of an alloca into a vector | ||||||||||||
13 | /// or bitfield-style integer scalar if appropriate. | ||||||||||||
14 | /// | ||||||||||||
15 | /// It works to do this with minimal slicing of the alloca so that regions | ||||||||||||
16 | /// which are merely transferred in and out of external memory remain unchanged | ||||||||||||
17 | /// and are not decomposed to scalar code. | ||||||||||||
18 | /// | ||||||||||||
19 | /// Because this also performs alloca promotion, it can be thought of as also | ||||||||||||
20 | /// serving the purpose of SSA formation. The algorithm iterates on the | ||||||||||||
21 | /// function until all opportunities for promotion have been realized. | ||||||||||||
22 | /// | ||||||||||||
23 | //===----------------------------------------------------------------------===// | ||||||||||||
24 | |||||||||||||
25 | #include "llvm/Transforms/Scalar/SROA.h" | ||||||||||||
26 | #include "llvm/ADT/APInt.h" | ||||||||||||
27 | #include "llvm/ADT/ArrayRef.h" | ||||||||||||
28 | #include "llvm/ADT/DenseMap.h" | ||||||||||||
29 | #include "llvm/ADT/PointerIntPair.h" | ||||||||||||
30 | #include "llvm/ADT/STLExtras.h" | ||||||||||||
31 | #include "llvm/ADT/SetVector.h" | ||||||||||||
32 | #include "llvm/ADT/SmallBitVector.h" | ||||||||||||
33 | #include "llvm/ADT/SmallPtrSet.h" | ||||||||||||
34 | #include "llvm/ADT/SmallVector.h" | ||||||||||||
35 | #include "llvm/ADT/Statistic.h" | ||||||||||||
36 | #include "llvm/ADT/StringRef.h" | ||||||||||||
37 | #include "llvm/ADT/Twine.h" | ||||||||||||
38 | #include "llvm/ADT/iterator.h" | ||||||||||||
39 | #include "llvm/ADT/iterator_range.h" | ||||||||||||
40 | #include "llvm/Analysis/AssumptionCache.h" | ||||||||||||
41 | #include "llvm/Analysis/GlobalsModRef.h" | ||||||||||||
42 | #include "llvm/Analysis/Loads.h" | ||||||||||||
43 | #include "llvm/Analysis/PtrUseVisitor.h" | ||||||||||||
44 | #include "llvm/Transforms/Utils/Local.h" | ||||||||||||
45 | #include "llvm/Config/llvm-config.h" | ||||||||||||
46 | #include "llvm/IR/BasicBlock.h" | ||||||||||||
47 | #include "llvm/IR/Constant.h" | ||||||||||||
48 | #include "llvm/IR/ConstantFolder.h" | ||||||||||||
49 | #include "llvm/IR/Constants.h" | ||||||||||||
50 | #include "llvm/IR/DIBuilder.h" | ||||||||||||
51 | #include "llvm/IR/DataLayout.h" | ||||||||||||
52 | #include "llvm/IR/DebugInfoMetadata.h" | ||||||||||||
53 | #include "llvm/IR/DerivedTypes.h" | ||||||||||||
54 | #include "llvm/IR/Dominators.h" | ||||||||||||
55 | #include "llvm/IR/Function.h" | ||||||||||||
56 | #include "llvm/IR/GetElementPtrTypeIterator.h" | ||||||||||||
57 | #include "llvm/IR/GlobalAlias.h" | ||||||||||||
58 | #include "llvm/IR/IRBuilder.h" | ||||||||||||
59 | #include "llvm/IR/InstVisitor.h" | ||||||||||||
60 | #include "llvm/IR/InstrTypes.h" | ||||||||||||
61 | #include "llvm/IR/Instruction.h" | ||||||||||||
62 | #include "llvm/IR/Instructions.h" | ||||||||||||
63 | #include "llvm/IR/IntrinsicInst.h" | ||||||||||||
64 | #include "llvm/IR/Intrinsics.h" | ||||||||||||
65 | #include "llvm/IR/LLVMContext.h" | ||||||||||||
66 | #include "llvm/IR/Metadata.h" | ||||||||||||
67 | #include "llvm/IR/Module.h" | ||||||||||||
68 | #include "llvm/IR/Operator.h" | ||||||||||||
69 | #include "llvm/IR/PassManager.h" | ||||||||||||
70 | #include "llvm/IR/Type.h" | ||||||||||||
71 | #include "llvm/IR/Use.h" | ||||||||||||
72 | #include "llvm/IR/User.h" | ||||||||||||
73 | #include "llvm/IR/Value.h" | ||||||||||||
74 | #include "llvm/Pass.h" | ||||||||||||
75 | #include "llvm/Support/Casting.h" | ||||||||||||
76 | #include "llvm/Support/CommandLine.h" | ||||||||||||
77 | #include "llvm/Support/Compiler.h" | ||||||||||||
78 | #include "llvm/Support/Debug.h" | ||||||||||||
79 | #include "llvm/Support/ErrorHandling.h" | ||||||||||||
80 | #include "llvm/Support/MathExtras.h" | ||||||||||||
81 | #include "llvm/Support/raw_ostream.h" | ||||||||||||
82 | #include "llvm/Transforms/Scalar.h" | ||||||||||||
83 | #include "llvm/Transforms/Utils/PromoteMemToReg.h" | ||||||||||||
84 | #include <algorithm> | ||||||||||||
85 | #include <cassert> | ||||||||||||
86 | #include <chrono> | ||||||||||||
87 | #include <cstddef> | ||||||||||||
88 | #include <cstdint> | ||||||||||||
89 | #include <cstring> | ||||||||||||
90 | #include <iterator> | ||||||||||||
91 | #include <string> | ||||||||||||
92 | #include <tuple> | ||||||||||||
93 | #include <utility> | ||||||||||||
94 | #include <vector> | ||||||||||||
95 | |||||||||||||
96 | #ifndef NDEBUG | ||||||||||||
97 | // We only use this for a debug check. | ||||||||||||
98 | #include <random> | ||||||||||||
99 | #endif | ||||||||||||
100 | |||||||||||||
101 | using namespace llvm; | ||||||||||||
102 | using namespace llvm::sroa; | ||||||||||||
103 | |||||||||||||
104 | #define DEBUG_TYPE"sroa" "sroa" | ||||||||||||
105 | |||||||||||||
106 | STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement")static llvm::Statistic NumAllocasAnalyzed = {"sroa", "NumAllocasAnalyzed" , "Number of allocas analyzed for replacement"}; | ||||||||||||
107 | STATISTIC(NumAllocaPartitions, "Number of alloca partitions formed")static llvm::Statistic NumAllocaPartitions = {"sroa", "NumAllocaPartitions" , "Number of alloca partitions formed"}; | ||||||||||||
108 | STATISTIC(MaxPartitionsPerAlloca, "Maximum number of partitions per alloca")static llvm::Statistic MaxPartitionsPerAlloca = {"sroa", "MaxPartitionsPerAlloca" , "Maximum number of partitions per alloca"}; | ||||||||||||
109 | STATISTIC(NumAllocaPartitionUses, "Number of alloca partition uses rewritten")static llvm::Statistic NumAllocaPartitionUses = {"sroa", "NumAllocaPartitionUses" , "Number of alloca partition uses rewritten"}; | ||||||||||||
110 | STATISTIC(MaxUsesPerAllocaPartition, "Maximum number of uses of a partition")static llvm::Statistic MaxUsesPerAllocaPartition = {"sroa", "MaxUsesPerAllocaPartition" , "Maximum number of uses of a partition"}; | ||||||||||||
111 | STATISTIC(NumNewAllocas, "Number of new, smaller allocas introduced")static llvm::Statistic NumNewAllocas = {"sroa", "NumNewAllocas" , "Number of new, smaller allocas introduced"}; | ||||||||||||
112 | STATISTIC(NumPromoted, "Number of allocas promoted to SSA values")static llvm::Statistic NumPromoted = {"sroa", "NumPromoted", "Number of allocas promoted to SSA values" }; | ||||||||||||
113 | STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion")static llvm::Statistic NumLoadsSpeculated = {"sroa", "NumLoadsSpeculated" , "Number of loads speculated to allow promotion"}; | ||||||||||||
114 | STATISTIC(NumDeleted, "Number of instructions deleted")static llvm::Statistic NumDeleted = {"sroa", "NumDeleted", "Number of instructions deleted" }; | ||||||||||||
115 | STATISTIC(NumVectorized, "Number of vectorized aggregates")static llvm::Statistic NumVectorized = {"sroa", "NumVectorized" , "Number of vectorized aggregates"}; | ||||||||||||
116 | |||||||||||||
117 | /// Hidden option to enable randomly shuffling the slices to help uncover | ||||||||||||
118 | /// instability in their order. | ||||||||||||
119 | static cl::opt<bool> SROARandomShuffleSlices("sroa-random-shuffle-slices", | ||||||||||||
120 | cl::init(false), cl::Hidden); | ||||||||||||
121 | |||||||||||||
122 | /// Hidden option to experiment with completely strict handling of inbounds | ||||||||||||
123 | /// GEPs. | ||||||||||||
124 | static cl::opt<bool> SROAStrictInbounds("sroa-strict-inbounds", cl::init(false), | ||||||||||||
125 | cl::Hidden); | ||||||||||||
126 | |||||||||||||
127 | namespace { | ||||||||||||
128 | |||||||||||||
129 | /// A custom IRBuilder inserter which prefixes all names, but only in | ||||||||||||
130 | /// Assert builds. | ||||||||||||
131 | class IRBuilderPrefixedInserter : public IRBuilderDefaultInserter { | ||||||||||||
132 | std::string Prefix; | ||||||||||||
133 | |||||||||||||
134 | const Twine getNameWithPrefix(const Twine &Name) const { | ||||||||||||
135 | return Name.isTriviallyEmpty() ? Name : Prefix + Name; | ||||||||||||
136 | } | ||||||||||||
137 | |||||||||||||
138 | public: | ||||||||||||
139 | void SetNamePrefix(const Twine &P) { Prefix = P.str(); } | ||||||||||||
140 | |||||||||||||
141 | protected: | ||||||||||||
142 | void InsertHelper(Instruction *I, const Twine &Name, BasicBlock *BB, | ||||||||||||
143 | BasicBlock::iterator InsertPt) const { | ||||||||||||
144 | IRBuilderDefaultInserter::InsertHelper(I, getNameWithPrefix(Name), BB, | ||||||||||||
145 | InsertPt); | ||||||||||||
146 | } | ||||||||||||
147 | }; | ||||||||||||
148 | |||||||||||||
149 | /// Provide a type for IRBuilder that drops names in release builds. | ||||||||||||
150 | using IRBuilderTy = IRBuilder<ConstantFolder, IRBuilderPrefixedInserter>; | ||||||||||||
151 | |||||||||||||
152 | /// A used slice of an alloca. | ||||||||||||
153 | /// | ||||||||||||
154 | /// This structure represents a slice of an alloca used by some instruction. It | ||||||||||||
155 | /// stores both the begin and end offsets of this use, a pointer to the use | ||||||||||||
156 | /// itself, and a flag indicating whether we can classify the use as splittable | ||||||||||||
157 | /// or not when forming partitions of the alloca. | ||||||||||||
158 | class Slice { | ||||||||||||
159 | /// The beginning offset of the range. | ||||||||||||
160 | uint64_t BeginOffset = 0; | ||||||||||||
161 | |||||||||||||
162 | /// The ending offset, not included in the range. | ||||||||||||
163 | uint64_t EndOffset = 0; | ||||||||||||
164 | |||||||||||||
165 | /// Storage for both the use of this slice and whether it can be | ||||||||||||
166 | /// split. | ||||||||||||
167 | PointerIntPair<Use *, 1, bool> UseAndIsSplittable; | ||||||||||||
168 | |||||||||||||
169 | public: | ||||||||||||
170 | Slice() = default; | ||||||||||||
171 | |||||||||||||
172 | Slice(uint64_t BeginOffset, uint64_t EndOffset, Use *U, bool IsSplittable) | ||||||||||||
173 | : BeginOffset(BeginOffset), EndOffset(EndOffset), | ||||||||||||
174 | UseAndIsSplittable(U, IsSplittable) {} | ||||||||||||
175 | |||||||||||||
176 | uint64_t beginOffset() const { return BeginOffset; } | ||||||||||||
177 | uint64_t endOffset() const { return EndOffset; } | ||||||||||||
178 | |||||||||||||
179 | bool isSplittable() const { return UseAndIsSplittable.getInt(); } | ||||||||||||
180 | void makeUnsplittable() { UseAndIsSplittable.setInt(false); } | ||||||||||||
181 | |||||||||||||
182 | Use *getUse() const { return UseAndIsSplittable.getPointer(); } | ||||||||||||
183 | |||||||||||||
184 | bool isDead() const { return getUse() == nullptr; } | ||||||||||||
185 | void kill() { UseAndIsSplittable.setPointer(nullptr); } | ||||||||||||
186 | |||||||||||||
187 | /// Support for ordering ranges. | ||||||||||||
188 | /// | ||||||||||||
189 | /// This provides an ordering over ranges such that start offsets are | ||||||||||||
190 | /// always increasing, and within equal start offsets, the end offsets are | ||||||||||||
191 | /// decreasing. Thus the spanning range comes first in a cluster with the | ||||||||||||
192 | /// same start position. | ||||||||||||
193 | bool operator<(const Slice &RHS) const { | ||||||||||||
194 | if (beginOffset() < RHS.beginOffset()) | ||||||||||||
195 | return true; | ||||||||||||
196 | if (beginOffset() > RHS.beginOffset()) | ||||||||||||
197 | return false; | ||||||||||||
198 | if (isSplittable() != RHS.isSplittable()) | ||||||||||||
199 | return !isSplittable(); | ||||||||||||
200 | if (endOffset() > RHS.endOffset()) | ||||||||||||
201 | return true; | ||||||||||||
202 | return false; | ||||||||||||
203 | } | ||||||||||||
204 | |||||||||||||
205 | /// Support comparison with a single offset to allow binary searches. | ||||||||||||
206 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(const Slice &LHS, | ||||||||||||
207 | uint64_t RHSOffset) { | ||||||||||||
208 | return LHS.beginOffset() < RHSOffset; | ||||||||||||
209 | } | ||||||||||||
210 | friend LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) bool operator<(uint64_t LHSOffset, | ||||||||||||
211 | const Slice &RHS) { | ||||||||||||
212 | return LHSOffset < RHS.beginOffset(); | ||||||||||||
213 | } | ||||||||||||
214 | |||||||||||||
215 | bool operator==(const Slice &RHS) const { | ||||||||||||
216 | return isSplittable() == RHS.isSplittable() && | ||||||||||||
217 | beginOffset() == RHS.beginOffset() && endOffset() == RHS.endOffset(); | ||||||||||||
218 | } | ||||||||||||
219 | bool operator!=(const Slice &RHS) const { return !operator==(RHS); } | ||||||||||||
220 | }; | ||||||||||||
221 | |||||||||||||
222 | } // end anonymous namespace | ||||||||||||
223 | |||||||||||||
224 | /// Representation of the alloca slices. | ||||||||||||
225 | /// | ||||||||||||
226 | /// This class represents the slices of an alloca which are formed by its | ||||||||||||
227 | /// various uses. If a pointer escapes, we can't fully build a representation | ||||||||||||
228 | /// for the slices used and we reflect that in this structure. The uses are | ||||||||||||
229 | /// stored, sorted by increasing beginning offset and with unsplittable slices | ||||||||||||
230 | /// starting at a particular offset before splittable slices. | ||||||||||||
231 | class llvm::sroa::AllocaSlices { | ||||||||||||
232 | public: | ||||||||||||
233 | /// Construct the slices of a particular alloca. | ||||||||||||
234 | AllocaSlices(const DataLayout &DL, AllocaInst &AI); | ||||||||||||
235 | |||||||||||||
236 | /// Test whether a pointer to the allocation escapes our analysis. | ||||||||||||
237 | /// | ||||||||||||
238 | /// If this is true, the slices are never fully built and should be | ||||||||||||
239 | /// ignored. | ||||||||||||
240 | bool isEscaped() const { return PointerEscapingInstr; } | ||||||||||||
241 | |||||||||||||
242 | /// Support for iterating over the slices. | ||||||||||||
243 | /// @{ | ||||||||||||
244 | using iterator = SmallVectorImpl<Slice>::iterator; | ||||||||||||
245 | using range = iterator_range<iterator>; | ||||||||||||
246 | |||||||||||||
247 | iterator begin() { return Slices.begin(); } | ||||||||||||
248 | iterator end() { return Slices.end(); } | ||||||||||||
249 | |||||||||||||
250 | using const_iterator = SmallVectorImpl<Slice>::const_iterator; | ||||||||||||
251 | using const_range = iterator_range<const_iterator>; | ||||||||||||
252 | |||||||||||||
253 | const_iterator begin() const { return Slices.begin(); } | ||||||||||||
254 | const_iterator end() const { return Slices.end(); } | ||||||||||||
255 | /// @} | ||||||||||||
256 | |||||||||||||
257 | /// Erase a range of slices. | ||||||||||||
258 | void erase(iterator Start, iterator Stop) { Slices.erase(Start, Stop); } | ||||||||||||
259 | |||||||||||||
260 | /// Insert new slices for this alloca. | ||||||||||||
261 | /// | ||||||||||||
262 | /// This moves the slices into the alloca's slices collection, and re-sorts | ||||||||||||
263 | /// everything so that the usual ordering properties of the alloca's slices | ||||||||||||
264 | /// hold. | ||||||||||||
265 | void insert(ArrayRef<Slice> NewSlices) { | ||||||||||||
266 | int OldSize = Slices.size(); | ||||||||||||
267 | Slices.append(NewSlices.begin(), NewSlices.end()); | ||||||||||||
268 | auto SliceI = Slices.begin() + OldSize; | ||||||||||||
269 | llvm::sort(SliceI, Slices.end()); | ||||||||||||
270 | std::inplace_merge(Slices.begin(), SliceI, Slices.end()); | ||||||||||||
271 | } | ||||||||||||
272 | |||||||||||||
273 | // Forward declare the iterator and range accessor for walking the | ||||||||||||
274 | // partitions. | ||||||||||||
275 | class partition_iterator; | ||||||||||||
276 | iterator_range<partition_iterator> partitions(); | ||||||||||||
277 | |||||||||||||
278 | /// Access the dead users for this alloca. | ||||||||||||
279 | ArrayRef<Instruction *> getDeadUsers() const { return DeadUsers; } | ||||||||||||
280 | |||||||||||||
281 | /// Access the dead operands referring to this alloca. | ||||||||||||
282 | /// | ||||||||||||
283 | /// These are operands which have cannot actually be used to refer to the | ||||||||||||
284 | /// alloca as they are outside its range and the user doesn't correct for | ||||||||||||
285 | /// that. These mostly consist of PHI node inputs and the like which we just | ||||||||||||
286 | /// need to replace with undef. | ||||||||||||
287 | ArrayRef<Use *> getDeadOperands() const { return DeadOperands; } | ||||||||||||
288 | |||||||||||||
289 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||||||||||||
290 | void print(raw_ostream &OS, const_iterator I, StringRef Indent = " ") const; | ||||||||||||
291 | void printSlice(raw_ostream &OS, const_iterator I, | ||||||||||||
292 | StringRef Indent = " ") const; | ||||||||||||
293 | void printUse(raw_ostream &OS, const_iterator I, | ||||||||||||
294 | StringRef Indent = " ") const; | ||||||||||||
295 | void print(raw_ostream &OS) const; | ||||||||||||
296 | void dump(const_iterator I) const; | ||||||||||||
297 | void dump() const; | ||||||||||||
298 | #endif | ||||||||||||
299 | |||||||||||||
300 | private: | ||||||||||||
301 | template <typename DerivedT, typename RetT = void> class BuilderBase; | ||||||||||||
302 | class SliceBuilder; | ||||||||||||
303 | |||||||||||||
304 | friend class AllocaSlices::SliceBuilder; | ||||||||||||
305 | |||||||||||||
306 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||||||||||||
307 | /// Handle to alloca instruction to simplify method interfaces. | ||||||||||||
308 | AllocaInst &AI; | ||||||||||||
309 | #endif | ||||||||||||
310 | |||||||||||||
311 | /// The instruction responsible for this alloca not having a known set | ||||||||||||
312 | /// of slices. | ||||||||||||
313 | /// | ||||||||||||
314 | /// When an instruction (potentially) escapes the pointer to the alloca, we | ||||||||||||
315 | /// store a pointer to that here and abort trying to form slices of the | ||||||||||||
316 | /// alloca. This will be null if the alloca slices are analyzed successfully. | ||||||||||||
317 | Instruction *PointerEscapingInstr; | ||||||||||||
318 | |||||||||||||
319 | /// The slices of the alloca. | ||||||||||||
320 | /// | ||||||||||||
321 | /// We store a vector of the slices formed by uses of the alloca here. This | ||||||||||||
322 | /// vector is sorted by increasing begin offset, and then the unsplittable | ||||||||||||
323 | /// slices before the splittable ones. See the Slice inner class for more | ||||||||||||
324 | /// details. | ||||||||||||
325 | SmallVector<Slice, 8> Slices; | ||||||||||||
326 | |||||||||||||
327 | /// Instructions which will become dead if we rewrite the alloca. | ||||||||||||
328 | /// | ||||||||||||
329 | /// Note that these are not separated by slice. This is because we expect an | ||||||||||||
330 | /// alloca to be completely rewritten or not rewritten at all. If rewritten, | ||||||||||||
331 | /// all these instructions can simply be removed and replaced with undef as | ||||||||||||
332 | /// they come from outside of the allocated space. | ||||||||||||
333 | SmallVector<Instruction *, 8> DeadUsers; | ||||||||||||
334 | |||||||||||||
335 | /// Operands which will become dead if we rewrite the alloca. | ||||||||||||
336 | /// | ||||||||||||
337 | /// These are operands that in their particular use can be replaced with | ||||||||||||
338 | /// undef when we rewrite the alloca. These show up in out-of-bounds inputs | ||||||||||||
339 | /// to PHI nodes and the like. They aren't entirely dead (there might be | ||||||||||||
340 | /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we | ||||||||||||
341 | /// want to swap this particular input for undef to simplify the use lists of | ||||||||||||
342 | /// the alloca. | ||||||||||||
343 | SmallVector<Use *, 8> DeadOperands; | ||||||||||||
344 | }; | ||||||||||||
345 | |||||||||||||
346 | /// A partition of the slices. | ||||||||||||
347 | /// | ||||||||||||
348 | /// An ephemeral representation for a range of slices which can be viewed as | ||||||||||||
349 | /// a partition of the alloca. This range represents a span of the alloca's | ||||||||||||
350 | /// memory which cannot be split, and provides access to all of the slices | ||||||||||||
351 | /// overlapping some part of the partition. | ||||||||||||
352 | /// | ||||||||||||
353 | /// Objects of this type are produced by traversing the alloca's slices, but | ||||||||||||
354 | /// are only ephemeral and not persistent. | ||||||||||||
355 | class llvm::sroa::Partition { | ||||||||||||
356 | private: | ||||||||||||
357 | friend class AllocaSlices; | ||||||||||||
358 | friend class AllocaSlices::partition_iterator; | ||||||||||||
359 | |||||||||||||
360 | using iterator = AllocaSlices::iterator; | ||||||||||||
361 | |||||||||||||
362 | /// The beginning and ending offsets of the alloca for this | ||||||||||||
363 | /// partition. | ||||||||||||
364 | uint64_t BeginOffset, EndOffset; | ||||||||||||
365 | |||||||||||||
366 | /// The start and end iterators of this partition. | ||||||||||||
367 | iterator SI, SJ; | ||||||||||||
368 | |||||||||||||
369 | /// A collection of split slice tails overlapping the partition. | ||||||||||||
370 | SmallVector<Slice *, 4> SplitTails; | ||||||||||||
371 | |||||||||||||
372 | /// Raw constructor builds an empty partition starting and ending at | ||||||||||||
373 | /// the given iterator. | ||||||||||||
374 | Partition(iterator SI) : SI(SI), SJ(SI) {} | ||||||||||||
375 | |||||||||||||
376 | public: | ||||||||||||
377 | /// The start offset of this partition. | ||||||||||||
378 | /// | ||||||||||||
379 | /// All of the contained slices start at or after this offset. | ||||||||||||
380 | uint64_t beginOffset() const { return BeginOffset; } | ||||||||||||
381 | |||||||||||||
382 | /// The end offset of this partition. | ||||||||||||
383 | /// | ||||||||||||
384 | /// All of the contained slices end at or before this offset. | ||||||||||||
385 | uint64_t endOffset() const { return EndOffset; } | ||||||||||||
386 | |||||||||||||
387 | /// The size of the partition. | ||||||||||||
388 | /// | ||||||||||||
389 | /// Note that this can never be zero. | ||||||||||||
390 | uint64_t size() const { | ||||||||||||
391 | assert(BeginOffset < EndOffset && "Partitions must span some bytes!")((BeginOffset < EndOffset && "Partitions must span some bytes!" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset < EndOffset && \"Partitions must span some bytes!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 391, __PRETTY_FUNCTION__)); | ||||||||||||
392 | return EndOffset - BeginOffset; | ||||||||||||
393 | } | ||||||||||||
394 | |||||||||||||
395 | /// Test whether this partition contains no slices, and merely spans | ||||||||||||
396 | /// a region occupied by split slices. | ||||||||||||
397 | bool empty() const { return SI == SJ; } | ||||||||||||
398 | |||||||||||||
399 | /// \name Iterate slices that start within the partition. | ||||||||||||
400 | /// These may be splittable or unsplittable. They have a begin offset >= the | ||||||||||||
401 | /// partition begin offset. | ||||||||||||
402 | /// @{ | ||||||||||||
403 | // FIXME: We should probably define a "concat_iterator" helper and use that | ||||||||||||
404 | // to stitch together pointee_iterators over the split tails and the | ||||||||||||
405 | // contiguous iterators of the partition. That would give a much nicer | ||||||||||||
406 | // interface here. We could then additionally expose filtered iterators for | ||||||||||||
407 | // split, unsplit, and unsplittable splices based on the usage patterns. | ||||||||||||
408 | iterator begin() const { return SI; } | ||||||||||||
409 | iterator end() const { return SJ; } | ||||||||||||
410 | /// @} | ||||||||||||
411 | |||||||||||||
412 | /// Get the sequence of split slice tails. | ||||||||||||
413 | /// | ||||||||||||
414 | /// These tails are of slices which start before this partition but are | ||||||||||||
415 | /// split and overlap into the partition. We accumulate these while forming | ||||||||||||
416 | /// partitions. | ||||||||||||
417 | ArrayRef<Slice *> splitSliceTails() const { return SplitTails; } | ||||||||||||
418 | }; | ||||||||||||
419 | |||||||||||||
420 | /// An iterator over partitions of the alloca's slices. | ||||||||||||
421 | /// | ||||||||||||
422 | /// This iterator implements the core algorithm for partitioning the alloca's | ||||||||||||
423 | /// slices. It is a forward iterator as we don't support backtracking for | ||||||||||||
424 | /// efficiency reasons, and re-use a single storage area to maintain the | ||||||||||||
425 | /// current set of split slices. | ||||||||||||
426 | /// | ||||||||||||
427 | /// It is templated on the slice iterator type to use so that it can operate | ||||||||||||
428 | /// with either const or non-const slice iterators. | ||||||||||||
429 | class AllocaSlices::partition_iterator | ||||||||||||
430 | : public iterator_facade_base<partition_iterator, std::forward_iterator_tag, | ||||||||||||
431 | Partition> { | ||||||||||||
432 | friend class AllocaSlices; | ||||||||||||
433 | |||||||||||||
434 | /// Most of the state for walking the partitions is held in a class | ||||||||||||
435 | /// with a nice interface for examining them. | ||||||||||||
436 | Partition P; | ||||||||||||
437 | |||||||||||||
438 | /// We need to keep the end of the slices to know when to stop. | ||||||||||||
439 | AllocaSlices::iterator SE; | ||||||||||||
440 | |||||||||||||
441 | /// We also need to keep track of the maximum split end offset seen. | ||||||||||||
442 | /// FIXME: Do we really? | ||||||||||||
443 | uint64_t MaxSplitSliceEndOffset = 0; | ||||||||||||
444 | |||||||||||||
445 | /// Sets the partition to be empty at given iterator, and sets the | ||||||||||||
446 | /// end iterator. | ||||||||||||
447 | partition_iterator(AllocaSlices::iterator SI, AllocaSlices::iterator SE) | ||||||||||||
448 | : P(SI), SE(SE) { | ||||||||||||
449 | // If not already at the end, advance our state to form the initial | ||||||||||||
450 | // partition. | ||||||||||||
451 | if (SI != SE) | ||||||||||||
452 | advance(); | ||||||||||||
453 | } | ||||||||||||
454 | |||||||||||||
455 | /// Advance the iterator to the next partition. | ||||||||||||
456 | /// | ||||||||||||
457 | /// Requires that the iterator not be at the end of the slices. | ||||||||||||
458 | void advance() { | ||||||||||||
459 | assert((P.SI != SE || !P.SplitTails.empty()) &&(((P.SI != SE || !P.SplitTails.empty()) && "Cannot advance past the end of the slices!" ) ? static_cast<void> (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 460, __PRETTY_FUNCTION__)) | ||||||||||||
460 | "Cannot advance past the end of the slices!")(((P.SI != SE || !P.SplitTails.empty()) && "Cannot advance past the end of the slices!" ) ? static_cast<void> (0) : __assert_fail ("(P.SI != SE || !P.SplitTails.empty()) && \"Cannot advance past the end of the slices!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 460, __PRETTY_FUNCTION__)); | ||||||||||||
461 | |||||||||||||
462 | // Clear out any split uses which have ended. | ||||||||||||
463 | if (!P.SplitTails.empty()) { | ||||||||||||
464 | if (P.EndOffset >= MaxSplitSliceEndOffset) { | ||||||||||||
465 | // If we've finished all splits, this is easy. | ||||||||||||
466 | P.SplitTails.clear(); | ||||||||||||
467 | MaxSplitSliceEndOffset = 0; | ||||||||||||
468 | } else { | ||||||||||||
469 | // Remove the uses which have ended in the prior partition. This | ||||||||||||
470 | // cannot change the max split slice end because we just checked that | ||||||||||||
471 | // the prior partition ended prior to that max. | ||||||||||||
472 | P.SplitTails.erase(llvm::remove_if(P.SplitTails, | ||||||||||||
473 | [&](Slice *S) { | ||||||||||||
474 | return S->endOffset() <= | ||||||||||||
475 | P.EndOffset; | ||||||||||||
476 | }), | ||||||||||||
477 | P.SplitTails.end()); | ||||||||||||
478 | assert(llvm::any_of(P.SplitTails,((llvm::any_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 482, __PRETTY_FUNCTION__)) | ||||||||||||
479 | [&](Slice *S) {((llvm::any_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 482, __PRETTY_FUNCTION__)) | ||||||||||||
480 | return S->endOffset() == MaxSplitSliceEndOffset;((llvm::any_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 482, __PRETTY_FUNCTION__)) | ||||||||||||
481 | }) &&((llvm::any_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 482, __PRETTY_FUNCTION__)) | ||||||||||||
482 | "Could not find the current max split slice offset!")((llvm::any_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() == MaxSplitSliceEndOffset; }) && "Could not find the current max split slice offset!" ) ? static_cast<void> (0) : __assert_fail ("llvm::any_of(P.SplitTails, [&](Slice *S) { return S->endOffset() == MaxSplitSliceEndOffset; }) && \"Could not find the current max split slice offset!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 482, __PRETTY_FUNCTION__)); | ||||||||||||
483 | assert(llvm::all_of(P.SplitTails,((llvm::all_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 487, __PRETTY_FUNCTION__)) | ||||||||||||
484 | [&](Slice *S) {((llvm::all_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 487, __PRETTY_FUNCTION__)) | ||||||||||||
485 | return S->endOffset() <= MaxSplitSliceEndOffset;((llvm::all_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 487, __PRETTY_FUNCTION__)) | ||||||||||||
486 | }) &&((llvm::all_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 487, __PRETTY_FUNCTION__)) | ||||||||||||
487 | "Max split slice end offset is not actually the max!")((llvm::all_of(P.SplitTails, [&](Slice *S) { return S-> endOffset() <= MaxSplitSliceEndOffset; }) && "Max split slice end offset is not actually the max!" ) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(P.SplitTails, [&](Slice *S) { return S->endOffset() <= MaxSplitSliceEndOffset; }) && \"Max split slice end offset is not actually the max!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 487, __PRETTY_FUNCTION__)); | ||||||||||||
488 | } | ||||||||||||
489 | } | ||||||||||||
490 | |||||||||||||
491 | // If P.SI is already at the end, then we've cleared the split tail and | ||||||||||||
492 | // now have an end iterator. | ||||||||||||
493 | if (P.SI == SE) { | ||||||||||||
494 | assert(P.SplitTails.empty() && "Failed to clear the split slices!")((P.SplitTails.empty() && "Failed to clear the split slices!" ) ? static_cast<void> (0) : __assert_fail ("P.SplitTails.empty() && \"Failed to clear the split slices!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 494, __PRETTY_FUNCTION__)); | ||||||||||||
495 | return; | ||||||||||||
496 | } | ||||||||||||
497 | |||||||||||||
498 | // If we had a non-empty partition previously, set up the state for | ||||||||||||
499 | // subsequent partitions. | ||||||||||||
500 | if (P.SI != P.SJ) { | ||||||||||||
501 | // Accumulate all the splittable slices which started in the old | ||||||||||||
502 | // partition into the split list. | ||||||||||||
503 | for (Slice &S : P) | ||||||||||||
504 | if (S.isSplittable() && S.endOffset() > P.EndOffset) { | ||||||||||||
505 | P.SplitTails.push_back(&S); | ||||||||||||
506 | MaxSplitSliceEndOffset = | ||||||||||||
507 | std::max(S.endOffset(), MaxSplitSliceEndOffset); | ||||||||||||
508 | } | ||||||||||||
509 | |||||||||||||
510 | // Start from the end of the previous partition. | ||||||||||||
511 | P.SI = P.SJ; | ||||||||||||
512 | |||||||||||||
513 | // If P.SI is now at the end, we at most have a tail of split slices. | ||||||||||||
514 | if (P.SI == SE) { | ||||||||||||
515 | P.BeginOffset = P.EndOffset; | ||||||||||||
516 | P.EndOffset = MaxSplitSliceEndOffset; | ||||||||||||
517 | return; | ||||||||||||
518 | } | ||||||||||||
519 | |||||||||||||
520 | // If the we have split slices and the next slice is after a gap and is | ||||||||||||
521 | // not splittable immediately form an empty partition for the split | ||||||||||||
522 | // slices up until the next slice begins. | ||||||||||||
523 | if (!P.SplitTails.empty() && P.SI->beginOffset() != P.EndOffset && | ||||||||||||
524 | !P.SI->isSplittable()) { | ||||||||||||
525 | P.BeginOffset = P.EndOffset; | ||||||||||||
526 | P.EndOffset = P.SI->beginOffset(); | ||||||||||||
527 | return; | ||||||||||||
528 | } | ||||||||||||
529 | } | ||||||||||||
530 | |||||||||||||
531 | // OK, we need to consume new slices. Set the end offset based on the | ||||||||||||
532 | // current slice, and step SJ past it. The beginning offset of the | ||||||||||||
533 | // partition is the beginning offset of the next slice unless we have | ||||||||||||
534 | // pre-existing split slices that are continuing, in which case we begin | ||||||||||||
535 | // at the prior end offset. | ||||||||||||
536 | P.BeginOffset = P.SplitTails.empty() ? P.SI->beginOffset() : P.EndOffset; | ||||||||||||
537 | P.EndOffset = P.SI->endOffset(); | ||||||||||||
538 | ++P.SJ; | ||||||||||||
539 | |||||||||||||
540 | // There are two strategies to form a partition based on whether the | ||||||||||||
541 | // partition starts with an unsplittable slice or a splittable slice. | ||||||||||||
542 | if (!P.SI->isSplittable()) { | ||||||||||||
543 | // When we're forming an unsplittable region, it must always start at | ||||||||||||
544 | // the first slice and will extend through its end. | ||||||||||||
545 | assert(P.BeginOffset == P.SI->beginOffset())((P.BeginOffset == P.SI->beginOffset()) ? static_cast<void > (0) : __assert_fail ("P.BeginOffset == P.SI->beginOffset()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 545, __PRETTY_FUNCTION__)); | ||||||||||||
546 | |||||||||||||
547 | // Form a partition including all of the overlapping slices with this | ||||||||||||
548 | // unsplittable slice. | ||||||||||||
549 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | ||||||||||||
550 | if (!P.SJ->isSplittable()) | ||||||||||||
551 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | ||||||||||||
552 | ++P.SJ; | ||||||||||||
553 | } | ||||||||||||
554 | |||||||||||||
555 | // We have a partition across a set of overlapping unsplittable | ||||||||||||
556 | // partitions. | ||||||||||||
557 | return; | ||||||||||||
558 | } | ||||||||||||
559 | |||||||||||||
560 | // If we're starting with a splittable slice, then we need to form | ||||||||||||
561 | // a synthetic partition spanning it and any other overlapping splittable | ||||||||||||
562 | // splices. | ||||||||||||
563 | assert(P.SI->isSplittable() && "Forming a splittable partition!")((P.SI->isSplittable() && "Forming a splittable partition!" ) ? static_cast<void> (0) : __assert_fail ("P.SI->isSplittable() && \"Forming a splittable partition!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 563, __PRETTY_FUNCTION__)); | ||||||||||||
564 | |||||||||||||
565 | // Collect all of the overlapping splittable slices. | ||||||||||||
566 | while (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset && | ||||||||||||
567 | P.SJ->isSplittable()) { | ||||||||||||
568 | P.EndOffset = std::max(P.EndOffset, P.SJ->endOffset()); | ||||||||||||
569 | ++P.SJ; | ||||||||||||
570 | } | ||||||||||||
571 | |||||||||||||
572 | // Back upiP.EndOffset if we ended the span early when encountering an | ||||||||||||
573 | // unsplittable slice. This synthesizes the early end offset of | ||||||||||||
574 | // a partition spanning only splittable slices. | ||||||||||||
575 | if (P.SJ != SE && P.SJ->beginOffset() < P.EndOffset) { | ||||||||||||
576 | assert(!P.SJ->isSplittable())((!P.SJ->isSplittable()) ? static_cast<void> (0) : __assert_fail ("!P.SJ->isSplittable()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 576, __PRETTY_FUNCTION__)); | ||||||||||||
577 | P.EndOffset = P.SJ->beginOffset(); | ||||||||||||
578 | } | ||||||||||||
579 | } | ||||||||||||
580 | |||||||||||||
581 | public: | ||||||||||||
582 | bool operator==(const partition_iterator &RHS) const { | ||||||||||||
583 | assert(SE == RHS.SE &&((SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? static_cast<void> (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 584, __PRETTY_FUNCTION__)) | ||||||||||||
584 | "End iterators don't match between compared partition iterators!")((SE == RHS.SE && "End iterators don't match between compared partition iterators!" ) ? static_cast<void> (0) : __assert_fail ("SE == RHS.SE && \"End iterators don't match between compared partition iterators!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 584, __PRETTY_FUNCTION__)); | ||||||||||||
585 | |||||||||||||
586 | // The observed positions of partitions is marked by the P.SI iterator and | ||||||||||||
587 | // the emptiness of the split slices. The latter is only relevant when | ||||||||||||
588 | // P.SI == SE, as the end iterator will additionally have an empty split | ||||||||||||
589 | // slices list, but the prior may have the same P.SI and a tail of split | ||||||||||||
590 | // slices. | ||||||||||||
591 | if (P.SI == RHS.P.SI && P.SplitTails.empty() == RHS.P.SplitTails.empty()) { | ||||||||||||
592 | assert(P.SJ == RHS.P.SJ &&((P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? static_cast<void> (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 593, __PRETTY_FUNCTION__)) | ||||||||||||
593 | "Same set of slices formed two different sized partitions!")((P.SJ == RHS.P.SJ && "Same set of slices formed two different sized partitions!" ) ? static_cast<void> (0) : __assert_fail ("P.SJ == RHS.P.SJ && \"Same set of slices formed two different sized partitions!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 593, __PRETTY_FUNCTION__)); | ||||||||||||
594 | assert(P.SplitTails.size() == RHS.P.SplitTails.size() &&((P.SplitTails.size() == RHS.P.SplitTails.size() && "Same slice position with differently sized non-empty split " "slice tails!") ? static_cast<void> (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 596, __PRETTY_FUNCTION__)) | ||||||||||||
595 | "Same slice position with differently sized non-empty split "((P.SplitTails.size() == RHS.P.SplitTails.size() && "Same slice position with differently sized non-empty split " "slice tails!") ? static_cast<void> (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 596, __PRETTY_FUNCTION__)) | ||||||||||||
596 | "slice tails!")((P.SplitTails.size() == RHS.P.SplitTails.size() && "Same slice position with differently sized non-empty split " "slice tails!") ? static_cast<void> (0) : __assert_fail ("P.SplitTails.size() == RHS.P.SplitTails.size() && \"Same slice position with differently sized non-empty split \" \"slice tails!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 596, __PRETTY_FUNCTION__)); | ||||||||||||
597 | return true; | ||||||||||||
598 | } | ||||||||||||
599 | return false; | ||||||||||||
600 | } | ||||||||||||
601 | |||||||||||||
602 | partition_iterator &operator++() { | ||||||||||||
603 | advance(); | ||||||||||||
604 | return *this; | ||||||||||||
605 | } | ||||||||||||
606 | |||||||||||||
607 | Partition &operator*() { return P; } | ||||||||||||
608 | }; | ||||||||||||
609 | |||||||||||||
610 | /// A forward range over the partitions of the alloca's slices. | ||||||||||||
611 | /// | ||||||||||||
612 | /// This accesses an iterator range over the partitions of the alloca's | ||||||||||||
613 | /// slices. It computes these partitions on the fly based on the overlapping | ||||||||||||
614 | /// offsets of the slices and the ability to split them. It will visit "empty" | ||||||||||||
615 | /// partitions to cover regions of the alloca only accessed via split | ||||||||||||
616 | /// slices. | ||||||||||||
617 | iterator_range<AllocaSlices::partition_iterator> AllocaSlices::partitions() { | ||||||||||||
618 | return make_range(partition_iterator(begin(), end()), | ||||||||||||
619 | partition_iterator(end(), end())); | ||||||||||||
620 | } | ||||||||||||
621 | |||||||||||||
622 | static Value *foldSelectInst(SelectInst &SI) { | ||||||||||||
623 | // If the condition being selected on is a constant or the same value is | ||||||||||||
624 | // being selected between, fold the select. Yes this does (rarely) happen | ||||||||||||
625 | // early on. | ||||||||||||
626 | if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition())) | ||||||||||||
627 | return SI.getOperand(1 + CI->isZero()); | ||||||||||||
628 | if (SI.getOperand(1) == SI.getOperand(2)) | ||||||||||||
629 | return SI.getOperand(1); | ||||||||||||
630 | |||||||||||||
631 | return nullptr; | ||||||||||||
632 | } | ||||||||||||
633 | |||||||||||||
634 | /// A helper that folds a PHI node or a select. | ||||||||||||
635 | static Value *foldPHINodeOrSelectInst(Instruction &I) { | ||||||||||||
636 | if (PHINode *PN = dyn_cast<PHINode>(&I)) { | ||||||||||||
637 | // If PN merges together the same value, return that value. | ||||||||||||
638 | return PN->hasConstantValue(); | ||||||||||||
639 | } | ||||||||||||
640 | return foldSelectInst(cast<SelectInst>(I)); | ||||||||||||
641 | } | ||||||||||||
642 | |||||||||||||
643 | /// Builder for the alloca slices. | ||||||||||||
644 | /// | ||||||||||||
645 | /// This class builds a set of alloca slices by recursively visiting the uses | ||||||||||||
646 | /// of an alloca and making a slice for each load and store at each offset. | ||||||||||||
647 | class AllocaSlices::SliceBuilder : public PtrUseVisitor<SliceBuilder> { | ||||||||||||
648 | friend class PtrUseVisitor<SliceBuilder>; | ||||||||||||
649 | friend class InstVisitor<SliceBuilder>; | ||||||||||||
650 | |||||||||||||
651 | using Base = PtrUseVisitor<SliceBuilder>; | ||||||||||||
652 | |||||||||||||
653 | const uint64_t AllocSize; | ||||||||||||
654 | AllocaSlices &AS; | ||||||||||||
655 | |||||||||||||
656 | SmallDenseMap<Instruction *, unsigned> MemTransferSliceMap; | ||||||||||||
657 | SmallDenseMap<Instruction *, uint64_t> PHIOrSelectSizes; | ||||||||||||
658 | |||||||||||||
659 | /// Set to de-duplicate dead instructions found in the use walk. | ||||||||||||
660 | SmallPtrSet<Instruction *, 4> VisitedDeadInsts; | ||||||||||||
661 | |||||||||||||
662 | public: | ||||||||||||
663 | SliceBuilder(const DataLayout &DL, AllocaInst &AI, AllocaSlices &AS) | ||||||||||||
664 | : PtrUseVisitor<SliceBuilder>(DL), | ||||||||||||
665 | AllocSize(DL.getTypeAllocSize(AI.getAllocatedType())), AS(AS) {} | ||||||||||||
666 | |||||||||||||
667 | private: | ||||||||||||
668 | void markAsDead(Instruction &I) { | ||||||||||||
669 | if (VisitedDeadInsts.insert(&I).second) | ||||||||||||
670 | AS.DeadUsers.push_back(&I); | ||||||||||||
671 | } | ||||||||||||
672 | |||||||||||||
673 | void insertUse(Instruction &I, const APInt &Offset, uint64_t Size, | ||||||||||||
674 | bool IsSplittable = false) { | ||||||||||||
675 | // Completely skip uses which have a zero size or start either before or | ||||||||||||
676 | // past the end of the allocation. | ||||||||||||
677 | if (Size == 0 || Offset.uge(AllocSize)) { | ||||||||||||
678 | LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
679 | << Offsetdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
680 | << " which has zero size or starts outside of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
681 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
682 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
683 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset << " which has zero size or starts outside of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | ||||||||||||
684 | return markAsDead(I); | ||||||||||||
685 | } | ||||||||||||
686 | |||||||||||||
687 | uint64_t BeginOffset = Offset.getZExtValue(); | ||||||||||||
688 | uint64_t EndOffset = BeginOffset + Size; | ||||||||||||
689 | |||||||||||||
690 | // Clamp the end offset to the end of the allocation. Note that this is | ||||||||||||
691 | // formulated to handle even the case where "BeginOffset + Size" overflows. | ||||||||||||
692 | // This may appear superficially to be something we could ignore entirely, | ||||||||||||
693 | // but that is not so! There may be widened loads or PHI-node uses where | ||||||||||||
694 | // some instructions are dead but not others. We can't completely ignore | ||||||||||||
695 | // them, and so have to record at least the information here. | ||||||||||||
696 | assert(AllocSize >= BeginOffset)((AllocSize >= BeginOffset) ? static_cast<void> (0) : __assert_fail ("AllocSize >= BeginOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 696, __PRETTY_FUNCTION__)); // Established above. | ||||||||||||
697 | if (Size > AllocSize - BeginOffset) { | ||||||||||||
698 | LLVM_DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
699 | << Offset << " to remain within the " << AllocSizedo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
700 | << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
701 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false) | ||||||||||||
702 | << " use: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset << " to remain within the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << I << "\n"; } } while (false); | ||||||||||||
703 | EndOffset = AllocSize; | ||||||||||||
704 | } | ||||||||||||
705 | |||||||||||||
706 | AS.Slices.push_back(Slice(BeginOffset, EndOffset, U, IsSplittable)); | ||||||||||||
707 | } | ||||||||||||
708 | |||||||||||||
709 | void visitBitCastInst(BitCastInst &BC) { | ||||||||||||
710 | if (BC.use_empty()) | ||||||||||||
711 | return markAsDead(BC); | ||||||||||||
712 | |||||||||||||
713 | return Base::visitBitCastInst(BC); | ||||||||||||
714 | } | ||||||||||||
715 | |||||||||||||
716 | void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { | ||||||||||||
717 | if (ASC.use_empty()) | ||||||||||||
718 | return markAsDead(ASC); | ||||||||||||
719 | |||||||||||||
720 | return Base::visitAddrSpaceCastInst(ASC); | ||||||||||||
721 | } | ||||||||||||
722 | |||||||||||||
723 | void visitGetElementPtrInst(GetElementPtrInst &GEPI) { | ||||||||||||
724 | if (GEPI.use_empty()) | ||||||||||||
725 | return markAsDead(GEPI); | ||||||||||||
726 | |||||||||||||
727 | if (SROAStrictInbounds && GEPI.isInBounds()) { | ||||||||||||
728 | // FIXME: This is a manually un-factored variant of the basic code inside | ||||||||||||
729 | // of GEPs with checking of the inbounds invariant specified in the | ||||||||||||
730 | // langref in a very strict sense. If we ever want to enable | ||||||||||||
731 | // SROAStrictInbounds, this code should be factored cleanly into | ||||||||||||
732 | // PtrUseVisitor, but it is easier to experiment with SROAStrictInbounds | ||||||||||||
733 | // by writing out the code here where we have the underlying allocation | ||||||||||||
734 | // size readily available. | ||||||||||||
735 | APInt GEPOffset = Offset; | ||||||||||||
736 | const DataLayout &DL = GEPI.getModule()->getDataLayout(); | ||||||||||||
737 | for (gep_type_iterator GTI = gep_type_begin(GEPI), | ||||||||||||
738 | GTE = gep_type_end(GEPI); | ||||||||||||
739 | GTI != GTE; ++GTI) { | ||||||||||||
740 | ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand()); | ||||||||||||
741 | if (!OpC) | ||||||||||||
742 | break; | ||||||||||||
743 | |||||||||||||
744 | // Handle a struct index, which adds its field offset to the pointer. | ||||||||||||
745 | if (StructType *STy = GTI.getStructTypeOrNull()) { | ||||||||||||
746 | unsigned ElementIdx = OpC->getZExtValue(); | ||||||||||||
747 | const StructLayout *SL = DL.getStructLayout(STy); | ||||||||||||
748 | GEPOffset += | ||||||||||||
749 | APInt(Offset.getBitWidth(), SL->getElementOffset(ElementIdx)); | ||||||||||||
750 | } else { | ||||||||||||
751 | // For array or vector indices, scale the index by the size of the | ||||||||||||
752 | // type. | ||||||||||||
753 | APInt Index = OpC->getValue().sextOrTrunc(Offset.getBitWidth()); | ||||||||||||
754 | GEPOffset += Index * APInt(Offset.getBitWidth(), | ||||||||||||
755 | DL.getTypeAllocSize(GTI.getIndexedType())); | ||||||||||||
756 | } | ||||||||||||
757 | |||||||||||||
758 | // If this index has computed an intermediate pointer which is not | ||||||||||||
759 | // inbounds, then the result of the GEP is a poison value and we can | ||||||||||||
760 | // delete it and all uses. | ||||||||||||
761 | if (GEPOffset.ugt(AllocSize)) | ||||||||||||
762 | return markAsDead(GEPI); | ||||||||||||
763 | } | ||||||||||||
764 | } | ||||||||||||
765 | |||||||||||||
766 | return Base::visitGetElementPtrInst(GEPI); | ||||||||||||
767 | } | ||||||||||||
768 | |||||||||||||
769 | void handleLoadOrStore(Type *Ty, Instruction &I, const APInt &Offset, | ||||||||||||
770 | uint64_t Size, bool IsVolatile) { | ||||||||||||
771 | // We allow splitting of non-volatile loads and stores where the type is an | ||||||||||||
772 | // integer type. These may be used to implement 'memcpy' or other "transfer | ||||||||||||
773 | // of bits" patterns. | ||||||||||||
774 | bool IsSplittable = Ty->isIntegerTy() && !IsVolatile; | ||||||||||||
775 | |||||||||||||
776 | insertUse(I, Offset, Size, IsSplittable); | ||||||||||||
777 | } | ||||||||||||
778 | |||||||||||||
779 | void visitLoadInst(LoadInst &LI) { | ||||||||||||
780 | assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&(((!LI.isSimple() || LI.getType()->isSingleValueType()) && "All simple FCA loads should have been pre-split") ? static_cast <void> (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 781, __PRETTY_FUNCTION__)) | ||||||||||||
781 | "All simple FCA loads should have been pre-split")(((!LI.isSimple() || LI.getType()->isSingleValueType()) && "All simple FCA loads should have been pre-split") ? static_cast <void> (0) : __assert_fail ("(!LI.isSimple() || LI.getType()->isSingleValueType()) && \"All simple FCA loads should have been pre-split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 781, __PRETTY_FUNCTION__)); | ||||||||||||
782 | |||||||||||||
783 | if (!IsOffsetKnown) | ||||||||||||
784 | return PI.setAborted(&LI); | ||||||||||||
785 | |||||||||||||
786 | if (LI.isVolatile() && | ||||||||||||
787 | LI.getPointerAddressSpace() != DL.getAllocaAddrSpace()) | ||||||||||||
788 | return PI.setAborted(&LI); | ||||||||||||
789 | |||||||||||||
790 | uint64_t Size = DL.getTypeStoreSize(LI.getType()); | ||||||||||||
791 | return handleLoadOrStore(LI.getType(), LI, Offset, Size, LI.isVolatile()); | ||||||||||||
792 | } | ||||||||||||
793 | |||||||||||||
794 | void visitStoreInst(StoreInst &SI) { | ||||||||||||
795 | Value *ValOp = SI.getValueOperand(); | ||||||||||||
796 | if (ValOp == *U) | ||||||||||||
797 | return PI.setEscapedAndAborted(&SI); | ||||||||||||
798 | if (!IsOffsetKnown) | ||||||||||||
799 | return PI.setAborted(&SI); | ||||||||||||
800 | |||||||||||||
801 | if (SI.isVolatile() && | ||||||||||||
802 | SI.getPointerAddressSpace() != DL.getAllocaAddrSpace()) | ||||||||||||
803 | return PI.setAborted(&SI); | ||||||||||||
804 | |||||||||||||
805 | uint64_t Size = DL.getTypeStoreSize(ValOp->getType()); | ||||||||||||
806 | |||||||||||||
807 | // If this memory access can be shown to *statically* extend outside the | ||||||||||||
808 | // bounds of the allocation, it's behavior is undefined, so simply | ||||||||||||
809 | // ignore it. Note that this is more strict than the generic clamping | ||||||||||||
810 | // behavior of insertUse. We also try to handle cases which might run the | ||||||||||||
811 | // risk of overflow. | ||||||||||||
812 | // FIXME: We should instead consider the pointer to have escaped if this | ||||||||||||
813 | // function is being instrumented for addressing bugs or race conditions. | ||||||||||||
814 | if (Size > AllocSize || Offset.ugt(AllocSize - Size)) { | ||||||||||||
815 | LLVM_DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte store @"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | ||||||||||||
816 | << Offset << " which extends past the end of the "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | ||||||||||||
817 | << AllocSize << " byte alloca:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | ||||||||||||
818 | << " alloca: " << AS.AI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false) | ||||||||||||
819 | << " use: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "WARNING: Ignoring " << Size << " byte store @" << Offset << " which extends past the end of the " << AllocSize << " byte alloca:\n" << " alloca: " << AS.AI << "\n" << " use: " << SI << "\n"; } } while (false); | ||||||||||||
820 | return markAsDead(SI); | ||||||||||||
821 | } | ||||||||||||
822 | |||||||||||||
823 | assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&(((!SI.isSimple() || ValOp->getType()->isSingleValueType ()) && "All simple FCA stores should have been pre-split" ) ? static_cast<void> (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 824, __PRETTY_FUNCTION__)) | ||||||||||||
824 | "All simple FCA stores should have been pre-split")(((!SI.isSimple() || ValOp->getType()->isSingleValueType ()) && "All simple FCA stores should have been pre-split" ) ? static_cast<void> (0) : __assert_fail ("(!SI.isSimple() || ValOp->getType()->isSingleValueType()) && \"All simple FCA stores should have been pre-split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 824, __PRETTY_FUNCTION__)); | ||||||||||||
825 | handleLoadOrStore(ValOp->getType(), SI, Offset, Size, SI.isVolatile()); | ||||||||||||
826 | } | ||||||||||||
827 | |||||||||||||
828 | void visitMemSetInst(MemSetInst &II) { | ||||||||||||
829 | assert(II.getRawDest() == *U && "Pointer use is not the destination?")((II.getRawDest() == *U && "Pointer use is not the destination?" ) ? static_cast<void> (0) : __assert_fail ("II.getRawDest() == *U && \"Pointer use is not the destination?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 829, __PRETTY_FUNCTION__)); | ||||||||||||
830 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | ||||||||||||
831 | if ((Length && Length->getValue() == 0) || | ||||||||||||
832 | (IsOffsetKnown && Offset.uge(AllocSize))) | ||||||||||||
833 | // Zero-length mem transfer intrinsics can be ignored entirely. | ||||||||||||
834 | return markAsDead(II); | ||||||||||||
835 | |||||||||||||
836 | if (!IsOffsetKnown) | ||||||||||||
837 | return PI.setAborted(&II); | ||||||||||||
838 | |||||||||||||
839 | // Don't replace this with a store with a different address space. TODO: | ||||||||||||
840 | // Use a store with the casted new alloca? | ||||||||||||
841 | if (II.isVolatile() && II.getDestAddressSpace() != DL.getAllocaAddrSpace()) | ||||||||||||
842 | return PI.setAborted(&II); | ||||||||||||
843 | |||||||||||||
844 | insertUse(II, Offset, Length ? Length->getLimitedValue() | ||||||||||||
845 | : AllocSize - Offset.getLimitedValue(), | ||||||||||||
846 | (bool)Length); | ||||||||||||
847 | } | ||||||||||||
848 | |||||||||||||
849 | void visitMemTransferInst(MemTransferInst &II) { | ||||||||||||
850 | ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength()); | ||||||||||||
851 | if (Length && Length->getValue() == 0) | ||||||||||||
852 | // Zero-length mem transfer intrinsics can be ignored entirely. | ||||||||||||
853 | return markAsDead(II); | ||||||||||||
854 | |||||||||||||
855 | // Because we can visit these intrinsics twice, also check to see if the | ||||||||||||
856 | // first time marked this instruction as dead. If so, skip it. | ||||||||||||
857 | if (VisitedDeadInsts.count(&II)) | ||||||||||||
858 | return; | ||||||||||||
859 | |||||||||||||
860 | if (!IsOffsetKnown) | ||||||||||||
861 | return PI.setAborted(&II); | ||||||||||||
862 | |||||||||||||
863 | // Don't replace this with a load/store with a different address space. | ||||||||||||
864 | // TODO: Use a store with the casted new alloca? | ||||||||||||
865 | if (II.isVolatile() && | ||||||||||||
866 | (II.getDestAddressSpace() != DL.getAllocaAddrSpace() || | ||||||||||||
867 | II.getSourceAddressSpace() != DL.getAllocaAddrSpace())) | ||||||||||||
868 | return PI.setAborted(&II); | ||||||||||||
869 | |||||||||||||
870 | // This side of the transfer is completely out-of-bounds, and so we can | ||||||||||||
871 | // nuke the entire transfer. However, we also need to nuke the other side | ||||||||||||
872 | // if already added to our partitions. | ||||||||||||
873 | // FIXME: Yet another place we really should bypass this when | ||||||||||||
874 | // instrumenting for ASan. | ||||||||||||
875 | if (Offset.uge(AllocSize)) { | ||||||||||||
876 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI = | ||||||||||||
877 | MemTransferSliceMap.find(&II); | ||||||||||||
878 | if (MTPI != MemTransferSliceMap.end()) | ||||||||||||
879 | AS.Slices[MTPI->second].kill(); | ||||||||||||
880 | return markAsDead(II); | ||||||||||||
881 | } | ||||||||||||
882 | |||||||||||||
883 | uint64_t RawOffset = Offset.getLimitedValue(); | ||||||||||||
884 | uint64_t Size = Length ? Length->getLimitedValue() : AllocSize - RawOffset; | ||||||||||||
885 | |||||||||||||
886 | // Check for the special case where the same exact value is used for both | ||||||||||||
887 | // source and dest. | ||||||||||||
888 | if (*U == II.getRawDest() && *U == II.getRawSource()) { | ||||||||||||
889 | // For non-volatile transfers this is a no-op. | ||||||||||||
890 | if (!II.isVolatile()) | ||||||||||||
891 | return markAsDead(II); | ||||||||||||
892 | |||||||||||||
893 | return insertUse(II, Offset, Size, /*IsSplittable=*/false); | ||||||||||||
894 | } | ||||||||||||
895 | |||||||||||||
896 | // If we have seen both source and destination for a mem transfer, then | ||||||||||||
897 | // they both point to the same alloca. | ||||||||||||
898 | bool Inserted; | ||||||||||||
899 | SmallDenseMap<Instruction *, unsigned>::iterator MTPI; | ||||||||||||
900 | std::tie(MTPI, Inserted) = | ||||||||||||
901 | MemTransferSliceMap.insert(std::make_pair(&II, AS.Slices.size())); | ||||||||||||
902 | unsigned PrevIdx = MTPI->second; | ||||||||||||
903 | if (!Inserted) { | ||||||||||||
904 | Slice &PrevP = AS.Slices[PrevIdx]; | ||||||||||||
905 | |||||||||||||
906 | // Check if the begin offsets match and this is a non-volatile transfer. | ||||||||||||
907 | // In that case, we can completely elide the transfer. | ||||||||||||
908 | if (!II.isVolatile() && PrevP.beginOffset() == RawOffset) { | ||||||||||||
909 | PrevP.kill(); | ||||||||||||
910 | return markAsDead(II); | ||||||||||||
911 | } | ||||||||||||
912 | |||||||||||||
913 | // Otherwise we have an offset transfer within the same alloca. We can't | ||||||||||||
914 | // split those. | ||||||||||||
915 | PrevP.makeUnsplittable(); | ||||||||||||
916 | } | ||||||||||||
917 | |||||||||||||
918 | // Insert the use now that we've fixed up the splittable nature. | ||||||||||||
919 | insertUse(II, Offset, Size, /*IsSplittable=*/Inserted && Length); | ||||||||||||
920 | |||||||||||||
921 | // Check that we ended up with a valid index in the map. | ||||||||||||
922 | assert(AS.Slices[PrevIdx].getUse()->getUser() == &II &&((AS.Slices[PrevIdx].getUse()->getUser() == &II && "Map index doesn't point back to a slice with this user.") ? static_cast<void> (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 923, __PRETTY_FUNCTION__)) | ||||||||||||
923 | "Map index doesn't point back to a slice with this user.")((AS.Slices[PrevIdx].getUse()->getUser() == &II && "Map index doesn't point back to a slice with this user.") ? static_cast<void> (0) : __assert_fail ("AS.Slices[PrevIdx].getUse()->getUser() == &II && \"Map index doesn't point back to a slice with this user.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 923, __PRETTY_FUNCTION__)); | ||||||||||||
924 | } | ||||||||||||
925 | |||||||||||||
926 | // Disable SRoA for any intrinsics except for lifetime invariants. | ||||||||||||
927 | // FIXME: What about debug intrinsics? This matches old behavior, but | ||||||||||||
928 | // doesn't make sense. | ||||||||||||
929 | void visitIntrinsicInst(IntrinsicInst &II) { | ||||||||||||
930 | if (!IsOffsetKnown) | ||||||||||||
931 | return PI.setAborted(&II); | ||||||||||||
932 | |||||||||||||
933 | if (II.isLifetimeStartOrEnd()) { | ||||||||||||
934 | ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0)); | ||||||||||||
935 | uint64_t Size = std::min(AllocSize - Offset.getLimitedValue(), | ||||||||||||
936 | Length->getLimitedValue()); | ||||||||||||
937 | insertUse(II, Offset, Size, true); | ||||||||||||
938 | return; | ||||||||||||
939 | } | ||||||||||||
940 | |||||||||||||
941 | Base::visitIntrinsicInst(II); | ||||||||||||
942 | } | ||||||||||||
943 | |||||||||||||
944 | Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) { | ||||||||||||
945 | // We consider any PHI or select that results in a direct load or store of | ||||||||||||
946 | // the same offset to be a viable use for slicing purposes. These uses | ||||||||||||
947 | // are considered unsplittable and the size is the maximum loaded or stored | ||||||||||||
948 | // size. | ||||||||||||
949 | SmallPtrSet<Instruction *, 4> Visited; | ||||||||||||
950 | SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses; | ||||||||||||
951 | Visited.insert(Root); | ||||||||||||
952 | Uses.push_back(std::make_pair(cast<Instruction>(*U), Root)); | ||||||||||||
953 | const DataLayout &DL = Root->getModule()->getDataLayout(); | ||||||||||||
954 | // If there are no loads or stores, the access is dead. We mark that as | ||||||||||||
955 | // a size zero access. | ||||||||||||
956 | Size = 0; | ||||||||||||
957 | do { | ||||||||||||
958 | Instruction *I, *UsedI; | ||||||||||||
959 | std::tie(UsedI, I) = Uses.pop_back_val(); | ||||||||||||
960 | |||||||||||||
961 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | ||||||||||||
962 | Size = std::max(Size, | ||||||||||||
963 | DL.getTypeStoreSize(LI->getType()).getFixedSize()); | ||||||||||||
964 | continue; | ||||||||||||
965 | } | ||||||||||||
966 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | ||||||||||||
967 | Value *Op = SI->getOperand(0); | ||||||||||||
968 | if (Op == UsedI) | ||||||||||||
969 | return SI; | ||||||||||||
970 | Size = std::max(Size, | ||||||||||||
971 | DL.getTypeStoreSize(Op->getType()).getFixedSize()); | ||||||||||||
972 | continue; | ||||||||||||
973 | } | ||||||||||||
974 | |||||||||||||
975 | if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) { | ||||||||||||
976 | if (!GEP->hasAllZeroIndices()) | ||||||||||||
977 | return GEP; | ||||||||||||
978 | } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) && | ||||||||||||
979 | !isa<SelectInst>(I) && !isa<AddrSpaceCastInst>(I)) { | ||||||||||||
980 | return I; | ||||||||||||
981 | } | ||||||||||||
982 | |||||||||||||
983 | for (User *U : I->users()) | ||||||||||||
984 | if (Visited.insert(cast<Instruction>(U)).second) | ||||||||||||
985 | Uses.push_back(std::make_pair(I, cast<Instruction>(U))); | ||||||||||||
986 | } while (!Uses.empty()); | ||||||||||||
987 | |||||||||||||
988 | return nullptr; | ||||||||||||
989 | } | ||||||||||||
990 | |||||||||||||
991 | void visitPHINodeOrSelectInst(Instruction &I) { | ||||||||||||
992 | assert(isa<PHINode>(I) || isa<SelectInst>(I))((isa<PHINode>(I) || isa<SelectInst>(I)) ? static_cast <void> (0) : __assert_fail ("isa<PHINode>(I) || isa<SelectInst>(I)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 992, __PRETTY_FUNCTION__)); | ||||||||||||
993 | if (I.use_empty()) | ||||||||||||
994 | return markAsDead(I); | ||||||||||||
995 | |||||||||||||
996 | // TODO: We could use SimplifyInstruction here to fold PHINodes and | ||||||||||||
997 | // SelectInsts. However, doing so requires to change the current | ||||||||||||
998 | // dead-operand-tracking mechanism. For instance, suppose neither loading | ||||||||||||
999 | // from %U nor %other traps. Then "load (select undef, %U, %other)" does not | ||||||||||||
1000 | // trap either. However, if we simply replace %U with undef using the | ||||||||||||
1001 | // current dead-operand-tracking mechanism, "load (select undef, undef, | ||||||||||||
1002 | // %other)" may trap because the select may return the first operand | ||||||||||||
1003 | // "undef". | ||||||||||||
1004 | if (Value *Result = foldPHINodeOrSelectInst(I)) { | ||||||||||||
1005 | if (Result == *U) | ||||||||||||
1006 | // If the result of the constant fold will be the pointer, recurse | ||||||||||||
1007 | // through the PHI/select as if we had RAUW'ed it. | ||||||||||||
1008 | enqueueUsers(I); | ||||||||||||
1009 | else | ||||||||||||
1010 | // Otherwise the operand to the PHI/select is dead, and we can replace | ||||||||||||
1011 | // it with undef. | ||||||||||||
1012 | AS.DeadOperands.push_back(U); | ||||||||||||
1013 | |||||||||||||
1014 | return; | ||||||||||||
1015 | } | ||||||||||||
1016 | |||||||||||||
1017 | if (!IsOffsetKnown) | ||||||||||||
1018 | return PI.setAborted(&I); | ||||||||||||
1019 | |||||||||||||
1020 | // See if we already have computed info on this node. | ||||||||||||
1021 | uint64_t &Size = PHIOrSelectSizes[&I]; | ||||||||||||
1022 | if (!Size) { | ||||||||||||
1023 | // This is a new PHI/Select, check for an unsafe use of it. | ||||||||||||
1024 | if (Instruction *UnsafeI = hasUnsafePHIOrSelectUse(&I, Size)) | ||||||||||||
1025 | return PI.setAborted(UnsafeI); | ||||||||||||
1026 | } | ||||||||||||
1027 | |||||||||||||
1028 | // For PHI and select operands outside the alloca, we can't nuke the entire | ||||||||||||
1029 | // phi or select -- the other side might still be relevant, so we special | ||||||||||||
1030 | // case them here and use a separate structure to track the operands | ||||||||||||
1031 | // themselves which should be replaced with undef. | ||||||||||||
1032 | // FIXME: This should instead be escaped in the event we're instrumenting | ||||||||||||
1033 | // for address sanitization. | ||||||||||||
1034 | if (Offset.uge(AllocSize)) { | ||||||||||||
1035 | AS.DeadOperands.push_back(U); | ||||||||||||
1036 | return; | ||||||||||||
1037 | } | ||||||||||||
1038 | |||||||||||||
1039 | insertUse(I, Offset, Size); | ||||||||||||
1040 | } | ||||||||||||
1041 | |||||||||||||
1042 | void visitPHINode(PHINode &PN) { visitPHINodeOrSelectInst(PN); } | ||||||||||||
1043 | |||||||||||||
1044 | void visitSelectInst(SelectInst &SI) { visitPHINodeOrSelectInst(SI); } | ||||||||||||
1045 | |||||||||||||
1046 | /// Disable SROA entirely if there are unhandled users of the alloca. | ||||||||||||
1047 | void visitInstruction(Instruction &I) { PI.setAborted(&I); } | ||||||||||||
1048 | }; | ||||||||||||
1049 | |||||||||||||
1050 | AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI) | ||||||||||||
1051 | : | ||||||||||||
1052 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||||||||||||
1053 | AI(AI), | ||||||||||||
1054 | #endif | ||||||||||||
1055 | PointerEscapingInstr(nullptr) { | ||||||||||||
1056 | SliceBuilder PB(DL, AI, *this); | ||||||||||||
1057 | SliceBuilder::PtrInfo PtrI = PB.visitPtr(AI); | ||||||||||||
1058 | if (PtrI.isEscaped() || PtrI.isAborted()) { | ||||||||||||
1059 | // FIXME: We should sink the escape vs. abort info into the caller nicely, | ||||||||||||
1060 | // possibly by just storing the PtrInfo in the AllocaSlices. | ||||||||||||
1061 | PointerEscapingInstr = PtrI.getEscapingInst() ? PtrI.getEscapingInst() | ||||||||||||
1062 | : PtrI.getAbortingInst(); | ||||||||||||
1063 | assert(PointerEscapingInstr && "Did not track a bad instruction")((PointerEscapingInstr && "Did not track a bad instruction" ) ? static_cast<void> (0) : __assert_fail ("PointerEscapingInstr && \"Did not track a bad instruction\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1063, __PRETTY_FUNCTION__)); | ||||||||||||
1064 | return; | ||||||||||||
1065 | } | ||||||||||||
1066 | |||||||||||||
1067 | Slices.erase( | ||||||||||||
1068 | llvm::remove_if(Slices, [](const Slice &S) { return S.isDead(); }), | ||||||||||||
1069 | Slices.end()); | ||||||||||||
1070 | |||||||||||||
1071 | #ifndef NDEBUG | ||||||||||||
1072 | if (SROARandomShuffleSlices) { | ||||||||||||
1073 | std::mt19937 MT(static_cast<unsigned>( | ||||||||||||
1074 | std::chrono::system_clock::now().time_since_epoch().count())); | ||||||||||||
1075 | std::shuffle(Slices.begin(), Slices.end(), MT); | ||||||||||||
1076 | } | ||||||||||||
1077 | #endif | ||||||||||||
1078 | |||||||||||||
1079 | // Sort the uses. This arranges for the offsets to be in ascending order, | ||||||||||||
1080 | // and the sizes to be in descending order. | ||||||||||||
1081 | llvm::sort(Slices); | ||||||||||||
1082 | } | ||||||||||||
1083 | |||||||||||||
1084 | #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||||||||||||
1085 | |||||||||||||
1086 | void AllocaSlices::print(raw_ostream &OS, const_iterator I, | ||||||||||||
1087 | StringRef Indent) const { | ||||||||||||
1088 | printSlice(OS, I, Indent); | ||||||||||||
1089 | OS << "\n"; | ||||||||||||
1090 | printUse(OS, I, Indent); | ||||||||||||
1091 | } | ||||||||||||
1092 | |||||||||||||
1093 | void AllocaSlices::printSlice(raw_ostream &OS, const_iterator I, | ||||||||||||
1094 | StringRef Indent) const { | ||||||||||||
1095 | OS << Indent << "[" << I->beginOffset() << "," << I->endOffset() << ")" | ||||||||||||
1096 | << " slice #" << (I - begin()) | ||||||||||||
1097 | << (I->isSplittable() ? " (splittable)" : ""); | ||||||||||||
1098 | } | ||||||||||||
1099 | |||||||||||||
1100 | void AllocaSlices::printUse(raw_ostream &OS, const_iterator I, | ||||||||||||
1101 | StringRef Indent) const { | ||||||||||||
1102 | OS << Indent << " used by: " << *I->getUse()->getUser() << "\n"; | ||||||||||||
1103 | } | ||||||||||||
1104 | |||||||||||||
1105 | void AllocaSlices::print(raw_ostream &OS) const { | ||||||||||||
1106 | if (PointerEscapingInstr) { | ||||||||||||
1107 | OS << "Can't analyze slices for alloca: " << AI << "\n" | ||||||||||||
1108 | << " A pointer to this alloca escaped by:\n" | ||||||||||||
1109 | << " " << *PointerEscapingInstr << "\n"; | ||||||||||||
1110 | return; | ||||||||||||
1111 | } | ||||||||||||
1112 | |||||||||||||
1113 | OS << "Slices of alloca: " << AI << "\n"; | ||||||||||||
1114 | for (const_iterator I = begin(), E = end(); I != E; ++I) | ||||||||||||
1115 | print(OS, I); | ||||||||||||
1116 | } | ||||||||||||
1117 | |||||||||||||
1118 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump(const_iterator I) const { | ||||||||||||
1119 | print(dbgs(), I); | ||||||||||||
1120 | } | ||||||||||||
1121 | LLVM_DUMP_METHOD__attribute__((noinline)) __attribute__((__used__)) void AllocaSlices::dump() const { print(dbgs()); } | ||||||||||||
1122 | |||||||||||||
1123 | #endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) | ||||||||||||
1124 | |||||||||||||
1125 | /// Walk the range of a partitioning looking for a common type to cover this | ||||||||||||
1126 | /// sequence of slices. | ||||||||||||
1127 | static Type *findCommonType(AllocaSlices::const_iterator B, | ||||||||||||
1128 | AllocaSlices::const_iterator E, | ||||||||||||
1129 | uint64_t EndOffset) { | ||||||||||||
1130 | Type *Ty = nullptr; | ||||||||||||
1131 | bool TyIsCommon = true; | ||||||||||||
1132 | IntegerType *ITy = nullptr; | ||||||||||||
1133 | |||||||||||||
1134 | // Note that we need to look at *every* alloca slice's Use to ensure we | ||||||||||||
1135 | // always get consistent results regardless of the order of slices. | ||||||||||||
1136 | for (AllocaSlices::const_iterator I = B; I != E; ++I) { | ||||||||||||
1137 | Use *U = I->getUse(); | ||||||||||||
1138 | if (isa<IntrinsicInst>(*U->getUser())) | ||||||||||||
1139 | continue; | ||||||||||||
1140 | if (I->beginOffset() != B->beginOffset() || I->endOffset() != EndOffset) | ||||||||||||
1141 | continue; | ||||||||||||
1142 | |||||||||||||
1143 | Type *UserTy = nullptr; | ||||||||||||
1144 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | ||||||||||||
1145 | UserTy = LI->getType(); | ||||||||||||
1146 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | ||||||||||||
1147 | UserTy = SI->getValueOperand()->getType(); | ||||||||||||
1148 | } | ||||||||||||
1149 | |||||||||||||
1150 | if (IntegerType *UserITy = dyn_cast_or_null<IntegerType>(UserTy)) { | ||||||||||||
1151 | // If the type is larger than the partition, skip it. We only encounter | ||||||||||||
1152 | // this for split integer operations where we want to use the type of the | ||||||||||||
1153 | // entity causing the split. Also skip if the type is not a byte width | ||||||||||||
1154 | // multiple. | ||||||||||||
1155 | if (UserITy->getBitWidth() % 8 != 0 || | ||||||||||||
1156 | UserITy->getBitWidth() / 8 > (EndOffset - B->beginOffset())) | ||||||||||||
1157 | continue; | ||||||||||||
1158 | |||||||||||||
1159 | // Track the largest bitwidth integer type used in this way in case there | ||||||||||||
1160 | // is no common type. | ||||||||||||
1161 | if (!ITy || ITy->getBitWidth() < UserITy->getBitWidth()) | ||||||||||||
1162 | ITy = UserITy; | ||||||||||||
1163 | } | ||||||||||||
1164 | |||||||||||||
1165 | // To avoid depending on the order of slices, Ty and TyIsCommon must not | ||||||||||||
1166 | // depend on types skipped above. | ||||||||||||
1167 | if (!UserTy || (Ty && Ty != UserTy)) | ||||||||||||
1168 | TyIsCommon = false; // Give up on anything but an iN type. | ||||||||||||
1169 | else | ||||||||||||
1170 | Ty = UserTy; | ||||||||||||
1171 | } | ||||||||||||
1172 | |||||||||||||
1173 | return TyIsCommon ? Ty : ITy; | ||||||||||||
1174 | } | ||||||||||||
1175 | |||||||||||||
1176 | /// PHI instructions that use an alloca and are subsequently loaded can be | ||||||||||||
1177 | /// rewritten to load both input pointers in the pred blocks and then PHI the | ||||||||||||
1178 | /// results, allowing the load of the alloca to be promoted. | ||||||||||||
1179 | /// From this: | ||||||||||||
1180 | /// %P2 = phi [i32* %Alloca, i32* %Other] | ||||||||||||
1181 | /// %V = load i32* %P2 | ||||||||||||
1182 | /// to: | ||||||||||||
1183 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | ||||||||||||
1184 | /// ... | ||||||||||||
1185 | /// %V2 = load i32* %Other | ||||||||||||
1186 | /// ... | ||||||||||||
1187 | /// %V = phi [i32 %V1, i32 %V2] | ||||||||||||
1188 | /// | ||||||||||||
1189 | /// We can do this to a select if its only uses are loads and if the operands | ||||||||||||
1190 | /// to the select can be loaded unconditionally. | ||||||||||||
1191 | /// | ||||||||||||
1192 | /// FIXME: This should be hoisted into a generic utility, likely in | ||||||||||||
1193 | /// Transforms/Util/Local.h | ||||||||||||
1194 | static bool isSafePHIToSpeculate(PHINode &PN) { | ||||||||||||
1195 | const DataLayout &DL = PN.getModule()->getDataLayout(); | ||||||||||||
1196 | |||||||||||||
1197 | // For now, we can only do this promotion if the load is in the same block | ||||||||||||
1198 | // as the PHI, and if there are no stores between the phi and load. | ||||||||||||
1199 | // TODO: Allow recursive phi users. | ||||||||||||
1200 | // TODO: Allow stores. | ||||||||||||
1201 | BasicBlock *BB = PN.getParent(); | ||||||||||||
1202 | MaybeAlign MaxAlign; | ||||||||||||
1203 | uint64_t APWidth = DL.getIndexTypeSizeInBits(PN.getType()); | ||||||||||||
1204 | APInt MaxSize(APWidth, 0); | ||||||||||||
1205 | bool HaveLoad = false; | ||||||||||||
1206 | for (User *U : PN.users()) { | ||||||||||||
1207 | LoadInst *LI = dyn_cast<LoadInst>(U); | ||||||||||||
1208 | if (!LI || !LI->isSimple()) | ||||||||||||
1209 | return false; | ||||||||||||
1210 | |||||||||||||
1211 | // For now we only allow loads in the same block as the PHI. This is | ||||||||||||
1212 | // a common case that happens when instcombine merges two loads through | ||||||||||||
1213 | // a PHI. | ||||||||||||
1214 | if (LI->getParent() != BB) | ||||||||||||
1215 | return false; | ||||||||||||
1216 | |||||||||||||
1217 | // Ensure that there are no instructions between the PHI and the load that | ||||||||||||
1218 | // could store. | ||||||||||||
1219 | for (BasicBlock::iterator BBI(PN); &*BBI != LI; ++BBI) | ||||||||||||
1220 | if (BBI->mayWriteToMemory()) | ||||||||||||
1221 | return false; | ||||||||||||
1222 | |||||||||||||
1223 | uint64_t Size = DL.getTypeStoreSize(LI->getType()); | ||||||||||||
1224 | MaxAlign = std::max(MaxAlign, MaybeAlign(LI->getAlignment())); | ||||||||||||
1225 | MaxSize = MaxSize.ult(Size) ? APInt(APWidth, Size) : MaxSize; | ||||||||||||
1226 | HaveLoad = true; | ||||||||||||
1227 | } | ||||||||||||
1228 | |||||||||||||
1229 | if (!HaveLoad) | ||||||||||||
1230 | return false; | ||||||||||||
1231 | |||||||||||||
1232 | // We can only transform this if it is safe to push the loads into the | ||||||||||||
1233 | // predecessor blocks. The only thing to watch out for is that we can't put | ||||||||||||
1234 | // a possibly trapping load in the predecessor if it is a critical edge. | ||||||||||||
1235 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | ||||||||||||
1236 | Instruction *TI = PN.getIncomingBlock(Idx)->getTerminator(); | ||||||||||||
1237 | Value *InVal = PN.getIncomingValue(Idx); | ||||||||||||
1238 | |||||||||||||
1239 | // If the value is produced by the terminator of the predecessor (an | ||||||||||||
1240 | // invoke) or it has side-effects, there is no valid place to put a load | ||||||||||||
1241 | // in the predecessor. | ||||||||||||
1242 | if (TI == InVal || TI->mayHaveSideEffects()) | ||||||||||||
1243 | return false; | ||||||||||||
1244 | |||||||||||||
1245 | // If the predecessor has a single successor, then the edge isn't | ||||||||||||
1246 | // critical. | ||||||||||||
1247 | if (TI->getNumSuccessors() == 1) | ||||||||||||
1248 | continue; | ||||||||||||
1249 | |||||||||||||
1250 | // If this pointer is always safe to load, or if we can prove that there | ||||||||||||
1251 | // is already a load in the block, then we can move the load to the pred | ||||||||||||
1252 | // block. | ||||||||||||
1253 | if (isSafeToLoadUnconditionally(InVal, MaxAlign, MaxSize, DL, TI)) | ||||||||||||
1254 | continue; | ||||||||||||
1255 | |||||||||||||
1256 | return false; | ||||||||||||
1257 | } | ||||||||||||
1258 | |||||||||||||
1259 | return true; | ||||||||||||
1260 | } | ||||||||||||
1261 | |||||||||||||
1262 | static void speculatePHINodeLoads(PHINode &PN) { | ||||||||||||
1263 | LLVM_DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | ||||||||||||
1264 | |||||||||||||
1265 | LoadInst *SomeLoad = cast<LoadInst>(PN.user_back()); | ||||||||||||
1266 | Type *LoadTy = SomeLoad->getType(); | ||||||||||||
1267 | IRBuilderTy PHIBuilder(&PN); | ||||||||||||
1268 | PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(), | ||||||||||||
1269 | PN.getName() + ".sroa.speculated"); | ||||||||||||
1270 | |||||||||||||
1271 | // Get the AA tags and alignment to use from one of the loads. It does not | ||||||||||||
1272 | // matter which one we get and if any differ. | ||||||||||||
1273 | AAMDNodes AATags; | ||||||||||||
1274 | SomeLoad->getAAMetadata(AATags); | ||||||||||||
1275 | const MaybeAlign Align = MaybeAlign(SomeLoad->getAlignment()); | ||||||||||||
1276 | |||||||||||||
1277 | // Rewrite all loads of the PN to use the new PHI. | ||||||||||||
1278 | while (!PN.use_empty()) { | ||||||||||||
1279 | LoadInst *LI = cast<LoadInst>(PN.user_back()); | ||||||||||||
1280 | LI->replaceAllUsesWith(NewPN); | ||||||||||||
1281 | LI->eraseFromParent(); | ||||||||||||
1282 | } | ||||||||||||
1283 | |||||||||||||
1284 | // Inject loads into all of the pred blocks. | ||||||||||||
1285 | DenseMap<BasicBlock*, Value*> InjectedLoads; | ||||||||||||
1286 | for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) { | ||||||||||||
1287 | BasicBlock *Pred = PN.getIncomingBlock(Idx); | ||||||||||||
1288 | Value *InVal = PN.getIncomingValue(Idx); | ||||||||||||
1289 | |||||||||||||
1290 | // A PHI node is allowed to have multiple (duplicated) entries for the same | ||||||||||||
1291 | // basic block, as long as the value is the same. So if we already injected | ||||||||||||
1292 | // a load in the predecessor, then we should reuse the same load for all | ||||||||||||
1293 | // duplicated entries. | ||||||||||||
1294 | if (Value* V = InjectedLoads.lookup(Pred)) { | ||||||||||||
1295 | NewPN->addIncoming(V, Pred); | ||||||||||||
1296 | continue; | ||||||||||||
1297 | } | ||||||||||||
1298 | |||||||||||||
1299 | Instruction *TI = Pred->getTerminator(); | ||||||||||||
1300 | IRBuilderTy PredBuilder(TI); | ||||||||||||
1301 | |||||||||||||
1302 | LoadInst *Load = PredBuilder.CreateLoad( | ||||||||||||
1303 | LoadTy, InVal, | ||||||||||||
1304 | (PN.getName() + ".sroa.speculate.load." + Pred->getName())); | ||||||||||||
1305 | ++NumLoadsSpeculated; | ||||||||||||
1306 | Load->setAlignment(Align); | ||||||||||||
1307 | if (AATags) | ||||||||||||
1308 | Load->setAAMetadata(AATags); | ||||||||||||
1309 | NewPN->addIncoming(Load, Pred); | ||||||||||||
1310 | InjectedLoads[Pred] = Load; | ||||||||||||
1311 | } | ||||||||||||
1312 | |||||||||||||
1313 | LLVM_DEBUG(dbgs() << " speculated to: " << *NewPN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *NewPN << "\n"; } } while (false); | ||||||||||||
1314 | PN.eraseFromParent(); | ||||||||||||
1315 | } | ||||||||||||
1316 | |||||||||||||
1317 | /// Select instructions that use an alloca and are subsequently loaded can be | ||||||||||||
1318 | /// rewritten to load both input pointers and then select between the result, | ||||||||||||
1319 | /// allowing the load of the alloca to be promoted. | ||||||||||||
1320 | /// From this: | ||||||||||||
1321 | /// %P2 = select i1 %cond, i32* %Alloca, i32* %Other | ||||||||||||
1322 | /// %V = load i32* %P2 | ||||||||||||
1323 | /// to: | ||||||||||||
1324 | /// %V1 = load i32* %Alloca -> will be mem2reg'd | ||||||||||||
1325 | /// %V2 = load i32* %Other | ||||||||||||
1326 | /// %V = select i1 %cond, i32 %V1, i32 %V2 | ||||||||||||
1327 | /// | ||||||||||||
1328 | /// We can do this to a select if its only uses are loads and if the operand | ||||||||||||
1329 | /// to the select can be loaded unconditionally. | ||||||||||||
1330 | static bool isSafeSelectToSpeculate(SelectInst &SI) { | ||||||||||||
1331 | Value *TValue = SI.getTrueValue(); | ||||||||||||
1332 | Value *FValue = SI.getFalseValue(); | ||||||||||||
1333 | const DataLayout &DL = SI.getModule()->getDataLayout(); | ||||||||||||
1334 | |||||||||||||
1335 | for (User *U : SI.users()) { | ||||||||||||
1336 | LoadInst *LI = dyn_cast<LoadInst>(U); | ||||||||||||
1337 | if (!LI || !LI->isSimple()) | ||||||||||||
1338 | return false; | ||||||||||||
1339 | |||||||||||||
1340 | // Both operands to the select need to be dereferenceable, either | ||||||||||||
1341 | // absolutely (e.g. allocas) or at this point because we can see other | ||||||||||||
1342 | // accesses to it. | ||||||||||||
1343 | if (!isSafeToLoadUnconditionally(TValue, LI->getType(), | ||||||||||||
1344 | MaybeAlign(LI->getAlignment()), DL, LI)) | ||||||||||||
1345 | return false; | ||||||||||||
1346 | if (!isSafeToLoadUnconditionally(FValue, LI->getType(), | ||||||||||||
1347 | MaybeAlign(LI->getAlignment()), DL, LI)) | ||||||||||||
1348 | return false; | ||||||||||||
1349 | } | ||||||||||||
1350 | |||||||||||||
1351 | return true; | ||||||||||||
1352 | } | ||||||||||||
1353 | |||||||||||||
1354 | static void speculateSelectInstLoads(SelectInst &SI) { | ||||||||||||
1355 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | ||||||||||||
1356 | |||||||||||||
1357 | IRBuilderTy IRB(&SI); | ||||||||||||
1358 | Value *TV = SI.getTrueValue(); | ||||||||||||
1359 | Value *FV = SI.getFalseValue(); | ||||||||||||
1360 | // Replace the loads of the select with a select of two loads. | ||||||||||||
1361 | while (!SI.use_empty()) { | ||||||||||||
1362 | LoadInst *LI = cast<LoadInst>(SI.user_back()); | ||||||||||||
1363 | assert(LI->isSimple() && "We only speculate simple loads")((LI->isSimple() && "We only speculate simple loads" ) ? static_cast<void> (0) : __assert_fail ("LI->isSimple() && \"We only speculate simple loads\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1363, __PRETTY_FUNCTION__)); | ||||||||||||
1364 | |||||||||||||
1365 | IRB.SetInsertPoint(LI); | ||||||||||||
1366 | LoadInst *TL = IRB.CreateLoad(LI->getType(), TV, | ||||||||||||
1367 | LI->getName() + ".sroa.speculate.load.true"); | ||||||||||||
1368 | LoadInst *FL = IRB.CreateLoad(LI->getType(), FV, | ||||||||||||
1369 | LI->getName() + ".sroa.speculate.load.false"); | ||||||||||||
1370 | NumLoadsSpeculated += 2; | ||||||||||||
1371 | |||||||||||||
1372 | // Transfer alignment and AA info if present. | ||||||||||||
1373 | TL->setAlignment(MaybeAlign(LI->getAlignment())); | ||||||||||||
1374 | FL->setAlignment(MaybeAlign(LI->getAlignment())); | ||||||||||||
1375 | |||||||||||||
1376 | AAMDNodes Tags; | ||||||||||||
1377 | LI->getAAMetadata(Tags); | ||||||||||||
1378 | if (Tags) { | ||||||||||||
1379 | TL->setAAMetadata(Tags); | ||||||||||||
1380 | FL->setAAMetadata(Tags); | ||||||||||||
1381 | } | ||||||||||||
1382 | |||||||||||||
1383 | Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL, | ||||||||||||
1384 | LI->getName() + ".sroa.speculated"); | ||||||||||||
1385 | |||||||||||||
1386 | LLVM_DEBUG(dbgs() << " speculated to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " speculated to: " << *V << "\n"; } } while (false); | ||||||||||||
1387 | LI->replaceAllUsesWith(V); | ||||||||||||
1388 | LI->eraseFromParent(); | ||||||||||||
1389 | } | ||||||||||||
1390 | SI.eraseFromParent(); | ||||||||||||
1391 | } | ||||||||||||
1392 | |||||||||||||
1393 | /// Build a GEP out of a base pointer and indices. | ||||||||||||
1394 | /// | ||||||||||||
1395 | /// This will return the BasePtr if that is valid, or build a new GEP | ||||||||||||
1396 | /// instruction using the IRBuilder if GEP-ing is needed. | ||||||||||||
1397 | static Value *buildGEP(IRBuilderTy &IRB, Value *BasePtr, | ||||||||||||
1398 | SmallVectorImpl<Value *> &Indices, Twine NamePrefix) { | ||||||||||||
1399 | if (Indices.empty()) | ||||||||||||
1400 | return BasePtr; | ||||||||||||
1401 | |||||||||||||
1402 | // A single zero index is a no-op, so check for this and avoid building a GEP | ||||||||||||
1403 | // in that case. | ||||||||||||
1404 | if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero()) | ||||||||||||
1405 | return BasePtr; | ||||||||||||
1406 | |||||||||||||
1407 | return IRB.CreateInBoundsGEP(BasePtr->getType()->getPointerElementType(), | ||||||||||||
1408 | BasePtr, Indices, NamePrefix + "sroa_idx"); | ||||||||||||
1409 | } | ||||||||||||
1410 | |||||||||||||
1411 | /// Get a natural GEP off of the BasePtr walking through Ty toward | ||||||||||||
1412 | /// TargetTy without changing the offset of the pointer. | ||||||||||||
1413 | /// | ||||||||||||
1414 | /// This routine assumes we've already established a properly offset GEP with | ||||||||||||
1415 | /// Indices, and arrived at the Ty type. The goal is to continue to GEP with | ||||||||||||
1416 | /// zero-indices down through type layers until we find one the same as | ||||||||||||
1417 | /// TargetTy. If we can't find one with the same type, we at least try to use | ||||||||||||
1418 | /// one with the same size. If none of that works, we just produce the GEP as | ||||||||||||
1419 | /// indicated by Indices to have the correct offset. | ||||||||||||
1420 | static Value *getNaturalGEPWithType(IRBuilderTy &IRB, const DataLayout &DL, | ||||||||||||
1421 | Value *BasePtr, Type *Ty, Type *TargetTy, | ||||||||||||
1422 | SmallVectorImpl<Value *> &Indices, | ||||||||||||
1423 | Twine NamePrefix) { | ||||||||||||
1424 | if (Ty == TargetTy) | ||||||||||||
1425 | return buildGEP(IRB, BasePtr, Indices, NamePrefix); | ||||||||||||
1426 | |||||||||||||
1427 | // Offset size to use for the indices. | ||||||||||||
1428 | unsigned OffsetSize = DL.getIndexTypeSizeInBits(BasePtr->getType()); | ||||||||||||
1429 | |||||||||||||
1430 | // See if we can descend into a struct and locate a field with the correct | ||||||||||||
1431 | // type. | ||||||||||||
1432 | unsigned NumLayers = 0; | ||||||||||||
1433 | Type *ElementTy = Ty; | ||||||||||||
1434 | do { | ||||||||||||
1435 | if (ElementTy->isPointerTy()) | ||||||||||||
1436 | break; | ||||||||||||
1437 | |||||||||||||
1438 | if (ArrayType *ArrayTy = dyn_cast<ArrayType>(ElementTy)) { | ||||||||||||
1439 | ElementTy = ArrayTy->getElementType(); | ||||||||||||
1440 | Indices.push_back(IRB.getIntN(OffsetSize, 0)); | ||||||||||||
1441 | } else if (VectorType *VectorTy = dyn_cast<VectorType>(ElementTy)) { | ||||||||||||
1442 | ElementTy = VectorTy->getElementType(); | ||||||||||||
1443 | Indices.push_back(IRB.getInt32(0)); | ||||||||||||
1444 | } else if (StructType *STy = dyn_cast<StructType>(ElementTy)) { | ||||||||||||
1445 | if (STy->element_begin() == STy->element_end()) | ||||||||||||
1446 | break; // Nothing left to descend into. | ||||||||||||
1447 | ElementTy = *STy->element_begin(); | ||||||||||||
1448 | Indices.push_back(IRB.getInt32(0)); | ||||||||||||
1449 | } else { | ||||||||||||
1450 | break; | ||||||||||||
1451 | } | ||||||||||||
1452 | ++NumLayers; | ||||||||||||
1453 | } while (ElementTy != TargetTy); | ||||||||||||
1454 | if (ElementTy != TargetTy) | ||||||||||||
1455 | Indices.erase(Indices.end() - NumLayers, Indices.end()); | ||||||||||||
1456 | |||||||||||||
1457 | return buildGEP(IRB, BasePtr, Indices, NamePrefix); | ||||||||||||
1458 | } | ||||||||||||
1459 | |||||||||||||
1460 | /// Recursively compute indices for a natural GEP. | ||||||||||||
1461 | /// | ||||||||||||
1462 | /// This is the recursive step for getNaturalGEPWithOffset that walks down the | ||||||||||||
1463 | /// element types adding appropriate indices for the GEP. | ||||||||||||
1464 | static Value *getNaturalGEPRecursively(IRBuilderTy &IRB, const DataLayout &DL, | ||||||||||||
1465 | Value *Ptr, Type *Ty, APInt &Offset, | ||||||||||||
1466 | Type *TargetTy, | ||||||||||||
1467 | SmallVectorImpl<Value *> &Indices, | ||||||||||||
1468 | Twine NamePrefix) { | ||||||||||||
1469 | if (Offset == 0) | ||||||||||||
1470 | return getNaturalGEPWithType(IRB, DL, Ptr, Ty, TargetTy, Indices, | ||||||||||||
1471 | NamePrefix); | ||||||||||||
1472 | |||||||||||||
1473 | // We can't recurse through pointer types. | ||||||||||||
1474 | if (Ty->isPointerTy()) | ||||||||||||
1475 | return nullptr; | ||||||||||||
1476 | |||||||||||||
1477 | // We try to analyze GEPs over vectors here, but note that these GEPs are | ||||||||||||
1478 | // extremely poorly defined currently. The long-term goal is to remove GEPing | ||||||||||||
1479 | // over a vector from the IR completely. | ||||||||||||
1480 | if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) { | ||||||||||||
1481 | unsigned ElementSizeInBits = DL.getTypeSizeInBits(VecTy->getScalarType()); | ||||||||||||
1482 | if (ElementSizeInBits % 8 != 0) { | ||||||||||||
1483 | // GEPs over non-multiple of 8 size vector elements are invalid. | ||||||||||||
1484 | return nullptr; | ||||||||||||
1485 | } | ||||||||||||
1486 | APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8); | ||||||||||||
1487 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | ||||||||||||
1488 | if (NumSkippedElements.ugt(VecTy->getNumElements())) | ||||||||||||
1489 | return nullptr; | ||||||||||||
1490 | Offset -= NumSkippedElements * ElementSize; | ||||||||||||
1491 | Indices.push_back(IRB.getInt(NumSkippedElements)); | ||||||||||||
1492 | return getNaturalGEPRecursively(IRB, DL, Ptr, VecTy->getElementType(), | ||||||||||||
1493 | Offset, TargetTy, Indices, NamePrefix); | ||||||||||||
1494 | } | ||||||||||||
1495 | |||||||||||||
1496 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | ||||||||||||
1497 | Type *ElementTy = ArrTy->getElementType(); | ||||||||||||
1498 | APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy)); | ||||||||||||
1499 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | ||||||||||||
1500 | if (NumSkippedElements.ugt(ArrTy->getNumElements())) | ||||||||||||
1501 | return nullptr; | ||||||||||||
1502 | |||||||||||||
1503 | Offset -= NumSkippedElements * ElementSize; | ||||||||||||
1504 | Indices.push_back(IRB.getInt(NumSkippedElements)); | ||||||||||||
1505 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | ||||||||||||
1506 | Indices, NamePrefix); | ||||||||||||
1507 | } | ||||||||||||
1508 | |||||||||||||
1509 | StructType *STy = dyn_cast<StructType>(Ty); | ||||||||||||
1510 | if (!STy) | ||||||||||||
1511 | return nullptr; | ||||||||||||
1512 | |||||||||||||
1513 | const StructLayout *SL = DL.getStructLayout(STy); | ||||||||||||
1514 | uint64_t StructOffset = Offset.getZExtValue(); | ||||||||||||
1515 | if (StructOffset >= SL->getSizeInBytes()) | ||||||||||||
1516 | return nullptr; | ||||||||||||
1517 | unsigned Index = SL->getElementContainingOffset(StructOffset); | ||||||||||||
1518 | Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index)); | ||||||||||||
1519 | Type *ElementTy = STy->getElementType(Index); | ||||||||||||
1520 | if (Offset.uge(DL.getTypeAllocSize(ElementTy))) | ||||||||||||
1521 | return nullptr; // The offset points into alignment padding. | ||||||||||||
1522 | |||||||||||||
1523 | Indices.push_back(IRB.getInt32(Index)); | ||||||||||||
1524 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | ||||||||||||
1525 | Indices, NamePrefix); | ||||||||||||
1526 | } | ||||||||||||
1527 | |||||||||||||
1528 | /// Get a natural GEP from a base pointer to a particular offset and | ||||||||||||
1529 | /// resulting in a particular type. | ||||||||||||
1530 | /// | ||||||||||||
1531 | /// The goal is to produce a "natural" looking GEP that works with the existing | ||||||||||||
1532 | /// composite types to arrive at the appropriate offset and element type for | ||||||||||||
1533 | /// a pointer. TargetTy is the element type the returned GEP should point-to if | ||||||||||||
1534 | /// possible. We recurse by decreasing Offset, adding the appropriate index to | ||||||||||||
1535 | /// Indices, and setting Ty to the result subtype. | ||||||||||||
1536 | /// | ||||||||||||
1537 | /// If no natural GEP can be constructed, this function returns null. | ||||||||||||
1538 | static Value *getNaturalGEPWithOffset(IRBuilderTy &IRB, const DataLayout &DL, | ||||||||||||
1539 | Value *Ptr, APInt Offset, Type *TargetTy, | ||||||||||||
1540 | SmallVectorImpl<Value *> &Indices, | ||||||||||||
1541 | Twine NamePrefix) { | ||||||||||||
1542 | PointerType *Ty = cast<PointerType>(Ptr->getType()); | ||||||||||||
1543 | |||||||||||||
1544 | // Don't consider any GEPs through an i8* as natural unless the TargetTy is | ||||||||||||
1545 | // an i8. | ||||||||||||
1546 | if (Ty == IRB.getInt8PtrTy(Ty->getAddressSpace()) && TargetTy->isIntegerTy(8)) | ||||||||||||
1547 | return nullptr; | ||||||||||||
1548 | |||||||||||||
1549 | Type *ElementTy = Ty->getElementType(); | ||||||||||||
1550 | if (!ElementTy->isSized()) | ||||||||||||
1551 | return nullptr; // We can't GEP through an unsized element. | ||||||||||||
1552 | APInt ElementSize(Offset.getBitWidth(), DL.getTypeAllocSize(ElementTy)); | ||||||||||||
1553 | if (ElementSize == 0) | ||||||||||||
1554 | return nullptr; // Zero-length arrays can't help us build a natural GEP. | ||||||||||||
1555 | APInt NumSkippedElements = Offset.sdiv(ElementSize); | ||||||||||||
1556 | |||||||||||||
1557 | Offset -= NumSkippedElements * ElementSize; | ||||||||||||
1558 | Indices.push_back(IRB.getInt(NumSkippedElements)); | ||||||||||||
1559 | return getNaturalGEPRecursively(IRB, DL, Ptr, ElementTy, Offset, TargetTy, | ||||||||||||
1560 | Indices, NamePrefix); | ||||||||||||
1561 | } | ||||||||||||
1562 | |||||||||||||
1563 | /// Compute an adjusted pointer from Ptr by Offset bytes where the | ||||||||||||
1564 | /// resulting pointer has PointerTy. | ||||||||||||
1565 | /// | ||||||||||||
1566 | /// This tries very hard to compute a "natural" GEP which arrives at the offset | ||||||||||||
1567 | /// and produces the pointer type desired. Where it cannot, it will try to use | ||||||||||||
1568 | /// the natural GEP to arrive at the offset and bitcast to the type. Where that | ||||||||||||
1569 | /// fails, it will try to use an existing i8* and GEP to the byte offset and | ||||||||||||
1570 | /// bitcast to the type. | ||||||||||||
1571 | /// | ||||||||||||
1572 | /// The strategy for finding the more natural GEPs is to peel off layers of the | ||||||||||||
1573 | /// pointer, walking back through bit casts and GEPs, searching for a base | ||||||||||||
1574 | /// pointer from which we can compute a natural GEP with the desired | ||||||||||||
1575 | /// properties. The algorithm tries to fold as many constant indices into | ||||||||||||
1576 | /// a single GEP as possible, thus making each GEP more independent of the | ||||||||||||
1577 | /// surrounding code. | ||||||||||||
1578 | static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr, | ||||||||||||
1579 | APInt Offset, Type *PointerTy, Twine NamePrefix) { | ||||||||||||
1580 | // Even though we don't look through PHI nodes, we could be called on an | ||||||||||||
1581 | // instruction in an unreachable block, which may be on a cycle. | ||||||||||||
1582 | SmallPtrSet<Value *, 4> Visited; | ||||||||||||
1583 | Visited.insert(Ptr); | ||||||||||||
1584 | SmallVector<Value *, 4> Indices; | ||||||||||||
1585 | |||||||||||||
1586 | // We may end up computing an offset pointer that has the wrong type. If we | ||||||||||||
1587 | // never are able to compute one directly that has the correct type, we'll | ||||||||||||
1588 | // fall back to it, so keep it and the base it was computed from around here. | ||||||||||||
1589 | Value *OffsetPtr = nullptr; | ||||||||||||
1590 | Value *OffsetBasePtr; | ||||||||||||
1591 | |||||||||||||
1592 | // Remember any i8 pointer we come across to re-use if we need to do a raw | ||||||||||||
1593 | // byte offset. | ||||||||||||
1594 | Value *Int8Ptr = nullptr; | ||||||||||||
1595 | APInt Int8PtrOffset(Offset.getBitWidth(), 0); | ||||||||||||
1596 | |||||||||||||
1597 | PointerType *TargetPtrTy = cast<PointerType>(PointerTy); | ||||||||||||
1598 | Type *TargetTy = TargetPtrTy->getElementType(); | ||||||||||||
1599 | |||||||||||||
1600 | // As `addrspacecast` is , `Ptr` (the storage pointer) may have different | ||||||||||||
1601 | // address space from the expected `PointerTy` (the pointer to be used). | ||||||||||||
1602 | // Adjust the pointer type based the original storage pointer. | ||||||||||||
1603 | auto AS = cast<PointerType>(Ptr->getType())->getAddressSpace(); | ||||||||||||
1604 | PointerTy = TargetTy->getPointerTo(AS); | ||||||||||||
1605 | |||||||||||||
1606 | do { | ||||||||||||
1607 | // First fold any existing GEPs into the offset. | ||||||||||||
1608 | while (GEPOperator *GEP = dyn_cast<GEPOperator>(Ptr)) { | ||||||||||||
1609 | APInt GEPOffset(Offset.getBitWidth(), 0); | ||||||||||||
1610 | if (!GEP->accumulateConstantOffset(DL, GEPOffset)) | ||||||||||||
1611 | break; | ||||||||||||
1612 | Offset += GEPOffset; | ||||||||||||
1613 | Ptr = GEP->getPointerOperand(); | ||||||||||||
1614 | if (!Visited.insert(Ptr).second) | ||||||||||||
1615 | break; | ||||||||||||
1616 | } | ||||||||||||
1617 | |||||||||||||
1618 | // See if we can perform a natural GEP here. | ||||||||||||
1619 | Indices.clear(); | ||||||||||||
1620 | if (Value *P
| ||||||||||||
1621 | Indices, NamePrefix)) { | ||||||||||||
1622 | // If we have a new natural pointer at the offset, clear out any old | ||||||||||||
1623 | // offset pointer we computed. Unless it is the base pointer or | ||||||||||||
1624 | // a non-instruction, we built a GEP we don't need. Zap it. | ||||||||||||
1625 | if (OffsetPtr && OffsetPtr != OffsetBasePtr) | ||||||||||||
1626 | if (Instruction *I = dyn_cast<Instruction>(OffsetPtr)) { | ||||||||||||
1627 | assert(I->use_empty() && "Built a GEP with uses some how!")((I->use_empty() && "Built a GEP with uses some how!" ) ? static_cast<void> (0) : __assert_fail ("I->use_empty() && \"Built a GEP with uses some how!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1627, __PRETTY_FUNCTION__)); | ||||||||||||
1628 | I->eraseFromParent(); | ||||||||||||
1629 | } | ||||||||||||
1630 | OffsetPtr = P; | ||||||||||||
1631 | OffsetBasePtr = Ptr; | ||||||||||||
1632 | // If we also found a pointer of the right type, we're done. | ||||||||||||
1633 | if (P->getType() == PointerTy) | ||||||||||||
1634 | break; | ||||||||||||
1635 | } | ||||||||||||
1636 | |||||||||||||
1637 | // Stash this pointer if we've found an i8*. | ||||||||||||
1638 | if (Ptr->getType()->isIntegerTy(8)) { | ||||||||||||
1639 | Int8Ptr = Ptr; | ||||||||||||
1640 | Int8PtrOffset = Offset; | ||||||||||||
1641 | } | ||||||||||||
1642 | |||||||||||||
1643 | // Peel off a layer of the pointer and update the offset appropriately. | ||||||||||||
1644 | if (Operator::getOpcode(Ptr) == Instruction::BitCast) { | ||||||||||||
1645 | Ptr = cast<Operator>(Ptr)->getOperand(0); | ||||||||||||
1646 | } else if (GlobalAlias *GA
| ||||||||||||
1647 | if (GA->isInterposable()) | ||||||||||||
1648 | break; | ||||||||||||
1649 | Ptr = GA->getAliasee(); | ||||||||||||
1650 | } else { | ||||||||||||
1651 | break; | ||||||||||||
1652 | } | ||||||||||||
1653 | assert(Ptr->getType()->isPointerTy() && "Unexpected operand type!")((Ptr->getType()->isPointerTy() && "Unexpected operand type!" ) ? static_cast<void> (0) : __assert_fail ("Ptr->getType()->isPointerTy() && \"Unexpected operand type!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1653, __PRETTY_FUNCTION__)); | ||||||||||||
1654 | } while (Visited.insert(Ptr).second); | ||||||||||||
1655 | |||||||||||||
1656 | if (!OffsetPtr
| ||||||||||||
1657 | if (!Int8Ptr
| ||||||||||||
1658 | Int8Ptr = IRB.CreateBitCast( | ||||||||||||
1659 | Ptr, IRB.getInt8PtrTy(PointerTy->getPointerAddressSpace()), | ||||||||||||
1660 | NamePrefix + "sroa_raw_cast"); | ||||||||||||
1661 | Int8PtrOffset = Offset; | ||||||||||||
1662 | } | ||||||||||||
1663 | |||||||||||||
1664 | OffsetPtr = Int8PtrOffset == 0 | ||||||||||||
1665 | ? Int8Ptr | ||||||||||||
1666 | : IRB.CreateInBoundsGEP(IRB.getInt8Ty(), Int8Ptr, | ||||||||||||
1667 | IRB.getInt(Int8PtrOffset), | ||||||||||||
1668 | NamePrefix + "sroa_raw_idx"); | ||||||||||||
1669 | } | ||||||||||||
1670 | Ptr = OffsetPtr; | ||||||||||||
1671 | |||||||||||||
1672 | // On the off chance we were targeting i8*, guard the bitcast here. | ||||||||||||
1673 | if (cast<PointerType>(Ptr->getType()) != TargetPtrTy) { | ||||||||||||
1674 | Ptr = IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, | ||||||||||||
1675 | TargetPtrTy, | ||||||||||||
1676 | NamePrefix + "sroa_cast"); | ||||||||||||
1677 | } | ||||||||||||
1678 | |||||||||||||
1679 | return Ptr; | ||||||||||||
1680 | } | ||||||||||||
1681 | |||||||||||||
1682 | /// Compute the adjusted alignment for a load or store from an offset. | ||||||||||||
1683 | static unsigned getAdjustedAlignment(Instruction *I, uint64_t Offset, | ||||||||||||
1684 | const DataLayout &DL) { | ||||||||||||
1685 | unsigned Alignment; | ||||||||||||
1686 | Type *Ty; | ||||||||||||
1687 | if (auto *LI = dyn_cast<LoadInst>(I)) { | ||||||||||||
1688 | Alignment = LI->getAlignment(); | ||||||||||||
1689 | Ty = LI->getType(); | ||||||||||||
1690 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | ||||||||||||
1691 | Alignment = SI->getAlignment(); | ||||||||||||
1692 | Ty = SI->getValueOperand()->getType(); | ||||||||||||
1693 | } else { | ||||||||||||
1694 | llvm_unreachable("Only loads and stores are allowed!")::llvm::llvm_unreachable_internal("Only loads and stores are allowed!" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1694); | ||||||||||||
1695 | } | ||||||||||||
1696 | |||||||||||||
1697 | if (!Alignment) | ||||||||||||
1698 | Alignment = DL.getABITypeAlignment(Ty); | ||||||||||||
1699 | |||||||||||||
1700 | return MinAlign(Alignment, Offset); | ||||||||||||
1701 | } | ||||||||||||
1702 | |||||||||||||
1703 | /// Test whether we can convert a value from the old to the new type. | ||||||||||||
1704 | /// | ||||||||||||
1705 | /// This predicate should be used to guard calls to convertValue in order to | ||||||||||||
1706 | /// ensure that we only try to convert viable values. The strategy is that we | ||||||||||||
1707 | /// will peel off single element struct and array wrappings to get to an | ||||||||||||
1708 | /// underlying value, and convert that value. | ||||||||||||
1709 | static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) { | ||||||||||||
1710 | if (OldTy == NewTy) | ||||||||||||
1711 | return true; | ||||||||||||
1712 | |||||||||||||
1713 | // For integer types, we can't handle any bit-width differences. This would | ||||||||||||
1714 | // break both vector conversions with extension and introduce endianness | ||||||||||||
1715 | // issues when in conjunction with loads and stores. | ||||||||||||
1716 | if (isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) { | ||||||||||||
1717 | assert(cast<IntegerType>(OldTy)->getBitWidth() !=((cast<IntegerType>(OldTy)->getBitWidth() != cast< IntegerType>(NewTy)->getBitWidth() && "We can't have the same bitwidth for different int types" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1719, __PRETTY_FUNCTION__)) | ||||||||||||
1718 | cast<IntegerType>(NewTy)->getBitWidth() &&((cast<IntegerType>(OldTy)->getBitWidth() != cast< IntegerType>(NewTy)->getBitWidth() && "We can't have the same bitwidth for different int types" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1719, __PRETTY_FUNCTION__)) | ||||||||||||
1719 | "We can't have the same bitwidth for different int types")((cast<IntegerType>(OldTy)->getBitWidth() != cast< IntegerType>(NewTy)->getBitWidth() && "We can't have the same bitwidth for different int types" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(OldTy)->getBitWidth() != cast<IntegerType>(NewTy)->getBitWidth() && \"We can't have the same bitwidth for different int types\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1719, __PRETTY_FUNCTION__)); | ||||||||||||
1720 | return false; | ||||||||||||
1721 | } | ||||||||||||
1722 | |||||||||||||
1723 | if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy)) | ||||||||||||
1724 | return false; | ||||||||||||
1725 | if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType()) | ||||||||||||
1726 | return false; | ||||||||||||
1727 | |||||||||||||
1728 | // We can convert pointers to integers and vice-versa. Same for vectors | ||||||||||||
1729 | // of pointers and integers. | ||||||||||||
1730 | OldTy = OldTy->getScalarType(); | ||||||||||||
1731 | NewTy = NewTy->getScalarType(); | ||||||||||||
1732 | if (NewTy->isPointerTy() || OldTy->isPointerTy()) { | ||||||||||||
1733 | if (NewTy->isPointerTy() && OldTy->isPointerTy()) { | ||||||||||||
1734 | return cast<PointerType>(NewTy)->getPointerAddressSpace() == | ||||||||||||
1735 | cast<PointerType>(OldTy)->getPointerAddressSpace(); | ||||||||||||
1736 | } | ||||||||||||
1737 | |||||||||||||
1738 | // We can convert integers to integral pointers, but not to non-integral | ||||||||||||
1739 | // pointers. | ||||||||||||
1740 | if (OldTy->isIntegerTy()) | ||||||||||||
1741 | return !DL.isNonIntegralPointerType(NewTy); | ||||||||||||
1742 | |||||||||||||
1743 | // We can convert integral pointers to integers, but non-integral pointers | ||||||||||||
1744 | // need to remain pointers. | ||||||||||||
1745 | if (!DL.isNonIntegralPointerType(OldTy)) | ||||||||||||
1746 | return NewTy->isIntegerTy(); | ||||||||||||
1747 | |||||||||||||
1748 | return false; | ||||||||||||
1749 | } | ||||||||||||
1750 | |||||||||||||
1751 | return true; | ||||||||||||
1752 | } | ||||||||||||
1753 | |||||||||||||
1754 | /// Generic routine to convert an SSA value to a value of a different | ||||||||||||
1755 | /// type. | ||||||||||||
1756 | /// | ||||||||||||
1757 | /// This will try various different casting techniques, such as bitcasts, | ||||||||||||
1758 | /// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test | ||||||||||||
1759 | /// two types for viability with this routine. | ||||||||||||
1760 | static Value *convertValue(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | ||||||||||||
1761 | Type *NewTy) { | ||||||||||||
1762 | Type *OldTy = V->getType(); | ||||||||||||
1763 | assert(canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type")((canConvertValue(DL, OldTy, NewTy) && "Value not convertable to type" ) ? static_cast<void> (0) : __assert_fail ("canConvertValue(DL, OldTy, NewTy) && \"Value not convertable to type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1763, __PRETTY_FUNCTION__)); | ||||||||||||
1764 | |||||||||||||
1765 | if (OldTy == NewTy) | ||||||||||||
1766 | return V; | ||||||||||||
1767 | |||||||||||||
1768 | assert(!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) &&((!(isa<IntegerType>(OldTy) && isa<IntegerType >(NewTy)) && "Integer types must be the exact same to convert." ) ? static_cast<void> (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1769, __PRETTY_FUNCTION__)) | ||||||||||||
1769 | "Integer types must be the exact same to convert.")((!(isa<IntegerType>(OldTy) && isa<IntegerType >(NewTy)) && "Integer types must be the exact same to convert." ) ? static_cast<void> (0) : __assert_fail ("!(isa<IntegerType>(OldTy) && isa<IntegerType>(NewTy)) && \"Integer types must be the exact same to convert.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1769, __PRETTY_FUNCTION__)); | ||||||||||||
1770 | |||||||||||||
1771 | // See if we need inttoptr for this type pair. A cast involving both scalars | ||||||||||||
1772 | // and vectors requires and additional bitcast. | ||||||||||||
1773 | if (OldTy->isIntOrIntVectorTy() && NewTy->isPtrOrPtrVectorTy()) { | ||||||||||||
1774 | // Expand <2 x i32> to i8* --> <2 x i32> to i64 to i8* | ||||||||||||
1775 | if (OldTy->isVectorTy() && !NewTy->isVectorTy()) | ||||||||||||
1776 | return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), | ||||||||||||
1777 | NewTy); | ||||||||||||
1778 | |||||||||||||
1779 | // Expand i128 to <2 x i8*> --> i128 to <2 x i64> to <2 x i8*> | ||||||||||||
1780 | if (!OldTy->isVectorTy() && NewTy->isVectorTy()) | ||||||||||||
1781 | return IRB.CreateIntToPtr(IRB.CreateBitCast(V, DL.getIntPtrType(NewTy)), | ||||||||||||
1782 | NewTy); | ||||||||||||
1783 | |||||||||||||
1784 | return IRB.CreateIntToPtr(V, NewTy); | ||||||||||||
1785 | } | ||||||||||||
1786 | |||||||||||||
1787 | // See if we need ptrtoint for this type pair. A cast involving both scalars | ||||||||||||
1788 | // and vectors requires and additional bitcast. | ||||||||||||
1789 | if (OldTy->isPtrOrPtrVectorTy() && NewTy->isIntOrIntVectorTy()) { | ||||||||||||
1790 | // Expand <2 x i8*> to i128 --> <2 x i8*> to <2 x i64> to i128 | ||||||||||||
1791 | if (OldTy->isVectorTy() && !NewTy->isVectorTy()) | ||||||||||||
1792 | return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | ||||||||||||
1793 | NewTy); | ||||||||||||
1794 | |||||||||||||
1795 | // Expand i8* to <2 x i32> --> i8* to i64 to <2 x i32> | ||||||||||||
1796 | if (!OldTy->isVectorTy() && NewTy->isVectorTy()) | ||||||||||||
1797 | return IRB.CreateBitCast(IRB.CreatePtrToInt(V, DL.getIntPtrType(OldTy)), | ||||||||||||
1798 | NewTy); | ||||||||||||
1799 | |||||||||||||
1800 | return IRB.CreatePtrToInt(V, NewTy); | ||||||||||||
1801 | } | ||||||||||||
1802 | |||||||||||||
1803 | return IRB.CreateBitCast(V, NewTy); | ||||||||||||
1804 | } | ||||||||||||
1805 | |||||||||||||
1806 | /// Test whether the given slice use can be promoted to a vector. | ||||||||||||
1807 | /// | ||||||||||||
1808 | /// This function is called to test each entry in a partition which is slated | ||||||||||||
1809 | /// for a single slice. | ||||||||||||
1810 | static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S, | ||||||||||||
1811 | VectorType *Ty, | ||||||||||||
1812 | uint64_t ElementSize, | ||||||||||||
1813 | const DataLayout &DL) { | ||||||||||||
1814 | // First validate the slice offsets. | ||||||||||||
1815 | uint64_t BeginOffset = | ||||||||||||
1816 | std::max(S.beginOffset(), P.beginOffset()) - P.beginOffset(); | ||||||||||||
1817 | uint64_t BeginIndex = BeginOffset / ElementSize; | ||||||||||||
1818 | if (BeginIndex * ElementSize != BeginOffset || | ||||||||||||
1819 | BeginIndex >= Ty->getNumElements()) | ||||||||||||
1820 | return false; | ||||||||||||
1821 | uint64_t EndOffset = | ||||||||||||
1822 | std::min(S.endOffset(), P.endOffset()) - P.beginOffset(); | ||||||||||||
1823 | uint64_t EndIndex = EndOffset / ElementSize; | ||||||||||||
1824 | if (EndIndex * ElementSize != EndOffset || EndIndex > Ty->getNumElements()) | ||||||||||||
1825 | return false; | ||||||||||||
1826 | |||||||||||||
1827 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1827, __PRETTY_FUNCTION__)); | ||||||||||||
1828 | uint64_t NumElements = EndIndex - BeginIndex; | ||||||||||||
1829 | Type *SliceTy = (NumElements == 1) | ||||||||||||
1830 | ? Ty->getElementType() | ||||||||||||
1831 | : VectorType::get(Ty->getElementType(), NumElements); | ||||||||||||
1832 | |||||||||||||
1833 | Type *SplitIntTy = | ||||||||||||
1834 | Type::getIntNTy(Ty->getContext(), NumElements * ElementSize * 8); | ||||||||||||
1835 | |||||||||||||
1836 | Use *U = S.getUse(); | ||||||||||||
1837 | |||||||||||||
1838 | if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | ||||||||||||
1839 | if (MI->isVolatile()) | ||||||||||||
1840 | return false; | ||||||||||||
1841 | if (!S.isSplittable()) | ||||||||||||
1842 | return false; // Skip any unsplittable intrinsics. | ||||||||||||
1843 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | ||||||||||||
1844 | if (!II->isLifetimeStartOrEnd()) | ||||||||||||
1845 | return false; | ||||||||||||
1846 | } else if (U->get()->getType()->getPointerElementType()->isStructTy()) { | ||||||||||||
1847 | // Disable vector promotion when there are loads or stores of an FCA. | ||||||||||||
1848 | return false; | ||||||||||||
1849 | } else if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | ||||||||||||
1850 | if (LI->isVolatile()) | ||||||||||||
1851 | return false; | ||||||||||||
1852 | Type *LTy = LI->getType(); | ||||||||||||
1853 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | ||||||||||||
1854 | assert(LTy->isIntegerTy())((LTy->isIntegerTy()) ? static_cast<void> (0) : __assert_fail ("LTy->isIntegerTy()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1854, __PRETTY_FUNCTION__)); | ||||||||||||
1855 | LTy = SplitIntTy; | ||||||||||||
1856 | } | ||||||||||||
1857 | if (!canConvertValue(DL, SliceTy, LTy)) | ||||||||||||
1858 | return false; | ||||||||||||
1859 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | ||||||||||||
1860 | if (SI->isVolatile()) | ||||||||||||
1861 | return false; | ||||||||||||
1862 | Type *STy = SI->getValueOperand()->getType(); | ||||||||||||
1863 | if (P.beginOffset() > S.beginOffset() || P.endOffset() < S.endOffset()) { | ||||||||||||
1864 | assert(STy->isIntegerTy())((STy->isIntegerTy()) ? static_cast<void> (0) : __assert_fail ("STy->isIntegerTy()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1864, __PRETTY_FUNCTION__)); | ||||||||||||
1865 | STy = SplitIntTy; | ||||||||||||
1866 | } | ||||||||||||
1867 | if (!canConvertValue(DL, STy, SliceTy)) | ||||||||||||
1868 | return false; | ||||||||||||
1869 | } else { | ||||||||||||
1870 | return false; | ||||||||||||
1871 | } | ||||||||||||
1872 | |||||||||||||
1873 | return true; | ||||||||||||
1874 | } | ||||||||||||
1875 | |||||||||||||
1876 | /// Test whether the given alloca partitioning and range of slices can be | ||||||||||||
1877 | /// promoted to a vector. | ||||||||||||
1878 | /// | ||||||||||||
1879 | /// This is a quick test to check whether we can rewrite a particular alloca | ||||||||||||
1880 | /// partition (and its newly formed alloca) into a vector alloca with only | ||||||||||||
1881 | /// whole-vector loads and stores such that it could be promoted to a vector | ||||||||||||
1882 | /// SSA value. We only can ensure this for a limited set of operations, and we | ||||||||||||
1883 | /// don't want to do the rewrites unless we are confident that the result will | ||||||||||||
1884 | /// be promotable, so we have an early test here. | ||||||||||||
1885 | static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) { | ||||||||||||
1886 | // Collect the candidate types for vector-based promotion. Also track whether | ||||||||||||
1887 | // we have different element types. | ||||||||||||
1888 | SmallVector<VectorType *, 4> CandidateTys; | ||||||||||||
1889 | Type *CommonEltTy = nullptr; | ||||||||||||
1890 | bool HaveCommonEltTy = true; | ||||||||||||
1891 | auto CheckCandidateType = [&](Type *Ty) { | ||||||||||||
1892 | if (auto *VTy = dyn_cast<VectorType>(Ty)) { | ||||||||||||
1893 | // Return if bitcast to vectors is different for total size in bits. | ||||||||||||
1894 | if (!CandidateTys.empty()) { | ||||||||||||
1895 | VectorType *V = CandidateTys[0]; | ||||||||||||
1896 | if (DL.getTypeSizeInBits(VTy) != DL.getTypeSizeInBits(V)) { | ||||||||||||
1897 | CandidateTys.clear(); | ||||||||||||
1898 | return; | ||||||||||||
1899 | } | ||||||||||||
1900 | } | ||||||||||||
1901 | CandidateTys.push_back(VTy); | ||||||||||||
1902 | if (!CommonEltTy) | ||||||||||||
1903 | CommonEltTy = VTy->getElementType(); | ||||||||||||
1904 | else if (CommonEltTy != VTy->getElementType()) | ||||||||||||
1905 | HaveCommonEltTy = false; | ||||||||||||
1906 | } | ||||||||||||
1907 | }; | ||||||||||||
1908 | // Consider any loads or stores that are the exact size of the slice. | ||||||||||||
1909 | for (const Slice &S : P) | ||||||||||||
1910 | if (S.beginOffset() == P.beginOffset() && | ||||||||||||
1911 | S.endOffset() == P.endOffset()) { | ||||||||||||
1912 | if (auto *LI = dyn_cast<LoadInst>(S.getUse()->getUser())) | ||||||||||||
1913 | CheckCandidateType(LI->getType()); | ||||||||||||
1914 | else if (auto *SI = dyn_cast<StoreInst>(S.getUse()->getUser())) | ||||||||||||
1915 | CheckCandidateType(SI->getValueOperand()->getType()); | ||||||||||||
1916 | } | ||||||||||||
1917 | |||||||||||||
1918 | // If we didn't find a vector type, nothing to do here. | ||||||||||||
1919 | if (CandidateTys.empty()) | ||||||||||||
1920 | return nullptr; | ||||||||||||
1921 | |||||||||||||
1922 | // Remove non-integer vector types if we had multiple common element types. | ||||||||||||
1923 | // FIXME: It'd be nice to replace them with integer vector types, but we can't | ||||||||||||
1924 | // do that until all the backends are known to produce good code for all | ||||||||||||
1925 | // integer vector types. | ||||||||||||
1926 | if (!HaveCommonEltTy) { | ||||||||||||
1927 | CandidateTys.erase( | ||||||||||||
1928 | llvm::remove_if(CandidateTys, | ||||||||||||
1929 | [](VectorType *VTy) { | ||||||||||||
1930 | return !VTy->getElementType()->isIntegerTy(); | ||||||||||||
1931 | }), | ||||||||||||
1932 | CandidateTys.end()); | ||||||||||||
1933 | |||||||||||||
1934 | // If there were no integer vector types, give up. | ||||||||||||
1935 | if (CandidateTys.empty()) | ||||||||||||
1936 | return nullptr; | ||||||||||||
1937 | |||||||||||||
1938 | // Rank the remaining candidate vector types. This is easy because we know | ||||||||||||
1939 | // they're all integer vectors. We sort by ascending number of elements. | ||||||||||||
1940 | auto RankVectorTypes = [&DL](VectorType *RHSTy, VectorType *LHSTy) { | ||||||||||||
1941 | (void)DL; | ||||||||||||
1942 | assert(DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) &&((DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && "Cannot have vector types of different sizes!") ? static_cast <void> (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && \"Cannot have vector types of different sizes!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1943, __PRETTY_FUNCTION__)) | ||||||||||||
1943 | "Cannot have vector types of different sizes!")((DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && "Cannot have vector types of different sizes!") ? static_cast <void> (0) : __assert_fail ("DL.getTypeSizeInBits(RHSTy) == DL.getTypeSizeInBits(LHSTy) && \"Cannot have vector types of different sizes!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1943, __PRETTY_FUNCTION__)); | ||||||||||||
1944 | assert(RHSTy->getElementType()->isIntegerTy() &&((RHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1945, __PRETTY_FUNCTION__)) | ||||||||||||
1945 | "All non-integer types eliminated!")((RHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("RHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1945, __PRETTY_FUNCTION__)); | ||||||||||||
1946 | assert(LHSTy->getElementType()->isIntegerTy() &&((LHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1947, __PRETTY_FUNCTION__)) | ||||||||||||
1947 | "All non-integer types eliminated!")((LHSTy->getElementType()->isIntegerTy() && "All non-integer types eliminated!" ) ? static_cast<void> (0) : __assert_fail ("LHSTy->getElementType()->isIntegerTy() && \"All non-integer types eliminated!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1947, __PRETTY_FUNCTION__)); | ||||||||||||
1948 | return RHSTy->getNumElements() < LHSTy->getNumElements(); | ||||||||||||
1949 | }; | ||||||||||||
1950 | llvm::sort(CandidateTys, RankVectorTypes); | ||||||||||||
1951 | CandidateTys.erase( | ||||||||||||
1952 | std::unique(CandidateTys.begin(), CandidateTys.end(), RankVectorTypes), | ||||||||||||
1953 | CandidateTys.end()); | ||||||||||||
1954 | } else { | ||||||||||||
1955 | // The only way to have the same element type in every vector type is to | ||||||||||||
1956 | // have the same vector type. Check that and remove all but one. | ||||||||||||
1957 | #ifndef NDEBUG | ||||||||||||
1958 | for (VectorType *VTy : CandidateTys) { | ||||||||||||
1959 | assert(VTy->getElementType() == CommonEltTy &&((VTy->getElementType() == CommonEltTy && "Unaccounted for element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1960, __PRETTY_FUNCTION__)) | ||||||||||||
1960 | "Unaccounted for element type!")((VTy->getElementType() == CommonEltTy && "Unaccounted for element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy->getElementType() == CommonEltTy && \"Unaccounted for element type!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1960, __PRETTY_FUNCTION__)); | ||||||||||||
1961 | assert(VTy == CandidateTys[0] &&((VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1962, __PRETTY_FUNCTION__)) | ||||||||||||
1962 | "Different vector types with the same element type!")((VTy == CandidateTys[0] && "Different vector types with the same element type!" ) ? static_cast<void> (0) : __assert_fail ("VTy == CandidateTys[0] && \"Different vector types with the same element type!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1962, __PRETTY_FUNCTION__)); | ||||||||||||
1963 | } | ||||||||||||
1964 | #endif | ||||||||||||
1965 | CandidateTys.resize(1); | ||||||||||||
1966 | } | ||||||||||||
1967 | |||||||||||||
1968 | // Try each vector type, and return the one which works. | ||||||||||||
1969 | auto CheckVectorTypeForPromotion = [&](VectorType *VTy) { | ||||||||||||
1970 | uint64_t ElementSize = DL.getTypeSizeInBits(VTy->getElementType()); | ||||||||||||
1971 | |||||||||||||
1972 | // While the definition of LLVM vectors is bitpacked, we don't support sizes | ||||||||||||
1973 | // that aren't byte sized. | ||||||||||||
1974 | if (ElementSize % 8) | ||||||||||||
1975 | return false; | ||||||||||||
1976 | assert((DL.getTypeSizeInBits(VTy) % 8) == 0 &&(((DL.getTypeSizeInBits(VTy) % 8) == 0 && "vector size not a multiple of element size?" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy) % 8) == 0 && \"vector size not a multiple of element size?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1977, __PRETTY_FUNCTION__)) | ||||||||||||
1977 | "vector size not a multiple of element size?")(((DL.getTypeSizeInBits(VTy) % 8) == 0 && "vector size not a multiple of element size?" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(VTy) % 8) == 0 && \"vector size not a multiple of element size?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 1977, __PRETTY_FUNCTION__)); | ||||||||||||
1978 | ElementSize /= 8; | ||||||||||||
1979 | |||||||||||||
1980 | for (const Slice &S : P) | ||||||||||||
1981 | if (!isVectorPromotionViableForSlice(P, S, VTy, ElementSize, DL)) | ||||||||||||
1982 | return false; | ||||||||||||
1983 | |||||||||||||
1984 | for (const Slice *S : P.splitSliceTails()) | ||||||||||||
1985 | if (!isVectorPromotionViableForSlice(P, *S, VTy, ElementSize, DL)) | ||||||||||||
1986 | return false; | ||||||||||||
1987 | |||||||||||||
1988 | return true; | ||||||||||||
1989 | }; | ||||||||||||
1990 | for (VectorType *VTy : CandidateTys) | ||||||||||||
1991 | if (CheckVectorTypeForPromotion(VTy)) | ||||||||||||
1992 | return VTy; | ||||||||||||
1993 | |||||||||||||
1994 | return nullptr; | ||||||||||||
1995 | } | ||||||||||||
1996 | |||||||||||||
1997 | /// Test whether a slice of an alloca is valid for integer widening. | ||||||||||||
1998 | /// | ||||||||||||
1999 | /// This implements the necessary checking for the \c isIntegerWideningViable | ||||||||||||
2000 | /// test below on a single slice of the alloca. | ||||||||||||
2001 | static bool isIntegerWideningViableForSlice(const Slice &S, | ||||||||||||
2002 | uint64_t AllocBeginOffset, | ||||||||||||
2003 | Type *AllocaTy, | ||||||||||||
2004 | const DataLayout &DL, | ||||||||||||
2005 | bool &WholeAllocaOp) { | ||||||||||||
2006 | uint64_t Size = DL.getTypeStoreSize(AllocaTy); | ||||||||||||
2007 | |||||||||||||
2008 | uint64_t RelBegin = S.beginOffset() - AllocBeginOffset; | ||||||||||||
2009 | uint64_t RelEnd = S.endOffset() - AllocBeginOffset; | ||||||||||||
2010 | |||||||||||||
2011 | // We can't reasonably handle cases where the load or store extends past | ||||||||||||
2012 | // the end of the alloca's type and into its padding. | ||||||||||||
2013 | if (RelEnd > Size) | ||||||||||||
2014 | return false; | ||||||||||||
2015 | |||||||||||||
2016 | Use *U = S.getUse(); | ||||||||||||
2017 | |||||||||||||
2018 | if (LoadInst *LI = dyn_cast<LoadInst>(U->getUser())) { | ||||||||||||
2019 | if (LI->isVolatile()) | ||||||||||||
2020 | return false; | ||||||||||||
2021 | // We can't handle loads that extend past the allocated memory. | ||||||||||||
2022 | if (DL.getTypeStoreSize(LI->getType()) > Size) | ||||||||||||
2023 | return false; | ||||||||||||
2024 | // So far, AllocaSliceRewriter does not support widening split slice tails | ||||||||||||
2025 | // in rewriteIntegerLoad. | ||||||||||||
2026 | if (S.beginOffset() < AllocBeginOffset) | ||||||||||||
2027 | return false; | ||||||||||||
2028 | // Note that we don't count vector loads or stores as whole-alloca | ||||||||||||
2029 | // operations which enable integer widening because we would prefer to use | ||||||||||||
2030 | // vector widening instead. | ||||||||||||
2031 | if (!isa<VectorType>(LI->getType()) && RelBegin == 0 && RelEnd == Size) | ||||||||||||
2032 | WholeAllocaOp = true; | ||||||||||||
2033 | if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) { | ||||||||||||
2034 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy)) | ||||||||||||
2035 | return false; | ||||||||||||
2036 | } else if (RelBegin != 0 || RelEnd != Size || | ||||||||||||
2037 | !canConvertValue(DL, AllocaTy, LI->getType())) { | ||||||||||||
2038 | // Non-integer loads need to be convertible from the alloca type so that | ||||||||||||
2039 | // they are promotable. | ||||||||||||
2040 | return false; | ||||||||||||
2041 | } | ||||||||||||
2042 | } else if (StoreInst *SI = dyn_cast<StoreInst>(U->getUser())) { | ||||||||||||
2043 | Type *ValueTy = SI->getValueOperand()->getType(); | ||||||||||||
2044 | if (SI->isVolatile()) | ||||||||||||
2045 | return false; | ||||||||||||
2046 | // We can't handle stores that extend past the allocated memory. | ||||||||||||
2047 | if (DL.getTypeStoreSize(ValueTy) > Size) | ||||||||||||
2048 | return false; | ||||||||||||
2049 | // So far, AllocaSliceRewriter does not support widening split slice tails | ||||||||||||
2050 | // in rewriteIntegerStore. | ||||||||||||
2051 | if (S.beginOffset() < AllocBeginOffset) | ||||||||||||
2052 | return false; | ||||||||||||
2053 | // Note that we don't count vector loads or stores as whole-alloca | ||||||||||||
2054 | // operations which enable integer widening because we would prefer to use | ||||||||||||
2055 | // vector widening instead. | ||||||||||||
2056 | if (!isa<VectorType>(ValueTy) && RelBegin == 0 && RelEnd == Size) | ||||||||||||
2057 | WholeAllocaOp = true; | ||||||||||||
2058 | if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) { | ||||||||||||
2059 | if (ITy->getBitWidth() < DL.getTypeStoreSizeInBits(ITy)) | ||||||||||||
2060 | return false; | ||||||||||||
2061 | } else if (RelBegin != 0 || RelEnd != Size || | ||||||||||||
2062 | !canConvertValue(DL, ValueTy, AllocaTy)) { | ||||||||||||
2063 | // Non-integer stores need to be convertible to the alloca type so that | ||||||||||||
2064 | // they are promotable. | ||||||||||||
2065 | return false; | ||||||||||||
2066 | } | ||||||||||||
2067 | } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(U->getUser())) { | ||||||||||||
2068 | if (MI->isVolatile() || !isa<Constant>(MI->getLength())) | ||||||||||||
2069 | return false; | ||||||||||||
2070 | if (!S.isSplittable()) | ||||||||||||
2071 | return false; // Skip any unsplittable intrinsics. | ||||||||||||
2072 | } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U->getUser())) { | ||||||||||||
2073 | if (!II->isLifetimeStartOrEnd()) | ||||||||||||
2074 | return false; | ||||||||||||
2075 | } else { | ||||||||||||
2076 | return false; | ||||||||||||
2077 | } | ||||||||||||
2078 | |||||||||||||
2079 | return true; | ||||||||||||
2080 | } | ||||||||||||
2081 | |||||||||||||
2082 | /// Test whether the given alloca partition's integer operations can be | ||||||||||||
2083 | /// widened to promotable ones. | ||||||||||||
2084 | /// | ||||||||||||
2085 | /// This is a quick test to check whether we can rewrite the integer loads and | ||||||||||||
2086 | /// stores to a particular alloca into wider loads and stores and be able to | ||||||||||||
2087 | /// promote the resulting alloca. | ||||||||||||
2088 | static bool isIntegerWideningViable(Partition &P, Type *AllocaTy, | ||||||||||||
2089 | const DataLayout &DL) { | ||||||||||||
2090 | uint64_t SizeInBits = DL.getTypeSizeInBits(AllocaTy); | ||||||||||||
2091 | // Don't create integer types larger than the maximum bitwidth. | ||||||||||||
2092 | if (SizeInBits > IntegerType::MAX_INT_BITS) | ||||||||||||
2093 | return false; | ||||||||||||
2094 | |||||||||||||
2095 | // Don't try to handle allocas with bit-padding. | ||||||||||||
2096 | if (SizeInBits != DL.getTypeStoreSizeInBits(AllocaTy)) | ||||||||||||
2097 | return false; | ||||||||||||
2098 | |||||||||||||
2099 | // We need to ensure that an integer type with the appropriate bitwidth can | ||||||||||||
2100 | // be converted to the alloca type, whatever that is. We don't want to force | ||||||||||||
2101 | // the alloca itself to have an integer type if there is a more suitable one. | ||||||||||||
2102 | Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits); | ||||||||||||
2103 | if (!canConvertValue(DL, AllocaTy, IntTy) || | ||||||||||||
2104 | !canConvertValue(DL, IntTy, AllocaTy)) | ||||||||||||
2105 | return false; | ||||||||||||
2106 | |||||||||||||
2107 | // While examining uses, we ensure that the alloca has a covering load or | ||||||||||||
2108 | // store. We don't want to widen the integer operations only to fail to | ||||||||||||
2109 | // promote due to some other unsplittable entry (which we may make splittable | ||||||||||||
2110 | // later). However, if there are only splittable uses, go ahead and assume | ||||||||||||
2111 | // that we cover the alloca. | ||||||||||||
2112 | // FIXME: We shouldn't consider split slices that happen to start in the | ||||||||||||
2113 | // partition here... | ||||||||||||
2114 | bool WholeAllocaOp = | ||||||||||||
2115 | P.begin() != P.end() ? false : DL.isLegalInteger(SizeInBits); | ||||||||||||
2116 | |||||||||||||
2117 | for (const Slice &S : P) | ||||||||||||
2118 | if (!isIntegerWideningViableForSlice(S, P.beginOffset(), AllocaTy, DL, | ||||||||||||
2119 | WholeAllocaOp)) | ||||||||||||
2120 | return false; | ||||||||||||
2121 | |||||||||||||
2122 | for (const Slice *S : P.splitSliceTails()) | ||||||||||||
2123 | if (!isIntegerWideningViableForSlice(*S, P.beginOffset(), AllocaTy, DL, | ||||||||||||
2124 | WholeAllocaOp)) | ||||||||||||
2125 | return false; | ||||||||||||
2126 | |||||||||||||
2127 | return WholeAllocaOp; | ||||||||||||
2128 | } | ||||||||||||
2129 | |||||||||||||
2130 | static Value *extractInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *V, | ||||||||||||
2131 | IntegerType *Ty, uint64_t Offset, | ||||||||||||
2132 | const Twine &Name) { | ||||||||||||
2133 | LLVM_DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | ||||||||||||
2134 | IntegerType *IntTy = cast<IntegerType>(V->getType()); | ||||||||||||
2135 | assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2136, __PRETTY_FUNCTION__)) | ||||||||||||
2136 | "Element extends past full value")((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2136, __PRETTY_FUNCTION__)); | ||||||||||||
2137 | uint64_t ShAmt = 8 * Offset; | ||||||||||||
2138 | if (DL.isBigEndian()) | ||||||||||||
2139 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset); | ||||||||||||
2140 | if (ShAmt) { | ||||||||||||
2141 | V = IRB.CreateLShr(V, ShAmt, Name + ".shift"); | ||||||||||||
2142 | LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | ||||||||||||
2143 | } | ||||||||||||
2144 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast<void > (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2145, __PRETTY_FUNCTION__)) | ||||||||||||
| |||||||||||||
2145 | "Cannot extract to a larger integer!")((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast<void > (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2145, __PRETTY_FUNCTION__)); | ||||||||||||
2146 | if (Ty != IntTy) { | ||||||||||||
2147 | V = IRB.CreateTrunc(V, Ty, Name + ".trunc"); | ||||||||||||
2148 | LLVM_DEBUG(dbgs() << " trunced: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " trunced: " << *V << "\n"; } } while (false); | ||||||||||||
2149 | } | ||||||||||||
2150 | return V; | ||||||||||||
2151 | } | ||||||||||||
2152 | |||||||||||||
2153 | static Value *insertInteger(const DataLayout &DL, IRBuilderTy &IRB, Value *Old, | ||||||||||||
2154 | Value *V, uint64_t Offset, const Twine &Name) { | ||||||||||||
2155 | IntegerType *IntTy = cast<IntegerType>(Old->getType()); | ||||||||||||
2156 | IntegerType *Ty = cast<IntegerType>(V->getType()); | ||||||||||||
2157 | assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot insert a larger integer!") ? static_cast<void> (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2158, __PRETTY_FUNCTION__)) | ||||||||||||
2158 | "Cannot insert a larger integer!")((Ty->getBitWidth() <= IntTy->getBitWidth() && "Cannot insert a larger integer!") ? static_cast<void> (0) : __assert_fail ("Ty->getBitWidth() <= IntTy->getBitWidth() && \"Cannot insert a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2158, __PRETTY_FUNCTION__)); | ||||||||||||
2159 | LLVM_DEBUG(dbgs() << " start: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " start: " << *V << "\n"; } } while (false); | ||||||||||||
2160 | if (Ty != IntTy) { | ||||||||||||
2161 | V = IRB.CreateZExt(V, IntTy, Name + ".ext"); | ||||||||||||
2162 | LLVM_DEBUG(dbgs() << " extended: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extended: " << *V << "\n"; } } while (false); | ||||||||||||
2163 | } | ||||||||||||
2164 | assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element store outside of alloca store") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element store outside of alloca store\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2165, __PRETTY_FUNCTION__)) | ||||||||||||
2165 | "Element store outside of alloca store")((DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize( IntTy) && "Element store outside of alloca store") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element store outside of alloca store\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2165, __PRETTY_FUNCTION__)); | ||||||||||||
2166 | uint64_t ShAmt = 8 * Offset; | ||||||||||||
2167 | if (DL.isBigEndian()) | ||||||||||||
2168 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset); | ||||||||||||
2169 | if (ShAmt) { | ||||||||||||
2170 | V = IRB.CreateShl(V, ShAmt, Name + ".shift"); | ||||||||||||
2171 | LLVM_DEBUG(dbgs() << " shifted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shifted: " << *V << "\n"; } } while (false); | ||||||||||||
2172 | } | ||||||||||||
2173 | |||||||||||||
2174 | if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) { | ||||||||||||
2175 | APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt); | ||||||||||||
2176 | Old = IRB.CreateAnd(Old, Mask, Name + ".mask"); | ||||||||||||
2177 | LLVM_DEBUG(dbgs() << " masked: " << *Old << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " masked: " << *Old << "\n"; } } while (false); | ||||||||||||
2178 | V = IRB.CreateOr(Old, V, Name + ".insert"); | ||||||||||||
2179 | LLVM_DEBUG(dbgs() << " inserted: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " inserted: " << *V << "\n"; } } while (false); | ||||||||||||
2180 | } | ||||||||||||
2181 | return V; | ||||||||||||
2182 | } | ||||||||||||
2183 | |||||||||||||
2184 | static Value *extractVector(IRBuilderTy &IRB, Value *V, unsigned BeginIndex, | ||||||||||||
2185 | unsigned EndIndex, const Twine &Name) { | ||||||||||||
2186 | VectorType *VecTy = cast<VectorType>(V->getType()); | ||||||||||||
2187 | unsigned NumElements = EndIndex - BeginIndex; | ||||||||||||
2188 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")((NumElements <= VecTy->getNumElements() && "Too many elements!" ) ? static_cast<void> (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2188, __PRETTY_FUNCTION__)); | ||||||||||||
2189 | |||||||||||||
2190 | if (NumElements == VecTy->getNumElements()) | ||||||||||||
2191 | return V; | ||||||||||||
2192 | |||||||||||||
2193 | if (NumElements == 1) { | ||||||||||||
2194 | V = IRB.CreateExtractElement(V, IRB.getInt32(BeginIndex), | ||||||||||||
2195 | Name + ".extract"); | ||||||||||||
2196 | LLVM_DEBUG(dbgs() << " extract: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " extract: " << *V << "\n"; } } while (false); | ||||||||||||
2197 | return V; | ||||||||||||
2198 | } | ||||||||||||
2199 | |||||||||||||
2200 | SmallVector<Constant *, 8> Mask; | ||||||||||||
2201 | Mask.reserve(NumElements); | ||||||||||||
2202 | for (unsigned i = BeginIndex; i != EndIndex; ++i) | ||||||||||||
2203 | Mask.push_back(IRB.getInt32(i)); | ||||||||||||
2204 | V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()), | ||||||||||||
2205 | ConstantVector::get(Mask), Name + ".extract"); | ||||||||||||
2206 | LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | ||||||||||||
2207 | return V; | ||||||||||||
2208 | } | ||||||||||||
2209 | |||||||||||||
2210 | static Value *insertVector(IRBuilderTy &IRB, Value *Old, Value *V, | ||||||||||||
2211 | unsigned BeginIndex, const Twine &Name) { | ||||||||||||
2212 | VectorType *VecTy = cast<VectorType>(Old->getType()); | ||||||||||||
2213 | assert(VecTy && "Can only insert a vector into a vector")((VecTy && "Can only insert a vector into a vector") ? static_cast<void> (0) : __assert_fail ("VecTy && \"Can only insert a vector into a vector\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2213, __PRETTY_FUNCTION__)); | ||||||||||||
2214 | |||||||||||||
2215 | VectorType *Ty = dyn_cast<VectorType>(V->getType()); | ||||||||||||
2216 | if (!Ty) { | ||||||||||||
2217 | // Single element to insert. | ||||||||||||
2218 | V = IRB.CreateInsertElement(Old, V, IRB.getInt32(BeginIndex), | ||||||||||||
2219 | Name + ".insert"); | ||||||||||||
2220 | LLVM_DEBUG(dbgs() << " insert: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " insert: " << *V << "\n"; } } while (false); | ||||||||||||
2221 | return V; | ||||||||||||
2222 | } | ||||||||||||
2223 | |||||||||||||
2224 | assert(Ty->getNumElements() <= VecTy->getNumElements() &&((Ty->getNumElements() <= VecTy->getNumElements() && "Too many elements!") ? static_cast<void> (0) : __assert_fail ("Ty->getNumElements() <= VecTy->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2225, __PRETTY_FUNCTION__)) | ||||||||||||
2225 | "Too many elements!")((Ty->getNumElements() <= VecTy->getNumElements() && "Too many elements!") ? static_cast<void> (0) : __assert_fail ("Ty->getNumElements() <= VecTy->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2225, __PRETTY_FUNCTION__)); | ||||||||||||
2226 | if (Ty->getNumElements() == VecTy->getNumElements()) { | ||||||||||||
2227 | assert(V->getType() == VecTy && "Vector type mismatch")((V->getType() == VecTy && "Vector type mismatch") ? static_cast<void> (0) : __assert_fail ("V->getType() == VecTy && \"Vector type mismatch\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2227, __PRETTY_FUNCTION__)); | ||||||||||||
2228 | return V; | ||||||||||||
2229 | } | ||||||||||||
2230 | unsigned EndIndex = BeginIndex + Ty->getNumElements(); | ||||||||||||
2231 | |||||||||||||
2232 | // When inserting a smaller vector into the larger to store, we first | ||||||||||||
2233 | // use a shuffle vector to widen it with undef elements, and then | ||||||||||||
2234 | // a second shuffle vector to select between the loaded vector and the | ||||||||||||
2235 | // incoming vector. | ||||||||||||
2236 | SmallVector<Constant *, 8> Mask; | ||||||||||||
2237 | Mask.reserve(VecTy->getNumElements()); | ||||||||||||
2238 | for (unsigned i = 0; i != VecTy->getNumElements(); ++i) | ||||||||||||
2239 | if (i >= BeginIndex && i < EndIndex) | ||||||||||||
2240 | Mask.push_back(IRB.getInt32(i - BeginIndex)); | ||||||||||||
2241 | else | ||||||||||||
2242 | Mask.push_back(UndefValue::get(IRB.getInt32Ty())); | ||||||||||||
2243 | V = IRB.CreateShuffleVector(V, UndefValue::get(V->getType()), | ||||||||||||
2244 | ConstantVector::get(Mask), Name + ".expand"); | ||||||||||||
2245 | LLVM_DEBUG(dbgs() << " shuffle: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " shuffle: " << *V << "\n"; } } while (false); | ||||||||||||
2246 | |||||||||||||
2247 | Mask.clear(); | ||||||||||||
2248 | for (unsigned i = 0; i != VecTy->getNumElements(); ++i) | ||||||||||||
2249 | Mask.push_back(IRB.getInt1(i >= BeginIndex && i < EndIndex)); | ||||||||||||
2250 | |||||||||||||
2251 | V = IRB.CreateSelect(ConstantVector::get(Mask), V, Old, Name + "blend"); | ||||||||||||
2252 | |||||||||||||
2253 | LLVM_DEBUG(dbgs() << " blend: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " blend: " << *V << "\n"; } } while (false); | ||||||||||||
2254 | return V; | ||||||||||||
2255 | } | ||||||||||||
2256 | |||||||||||||
2257 | /// Visitor to rewrite instructions using p particular slice of an alloca | ||||||||||||
2258 | /// to use a new alloca. | ||||||||||||
2259 | /// | ||||||||||||
2260 | /// Also implements the rewriting to vector-based accesses when the partition | ||||||||||||
2261 | /// passes the isVectorPromotionViable predicate. Most of the rewriting logic | ||||||||||||
2262 | /// lives here. | ||||||||||||
2263 | class llvm::sroa::AllocaSliceRewriter | ||||||||||||
2264 | : public InstVisitor<AllocaSliceRewriter, bool> { | ||||||||||||
2265 | // Befriend the base class so it can delegate to private visit methods. | ||||||||||||
2266 | friend class InstVisitor<AllocaSliceRewriter, bool>; | ||||||||||||
2267 | |||||||||||||
2268 | using Base = InstVisitor<AllocaSliceRewriter, bool>; | ||||||||||||
2269 | |||||||||||||
2270 | const DataLayout &DL; | ||||||||||||
2271 | AllocaSlices &AS; | ||||||||||||
2272 | SROA &Pass; | ||||||||||||
2273 | AllocaInst &OldAI, &NewAI; | ||||||||||||
2274 | const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset; | ||||||||||||
2275 | Type *NewAllocaTy; | ||||||||||||
2276 | |||||||||||||
2277 | // This is a convenience and flag variable that will be null unless the new | ||||||||||||
2278 | // alloca's integer operations should be widened to this integer type due to | ||||||||||||
2279 | // passing isIntegerWideningViable above. If it is non-null, the desired | ||||||||||||
2280 | // integer type will be stored here for easy access during rewriting. | ||||||||||||
2281 | IntegerType *IntTy; | ||||||||||||
2282 | |||||||||||||
2283 | // If we are rewriting an alloca partition which can be written as pure | ||||||||||||
2284 | // vector operations, we stash extra information here. When VecTy is | ||||||||||||
2285 | // non-null, we have some strict guarantees about the rewritten alloca: | ||||||||||||
2286 | // - The new alloca is exactly the size of the vector type here. | ||||||||||||
2287 | // - The accesses all either map to the entire vector or to a single | ||||||||||||
2288 | // element. | ||||||||||||
2289 | // - The set of accessing instructions is only one of those handled above | ||||||||||||
2290 | // in isVectorPromotionViable. Generally these are the same access kinds | ||||||||||||
2291 | // which are promotable via mem2reg. | ||||||||||||
2292 | VectorType *VecTy; | ||||||||||||
2293 | Type *ElementTy; | ||||||||||||
2294 | uint64_t ElementSize; | ||||||||||||
2295 | |||||||||||||
2296 | // The original offset of the slice currently being rewritten relative to | ||||||||||||
2297 | // the original alloca. | ||||||||||||
2298 | uint64_t BeginOffset = 0; | ||||||||||||
2299 | uint64_t EndOffset = 0; | ||||||||||||
2300 | |||||||||||||
2301 | // The new offsets of the slice currently being rewritten relative to the | ||||||||||||
2302 | // original alloca. | ||||||||||||
2303 | uint64_t NewBeginOffset, NewEndOffset; | ||||||||||||
2304 | |||||||||||||
2305 | uint64_t SliceSize; | ||||||||||||
2306 | bool IsSplittable = false; | ||||||||||||
2307 | bool IsSplit = false; | ||||||||||||
2308 | Use *OldUse = nullptr; | ||||||||||||
2309 | Instruction *OldPtr = nullptr; | ||||||||||||
2310 | |||||||||||||
2311 | // Track post-rewrite users which are PHI nodes and Selects. | ||||||||||||
2312 | SmallSetVector<PHINode *, 8> &PHIUsers; | ||||||||||||
2313 | SmallSetVector<SelectInst *, 8> &SelectUsers; | ||||||||||||
2314 | |||||||||||||
2315 | // Utility IR builder, whose name prefix is setup for each visited use, and | ||||||||||||
2316 | // the insertion point is set to point to the user. | ||||||||||||
2317 | IRBuilderTy IRB; | ||||||||||||
2318 | |||||||||||||
2319 | public: | ||||||||||||
2320 | AllocaSliceRewriter(const DataLayout &DL, AllocaSlices &AS, SROA &Pass, | ||||||||||||
2321 | AllocaInst &OldAI, AllocaInst &NewAI, | ||||||||||||
2322 | uint64_t NewAllocaBeginOffset, | ||||||||||||
2323 | uint64_t NewAllocaEndOffset, bool IsIntegerPromotable, | ||||||||||||
2324 | VectorType *PromotableVecTy, | ||||||||||||
2325 | SmallSetVector<PHINode *, 8> &PHIUsers, | ||||||||||||
2326 | SmallSetVector<SelectInst *, 8> &SelectUsers) | ||||||||||||
2327 | : DL(DL), AS(AS), Pass(Pass), OldAI(OldAI), NewAI(NewAI), | ||||||||||||
2328 | NewAllocaBeginOffset(NewAllocaBeginOffset), | ||||||||||||
2329 | NewAllocaEndOffset(NewAllocaEndOffset), | ||||||||||||
2330 | NewAllocaTy(NewAI.getAllocatedType()), | ||||||||||||
2331 | IntTy(IsIntegerPromotable | ||||||||||||
2332 | ? Type::getIntNTy( | ||||||||||||
2333 | NewAI.getContext(), | ||||||||||||
2334 | DL.getTypeSizeInBits(NewAI.getAllocatedType())) | ||||||||||||
2335 | : nullptr), | ||||||||||||
2336 | VecTy(PromotableVecTy), | ||||||||||||
2337 | ElementTy(VecTy ? VecTy->getElementType() : nullptr), | ||||||||||||
2338 | ElementSize(VecTy ? DL.getTypeSizeInBits(ElementTy) / 8 : 0), | ||||||||||||
2339 | PHIUsers(PHIUsers), SelectUsers(SelectUsers), | ||||||||||||
2340 | IRB(NewAI.getContext(), ConstantFolder()) { | ||||||||||||
2341 | if (VecTy) { | ||||||||||||
2342 | assert((DL.getTypeSizeInBits(ElementTy) % 8) == 0 &&(((DL.getTypeSizeInBits(ElementTy) % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy) % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2343, __PRETTY_FUNCTION__)) | ||||||||||||
2343 | "Only multiple-of-8 sized vector elements are viable")(((DL.getTypeSizeInBits(ElementTy) % 8) == 0 && "Only multiple-of-8 sized vector elements are viable" ) ? static_cast<void> (0) : __assert_fail ("(DL.getTypeSizeInBits(ElementTy) % 8) == 0 && \"Only multiple-of-8 sized vector elements are viable\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2343, __PRETTY_FUNCTION__)); | ||||||||||||
2344 | ++NumVectorized; | ||||||||||||
2345 | } | ||||||||||||
2346 | assert((!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy))(((!IntTy && !VecTy) || (IntTy && !VecTy) || ( !IntTy && VecTy)) ? static_cast<void> (0) : __assert_fail ("(!IntTy && !VecTy) || (IntTy && !VecTy) || (!IntTy && VecTy)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2346, __PRETTY_FUNCTION__)); | ||||||||||||
2347 | } | ||||||||||||
2348 | |||||||||||||
2349 | bool visit(AllocaSlices::const_iterator I) { | ||||||||||||
2350 | bool CanSROA = true; | ||||||||||||
2351 | BeginOffset = I->beginOffset(); | ||||||||||||
2352 | EndOffset = I->endOffset(); | ||||||||||||
2353 | IsSplittable = I->isSplittable(); | ||||||||||||
2354 | IsSplit = | ||||||||||||
2355 | BeginOffset < NewAllocaBeginOffset || EndOffset > NewAllocaEndOffset; | ||||||||||||
2356 | LLVM_DEBUG(dbgs() << " rewriting " << (IsSplit ? "split " : ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " rewriting " << (IsSplit ? "split " : ""); } } while (false); | ||||||||||||
2357 | LLVM_DEBUG(AS.printSlice(dbgs(), I, ""))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.printSlice(dbgs(), I, ""); } } while (false); | ||||||||||||
2358 | LLVM_DEBUG(dbgs() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "\n"; } } while (false); | ||||||||||||
2359 | |||||||||||||
2360 | // Compute the intersecting offset range. | ||||||||||||
2361 | assert(BeginOffset < NewAllocaEndOffset)((BeginOffset < NewAllocaEndOffset) ? static_cast<void> (0) : __assert_fail ("BeginOffset < NewAllocaEndOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2361, __PRETTY_FUNCTION__)); | ||||||||||||
2362 | assert(EndOffset > NewAllocaBeginOffset)((EndOffset > NewAllocaBeginOffset) ? static_cast<void> (0) : __assert_fail ("EndOffset > NewAllocaBeginOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2362, __PRETTY_FUNCTION__)); | ||||||||||||
2363 | NewBeginOffset = std::max(BeginOffset, NewAllocaBeginOffset); | ||||||||||||
2364 | NewEndOffset = std::min(EndOffset, NewAllocaEndOffset); | ||||||||||||
2365 | |||||||||||||
2366 | SliceSize = NewEndOffset - NewBeginOffset; | ||||||||||||
2367 | |||||||||||||
2368 | OldUse = I->getUse(); | ||||||||||||
2369 | OldPtr = cast<Instruction>(OldUse->get()); | ||||||||||||
2370 | |||||||||||||
2371 | Instruction *OldUserI = cast<Instruction>(OldUse->getUser()); | ||||||||||||
2372 | IRB.SetInsertPoint(OldUserI); | ||||||||||||
2373 | IRB.SetCurrentDebugLocation(OldUserI->getDebugLoc()); | ||||||||||||
2374 | IRB.SetNamePrefix(Twine(NewAI.getName()) + "." + Twine(BeginOffset) + "."); | ||||||||||||
2375 | |||||||||||||
2376 | CanSROA &= visit(cast<Instruction>(OldUse->getUser())); | ||||||||||||
2377 | if (VecTy || IntTy) | ||||||||||||
2378 | assert(CanSROA)((CanSROA) ? static_cast<void> (0) : __assert_fail ("CanSROA" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2378, __PRETTY_FUNCTION__)); | ||||||||||||
2379 | return CanSROA; | ||||||||||||
2380 | } | ||||||||||||
2381 | |||||||||||||
2382 | private: | ||||||||||||
2383 | // Make sure the other visit overloads are visible. | ||||||||||||
2384 | using Base::visit; | ||||||||||||
2385 | |||||||||||||
2386 | // Every instruction which can end up as a user must have a rewrite rule. | ||||||||||||
2387 | bool visitInstruction(Instruction &I) { | ||||||||||||
2388 | LLVM_DEBUG(dbgs() << " !!!! Cannot rewrite: " << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " !!!! Cannot rewrite: " << I << "\n"; } } while (false); | ||||||||||||
2389 | llvm_unreachable("No rewrite rule for this instruction!")::llvm::llvm_unreachable_internal("No rewrite rule for this instruction!" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2389); | ||||||||||||
2390 | } | ||||||||||||
2391 | |||||||||||||
2392 | Value *getNewAllocaSlicePtr(IRBuilderTy &IRB, Type *PointerTy) { | ||||||||||||
2393 | // Note that the offset computation can use BeginOffset or NewBeginOffset | ||||||||||||
2394 | // interchangeably for unsplit slices. | ||||||||||||
2395 | assert(IsSplit || BeginOffset == NewBeginOffset)((IsSplit || BeginOffset == NewBeginOffset) ? static_cast< void> (0) : __assert_fail ("IsSplit || BeginOffset == NewBeginOffset" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2395, __PRETTY_FUNCTION__)); | ||||||||||||
2396 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | ||||||||||||
2397 | |||||||||||||
2398 | #ifndef NDEBUG | ||||||||||||
2399 | StringRef OldName = OldPtr->getName(); | ||||||||||||
2400 | // Skip through the last '.sroa.' component of the name. | ||||||||||||
2401 | size_t LastSROAPrefix = OldName.rfind(".sroa."); | ||||||||||||
2402 | if (LastSROAPrefix != StringRef::npos) { | ||||||||||||
2403 | OldName = OldName.substr(LastSROAPrefix + strlen(".sroa.")); | ||||||||||||
2404 | // Look for an SROA slice index. | ||||||||||||
2405 | size_t IndexEnd = OldName.find_first_not_of("0123456789"); | ||||||||||||
2406 | if (IndexEnd != StringRef::npos && OldName[IndexEnd] == '.') { | ||||||||||||
2407 | // Strip the index and look for the offset. | ||||||||||||
2408 | OldName = OldName.substr(IndexEnd + 1); | ||||||||||||
2409 | size_t OffsetEnd = OldName.find_first_not_of("0123456789"); | ||||||||||||
2410 | if (OffsetEnd != StringRef::npos && OldName[OffsetEnd] == '.') | ||||||||||||
2411 | // Strip the offset. | ||||||||||||
2412 | OldName = OldName.substr(OffsetEnd + 1); | ||||||||||||
2413 | } | ||||||||||||
2414 | } | ||||||||||||
2415 | // Strip any SROA suffixes as well. | ||||||||||||
2416 | OldName = OldName.substr(0, OldName.find(".sroa_")); | ||||||||||||
2417 | #endif | ||||||||||||
2418 | |||||||||||||
2419 | return getAdjustedPtr(IRB, DL, &NewAI, | ||||||||||||
2420 | APInt(DL.getIndexTypeSizeInBits(PointerTy), Offset), | ||||||||||||
2421 | PointerTy, | ||||||||||||
2422 | #ifndef NDEBUG | ||||||||||||
2423 | Twine(OldName) + "." | ||||||||||||
2424 | #else | ||||||||||||
2425 | Twine() | ||||||||||||
2426 | #endif | ||||||||||||
2427 | ); | ||||||||||||
2428 | } | ||||||||||||
2429 | |||||||||||||
2430 | /// Compute suitable alignment to access this slice of the *new* | ||||||||||||
2431 | /// alloca. | ||||||||||||
2432 | /// | ||||||||||||
2433 | /// You can optionally pass a type to this routine and if that type's ABI | ||||||||||||
2434 | /// alignment is itself suitable, this will return zero. | ||||||||||||
2435 | unsigned getSliceAlign(Type *Ty = nullptr) { | ||||||||||||
2436 | unsigned NewAIAlign = NewAI.getAlignment(); | ||||||||||||
2437 | if (!NewAIAlign) | ||||||||||||
2438 | NewAIAlign = DL.getABITypeAlignment(NewAI.getAllocatedType()); | ||||||||||||
2439 | unsigned Align = | ||||||||||||
2440 | MinAlign(NewAIAlign, NewBeginOffset - NewAllocaBeginOffset); | ||||||||||||
2441 | return (Ty && Align == DL.getABITypeAlignment(Ty)) ? 0 : Align; | ||||||||||||
2442 | } | ||||||||||||
2443 | |||||||||||||
2444 | unsigned getIndex(uint64_t Offset) { | ||||||||||||
2445 | assert(VecTy && "Can only call getIndex when rewriting a vector")((VecTy && "Can only call getIndex when rewriting a vector" ) ? static_cast<void> (0) : __assert_fail ("VecTy && \"Can only call getIndex when rewriting a vector\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2445, __PRETTY_FUNCTION__)); | ||||||||||||
2446 | uint64_t RelOffset = Offset - NewAllocaBeginOffset; | ||||||||||||
2447 | assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds")((RelOffset / ElementSize < (4294967295U) && "Index out of bounds" ) ? static_cast<void> (0) : __assert_fail ("RelOffset / ElementSize < UINT32_MAX && \"Index out of bounds\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2447, __PRETTY_FUNCTION__)); | ||||||||||||
2448 | uint32_t Index = RelOffset / ElementSize; | ||||||||||||
2449 | assert(Index * ElementSize == RelOffset)((Index * ElementSize == RelOffset) ? static_cast<void> (0) : __assert_fail ("Index * ElementSize == RelOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2449, __PRETTY_FUNCTION__)); | ||||||||||||
2450 | return Index; | ||||||||||||
2451 | } | ||||||||||||
2452 | |||||||||||||
2453 | void deleteIfTriviallyDead(Value *V) { | ||||||||||||
2454 | Instruction *I = cast<Instruction>(V); | ||||||||||||
2455 | if (isInstructionTriviallyDead(I)) | ||||||||||||
2456 | Pass.DeadInsts.insert(I); | ||||||||||||
2457 | } | ||||||||||||
2458 | |||||||||||||
2459 | Value *rewriteVectorizedLoadInst() { | ||||||||||||
2460 | unsigned BeginIndex = getIndex(NewBeginOffset); | ||||||||||||
2461 | unsigned EndIndex = getIndex(NewEndOffset); | ||||||||||||
2462 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2462, __PRETTY_FUNCTION__)); | ||||||||||||
2463 | |||||||||||||
2464 | Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2465 | NewAI.getAlignment(), "load"); | ||||||||||||
2466 | return extractVector(IRB, V, BeginIndex, EndIndex, "vec"); | ||||||||||||
2467 | } | ||||||||||||
2468 | |||||||||||||
2469 | Value *rewriteIntegerLoad(LoadInst &LI) { | ||||||||||||
2470 | assert(IntTy && "We cannot insert an integer to the alloca")((IntTy && "We cannot insert an integer to the alloca" ) ? static_cast<void> (0) : __assert_fail ("IntTy && \"We cannot insert an integer to the alloca\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2470, __PRETTY_FUNCTION__)); | ||||||||||||
2471 | assert(!LI.isVolatile())((!LI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!LI.isVolatile()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2471, __PRETTY_FUNCTION__)); | ||||||||||||
2472 | Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2473 | NewAI.getAlignment(), "load"); | ||||||||||||
2474 | V = convertValue(DL, IRB, V, IntTy); | ||||||||||||
2475 | assert(NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")((NewBeginOffset >= NewAllocaBeginOffset && "Out of bounds offset" ) ? static_cast<void> (0) : __assert_fail ("NewBeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2475, __PRETTY_FUNCTION__)); | ||||||||||||
2476 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | ||||||||||||
2477 | if (Offset > 0 || NewEndOffset < NewAllocaEndOffset) { | ||||||||||||
2478 | IntegerType *ExtractTy = Type::getIntNTy(LI.getContext(), SliceSize * 8); | ||||||||||||
2479 | V = extractInteger(DL, IRB, V, ExtractTy, Offset, "extract"); | ||||||||||||
2480 | } | ||||||||||||
2481 | // It is possible that the extracted type is not the load type. This | ||||||||||||
2482 | // happens if there is a load past the end of the alloca, and as | ||||||||||||
2483 | // a consequence the slice is narrower but still a candidate for integer | ||||||||||||
2484 | // lowering. To handle this case, we just zero extend the extracted | ||||||||||||
2485 | // integer. | ||||||||||||
2486 | assert(cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 &&((cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2487, __PRETTY_FUNCTION__)) | ||||||||||||
2487 | "Can only handle an extract for an overly wide load")((cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && "Can only handle an extract for an overly wide load" ) ? static_cast<void> (0) : __assert_fail ("cast<IntegerType>(LI.getType())->getBitWidth() >= SliceSize * 8 && \"Can only handle an extract for an overly wide load\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2487, __PRETTY_FUNCTION__)); | ||||||||||||
2488 | if (cast<IntegerType>(LI.getType())->getBitWidth() > SliceSize * 8) | ||||||||||||
2489 | V = IRB.CreateZExt(V, LI.getType()); | ||||||||||||
2490 | return V; | ||||||||||||
2491 | } | ||||||||||||
2492 | |||||||||||||
2493 | bool visitLoadInst(LoadInst &LI) { | ||||||||||||
2494 | LLVM_DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | ||||||||||||
2495 | Value *OldOp = LI.getOperand(0); | ||||||||||||
2496 | assert(OldOp == OldPtr)((OldOp == OldPtr) ? static_cast<void> (0) : __assert_fail ("OldOp == OldPtr", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2496, __PRETTY_FUNCTION__)); | ||||||||||||
2497 | |||||||||||||
2498 | AAMDNodes AATags; | ||||||||||||
2499 | LI.getAAMetadata(AATags); | ||||||||||||
2500 | |||||||||||||
2501 | unsigned AS = LI.getPointerAddressSpace(); | ||||||||||||
2502 | |||||||||||||
2503 | Type *TargetTy = IsSplit ? Type::getIntNTy(LI.getContext(), SliceSize * 8) | ||||||||||||
2504 | : LI.getType(); | ||||||||||||
2505 | const bool IsLoadPastEnd = DL.getTypeStoreSize(TargetTy) > SliceSize; | ||||||||||||
2506 | bool IsPtrAdjusted = false; | ||||||||||||
2507 | Value *V; | ||||||||||||
2508 | if (VecTy) { | ||||||||||||
2509 | V = rewriteVectorizedLoadInst(); | ||||||||||||
2510 | } else if (IntTy && LI.getType()->isIntegerTy()) { | ||||||||||||
2511 | V = rewriteIntegerLoad(LI); | ||||||||||||
2512 | } else if (NewBeginOffset == NewAllocaBeginOffset && | ||||||||||||
2513 | NewEndOffset == NewAllocaEndOffset && | ||||||||||||
2514 | (canConvertValue(DL, NewAllocaTy, TargetTy) || | ||||||||||||
2515 | (IsLoadPastEnd && NewAllocaTy->isIntegerTy() && | ||||||||||||
2516 | TargetTy->isIntegerTy()))) { | ||||||||||||
2517 | LoadInst *NewLI = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2518 | NewAI.getAlignment(), | ||||||||||||
2519 | LI.isVolatile(), LI.getName()); | ||||||||||||
2520 | if (AATags) | ||||||||||||
2521 | NewLI->setAAMetadata(AATags); | ||||||||||||
2522 | if (LI.isVolatile()) | ||||||||||||
2523 | NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); | ||||||||||||
2524 | |||||||||||||
2525 | // Any !nonnull metadata or !range metadata on the old load is also valid | ||||||||||||
2526 | // on the new load. This is even true in some cases even when the loads | ||||||||||||
2527 | // are different types, for example by mapping !nonnull metadata to | ||||||||||||
2528 | // !range metadata by modeling the null pointer constant converted to the | ||||||||||||
2529 | // integer type. | ||||||||||||
2530 | // FIXME: Add support for range metadata here. Currently the utilities | ||||||||||||
2531 | // for this don't propagate range metadata in trivial cases from one | ||||||||||||
2532 | // integer load to another, don't handle non-addrspace-0 null pointers | ||||||||||||
2533 | // correctly, and don't have any support for mapping ranges as the | ||||||||||||
2534 | // integer type becomes winder or narrower. | ||||||||||||
2535 | if (MDNode *N = LI.getMetadata(LLVMContext::MD_nonnull)) | ||||||||||||
2536 | copyNonnullMetadata(LI, N, *NewLI); | ||||||||||||
2537 | |||||||||||||
2538 | // Try to preserve nonnull metadata | ||||||||||||
2539 | V = NewLI; | ||||||||||||
2540 | |||||||||||||
2541 | // If this is an integer load past the end of the slice (which means the | ||||||||||||
2542 | // bytes outside the slice are undef or this load is dead) just forcibly | ||||||||||||
2543 | // fix the integer size with correct handling of endianness. | ||||||||||||
2544 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | ||||||||||||
2545 | if (auto *TITy = dyn_cast<IntegerType>(TargetTy)) | ||||||||||||
2546 | if (AITy->getBitWidth() < TITy->getBitWidth()) { | ||||||||||||
2547 | V = IRB.CreateZExt(V, TITy, "load.ext"); | ||||||||||||
2548 | if (DL.isBigEndian()) | ||||||||||||
2549 | V = IRB.CreateShl(V, TITy->getBitWidth() - AITy->getBitWidth(), | ||||||||||||
2550 | "endian_shift"); | ||||||||||||
2551 | } | ||||||||||||
2552 | } else { | ||||||||||||
2553 | Type *LTy = TargetTy->getPointerTo(AS); | ||||||||||||
2554 | LoadInst *NewLI = IRB.CreateAlignedLoad( | ||||||||||||
2555 | TargetTy, getNewAllocaSlicePtr(IRB, LTy), getSliceAlign(TargetTy), | ||||||||||||
2556 | LI.isVolatile(), LI.getName()); | ||||||||||||
2557 | if (AATags) | ||||||||||||
2558 | NewLI->setAAMetadata(AATags); | ||||||||||||
2559 | if (LI.isVolatile()) | ||||||||||||
2560 | NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID()); | ||||||||||||
2561 | |||||||||||||
2562 | V = NewLI; | ||||||||||||
2563 | IsPtrAdjusted = true; | ||||||||||||
2564 | } | ||||||||||||
2565 | V = convertValue(DL, IRB, V, TargetTy); | ||||||||||||
2566 | |||||||||||||
2567 | if (IsSplit) { | ||||||||||||
2568 | assert(!LI.isVolatile())((!LI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!LI.isVolatile()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2568, __PRETTY_FUNCTION__)); | ||||||||||||
2569 | assert(LI.getType()->isIntegerTy() &&((LI.getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2570, __PRETTY_FUNCTION__)) | ||||||||||||
2570 | "Only integer type loads and stores are split")((LI.getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("LI.getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2570, __PRETTY_FUNCTION__)); | ||||||||||||
2571 | assert(SliceSize < DL.getTypeStoreSize(LI.getType()) &&((SliceSize < DL.getTypeStoreSize(LI.getType()) && "Split load isn't smaller than original load") ? static_cast <void> (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()) && \"Split load isn't smaller than original load\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2572, __PRETTY_FUNCTION__)) | ||||||||||||
2572 | "Split load isn't smaller than original load")((SliceSize < DL.getTypeStoreSize(LI.getType()) && "Split load isn't smaller than original load") ? static_cast <void> (0) : __assert_fail ("SliceSize < DL.getTypeStoreSize(LI.getType()) && \"Split load isn't smaller than original load\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2572, __PRETTY_FUNCTION__)); | ||||||||||||
2573 | assert(DL.typeSizeEqualsStoreSize(LI.getType()) &&((DL.typeSizeEqualsStoreSize(LI.getType()) && "Non-byte-multiple bit width" ) ? static_cast<void> (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(LI.getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2574, __PRETTY_FUNCTION__)) | ||||||||||||
2574 | "Non-byte-multiple bit width")((DL.typeSizeEqualsStoreSize(LI.getType()) && "Non-byte-multiple bit width" ) ? static_cast<void> (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(LI.getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2574, __PRETTY_FUNCTION__)); | ||||||||||||
2575 | // Move the insertion point just past the load so that we can refer to it. | ||||||||||||
2576 | IRB.SetInsertPoint(&*std::next(BasicBlock::iterator(&LI))); | ||||||||||||
2577 | // Create a placeholder value with the same type as LI to use as the | ||||||||||||
2578 | // basis for the new value. This allows us to replace the uses of LI with | ||||||||||||
2579 | // the computed value, and then replace the placeholder with LI, leaving | ||||||||||||
2580 | // LI only used for this computation. | ||||||||||||
2581 | Value *Placeholder = new LoadInst( | ||||||||||||
2582 | LI.getType(), UndefValue::get(LI.getType()->getPointerTo(AS))); | ||||||||||||
2583 | V = insertInteger(DL, IRB, Placeholder, V, NewBeginOffset - BeginOffset, | ||||||||||||
2584 | "insert"); | ||||||||||||
2585 | LI.replaceAllUsesWith(V); | ||||||||||||
2586 | Placeholder->replaceAllUsesWith(&LI); | ||||||||||||
2587 | Placeholder->deleteValue(); | ||||||||||||
2588 | } else { | ||||||||||||
2589 | LI.replaceAllUsesWith(V); | ||||||||||||
2590 | } | ||||||||||||
2591 | |||||||||||||
2592 | Pass.DeadInsts.insert(&LI); | ||||||||||||
2593 | deleteIfTriviallyDead(OldOp); | ||||||||||||
2594 | LLVM_DEBUG(dbgs() << " to: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *V << "\n"; } } while (false); | ||||||||||||
2595 | return !LI.isVolatile() && !IsPtrAdjusted; | ||||||||||||
2596 | } | ||||||||||||
2597 | |||||||||||||
2598 | bool rewriteVectorizedStoreInst(Value *V, StoreInst &SI, Value *OldOp, | ||||||||||||
2599 | AAMDNodes AATags) { | ||||||||||||
2600 | if (V->getType() != VecTy) { | ||||||||||||
2601 | unsigned BeginIndex = getIndex(NewBeginOffset); | ||||||||||||
2602 | unsigned EndIndex = getIndex(NewEndOffset); | ||||||||||||
2603 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2603, __PRETTY_FUNCTION__)); | ||||||||||||
2604 | unsigned NumElements = EndIndex - BeginIndex; | ||||||||||||
2605 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")((NumElements <= VecTy->getNumElements() && "Too many elements!" ) ? static_cast<void> (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2605, __PRETTY_FUNCTION__)); | ||||||||||||
2606 | Type *SliceTy = (NumElements == 1) | ||||||||||||
2607 | ? ElementTy | ||||||||||||
2608 | : VectorType::get(ElementTy, NumElements); | ||||||||||||
2609 | if (V->getType() != SliceTy) | ||||||||||||
2610 | V = convertValue(DL, IRB, V, SliceTy); | ||||||||||||
2611 | |||||||||||||
2612 | // Mix in the existing elements. | ||||||||||||
2613 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2614 | NewAI.getAlignment(), "load"); | ||||||||||||
2615 | V = insertVector(IRB, Old, V, BeginIndex, "vec"); | ||||||||||||
2616 | } | ||||||||||||
2617 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); | ||||||||||||
2618 | if (AATags) | ||||||||||||
2619 | Store->setAAMetadata(AATags); | ||||||||||||
2620 | Pass.DeadInsts.insert(&SI); | ||||||||||||
2621 | |||||||||||||
2622 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | ||||||||||||
2623 | return true; | ||||||||||||
2624 | } | ||||||||||||
2625 | |||||||||||||
2626 | bool rewriteIntegerStore(Value *V, StoreInst &SI, AAMDNodes AATags) { | ||||||||||||
2627 | assert(IntTy && "We cannot extract an integer from the alloca")((IntTy && "We cannot extract an integer from the alloca" ) ? static_cast<void> (0) : __assert_fail ("IntTy && \"We cannot extract an integer from the alloca\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2627, __PRETTY_FUNCTION__)); | ||||||||||||
2628 | assert(!SI.isVolatile())((!SI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!SI.isVolatile()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2628, __PRETTY_FUNCTION__)); | ||||||||||||
2629 | if (DL.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) { | ||||||||||||
2630 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2631 | NewAI.getAlignment(), "oldload"); | ||||||||||||
2632 | Old = convertValue(DL, IRB, Old, IntTy); | ||||||||||||
2633 | assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset")((BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Out of bounds offset\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2633, __PRETTY_FUNCTION__)); | ||||||||||||
2634 | uint64_t Offset = BeginOffset - NewAllocaBeginOffset; | ||||||||||||
2635 | V = insertInteger(DL, IRB, Old, SI.getValueOperand(), Offset, "insert"); | ||||||||||||
2636 | } | ||||||||||||
2637 | V = convertValue(DL, IRB, V, NewAllocaTy); | ||||||||||||
2638 | StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment()); | ||||||||||||
2639 | Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | ||||||||||||
2640 | LLVMContext::MD_access_group}); | ||||||||||||
2641 | if (AATags) | ||||||||||||
2642 | Store->setAAMetadata(AATags); | ||||||||||||
2643 | Pass.DeadInsts.insert(&SI); | ||||||||||||
2644 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | ||||||||||||
2645 | return true; | ||||||||||||
2646 | } | ||||||||||||
2647 | |||||||||||||
2648 | bool visitStoreInst(StoreInst &SI) { | ||||||||||||
2649 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | ||||||||||||
2650 | Value *OldOp = SI.getOperand(1); | ||||||||||||
2651 | assert(OldOp == OldPtr)((OldOp == OldPtr) ? static_cast<void> (0) : __assert_fail ("OldOp == OldPtr", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2651, __PRETTY_FUNCTION__)); | ||||||||||||
2652 | |||||||||||||
2653 | AAMDNodes AATags; | ||||||||||||
2654 | SI.getAAMetadata(AATags); | ||||||||||||
2655 | |||||||||||||
2656 | Value *V = SI.getValueOperand(); | ||||||||||||
2657 | |||||||||||||
2658 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | ||||||||||||
2659 | // alloca that should be re-examined after promoting this alloca. | ||||||||||||
2660 | if (V->getType()->isPointerTy()) | ||||||||||||
2661 | if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets())) | ||||||||||||
2662 | Pass.PostPromotionWorklist.insert(AI); | ||||||||||||
2663 | |||||||||||||
2664 | if (SliceSize < DL.getTypeStoreSize(V->getType())) { | ||||||||||||
2665 | assert(!SI.isVolatile())((!SI.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!SI.isVolatile()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2665, __PRETTY_FUNCTION__)); | ||||||||||||
2666 | assert(V->getType()->isIntegerTy() &&((V->getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2667, __PRETTY_FUNCTION__)) | ||||||||||||
2667 | "Only integer type loads and stores are split")((V->getType()->isIntegerTy() && "Only integer type loads and stores are split" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntegerTy() && \"Only integer type loads and stores are split\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2667, __PRETTY_FUNCTION__)); | ||||||||||||
2668 | assert(DL.typeSizeEqualsStoreSize(V->getType()) &&((DL.typeSizeEqualsStoreSize(V->getType()) && "Non-byte-multiple bit width" ) ? static_cast<void> (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(V->getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2669, __PRETTY_FUNCTION__)) | ||||||||||||
2669 | "Non-byte-multiple bit width")((DL.typeSizeEqualsStoreSize(V->getType()) && "Non-byte-multiple bit width" ) ? static_cast<void> (0) : __assert_fail ("DL.typeSizeEqualsStoreSize(V->getType()) && \"Non-byte-multiple bit width\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2669, __PRETTY_FUNCTION__)); | ||||||||||||
2670 | IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), SliceSize * 8); | ||||||||||||
2671 | V = extractInteger(DL, IRB, V, NarrowTy, NewBeginOffset - BeginOffset, | ||||||||||||
2672 | "extract"); | ||||||||||||
2673 | } | ||||||||||||
2674 | |||||||||||||
2675 | if (VecTy) | ||||||||||||
2676 | return rewriteVectorizedStoreInst(V, SI, OldOp, AATags); | ||||||||||||
2677 | if (IntTy && V->getType()->isIntegerTy()) | ||||||||||||
2678 | return rewriteIntegerStore(V, SI, AATags); | ||||||||||||
2679 | |||||||||||||
2680 | const bool IsStorePastEnd = DL.getTypeStoreSize(V->getType()) > SliceSize; | ||||||||||||
2681 | StoreInst *NewSI; | ||||||||||||
2682 | if (NewBeginOffset == NewAllocaBeginOffset && | ||||||||||||
2683 | NewEndOffset == NewAllocaEndOffset && | ||||||||||||
2684 | (canConvertValue(DL, V->getType(), NewAllocaTy) || | ||||||||||||
2685 | (IsStorePastEnd && NewAllocaTy->isIntegerTy() && | ||||||||||||
2686 | V->getType()->isIntegerTy()))) { | ||||||||||||
2687 | // If this is an integer store past the end of slice (and thus the bytes | ||||||||||||
2688 | // past that point are irrelevant or this is unreachable), truncate the | ||||||||||||
2689 | // value prior to storing. | ||||||||||||
2690 | if (auto *VITy = dyn_cast<IntegerType>(V->getType())) | ||||||||||||
2691 | if (auto *AITy = dyn_cast<IntegerType>(NewAllocaTy)) | ||||||||||||
2692 | if (VITy->getBitWidth() > AITy->getBitWidth()) { | ||||||||||||
2693 | if (DL.isBigEndian()) | ||||||||||||
2694 | V = IRB.CreateLShr(V, VITy->getBitWidth() - AITy->getBitWidth(), | ||||||||||||
2695 | "endian_shift"); | ||||||||||||
2696 | V = IRB.CreateTrunc(V, AITy, "load.trunc"); | ||||||||||||
2697 | } | ||||||||||||
2698 | |||||||||||||
2699 | V = convertValue(DL, IRB, V, NewAllocaTy); | ||||||||||||
2700 | NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), | ||||||||||||
2701 | SI.isVolatile()); | ||||||||||||
2702 | } else { | ||||||||||||
2703 | unsigned AS = SI.getPointerAddressSpace(); | ||||||||||||
2704 | Value *NewPtr = getNewAllocaSlicePtr(IRB, V->getType()->getPointerTo(AS)); | ||||||||||||
2705 | NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()), | ||||||||||||
2706 | SI.isVolatile()); | ||||||||||||
2707 | } | ||||||||||||
2708 | NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access, | ||||||||||||
2709 | LLVMContext::MD_access_group}); | ||||||||||||
2710 | if (AATags) | ||||||||||||
2711 | NewSI->setAAMetadata(AATags); | ||||||||||||
2712 | if (SI.isVolatile()) | ||||||||||||
2713 | NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID()); | ||||||||||||
2714 | Pass.DeadInsts.insert(&SI); | ||||||||||||
2715 | deleteIfTriviallyDead(OldOp); | ||||||||||||
2716 | |||||||||||||
2717 | LLVM_DEBUG(dbgs() << " to: " << *NewSI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *NewSI << "\n"; } } while (false); | ||||||||||||
2718 | return NewSI->getPointerOperand() == &NewAI && !SI.isVolatile(); | ||||||||||||
2719 | } | ||||||||||||
2720 | |||||||||||||
2721 | /// Compute an integer value from splatting an i8 across the given | ||||||||||||
2722 | /// number of bytes. | ||||||||||||
2723 | /// | ||||||||||||
2724 | /// Note that this routine assumes an i8 is a byte. If that isn't true, don't | ||||||||||||
2725 | /// call this routine. | ||||||||||||
2726 | /// FIXME: Heed the advice above. | ||||||||||||
2727 | /// | ||||||||||||
2728 | /// \param V The i8 value to splat. | ||||||||||||
2729 | /// \param Size The number of bytes in the output (assuming i8 is one byte) | ||||||||||||
2730 | Value *getIntegerSplat(Value *V, unsigned Size) { | ||||||||||||
2731 | assert(Size > 0 && "Expected a positive number of bytes.")((Size > 0 && "Expected a positive number of bytes." ) ? static_cast<void> (0) : __assert_fail ("Size > 0 && \"Expected a positive number of bytes.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2731, __PRETTY_FUNCTION__)); | ||||||||||||
2732 | IntegerType *VTy = cast<IntegerType>(V->getType()); | ||||||||||||
2733 | assert(VTy->getBitWidth() == 8 && "Expected an i8 value for the byte")((VTy->getBitWidth() == 8 && "Expected an i8 value for the byte" ) ? static_cast<void> (0) : __assert_fail ("VTy->getBitWidth() == 8 && \"Expected an i8 value for the byte\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2733, __PRETTY_FUNCTION__)); | ||||||||||||
2734 | if (Size == 1) | ||||||||||||
2735 | return V; | ||||||||||||
2736 | |||||||||||||
2737 | Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size * 8); | ||||||||||||
2738 | V = IRB.CreateMul( | ||||||||||||
2739 | IRB.CreateZExt(V, SplatIntTy, "zext"), | ||||||||||||
2740 | ConstantExpr::getUDiv( | ||||||||||||
2741 | Constant::getAllOnesValue(SplatIntTy), | ||||||||||||
2742 | ConstantExpr::getZExt(Constant::getAllOnesValue(V->getType()), | ||||||||||||
2743 | SplatIntTy)), | ||||||||||||
2744 | "isplat"); | ||||||||||||
2745 | return V; | ||||||||||||
2746 | } | ||||||||||||
2747 | |||||||||||||
2748 | /// Compute a vector splat for a given element value. | ||||||||||||
2749 | Value *getVectorSplat(Value *V, unsigned NumElements) { | ||||||||||||
2750 | V = IRB.CreateVectorSplat(NumElements, V, "vsplat"); | ||||||||||||
2751 | LLVM_DEBUG(dbgs() << " splat: " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " splat: " << *V << "\n"; } } while (false); | ||||||||||||
2752 | return V; | ||||||||||||
2753 | } | ||||||||||||
2754 | |||||||||||||
2755 | bool visitMemSetInst(MemSetInst &II) { | ||||||||||||
2756 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | ||||||||||||
2757 | assert(II.getRawDest() == OldPtr)((II.getRawDest() == OldPtr) ? static_cast<void> (0) : __assert_fail ("II.getRawDest() == OldPtr", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2757, __PRETTY_FUNCTION__)); | ||||||||||||
2758 | |||||||||||||
2759 | AAMDNodes AATags; | ||||||||||||
2760 | II.getAAMetadata(AATags); | ||||||||||||
2761 | |||||||||||||
2762 | // If the memset has a variable size, it cannot be split, just adjust the | ||||||||||||
2763 | // pointer to the new alloca. | ||||||||||||
2764 | if (!isa<Constant>(II.getLength())) { | ||||||||||||
2765 | assert(!IsSplit)((!IsSplit) ? static_cast<void> (0) : __assert_fail ("!IsSplit" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2765, __PRETTY_FUNCTION__)); | ||||||||||||
2766 | assert(NewBeginOffset == BeginOffset)((NewBeginOffset == BeginOffset) ? static_cast<void> (0 ) : __assert_fail ("NewBeginOffset == BeginOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2766, __PRETTY_FUNCTION__)); | ||||||||||||
2767 | II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType())); | ||||||||||||
2768 | II.setDestAlignment(getSliceAlign()); | ||||||||||||
2769 | |||||||||||||
2770 | deleteIfTriviallyDead(OldPtr); | ||||||||||||
2771 | return false; | ||||||||||||
2772 | } | ||||||||||||
2773 | |||||||||||||
2774 | // Record this instruction for deletion. | ||||||||||||
2775 | Pass.DeadInsts.insert(&II); | ||||||||||||
2776 | |||||||||||||
2777 | Type *AllocaTy = NewAI.getAllocatedType(); | ||||||||||||
2778 | Type *ScalarTy = AllocaTy->getScalarType(); | ||||||||||||
2779 | |||||||||||||
2780 | const bool CanContinue = [&]() { | ||||||||||||
2781 | if (VecTy || IntTy) | ||||||||||||
2782 | return true; | ||||||||||||
2783 | if (BeginOffset > NewAllocaBeginOffset || | ||||||||||||
2784 | EndOffset < NewAllocaEndOffset) | ||||||||||||
2785 | return false; | ||||||||||||
2786 | auto *C = cast<ConstantInt>(II.getLength()); | ||||||||||||
2787 | if (C->getBitWidth() > 64) | ||||||||||||
2788 | return false; | ||||||||||||
2789 | const auto Len = C->getZExtValue(); | ||||||||||||
2790 | auto *Int8Ty = IntegerType::getInt8Ty(NewAI.getContext()); | ||||||||||||
2791 | auto *SrcTy = VectorType::get(Int8Ty, Len); | ||||||||||||
2792 | return canConvertValue(DL, SrcTy, AllocaTy) && | ||||||||||||
2793 | DL.isLegalInteger(DL.getTypeSizeInBits(ScalarTy)); | ||||||||||||
2794 | }(); | ||||||||||||
2795 | |||||||||||||
2796 | // If this doesn't map cleanly onto the alloca type, and that type isn't | ||||||||||||
2797 | // a single value type, just emit a memset. | ||||||||||||
2798 | if (!CanContinue) { | ||||||||||||
2799 | Type *SizeTy = II.getLength()->getType(); | ||||||||||||
2800 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | ||||||||||||
2801 | CallInst *New = IRB.CreateMemSet( | ||||||||||||
2802 | getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size, | ||||||||||||
2803 | getSliceAlign(), II.isVolatile()); | ||||||||||||
2804 | if (AATags) | ||||||||||||
2805 | New->setAAMetadata(AATags); | ||||||||||||
2806 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | ||||||||||||
2807 | return false; | ||||||||||||
2808 | } | ||||||||||||
2809 | |||||||||||||
2810 | // If we can represent this as a simple value, we have to build the actual | ||||||||||||
2811 | // value to store, which requires expanding the byte present in memset to | ||||||||||||
2812 | // a sensible representation for the alloca type. This is essentially | ||||||||||||
2813 | // splatting the byte to a sufficiently wide integer, splatting it across | ||||||||||||
2814 | // any desired vector width, and bitcasting to the final type. | ||||||||||||
2815 | Value *V; | ||||||||||||
2816 | |||||||||||||
2817 | if (VecTy) { | ||||||||||||
2818 | // If this is a memset of a vectorized alloca, insert it. | ||||||||||||
2819 | assert(ElementTy == ScalarTy)((ElementTy == ScalarTy) ? static_cast<void> (0) : __assert_fail ("ElementTy == ScalarTy", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2819, __PRETTY_FUNCTION__)); | ||||||||||||
2820 | |||||||||||||
2821 | unsigned BeginIndex = getIndex(NewBeginOffset); | ||||||||||||
2822 | unsigned EndIndex = getIndex(NewEndOffset); | ||||||||||||
2823 | assert(EndIndex > BeginIndex && "Empty vector!")((EndIndex > BeginIndex && "Empty vector!") ? static_cast <void> (0) : __assert_fail ("EndIndex > BeginIndex && \"Empty vector!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2823, __PRETTY_FUNCTION__)); | ||||||||||||
2824 | unsigned NumElements = EndIndex - BeginIndex; | ||||||||||||
2825 | assert(NumElements <= VecTy->getNumElements() && "Too many elements!")((NumElements <= VecTy->getNumElements() && "Too many elements!" ) ? static_cast<void> (0) : __assert_fail ("NumElements <= VecTy->getNumElements() && \"Too many elements!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2825, __PRETTY_FUNCTION__)); | ||||||||||||
2826 | |||||||||||||
2827 | Value *Splat = | ||||||||||||
2828 | getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ElementTy) / 8); | ||||||||||||
2829 | Splat = convertValue(DL, IRB, Splat, ElementTy); | ||||||||||||
2830 | if (NumElements > 1) | ||||||||||||
2831 | Splat = getVectorSplat(Splat, NumElements); | ||||||||||||
2832 | |||||||||||||
2833 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2834 | NewAI.getAlignment(), "oldload"); | ||||||||||||
2835 | V = insertVector(IRB, Old, Splat, BeginIndex, "vec"); | ||||||||||||
2836 | } else if (IntTy) { | ||||||||||||
2837 | // If this is a memset on an alloca where we can widen stores, insert the | ||||||||||||
2838 | // set integer. | ||||||||||||
2839 | assert(!II.isVolatile())((!II.isVolatile()) ? static_cast<void> (0) : __assert_fail ("!II.isVolatile()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2839, __PRETTY_FUNCTION__)); | ||||||||||||
2840 | |||||||||||||
2841 | uint64_t Size = NewEndOffset - NewBeginOffset; | ||||||||||||
2842 | V = getIntegerSplat(II.getValue(), Size); | ||||||||||||
2843 | |||||||||||||
2844 | if (IntTy && (BeginOffset != NewAllocaBeginOffset || | ||||||||||||
2845 | EndOffset != NewAllocaBeginOffset)) { | ||||||||||||
2846 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
2847 | NewAI.getAlignment(), "oldload"); | ||||||||||||
2848 | Old = convertValue(DL, IRB, Old, IntTy); | ||||||||||||
2849 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | ||||||||||||
2850 | V = insertInteger(DL, IRB, Old, V, Offset, "insert"); | ||||||||||||
2851 | } else { | ||||||||||||
2852 | assert(V->getType() == IntTy &&((V->getType() == IntTy && "Wrong type for an alloca wide integer!" ) ? static_cast<void> (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2853, __PRETTY_FUNCTION__)) | ||||||||||||
2853 | "Wrong type for an alloca wide integer!")((V->getType() == IntTy && "Wrong type for an alloca wide integer!" ) ? static_cast<void> (0) : __assert_fail ("V->getType() == IntTy && \"Wrong type for an alloca wide integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2853, __PRETTY_FUNCTION__)); | ||||||||||||
2854 | } | ||||||||||||
2855 | V = convertValue(DL, IRB, V, AllocaTy); | ||||||||||||
2856 | } else { | ||||||||||||
2857 | // Established these invariants above. | ||||||||||||
2858 | assert(NewBeginOffset == NewAllocaBeginOffset)((NewBeginOffset == NewAllocaBeginOffset) ? static_cast<void > (0) : __assert_fail ("NewBeginOffset == NewAllocaBeginOffset" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2858, __PRETTY_FUNCTION__)); | ||||||||||||
2859 | assert(NewEndOffset == NewAllocaEndOffset)((NewEndOffset == NewAllocaEndOffset) ? static_cast<void> (0) : __assert_fail ("NewEndOffset == NewAllocaEndOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2859, __PRETTY_FUNCTION__)); | ||||||||||||
2860 | |||||||||||||
2861 | V = getIntegerSplat(II.getValue(), DL.getTypeSizeInBits(ScalarTy) / 8); | ||||||||||||
2862 | if (VectorType *AllocaVecTy = dyn_cast<VectorType>(AllocaTy)) | ||||||||||||
2863 | V = getVectorSplat(V, AllocaVecTy->getNumElements()); | ||||||||||||
2864 | |||||||||||||
2865 | V = convertValue(DL, IRB, V, AllocaTy); | ||||||||||||
2866 | } | ||||||||||||
2867 | |||||||||||||
2868 | StoreInst *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(), | ||||||||||||
2869 | II.isVolatile()); | ||||||||||||
2870 | if (AATags) | ||||||||||||
2871 | New->setAAMetadata(AATags); | ||||||||||||
2872 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | ||||||||||||
2873 | return !II.isVolatile(); | ||||||||||||
2874 | } | ||||||||||||
2875 | |||||||||||||
2876 | bool visitMemTransferInst(MemTransferInst &II) { | ||||||||||||
2877 | // Rewriting of memory transfer instructions can be a bit tricky. We break | ||||||||||||
2878 | // them into two categories: split intrinsics and unsplit intrinsics. | ||||||||||||
2879 | |||||||||||||
2880 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | ||||||||||||
| |||||||||||||
2881 | |||||||||||||
2882 | AAMDNodes AATags; | ||||||||||||
2883 | II.getAAMetadata(AATags); | ||||||||||||
2884 | |||||||||||||
2885 | bool IsDest = &II.getRawDestUse() == OldUse; | ||||||||||||
2886 | assert
II.getRawSource() == OldPtr)) ? static_cast<void> (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2887, __PRETTY_FUNCTION__)) | ||||||||||||
2887 | (!IsDest && II.getRawSource() == OldPtr))(((IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)) ? static_cast<void> (0) : __assert_fail ("(IsDest && II.getRawDest() == OldPtr) || (!IsDest && II.getRawSource() == OldPtr)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2887, __PRETTY_FUNCTION__)); | ||||||||||||
2888 | |||||||||||||
2889 | unsigned SliceAlign = getSliceAlign(); | ||||||||||||
2890 | |||||||||||||
2891 | // For unsplit intrinsics, we simply modify the source and destination | ||||||||||||
2892 | // pointers in place. This isn't just an optimization, it is a matter of | ||||||||||||
2893 | // correctness. With unsplit intrinsics we may be dealing with transfers | ||||||||||||
2894 | // within a single alloca before SROA ran, or with transfers that have | ||||||||||||
2895 | // a variable length. We may also be dealing with memmove instead of | ||||||||||||
2896 | // memcpy, and so simply updating the pointers is the necessary for us to | ||||||||||||
2897 | // update both source and dest of a single call. | ||||||||||||
2898 | if (!IsSplittable) { | ||||||||||||
2899 | Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | ||||||||||||
2900 | if (IsDest) { | ||||||||||||
2901 | II.setDest(AdjustedPtr); | ||||||||||||
2902 | II.setDestAlignment(SliceAlign); | ||||||||||||
2903 | } | ||||||||||||
2904 | else { | ||||||||||||
2905 | II.setSource(AdjustedPtr); | ||||||||||||
2906 | II.setSourceAlignment(SliceAlign); | ||||||||||||
2907 | } | ||||||||||||
2908 | |||||||||||||
2909 | LLVM_DEBUG(dbgs() << " to: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << II << "\n"; } } while (false); | ||||||||||||
2910 | deleteIfTriviallyDead(OldPtr); | ||||||||||||
2911 | return false; | ||||||||||||
2912 | } | ||||||||||||
2913 | // For split transfer intrinsics we have an incredibly useful assurance: | ||||||||||||
2914 | // the source and destination do not reside within the same alloca, and at | ||||||||||||
2915 | // least one of them does not escape. This means that we can replace | ||||||||||||
2916 | // memmove with memcpy, and we don't need to worry about all manner of | ||||||||||||
2917 | // downsides to splitting and transforming the operations. | ||||||||||||
2918 | |||||||||||||
2919 | // If this doesn't map cleanly onto the alloca type, and that type isn't | ||||||||||||
2920 | // a single value type, just emit a memcpy. | ||||||||||||
2921 | bool EmitMemCpy = | ||||||||||||
2922 | !VecTy && !IntTy && | ||||||||||||
2923 | (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset || | ||||||||||||
2924 | SliceSize != DL.getTypeStoreSize(NewAI.getAllocatedType()) || | ||||||||||||
2925 | !NewAI.getAllocatedType()->isSingleValueType()); | ||||||||||||
2926 | |||||||||||||
2927 | // If we're just going to emit a memcpy, the alloca hasn't changed, and the | ||||||||||||
2928 | // size hasn't been shrunk based on analysis of the viable range, this is | ||||||||||||
2929 | // a no-op. | ||||||||||||
2930 | if (EmitMemCpy
| ||||||||||||
2931 | // Ensure the start lines up. | ||||||||||||
2932 | assert(NewBeginOffset == BeginOffset)((NewBeginOffset == BeginOffset) ? static_cast<void> (0 ) : __assert_fail ("NewBeginOffset == BeginOffset", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2932, __PRETTY_FUNCTION__)); | ||||||||||||
2933 | |||||||||||||
2934 | // Rewrite the size as needed. | ||||||||||||
2935 | if (NewEndOffset != EndOffset) | ||||||||||||
2936 | II.setLength(ConstantInt::get(II.getLength()->getType(), | ||||||||||||
2937 | NewEndOffset - NewBeginOffset)); | ||||||||||||
2938 | return false; | ||||||||||||
2939 | } | ||||||||||||
2940 | // Record this instruction for deletion. | ||||||||||||
2941 | Pass.DeadInsts.insert(&II); | ||||||||||||
2942 | |||||||||||||
2943 | // Strip all inbounds GEPs and pointer casts to try to dig out any root | ||||||||||||
2944 | // alloca that should be re-examined after rewriting this instruction. | ||||||||||||
2945 | Value *OtherPtr = IsDest
| ||||||||||||
2946 | if (AllocaInst *AI
| ||||||||||||
2947 | dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets())) { | ||||||||||||
2948 | assert(AI != &OldAI && AI != &NewAI &&((AI != &OldAI && AI != &NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? static_cast<void> (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2949, __PRETTY_FUNCTION__)) | ||||||||||||
2949 | "Splittable transfers cannot reach the same alloca on both ends.")((AI != &OldAI && AI != &NewAI && "Splittable transfers cannot reach the same alloca on both ends." ) ? static_cast<void> (0) : __assert_fail ("AI != &OldAI && AI != &NewAI && \"Splittable transfers cannot reach the same alloca on both ends.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 2949, __PRETTY_FUNCTION__)); | ||||||||||||
2950 | Pass.Worklist.insert(AI); | ||||||||||||
2951 | } | ||||||||||||
2952 | |||||||||||||
2953 | Type *OtherPtrTy = OtherPtr->getType(); | ||||||||||||
2954 | unsigned OtherAS = OtherPtrTy->getPointerAddressSpace(); | ||||||||||||
2955 | |||||||||||||
2956 | // Compute the relative offset for the other pointer within the transfer. | ||||||||||||
2957 | unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS); | ||||||||||||
2958 | APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset); | ||||||||||||
2959 | unsigned OtherAlign = | ||||||||||||
2960 | IsDest
| ||||||||||||
2961 | OtherAlign = MinAlign(OtherAlign
| ||||||||||||
2962 | OtherOffset.zextOrTrunc(64).getZExtValue()); | ||||||||||||
2963 | |||||||||||||
2964 | if (EmitMemCpy
| ||||||||||||
2965 | // Compute the other pointer, folding as much as possible to produce | ||||||||||||
2966 | // a single, simple GEP in most cases. | ||||||||||||
2967 | OtherPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | ||||||||||||
2968 | OtherPtr->getName() + "."); | ||||||||||||
2969 | |||||||||||||
2970 | Value *OurPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | ||||||||||||
2971 | Type *SizeTy = II.getLength()->getType(); | ||||||||||||
2972 | Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset); | ||||||||||||
2973 | |||||||||||||
2974 | Value *DestPtr, *SrcPtr; | ||||||||||||
2975 | unsigned DestAlign, SrcAlign; | ||||||||||||
2976 | // Note: IsDest is true iff we're copying into the new alloca slice | ||||||||||||
2977 | if (IsDest) { | ||||||||||||
2978 | DestPtr = OurPtr; | ||||||||||||
2979 | DestAlign = SliceAlign; | ||||||||||||
2980 | SrcPtr = OtherPtr; | ||||||||||||
2981 | SrcAlign = OtherAlign; | ||||||||||||
2982 | } else { | ||||||||||||
2983 | DestPtr = OtherPtr; | ||||||||||||
2984 | DestAlign = OtherAlign; | ||||||||||||
2985 | SrcPtr = OurPtr; | ||||||||||||
2986 | SrcAlign = SliceAlign; | ||||||||||||
2987 | } | ||||||||||||
2988 | CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign, | ||||||||||||
2989 | Size, II.isVolatile()); | ||||||||||||
2990 | if (AATags) | ||||||||||||
2991 | New->setAAMetadata(AATags); | ||||||||||||
2992 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | ||||||||||||
2993 | return false; | ||||||||||||
2994 | } | ||||||||||||
2995 | |||||||||||||
2996 | bool IsWholeAlloca = NewBeginOffset == NewAllocaBeginOffset && | ||||||||||||
2997 | NewEndOffset == NewAllocaEndOffset; | ||||||||||||
2998 | uint64_t Size = NewEndOffset - NewBeginOffset; | ||||||||||||
2999 | unsigned BeginIndex = VecTy
| ||||||||||||
3000 | unsigned EndIndex = VecTy
| ||||||||||||
3001 | unsigned NumElements = EndIndex - BeginIndex; | ||||||||||||
3002 | IntegerType *SubIntTy = | ||||||||||||
3003 | IntTy ? Type::getIntNTy(IntTy->getContext(), Size * 8) : nullptr; | ||||||||||||
3004 | |||||||||||||
3005 | // Reset the other pointer type to match the register type we're going to | ||||||||||||
3006 | // use, but using the address space of the original other pointer. | ||||||||||||
3007 | Type *OtherTy; | ||||||||||||
3008 | if (VecTy
| ||||||||||||
3009 | if (NumElements == 1) | ||||||||||||
3010 | OtherTy = VecTy->getElementType(); | ||||||||||||
3011 | else | ||||||||||||
3012 | OtherTy = VectorType::get(VecTy->getElementType(), NumElements); | ||||||||||||
3013 | } else if (IntTy && !IsWholeAlloca) { | ||||||||||||
3014 | OtherTy = SubIntTy; | ||||||||||||
3015 | } else { | ||||||||||||
3016 | OtherTy = NewAllocaTy; | ||||||||||||
3017 | } | ||||||||||||
3018 | OtherPtrTy = OtherTy->getPointerTo(OtherAS); | ||||||||||||
3019 | |||||||||||||
3020 | Value *SrcPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy, | ||||||||||||
3021 | OtherPtr->getName() + "."); | ||||||||||||
3022 | unsigned SrcAlign = OtherAlign; | ||||||||||||
3023 | Value *DstPtr = &NewAI; | ||||||||||||
3024 | unsigned DstAlign = SliceAlign; | ||||||||||||
3025 | if (!IsDest
| ||||||||||||
3026 | std::swap(SrcPtr, DstPtr); | ||||||||||||
3027 | std::swap(SrcAlign, DstAlign); | ||||||||||||
3028 | } | ||||||||||||
3029 | |||||||||||||
3030 | Value *Src; | ||||||||||||
3031 | if (VecTy && !IsWholeAlloca && !IsDest) { | ||||||||||||
3032 | Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
3033 | NewAI.getAlignment(), "load"); | ||||||||||||
3034 | Src = extractVector(IRB, Src, BeginIndex, EndIndex, "vec"); | ||||||||||||
3035 | } else if (IntTy && !IsWholeAlloca
| ||||||||||||
3036 | Src = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
3037 | NewAI.getAlignment(), "load"); | ||||||||||||
3038 | Src = convertValue(DL, IRB, Src, IntTy); | ||||||||||||
3039 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | ||||||||||||
3040 | Src = extractInteger(DL, IRB, Src, SubIntTy, Offset, "extract"); | ||||||||||||
3041 | } else { | ||||||||||||
3042 | LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign, | ||||||||||||
3043 | II.isVolatile(), "copyload"); | ||||||||||||
3044 | if (AATags) | ||||||||||||
3045 | Load->setAAMetadata(AATags); | ||||||||||||
3046 | Src = Load; | ||||||||||||
3047 | } | ||||||||||||
3048 | |||||||||||||
3049 | if (VecTy && !IsWholeAlloca && IsDest) { | ||||||||||||
3050 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
3051 | NewAI.getAlignment(), "oldload"); | ||||||||||||
3052 | Src = insertVector(IRB, Old, Src, BeginIndex, "vec"); | ||||||||||||
3053 | } else if (IntTy && !IsWholeAlloca && IsDest) { | ||||||||||||
3054 | Value *Old = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI, | ||||||||||||
3055 | NewAI.getAlignment(), "oldload"); | ||||||||||||
3056 | Old = convertValue(DL, IRB, Old, IntTy); | ||||||||||||
3057 | uint64_t Offset = NewBeginOffset - NewAllocaBeginOffset; | ||||||||||||
3058 | Src = insertInteger(DL, IRB, Old, Src, Offset, "insert"); | ||||||||||||
3059 | Src = convertValue(DL, IRB, Src, NewAllocaTy); | ||||||||||||
3060 | } | ||||||||||||
3061 | |||||||||||||
3062 | StoreInst *Store = cast<StoreInst>( | ||||||||||||
3063 | IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile())); | ||||||||||||
3064 | if (AATags) | ||||||||||||
3065 | Store->setAAMetadata(AATags); | ||||||||||||
3066 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | ||||||||||||
3067 | return !II.isVolatile(); | ||||||||||||
3068 | } | ||||||||||||
3069 | |||||||||||||
3070 | bool visitIntrinsicInst(IntrinsicInst &II) { | ||||||||||||
3071 | assert(II.isLifetimeStartOrEnd())((II.isLifetimeStartOrEnd()) ? static_cast<void> (0) : __assert_fail ("II.isLifetimeStartOrEnd()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3071, __PRETTY_FUNCTION__)); | ||||||||||||
3072 | LLVM_DEBUG(dbgs() << " original: " << II << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << II << "\n"; } } while (false); | ||||||||||||
3073 | assert(II.getArgOperand(1) == OldPtr)((II.getArgOperand(1) == OldPtr) ? static_cast<void> (0 ) : __assert_fail ("II.getArgOperand(1) == OldPtr", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3073, __PRETTY_FUNCTION__)); | ||||||||||||
3074 | |||||||||||||
3075 | // Record this instruction for deletion. | ||||||||||||
3076 | Pass.DeadInsts.insert(&II); | ||||||||||||
3077 | |||||||||||||
3078 | // Lifetime intrinsics are only promotable if they cover the whole alloca. | ||||||||||||
3079 | // Therefore, we drop lifetime intrinsics which don't cover the whole | ||||||||||||
3080 | // alloca. | ||||||||||||
3081 | // (In theory, intrinsics which partially cover an alloca could be | ||||||||||||
3082 | // promoted, but PromoteMemToReg doesn't handle that case.) | ||||||||||||
3083 | // FIXME: Check whether the alloca is promotable before dropping the | ||||||||||||
3084 | // lifetime intrinsics? | ||||||||||||
3085 | if (NewBeginOffset != NewAllocaBeginOffset || | ||||||||||||
3086 | NewEndOffset != NewAllocaEndOffset) | ||||||||||||
3087 | return true; | ||||||||||||
3088 | |||||||||||||
3089 | ConstantInt *Size = | ||||||||||||
3090 | ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()), | ||||||||||||
3091 | NewEndOffset - NewBeginOffset); | ||||||||||||
3092 | // Lifetime intrinsics always expect an i8* so directly get such a pointer | ||||||||||||
3093 | // for the new alloca slice. | ||||||||||||
3094 | Type *PointerTy = IRB.getInt8PtrTy(OldPtr->getType()->getPointerAddressSpace()); | ||||||||||||
3095 | Value *Ptr = getNewAllocaSlicePtr(IRB, PointerTy); | ||||||||||||
3096 | Value *New; | ||||||||||||
3097 | if (II.getIntrinsicID() == Intrinsic::lifetime_start) | ||||||||||||
3098 | New = IRB.CreateLifetimeStart(Ptr, Size); | ||||||||||||
3099 | else | ||||||||||||
3100 | New = IRB.CreateLifetimeEnd(Ptr, Size); | ||||||||||||
3101 | |||||||||||||
3102 | (void)New; | ||||||||||||
3103 | LLVM_DEBUG(dbgs() << " to: " << *New << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *New << "\n"; } } while (false); | ||||||||||||
3104 | |||||||||||||
3105 | return true; | ||||||||||||
3106 | } | ||||||||||||
3107 | |||||||||||||
3108 | void fixLoadStoreAlign(Instruction &Root) { | ||||||||||||
3109 | // This algorithm implements the same visitor loop as | ||||||||||||
3110 | // hasUnsafePHIOrSelectUse, and fixes the alignment of each load | ||||||||||||
3111 | // or store found. | ||||||||||||
3112 | SmallPtrSet<Instruction *, 4> Visited; | ||||||||||||
3113 | SmallVector<Instruction *, 4> Uses; | ||||||||||||
3114 | Visited.insert(&Root); | ||||||||||||
3115 | Uses.push_back(&Root); | ||||||||||||
3116 | do { | ||||||||||||
3117 | Instruction *I = Uses.pop_back_val(); | ||||||||||||
3118 | |||||||||||||
3119 | if (LoadInst *LI = dyn_cast<LoadInst>(I)) { | ||||||||||||
3120 | unsigned LoadAlign = LI->getAlignment(); | ||||||||||||
3121 | if (!LoadAlign) | ||||||||||||
3122 | LoadAlign = DL.getABITypeAlignment(LI->getType()); | ||||||||||||
3123 | LI->setAlignment(MaybeAlign(std::min(LoadAlign, getSliceAlign()))); | ||||||||||||
3124 | continue; | ||||||||||||
3125 | } | ||||||||||||
3126 | if (StoreInst *SI = dyn_cast<StoreInst>(I)) { | ||||||||||||
3127 | unsigned StoreAlign = SI->getAlignment(); | ||||||||||||
3128 | if (!StoreAlign) { | ||||||||||||
3129 | Value *Op = SI->getOperand(0); | ||||||||||||
3130 | StoreAlign = DL.getABITypeAlignment(Op->getType()); | ||||||||||||
3131 | } | ||||||||||||
3132 | SI->setAlignment(MaybeAlign(std::min(StoreAlign, getSliceAlign()))); | ||||||||||||
3133 | continue; | ||||||||||||
3134 | } | ||||||||||||
3135 | |||||||||||||
3136 | assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||((isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I ) || isa<PHINode>(I) || isa<SelectInst>(I) || isa <GetElementPtrInst>(I)) ? static_cast<void> (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3138, __PRETTY_FUNCTION__)) | ||||||||||||
3137 | isa<PHINode>(I) || isa<SelectInst>(I) ||((isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I ) || isa<PHINode>(I) || isa<SelectInst>(I) || isa <GetElementPtrInst>(I)) ? static_cast<void> (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3138, __PRETTY_FUNCTION__)) | ||||||||||||
3138 | isa<GetElementPtrInst>(I))((isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I ) || isa<PHINode>(I) || isa<SelectInst>(I) || isa <GetElementPtrInst>(I)) ? static_cast<void> (0) : __assert_fail ("isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) || isa<PHINode>(I) || isa<SelectInst>(I) || isa<GetElementPtrInst>(I)" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3138, __PRETTY_FUNCTION__)); | ||||||||||||
3139 | for (User *U : I->users()) | ||||||||||||
3140 | if (Visited.insert(cast<Instruction>(U)).second) | ||||||||||||
3141 | Uses.push_back(cast<Instruction>(U)); | ||||||||||||
3142 | } while (!Uses.empty()); | ||||||||||||
3143 | } | ||||||||||||
3144 | |||||||||||||
3145 | bool visitPHINode(PHINode &PN) { | ||||||||||||
3146 | LLVM_DEBUG(dbgs() << " original: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << PN << "\n"; } } while (false); | ||||||||||||
3147 | assert(BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable")((BeginOffset >= NewAllocaBeginOffset && "PHIs are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"PHIs are unsplittable\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3147, __PRETTY_FUNCTION__)); | ||||||||||||
3148 | assert(EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable")((EndOffset <= NewAllocaEndOffset && "PHIs are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"PHIs are unsplittable\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3148, __PRETTY_FUNCTION__)); | ||||||||||||
3149 | |||||||||||||
3150 | // We would like to compute a new pointer in only one place, but have it be | ||||||||||||
3151 | // as local as possible to the PHI. To do that, we re-use the location of | ||||||||||||
3152 | // the old pointer, which necessarily must be in the right position to | ||||||||||||
3153 | // dominate the PHI. | ||||||||||||
3154 | IRBuilderTy PtrBuilder(IRB); | ||||||||||||
3155 | if (isa<PHINode>(OldPtr)) | ||||||||||||
3156 | PtrBuilder.SetInsertPoint(&*OldPtr->getParent()->getFirstInsertionPt()); | ||||||||||||
3157 | else | ||||||||||||
3158 | PtrBuilder.SetInsertPoint(OldPtr); | ||||||||||||
3159 | PtrBuilder.SetCurrentDebugLocation(OldPtr->getDebugLoc()); | ||||||||||||
3160 | |||||||||||||
3161 | Value *NewPtr = getNewAllocaSlicePtr(PtrBuilder, OldPtr->getType()); | ||||||||||||
3162 | // Replace the operands which were using the old pointer. | ||||||||||||
3163 | std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr); | ||||||||||||
3164 | |||||||||||||
3165 | LLVM_DEBUG(dbgs() << " to: " << PN << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << PN << "\n"; } } while (false); | ||||||||||||
3166 | deleteIfTriviallyDead(OldPtr); | ||||||||||||
3167 | |||||||||||||
3168 | // Fix the alignment of any loads or stores using this PHI node. | ||||||||||||
3169 | fixLoadStoreAlign(PN); | ||||||||||||
3170 | |||||||||||||
3171 | // PHIs can't be promoted on their own, but often can be speculated. We | ||||||||||||
3172 | // check the speculation outside of the rewriter so that we see the | ||||||||||||
3173 | // fully-rewritten alloca. | ||||||||||||
3174 | PHIUsers.insert(&PN); | ||||||||||||
3175 | return true; | ||||||||||||
3176 | } | ||||||||||||
3177 | |||||||||||||
3178 | bool visitSelectInst(SelectInst &SI) { | ||||||||||||
3179 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | ||||||||||||
3180 | assert((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) &&(((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr ) && "Pointer isn't an operand!") ? static_cast<void > (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3181, __PRETTY_FUNCTION__)) | ||||||||||||
3181 | "Pointer isn't an operand!")(((SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr ) && "Pointer isn't an operand!") ? static_cast<void > (0) : __assert_fail ("(SI.getTrueValue() == OldPtr || SI.getFalseValue() == OldPtr) && \"Pointer isn't an operand!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3181, __PRETTY_FUNCTION__)); | ||||||||||||
3182 | assert(BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable")((BeginOffset >= NewAllocaBeginOffset && "Selects are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("BeginOffset >= NewAllocaBeginOffset && \"Selects are unsplittable\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3182, __PRETTY_FUNCTION__)); | ||||||||||||
3183 | assert(EndOffset <= NewAllocaEndOffset && "Selects are unsplittable")((EndOffset <= NewAllocaEndOffset && "Selects are unsplittable" ) ? static_cast<void> (0) : __assert_fail ("EndOffset <= NewAllocaEndOffset && \"Selects are unsplittable\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3183, __PRETTY_FUNCTION__)); | ||||||||||||
3184 | |||||||||||||
3185 | Value *NewPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType()); | ||||||||||||
3186 | // Replace the operands which were using the old pointer. | ||||||||||||
3187 | if (SI.getOperand(1) == OldPtr) | ||||||||||||
3188 | SI.setOperand(1, NewPtr); | ||||||||||||
3189 | if (SI.getOperand(2) == OldPtr) | ||||||||||||
3190 | SI.setOperand(2, NewPtr); | ||||||||||||
3191 | |||||||||||||
3192 | LLVM_DEBUG(dbgs() << " to: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << SI << "\n"; } } while (false); | ||||||||||||
3193 | deleteIfTriviallyDead(OldPtr); | ||||||||||||
3194 | |||||||||||||
3195 | // Fix the alignment of any loads or stores using this select. | ||||||||||||
3196 | fixLoadStoreAlign(SI); | ||||||||||||
3197 | |||||||||||||
3198 | // Selects can't be promoted on their own, but often can be speculated. We | ||||||||||||
3199 | // check the speculation outside of the rewriter so that we see the | ||||||||||||
3200 | // fully-rewritten alloca. | ||||||||||||
3201 | SelectUsers.insert(&SI); | ||||||||||||
3202 | return true; | ||||||||||||
3203 | } | ||||||||||||
3204 | }; | ||||||||||||
3205 | |||||||||||||
3206 | namespace { | ||||||||||||
3207 | |||||||||||||
3208 | /// Visitor to rewrite aggregate loads and stores as scalar. | ||||||||||||
3209 | /// | ||||||||||||
3210 | /// This pass aggressively rewrites all aggregate loads and stores on | ||||||||||||
3211 | /// a particular pointer (or any pointer derived from it which we can identify) | ||||||||||||
3212 | /// with scalar loads and stores. | ||||||||||||
3213 | class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> { | ||||||||||||
3214 | // Befriend the base class so it can delegate to private visit methods. | ||||||||||||
3215 | friend class InstVisitor<AggLoadStoreRewriter, bool>; | ||||||||||||
3216 | |||||||||||||
3217 | /// Queue of pointer uses to analyze and potentially rewrite. | ||||||||||||
3218 | SmallVector<Use *, 8> Queue; | ||||||||||||
3219 | |||||||||||||
3220 | /// Set to prevent us from cycling with phi nodes and loops. | ||||||||||||
3221 | SmallPtrSet<User *, 8> Visited; | ||||||||||||
3222 | |||||||||||||
3223 | /// The current pointer use being rewritten. This is used to dig up the used | ||||||||||||
3224 | /// value (as opposed to the user). | ||||||||||||
3225 | Use *U; | ||||||||||||
3226 | |||||||||||||
3227 | /// Used to calculate offsets, and hence alignment, of subobjects. | ||||||||||||
3228 | const DataLayout &DL; | ||||||||||||
3229 | |||||||||||||
3230 | public: | ||||||||||||
3231 | AggLoadStoreRewriter(const DataLayout &DL) : DL(DL) {} | ||||||||||||
3232 | |||||||||||||
3233 | /// Rewrite loads and stores through a pointer and all pointers derived from | ||||||||||||
3234 | /// it. | ||||||||||||
3235 | bool rewrite(Instruction &I) { | ||||||||||||
3236 | LLVM_DEBUG(dbgs() << " Rewriting FCA loads and stores...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Rewriting FCA loads and stores...\n" ; } } while (false); | ||||||||||||
3237 | enqueueUsers(I); | ||||||||||||
3238 | bool Changed = false; | ||||||||||||
3239 | while (!Queue.empty()) { | ||||||||||||
3240 | U = Queue.pop_back_val(); | ||||||||||||
3241 | Changed |= visit(cast<Instruction>(U->getUser())); | ||||||||||||
3242 | } | ||||||||||||
3243 | return Changed; | ||||||||||||
3244 | } | ||||||||||||
3245 | |||||||||||||
3246 | private: | ||||||||||||
3247 | /// Enqueue all the users of the given instruction for further processing. | ||||||||||||
3248 | /// This uses a set to de-duplicate users. | ||||||||||||
3249 | void enqueueUsers(Instruction &I) { | ||||||||||||
3250 | for (Use &U : I.uses()) | ||||||||||||
3251 | if (Visited.insert(U.getUser()).second) | ||||||||||||
3252 | Queue.push_back(&U); | ||||||||||||
3253 | } | ||||||||||||
3254 | |||||||||||||
3255 | // Conservative default is to not rewrite anything. | ||||||||||||
3256 | bool visitInstruction(Instruction &I) { return false; } | ||||||||||||
3257 | |||||||||||||
3258 | /// Generic recursive split emission class. | ||||||||||||
3259 | template <typename Derived> class OpSplitter { | ||||||||||||
3260 | protected: | ||||||||||||
3261 | /// The builder used to form new instructions. | ||||||||||||
3262 | IRBuilderTy IRB; | ||||||||||||
3263 | |||||||||||||
3264 | /// The indices which to be used with insert- or extractvalue to select the | ||||||||||||
3265 | /// appropriate value within the aggregate. | ||||||||||||
3266 | SmallVector<unsigned, 4> Indices; | ||||||||||||
3267 | |||||||||||||
3268 | /// The indices to a GEP instruction which will move Ptr to the correct slot | ||||||||||||
3269 | /// within the aggregate. | ||||||||||||
3270 | SmallVector<Value *, 4> GEPIndices; | ||||||||||||
3271 | |||||||||||||
3272 | /// The base pointer of the original op, used as a base for GEPing the | ||||||||||||
3273 | /// split operations. | ||||||||||||
3274 | Value *Ptr; | ||||||||||||
3275 | |||||||||||||
3276 | /// The base pointee type being GEPed into. | ||||||||||||
3277 | Type *BaseTy; | ||||||||||||
3278 | |||||||||||||
3279 | /// Known alignment of the base pointer. | ||||||||||||
3280 | unsigned BaseAlign; | ||||||||||||
3281 | |||||||||||||
3282 | /// To calculate offset of each component so we can correctly deduce | ||||||||||||
3283 | /// alignments. | ||||||||||||
3284 | const DataLayout &DL; | ||||||||||||
3285 | |||||||||||||
3286 | /// Initialize the splitter with an insertion point, Ptr and start with a | ||||||||||||
3287 | /// single zero GEP index. | ||||||||||||
3288 | OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | ||||||||||||
3289 | unsigned BaseAlign, const DataLayout &DL) | ||||||||||||
3290 | : IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr), | ||||||||||||
3291 | BaseTy(BaseTy), BaseAlign(BaseAlign), DL(DL) {} | ||||||||||||
3292 | |||||||||||||
3293 | public: | ||||||||||||
3294 | /// Generic recursive split emission routine. | ||||||||||||
3295 | /// | ||||||||||||
3296 | /// This method recursively splits an aggregate op (load or store) into | ||||||||||||
3297 | /// scalar or vector ops. It splits recursively until it hits a single value | ||||||||||||
3298 | /// and emits that single value operation via the template argument. | ||||||||||||
3299 | /// | ||||||||||||
3300 | /// The logic of this routine relies on GEPs and insertvalue and | ||||||||||||
3301 | /// extractvalue all operating with the same fundamental index list, merely | ||||||||||||
3302 | /// formatted differently (GEPs need actual values). | ||||||||||||
3303 | /// | ||||||||||||
3304 | /// \param Ty The type being split recursively into smaller ops. | ||||||||||||
3305 | /// \param Agg The aggregate value being built up or stored, depending on | ||||||||||||
3306 | /// whether this is splitting a load or a store respectively. | ||||||||||||
3307 | void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) { | ||||||||||||
3308 | if (Ty->isSingleValueType()) { | ||||||||||||
3309 | unsigned Offset = DL.getIndexedOffsetInType(BaseTy, GEPIndices); | ||||||||||||
3310 | return static_cast<Derived *>(this)->emitFunc( | ||||||||||||
3311 | Ty, Agg, MinAlign(BaseAlign, Offset), Name); | ||||||||||||
3312 | } | ||||||||||||
3313 | |||||||||||||
3314 | if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) { | ||||||||||||
3315 | unsigned OldSize = Indices.size(); | ||||||||||||
3316 | (void)OldSize; | ||||||||||||
3317 | for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size; | ||||||||||||
3318 | ++Idx) { | ||||||||||||
3319 | assert(Indices.size() == OldSize && "Did not return to the old size")((Indices.size() == OldSize && "Did not return to the old size" ) ? static_cast<void> (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3319, __PRETTY_FUNCTION__)); | ||||||||||||
3320 | Indices.push_back(Idx); | ||||||||||||
3321 | GEPIndices.push_back(IRB.getInt32(Idx)); | ||||||||||||
3322 | emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx)); | ||||||||||||
3323 | GEPIndices.pop_back(); | ||||||||||||
3324 | Indices.pop_back(); | ||||||||||||
3325 | } | ||||||||||||
3326 | return; | ||||||||||||
3327 | } | ||||||||||||
3328 | |||||||||||||
3329 | if (StructType *STy = dyn_cast<StructType>(Ty)) { | ||||||||||||
3330 | unsigned OldSize = Indices.size(); | ||||||||||||
3331 | (void)OldSize; | ||||||||||||
3332 | for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size; | ||||||||||||
3333 | ++Idx) { | ||||||||||||
3334 | assert(Indices.size() == OldSize && "Did not return to the old size")((Indices.size() == OldSize && "Did not return to the old size" ) ? static_cast<void> (0) : __assert_fail ("Indices.size() == OldSize && \"Did not return to the old size\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3334, __PRETTY_FUNCTION__)); | ||||||||||||
3335 | Indices.push_back(Idx); | ||||||||||||
3336 | GEPIndices.push_back(IRB.getInt32(Idx)); | ||||||||||||
3337 | emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx)); | ||||||||||||
3338 | GEPIndices.pop_back(); | ||||||||||||
3339 | Indices.pop_back(); | ||||||||||||
3340 | } | ||||||||||||
3341 | return; | ||||||||||||
3342 | } | ||||||||||||
3343 | |||||||||||||
3344 | llvm_unreachable("Only arrays and structs are aggregate loadable types")::llvm::llvm_unreachable_internal("Only arrays and structs are aggregate loadable types" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3344); | ||||||||||||
3345 | } | ||||||||||||
3346 | }; | ||||||||||||
3347 | |||||||||||||
3348 | struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> { | ||||||||||||
3349 | AAMDNodes AATags; | ||||||||||||
3350 | |||||||||||||
3351 | LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | ||||||||||||
3352 | AAMDNodes AATags, unsigned BaseAlign, const DataLayout &DL) | ||||||||||||
3353 | : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, | ||||||||||||
3354 | DL), AATags(AATags) {} | ||||||||||||
3355 | |||||||||||||
3356 | /// Emit a leaf load of a single value. This is called at the leaves of the | ||||||||||||
3357 | /// recursive emission to actually load values. | ||||||||||||
3358 | void emitFunc(Type *Ty, Value *&Agg, unsigned Align, const Twine &Name) { | ||||||||||||
3359 | assert(Ty->isSingleValueType())((Ty->isSingleValueType()) ? static_cast<void> (0) : __assert_fail ("Ty->isSingleValueType()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3359, __PRETTY_FUNCTION__)); | ||||||||||||
3360 | // Load the single value and insert it using the indices. | ||||||||||||
3361 | Value *GEP = | ||||||||||||
3362 | IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); | ||||||||||||
3363 | LoadInst *Load = IRB.CreateAlignedLoad(Ty, GEP, Align, Name + ".load"); | ||||||||||||
3364 | if (AATags) | ||||||||||||
3365 | Load->setAAMetadata(AATags); | ||||||||||||
3366 | Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert"); | ||||||||||||
3367 | LLVM_DEBUG(dbgs() << " to: " << *Load << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Load << "\n"; } } while (false); | ||||||||||||
3368 | } | ||||||||||||
3369 | }; | ||||||||||||
3370 | |||||||||||||
3371 | bool visitLoadInst(LoadInst &LI) { | ||||||||||||
3372 | assert(LI.getPointerOperand() == *U)((LI.getPointerOperand() == *U) ? static_cast<void> (0) : __assert_fail ("LI.getPointerOperand() == *U", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3372, __PRETTY_FUNCTION__)); | ||||||||||||
3373 | if (!LI.isSimple() || LI.getType()->isSingleValueType()) | ||||||||||||
3374 | return false; | ||||||||||||
3375 | |||||||||||||
3376 | // We have an aggregate being loaded, split it apart. | ||||||||||||
3377 | LLVM_DEBUG(dbgs() << " original: " << LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << LI << "\n"; } } while (false); | ||||||||||||
3378 | AAMDNodes AATags; | ||||||||||||
3379 | LI.getAAMetadata(AATags); | ||||||||||||
3380 | LoadOpSplitter Splitter(&LI, *U, LI.getType(), AATags, | ||||||||||||
3381 | getAdjustedAlignment(&LI, 0, DL), DL); | ||||||||||||
3382 | Value *V = UndefValue::get(LI.getType()); | ||||||||||||
3383 | Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca"); | ||||||||||||
3384 | LI.replaceAllUsesWith(V); | ||||||||||||
3385 | LI.eraseFromParent(); | ||||||||||||
3386 | return true; | ||||||||||||
3387 | } | ||||||||||||
3388 | |||||||||||||
3389 | struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> { | ||||||||||||
3390 | StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy, | ||||||||||||
3391 | AAMDNodes AATags, unsigned BaseAlign, const DataLayout &DL) | ||||||||||||
3392 | : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign, | ||||||||||||
3393 | DL), | ||||||||||||
3394 | AATags(AATags) {} | ||||||||||||
3395 | AAMDNodes AATags; | ||||||||||||
3396 | /// Emit a leaf store of a single value. This is called at the leaves of the | ||||||||||||
3397 | /// recursive emission to actually produce stores. | ||||||||||||
3398 | void emitFunc(Type *Ty, Value *&Agg, unsigned Align, const Twine &Name) { | ||||||||||||
3399 | assert(Ty->isSingleValueType())((Ty->isSingleValueType()) ? static_cast<void> (0) : __assert_fail ("Ty->isSingleValueType()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3399, __PRETTY_FUNCTION__)); | ||||||||||||
3400 | // Extract the single value and store it using the indices. | ||||||||||||
3401 | // | ||||||||||||
3402 | // The gep and extractvalue values are factored out of the CreateStore | ||||||||||||
3403 | // call to make the output independent of the argument evaluation order. | ||||||||||||
3404 | Value *ExtractValue = | ||||||||||||
3405 | IRB.CreateExtractValue(Agg, Indices, Name + ".extract"); | ||||||||||||
3406 | Value *InBoundsGEP = | ||||||||||||
3407 | IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep"); | ||||||||||||
3408 | StoreInst *Store = | ||||||||||||
3409 | IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Align); | ||||||||||||
3410 | if (AATags) | ||||||||||||
3411 | Store->setAAMetadata(AATags); | ||||||||||||
3412 | LLVM_DEBUG(dbgs() << " to: " << *Store << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " to: " << *Store << "\n"; } } while (false); | ||||||||||||
3413 | } | ||||||||||||
3414 | }; | ||||||||||||
3415 | |||||||||||||
3416 | bool visitStoreInst(StoreInst &SI) { | ||||||||||||
3417 | if (!SI.isSimple() || SI.getPointerOperand() != *U) | ||||||||||||
3418 | return false; | ||||||||||||
3419 | Value *V = SI.getValueOperand(); | ||||||||||||
3420 | if (V->getType()->isSingleValueType()) | ||||||||||||
3421 | return false; | ||||||||||||
3422 | |||||||||||||
3423 | // We have an aggregate being stored, split it apart. | ||||||||||||
3424 | LLVM_DEBUG(dbgs() << " original: " << SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " original: " << SI << "\n"; } } while (false); | ||||||||||||
3425 | AAMDNodes AATags; | ||||||||||||
3426 | SI.getAAMetadata(AATags); | ||||||||||||
3427 | StoreOpSplitter Splitter(&SI, *U, V->getType(), AATags, | ||||||||||||
3428 | getAdjustedAlignment(&SI, 0, DL), DL); | ||||||||||||
3429 | Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca"); | ||||||||||||
3430 | SI.eraseFromParent(); | ||||||||||||
3431 | return true; | ||||||||||||
3432 | } | ||||||||||||
3433 | |||||||||||||
3434 | bool visitBitCastInst(BitCastInst &BC) { | ||||||||||||
3435 | enqueueUsers(BC); | ||||||||||||
3436 | return false; | ||||||||||||
3437 | } | ||||||||||||
3438 | |||||||||||||
3439 | bool visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { | ||||||||||||
3440 | enqueueUsers(ASC); | ||||||||||||
3441 | return false; | ||||||||||||
3442 | } | ||||||||||||
3443 | |||||||||||||
3444 | bool visitGetElementPtrInst(GetElementPtrInst &GEPI) { | ||||||||||||
3445 | enqueueUsers(GEPI); | ||||||||||||
3446 | return false; | ||||||||||||
3447 | } | ||||||||||||
3448 | |||||||||||||
3449 | bool visitPHINode(PHINode &PN) { | ||||||||||||
3450 | enqueueUsers(PN); | ||||||||||||
3451 | return false; | ||||||||||||
3452 | } | ||||||||||||
3453 | |||||||||||||
3454 | bool visitSelectInst(SelectInst &SI) { | ||||||||||||
3455 | enqueueUsers(SI); | ||||||||||||
3456 | return false; | ||||||||||||
3457 | } | ||||||||||||
3458 | }; | ||||||||||||
3459 | |||||||||||||
3460 | } // end anonymous namespace | ||||||||||||
3461 | |||||||||||||
3462 | /// Strip aggregate type wrapping. | ||||||||||||
3463 | /// | ||||||||||||
3464 | /// This removes no-op aggregate types wrapping an underlying type. It will | ||||||||||||
3465 | /// strip as many layers of types as it can without changing either the type | ||||||||||||
3466 | /// size or the allocated size. | ||||||||||||
3467 | static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) { | ||||||||||||
3468 | if (Ty->isSingleValueType()) | ||||||||||||
3469 | return Ty; | ||||||||||||
3470 | |||||||||||||
3471 | uint64_t AllocSize = DL.getTypeAllocSize(Ty); | ||||||||||||
3472 | uint64_t TypeSize = DL.getTypeSizeInBits(Ty); | ||||||||||||
3473 | |||||||||||||
3474 | Type *InnerTy; | ||||||||||||
3475 | if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) { | ||||||||||||
3476 | InnerTy = ArrTy->getElementType(); | ||||||||||||
3477 | } else if (StructType *STy = dyn_cast<StructType>(Ty)) { | ||||||||||||
3478 | const StructLayout *SL = DL.getStructLayout(STy); | ||||||||||||
3479 | unsigned Index = SL->getElementContainingOffset(0); | ||||||||||||
3480 | InnerTy = STy->getElementType(Index); | ||||||||||||
3481 | } else { | ||||||||||||
3482 | return Ty; | ||||||||||||
3483 | } | ||||||||||||
3484 | |||||||||||||
3485 | if (AllocSize > DL.getTypeAllocSize(InnerTy) || | ||||||||||||
3486 | TypeSize > DL.getTypeSizeInBits(InnerTy)) | ||||||||||||
3487 | return Ty; | ||||||||||||
3488 | |||||||||||||
3489 | return stripAggregateTypeWrapping(DL, InnerTy); | ||||||||||||
3490 | } | ||||||||||||
3491 | |||||||||||||
3492 | /// Try to find a partition of the aggregate type passed in for a given | ||||||||||||
3493 | /// offset and size. | ||||||||||||
3494 | /// | ||||||||||||
3495 | /// This recurses through the aggregate type and tries to compute a subtype | ||||||||||||
3496 | /// based on the offset and size. When the offset and size span a sub-section | ||||||||||||
3497 | /// of an array, it will even compute a new array type for that sub-section, | ||||||||||||
3498 | /// and the same for structs. | ||||||||||||
3499 | /// | ||||||||||||
3500 | /// Note that this routine is very strict and tries to find a partition of the | ||||||||||||
3501 | /// type which produces the *exact* right offset and size. It is not forgiving | ||||||||||||
3502 | /// when the size or offset cause either end of type-based partition to be off. | ||||||||||||
3503 | /// Also, this is a best-effort routine. It is reasonable to give up and not | ||||||||||||
3504 | /// return a type if necessary. | ||||||||||||
3505 | static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset, | ||||||||||||
3506 | uint64_t Size) { | ||||||||||||
3507 | if (Offset == 0 && DL.getTypeAllocSize(Ty) == Size) | ||||||||||||
3508 | return stripAggregateTypeWrapping(DL, Ty); | ||||||||||||
3509 | if (Offset > DL.getTypeAllocSize(Ty) || | ||||||||||||
3510 | (DL.getTypeAllocSize(Ty) - Offset) < Size) | ||||||||||||
3511 | return nullptr; | ||||||||||||
3512 | |||||||||||||
3513 | if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) { | ||||||||||||
3514 | Type *ElementTy = SeqTy->getElementType(); | ||||||||||||
3515 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy); | ||||||||||||
3516 | uint64_t NumSkippedElements = Offset / ElementSize; | ||||||||||||
3517 | if (NumSkippedElements >= SeqTy->getNumElements()) | ||||||||||||
3518 | return nullptr; | ||||||||||||
3519 | Offset -= NumSkippedElements * ElementSize; | ||||||||||||
3520 | |||||||||||||
3521 | // First check if we need to recurse. | ||||||||||||
3522 | if (Offset > 0 || Size < ElementSize) { | ||||||||||||
3523 | // Bail if the partition ends in a different array element. | ||||||||||||
3524 | if ((Offset + Size) > ElementSize) | ||||||||||||
3525 | return nullptr; | ||||||||||||
3526 | // Recurse through the element type trying to peel off offset bytes. | ||||||||||||
3527 | return getTypePartition(DL, ElementTy, Offset, Size); | ||||||||||||
3528 | } | ||||||||||||
3529 | assert(Offset == 0)((Offset == 0) ? static_cast<void> (0) : __assert_fail ( "Offset == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3529, __PRETTY_FUNCTION__)); | ||||||||||||
3530 | |||||||||||||
3531 | if (Size == ElementSize) | ||||||||||||
3532 | return stripAggregateTypeWrapping(DL, ElementTy); | ||||||||||||
3533 | assert(Size > ElementSize)((Size > ElementSize) ? static_cast<void> (0) : __assert_fail ("Size > ElementSize", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3533, __PRETTY_FUNCTION__)); | ||||||||||||
3534 | uint64_t NumElements = Size / ElementSize; | ||||||||||||
3535 | if (NumElements * ElementSize != Size) | ||||||||||||
3536 | return nullptr; | ||||||||||||
3537 | return ArrayType::get(ElementTy, NumElements); | ||||||||||||
3538 | } | ||||||||||||
3539 | |||||||||||||
3540 | StructType *STy = dyn_cast<StructType>(Ty); | ||||||||||||
3541 | if (!STy) | ||||||||||||
3542 | return nullptr; | ||||||||||||
3543 | |||||||||||||
3544 | const StructLayout *SL = DL.getStructLayout(STy); | ||||||||||||
3545 | if (Offset >= SL->getSizeInBytes()) | ||||||||||||
3546 | return nullptr; | ||||||||||||
3547 | uint64_t EndOffset = Offset + Size; | ||||||||||||
3548 | if (EndOffset > SL->getSizeInBytes()) | ||||||||||||
3549 | return nullptr; | ||||||||||||
3550 | |||||||||||||
3551 | unsigned Index = SL->getElementContainingOffset(Offset); | ||||||||||||
3552 | Offset -= SL->getElementOffset(Index); | ||||||||||||
3553 | |||||||||||||
3554 | Type *ElementTy = STy->getElementType(Index); | ||||||||||||
3555 | uint64_t ElementSize = DL.getTypeAllocSize(ElementTy); | ||||||||||||
3556 | if (Offset >= ElementSize) | ||||||||||||
3557 | return nullptr; // The offset points into alignment padding. | ||||||||||||
3558 | |||||||||||||
3559 | // See if any partition must be contained by the element. | ||||||||||||
3560 | if (Offset > 0 || Size < ElementSize) { | ||||||||||||
3561 | if ((Offset + Size) > ElementSize) | ||||||||||||
3562 | return nullptr; | ||||||||||||
3563 | return getTypePartition(DL, ElementTy, Offset, Size); | ||||||||||||
3564 | } | ||||||||||||
3565 | assert(Offset == 0)((Offset == 0) ? static_cast<void> (0) : __assert_fail ( "Offset == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3565, __PRETTY_FUNCTION__)); | ||||||||||||
3566 | |||||||||||||
3567 | if (Size == ElementSize) | ||||||||||||
3568 | return stripAggregateTypeWrapping(DL, ElementTy); | ||||||||||||
3569 | |||||||||||||
3570 | StructType::element_iterator EI = STy->element_begin() + Index, | ||||||||||||
3571 | EE = STy->element_end(); | ||||||||||||
3572 | if (EndOffset < SL->getSizeInBytes()) { | ||||||||||||
3573 | unsigned EndIndex = SL->getElementContainingOffset(EndOffset); | ||||||||||||
3574 | if (Index == EndIndex) | ||||||||||||
3575 | return nullptr; // Within a single element and its padding. | ||||||||||||
3576 | |||||||||||||
3577 | // Don't try to form "natural" types if the elements don't line up with the | ||||||||||||
3578 | // expected size. | ||||||||||||
3579 | // FIXME: We could potentially recurse down through the last element in the | ||||||||||||
3580 | // sub-struct to find a natural end point. | ||||||||||||
3581 | if (SL->getElementOffset(EndIndex) != EndOffset) | ||||||||||||
3582 | return nullptr; | ||||||||||||
3583 | |||||||||||||
3584 | assert(Index < EndIndex)((Index < EndIndex) ? static_cast<void> (0) : __assert_fail ("Index < EndIndex", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3584, __PRETTY_FUNCTION__)); | ||||||||||||
3585 | EE = STy->element_begin() + EndIndex; | ||||||||||||
3586 | } | ||||||||||||
3587 | |||||||||||||
3588 | // Try to build up a sub-structure. | ||||||||||||
3589 | StructType *SubTy = | ||||||||||||
3590 | StructType::get(STy->getContext(), makeArrayRef(EI, EE), STy->isPacked()); | ||||||||||||
3591 | const StructLayout *SubSL = DL.getStructLayout(SubTy); | ||||||||||||
3592 | if (Size != SubSL->getSizeInBytes()) | ||||||||||||
3593 | return nullptr; // The sub-struct doesn't have quite the size needed. | ||||||||||||
3594 | |||||||||||||
3595 | return SubTy; | ||||||||||||
3596 | } | ||||||||||||
3597 | |||||||||||||
3598 | /// Pre-split loads and stores to simplify rewriting. | ||||||||||||
3599 | /// | ||||||||||||
3600 | /// We want to break up the splittable load+store pairs as much as | ||||||||||||
3601 | /// possible. This is important to do as a preprocessing step, as once we | ||||||||||||
3602 | /// start rewriting the accesses to partitions of the alloca we lose the | ||||||||||||
3603 | /// necessary information to correctly split apart paired loads and stores | ||||||||||||
3604 | /// which both point into this alloca. The case to consider is something like | ||||||||||||
3605 | /// the following: | ||||||||||||
3606 | /// | ||||||||||||
3607 | /// %a = alloca [12 x i8] | ||||||||||||
3608 | /// %gep1 = getelementptr [12 x i8]* %a, i32 0, i32 0 | ||||||||||||
3609 | /// %gep2 = getelementptr [12 x i8]* %a, i32 0, i32 4 | ||||||||||||
3610 | /// %gep3 = getelementptr [12 x i8]* %a, i32 0, i32 8 | ||||||||||||
3611 | /// %iptr1 = bitcast i8* %gep1 to i64* | ||||||||||||
3612 | /// %iptr2 = bitcast i8* %gep2 to i64* | ||||||||||||
3613 | /// %fptr1 = bitcast i8* %gep1 to float* | ||||||||||||
3614 | /// %fptr2 = bitcast i8* %gep2 to float* | ||||||||||||
3615 | /// %fptr3 = bitcast i8* %gep3 to float* | ||||||||||||
3616 | /// store float 0.0, float* %fptr1 | ||||||||||||
3617 | /// store float 1.0, float* %fptr2 | ||||||||||||
3618 | /// %v = load i64* %iptr1 | ||||||||||||
3619 | /// store i64 %v, i64* %iptr2 | ||||||||||||
3620 | /// %f1 = load float* %fptr2 | ||||||||||||
3621 | /// %f2 = load float* %fptr3 | ||||||||||||
3622 | /// | ||||||||||||
3623 | /// Here we want to form 3 partitions of the alloca, each 4 bytes large, and | ||||||||||||
3624 | /// promote everything so we recover the 2 SSA values that should have been | ||||||||||||
3625 | /// there all along. | ||||||||||||
3626 | /// | ||||||||||||
3627 | /// \returns true if any changes are made. | ||||||||||||
3628 | bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) { | ||||||||||||
3629 | LLVM_DEBUG(dbgs() << "Pre-splitting loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Pre-splitting loads and stores\n" ; } } while (false); | ||||||||||||
3630 | |||||||||||||
3631 | // Track the loads and stores which are candidates for pre-splitting here, in | ||||||||||||
3632 | // the order they first appear during the partition scan. These give stable | ||||||||||||
3633 | // iteration order and a basis for tracking which loads and stores we | ||||||||||||
3634 | // actually split. | ||||||||||||
3635 | SmallVector<LoadInst *, 4> Loads; | ||||||||||||
3636 | SmallVector<StoreInst *, 4> Stores; | ||||||||||||
3637 | |||||||||||||
3638 | // We need to accumulate the splits required of each load or store where we | ||||||||||||
3639 | // can find them via a direct lookup. This is important to cross-check loads | ||||||||||||
3640 | // and stores against each other. We also track the slice so that we can kill | ||||||||||||
3641 | // all the slices that end up split. | ||||||||||||
3642 | struct SplitOffsets { | ||||||||||||
3643 | Slice *S; | ||||||||||||
3644 | std::vector<uint64_t> Splits; | ||||||||||||
3645 | }; | ||||||||||||
3646 | SmallDenseMap<Instruction *, SplitOffsets, 8> SplitOffsetsMap; | ||||||||||||
3647 | |||||||||||||
3648 | // Track loads out of this alloca which cannot, for any reason, be pre-split. | ||||||||||||
3649 | // This is important as we also cannot pre-split stores of those loads! | ||||||||||||
3650 | // FIXME: This is all pretty gross. It means that we can be more aggressive | ||||||||||||
3651 | // in pre-splitting when the load feeding the store happens to come from | ||||||||||||
3652 | // a separate alloca. Put another way, the effectiveness of SROA would be | ||||||||||||
3653 | // decreased by a frontend which just concatenated all of its local allocas | ||||||||||||
3654 | // into one big flat alloca. But defeating such patterns is exactly the job | ||||||||||||
3655 | // SROA is tasked with! Sadly, to not have this discrepancy we would have | ||||||||||||
3656 | // change store pre-splitting to actually force pre-splitting of the load | ||||||||||||
3657 | // that feeds it *and all stores*. That makes pre-splitting much harder, but | ||||||||||||
3658 | // maybe it would make it more principled? | ||||||||||||
3659 | SmallPtrSet<LoadInst *, 8> UnsplittableLoads; | ||||||||||||
3660 | |||||||||||||
3661 | LLVM_DEBUG(dbgs() << " Searching for candidate loads and stores\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Searching for candidate loads and stores\n" ; } } while (false); | ||||||||||||
3662 | for (auto &P : AS.partitions()) { | ||||||||||||
3663 | for (Slice &S : P) { | ||||||||||||
3664 | Instruction *I = cast<Instruction>(S.getUse()->getUser()); | ||||||||||||
3665 | if (!S.isSplittable() || S.endOffset() <= P.endOffset()) { | ||||||||||||
3666 | // If this is a load we have to track that it can't participate in any | ||||||||||||
3667 | // pre-splitting. If this is a store of a load we have to track that | ||||||||||||
3668 | // that load also can't participate in any pre-splitting. | ||||||||||||
3669 | if (auto *LI = dyn_cast<LoadInst>(I)) | ||||||||||||
3670 | UnsplittableLoads.insert(LI); | ||||||||||||
3671 | else if (auto *SI = dyn_cast<StoreInst>(I)) | ||||||||||||
3672 | if (auto *LI = dyn_cast<LoadInst>(SI->getValueOperand())) | ||||||||||||
3673 | UnsplittableLoads.insert(LI); | ||||||||||||
3674 | continue; | ||||||||||||
3675 | } | ||||||||||||
3676 | assert(P.endOffset() > S.beginOffset() &&((P.endOffset() > S.beginOffset() && "Empty or backwards partition!" ) ? static_cast<void> (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3677, __PRETTY_FUNCTION__)) | ||||||||||||
3677 | "Empty or backwards partition!")((P.endOffset() > S.beginOffset() && "Empty or backwards partition!" ) ? static_cast<void> (0) : __assert_fail ("P.endOffset() > S.beginOffset() && \"Empty or backwards partition!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3677, __PRETTY_FUNCTION__)); | ||||||||||||
3678 | |||||||||||||
3679 | // Determine if this is a pre-splittable slice. | ||||||||||||
3680 | if (auto *LI = dyn_cast<LoadInst>(I)) { | ||||||||||||
3681 | assert(!LI->isVolatile() && "Cannot split volatile loads!")((!LI->isVolatile() && "Cannot split volatile loads!" ) ? static_cast<void> (0) : __assert_fail ("!LI->isVolatile() && \"Cannot split volatile loads!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3681, __PRETTY_FUNCTION__)); | ||||||||||||
3682 | |||||||||||||
3683 | // The load must be used exclusively to store into other pointers for | ||||||||||||
3684 | // us to be able to arbitrarily pre-split it. The stores must also be | ||||||||||||
3685 | // simple to avoid changing semantics. | ||||||||||||
3686 | auto IsLoadSimplyStored = [](LoadInst *LI) { | ||||||||||||
3687 | for (User *LU : LI->users()) { | ||||||||||||
3688 | auto *SI = dyn_cast<StoreInst>(LU); | ||||||||||||
3689 | if (!SI || !SI->isSimple()) | ||||||||||||
3690 | return false; | ||||||||||||
3691 | } | ||||||||||||
3692 | return true; | ||||||||||||
3693 | }; | ||||||||||||
3694 | if (!IsLoadSimplyStored(LI)) { | ||||||||||||
3695 | UnsplittableLoads.insert(LI); | ||||||||||||
3696 | continue; | ||||||||||||
3697 | } | ||||||||||||
3698 | |||||||||||||
3699 | Loads.push_back(LI); | ||||||||||||
3700 | } else if (auto *SI = dyn_cast<StoreInst>(I)) { | ||||||||||||
3701 | if (S.getUse() != &SI->getOperandUse(SI->getPointerOperandIndex())) | ||||||||||||
3702 | // Skip stores *of* pointers. FIXME: This shouldn't even be possible! | ||||||||||||
3703 | continue; | ||||||||||||
3704 | auto *StoredLoad = dyn_cast<LoadInst>(SI->getValueOperand()); | ||||||||||||
3705 | if (!StoredLoad || !StoredLoad->isSimple()) | ||||||||||||
3706 | continue; | ||||||||||||
3707 | assert(!SI->isVolatile() && "Cannot split volatile stores!")((!SI->isVolatile() && "Cannot split volatile stores!" ) ? static_cast<void> (0) : __assert_fail ("!SI->isVolatile() && \"Cannot split volatile stores!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3707, __PRETTY_FUNCTION__)); | ||||||||||||
3708 | |||||||||||||
3709 | Stores.push_back(SI); | ||||||||||||
3710 | } else { | ||||||||||||
3711 | // Other uses cannot be pre-split. | ||||||||||||
3712 | continue; | ||||||||||||
3713 | } | ||||||||||||
3714 | |||||||||||||
3715 | // Record the initial split. | ||||||||||||
3716 | LLVM_DEBUG(dbgs() << " Candidate: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Candidate: " << *I << "\n"; } } while (false); | ||||||||||||
3717 | auto &Offsets = SplitOffsetsMap[I]; | ||||||||||||
3718 | assert(Offsets.Splits.empty() &&((Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3719, __PRETTY_FUNCTION__)) | ||||||||||||
3719 | "Should not have splits the first time we see an instruction!")((Offsets.Splits.empty() && "Should not have splits the first time we see an instruction!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.empty() && \"Should not have splits the first time we see an instruction!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3719, __PRETTY_FUNCTION__)); | ||||||||||||
3720 | Offsets.S = &S; | ||||||||||||
3721 | Offsets.Splits.push_back(P.endOffset() - S.beginOffset()); | ||||||||||||
3722 | } | ||||||||||||
3723 | |||||||||||||
3724 | // Now scan the already split slices, and add a split for any of them which | ||||||||||||
3725 | // we're going to pre-split. | ||||||||||||
3726 | for (Slice *S : P.splitSliceTails()) { | ||||||||||||
3727 | auto SplitOffsetsMapI = | ||||||||||||
3728 | SplitOffsetsMap.find(cast<Instruction>(S->getUse()->getUser())); | ||||||||||||
3729 | if (SplitOffsetsMapI == SplitOffsetsMap.end()) | ||||||||||||
3730 | continue; | ||||||||||||
3731 | auto &Offsets = SplitOffsetsMapI->second; | ||||||||||||
3732 | |||||||||||||
3733 | assert(Offsets.S == S && "Found a mismatched slice!")((Offsets.S == S && "Found a mismatched slice!") ? static_cast <void> (0) : __assert_fail ("Offsets.S == S && \"Found a mismatched slice!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3733, __PRETTY_FUNCTION__)); | ||||||||||||
3734 | assert(!Offsets.Splits.empty() &&((!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? static_cast<void> (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3735, __PRETTY_FUNCTION__)) | ||||||||||||
3735 | "Cannot have an empty set of splits on the second partition!")((!Offsets.Splits.empty() && "Cannot have an empty set of splits on the second partition!" ) ? static_cast<void> (0) : __assert_fail ("!Offsets.Splits.empty() && \"Cannot have an empty set of splits on the second partition!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3735, __PRETTY_FUNCTION__)); | ||||||||||||
3736 | assert(Offsets.Splits.back() ==((Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset () && "Previous split does not end where this one begins!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3738, __PRETTY_FUNCTION__)) | ||||||||||||
3737 | P.beginOffset() - Offsets.S->beginOffset() &&((Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset () && "Previous split does not end where this one begins!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3738, __PRETTY_FUNCTION__)) | ||||||||||||
3738 | "Previous split does not end where this one begins!")((Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset () && "Previous split does not end where this one begins!" ) ? static_cast<void> (0) : __assert_fail ("Offsets.Splits.back() == P.beginOffset() - Offsets.S->beginOffset() && \"Previous split does not end where this one begins!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3738, __PRETTY_FUNCTION__)); | ||||||||||||
3739 | |||||||||||||
3740 | // Record each split. The last partition's end isn't needed as the size | ||||||||||||
3741 | // of the slice dictates that. | ||||||||||||
3742 | if (S->endOffset() > P.endOffset()) | ||||||||||||
3743 | Offsets.Splits.push_back(P.endOffset() - Offsets.S->beginOffset()); | ||||||||||||
3744 | } | ||||||||||||
3745 | } | ||||||||||||
3746 | |||||||||||||
3747 | // We may have split loads where some of their stores are split stores. For | ||||||||||||
3748 | // such loads and stores, we can only pre-split them if their splits exactly | ||||||||||||
3749 | // match relative to their starting offset. We have to verify this prior to | ||||||||||||
3750 | // any rewriting. | ||||||||||||
3751 | Stores.erase( | ||||||||||||
3752 | llvm::remove_if(Stores, | ||||||||||||
3753 | [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) { | ||||||||||||
3754 | // Lookup the load we are storing in our map of split | ||||||||||||
3755 | // offsets. | ||||||||||||
3756 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | ||||||||||||
3757 | // If it was completely unsplittable, then we're done, | ||||||||||||
3758 | // and this store can't be pre-split. | ||||||||||||
3759 | if (UnsplittableLoads.count(LI)) | ||||||||||||
3760 | return true; | ||||||||||||
3761 | |||||||||||||
3762 | auto LoadOffsetsI = SplitOffsetsMap.find(LI); | ||||||||||||
3763 | if (LoadOffsetsI == SplitOffsetsMap.end()) | ||||||||||||
3764 | return false; // Unrelated loads are definitely safe. | ||||||||||||
3765 | auto &LoadOffsets = LoadOffsetsI->second; | ||||||||||||
3766 | |||||||||||||
3767 | // Now lookup the store's offsets. | ||||||||||||
3768 | auto &StoreOffsets = SplitOffsetsMap[SI]; | ||||||||||||
3769 | |||||||||||||
3770 | // If the relative offsets of each split in the load and | ||||||||||||
3771 | // store match exactly, then we can split them and we | ||||||||||||
3772 | // don't need to remove them here. | ||||||||||||
3773 | if (LoadOffsets.Splits == StoreOffsets.Splits) | ||||||||||||
3774 | return false; | ||||||||||||
3775 | |||||||||||||
3776 | LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | ||||||||||||
3777 | dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | ||||||||||||
3778 | << " Mismatched splits for load and store:\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | ||||||||||||
3779 | << " " << *LI << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false) | ||||||||||||
3780 | << " " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Mismatched splits for load and store:\n" << " " << *LI << "\n" << " " << *SI << "\n"; } } while (false); | ||||||||||||
3781 | |||||||||||||
3782 | // We've found a store and load that we need to split | ||||||||||||
3783 | // with mismatched relative splits. Just give up on them | ||||||||||||
3784 | // and remove both instructions from our list of | ||||||||||||
3785 | // candidates. | ||||||||||||
3786 | UnsplittableLoads.insert(LI); | ||||||||||||
3787 | return true; | ||||||||||||
3788 | }), | ||||||||||||
3789 | Stores.end()); | ||||||||||||
3790 | // Now we have to go *back* through all the stores, because a later store may | ||||||||||||
3791 | // have caused an earlier store's load to become unsplittable and if it is | ||||||||||||
3792 | // unsplittable for the later store, then we can't rely on it being split in | ||||||||||||
3793 | // the earlier store either. | ||||||||||||
3794 | Stores.erase(llvm::remove_if(Stores, | ||||||||||||
3795 | [&UnsplittableLoads](StoreInst *SI) { | ||||||||||||
3796 | auto *LI = | ||||||||||||
3797 | cast<LoadInst>(SI->getValueOperand()); | ||||||||||||
3798 | return UnsplittableLoads.count(LI); | ||||||||||||
3799 | }), | ||||||||||||
3800 | Stores.end()); | ||||||||||||
3801 | // Once we've established all the loads that can't be split for some reason, | ||||||||||||
3802 | // filter any that made it into our list out. | ||||||||||||
3803 | Loads.erase(llvm::remove_if(Loads, | ||||||||||||
3804 | [&UnsplittableLoads](LoadInst *LI) { | ||||||||||||
3805 | return UnsplittableLoads.count(LI); | ||||||||||||
3806 | }), | ||||||||||||
3807 | Loads.end()); | ||||||||||||
3808 | |||||||||||||
3809 | // If no loads or stores are left, there is no pre-splitting to be done for | ||||||||||||
3810 | // this alloca. | ||||||||||||
3811 | if (Loads.empty() && Stores.empty()) | ||||||||||||
3812 | return false; | ||||||||||||
3813 | |||||||||||||
3814 | // From here on, we can't fail and will be building new accesses, so rig up | ||||||||||||
3815 | // an IR builder. | ||||||||||||
3816 | IRBuilderTy IRB(&AI); | ||||||||||||
3817 | |||||||||||||
3818 | // Collect the new slices which we will merge into the alloca slices. | ||||||||||||
3819 | SmallVector<Slice, 4> NewSlices; | ||||||||||||
3820 | |||||||||||||
3821 | // Track any allocas we end up splitting loads and stores for so we iterate | ||||||||||||
3822 | // on them. | ||||||||||||
3823 | SmallPtrSet<AllocaInst *, 4> ResplitPromotableAllocas; | ||||||||||||
3824 | |||||||||||||
3825 | // At this point, we have collected all of the loads and stores we can | ||||||||||||
3826 | // pre-split, and the specific splits needed for them. We actually do the | ||||||||||||
3827 | // splitting in a specific order in order to handle when one of the loads in | ||||||||||||
3828 | // the value operand to one of the stores. | ||||||||||||
3829 | // | ||||||||||||
3830 | // First, we rewrite all of the split loads, and just accumulate each split | ||||||||||||
3831 | // load in a parallel structure. We also build the slices for them and append | ||||||||||||
3832 | // them to the alloca slices. | ||||||||||||
3833 | SmallDenseMap<LoadInst *, std::vector<LoadInst *>, 1> SplitLoadsMap; | ||||||||||||
3834 | std::vector<LoadInst *> SplitLoads; | ||||||||||||
3835 | const DataLayout &DL = AI.getModule()->getDataLayout(); | ||||||||||||
3836 | for (LoadInst *LI : Loads) { | ||||||||||||
3837 | SplitLoads.clear(); | ||||||||||||
3838 | |||||||||||||
3839 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | ||||||||||||
3840 | uint64_t LoadSize = Ty->getBitWidth() / 8; | ||||||||||||
3841 | assert(LoadSize > 0 && "Cannot have a zero-sized integer load!")((LoadSize > 0 && "Cannot have a zero-sized integer load!" ) ? static_cast<void> (0) : __assert_fail ("LoadSize > 0 && \"Cannot have a zero-sized integer load!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3841, __PRETTY_FUNCTION__)); | ||||||||||||
3842 | |||||||||||||
3843 | auto &Offsets = SplitOffsetsMap[LI]; | ||||||||||||
3844 | assert(LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&((LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3845, __PRETTY_FUNCTION__)) | ||||||||||||
3845 | "Slice size should always match load size exactly!")((LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("LoadSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3845, __PRETTY_FUNCTION__)); | ||||||||||||
3846 | uint64_t BaseOffset = Offsets.S->beginOffset(); | ||||||||||||
3847 | assert(BaseOffset + LoadSize > BaseOffset &&((BaseOffset + LoadSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + LoadSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3848, __PRETTY_FUNCTION__)) | ||||||||||||
3848 | "Cannot represent alloca access size using 64-bit integers!")((BaseOffset + LoadSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + LoadSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3848, __PRETTY_FUNCTION__)); | ||||||||||||
3849 | |||||||||||||
3850 | Instruction *BasePtr = cast<Instruction>(LI->getPointerOperand()); | ||||||||||||
3851 | IRB.SetInsertPoint(LI); | ||||||||||||
3852 | |||||||||||||
3853 | LLVM_DEBUG(dbgs() << " Splitting load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting load: " << *LI << "\n"; } } while (false); | ||||||||||||
3854 | |||||||||||||
3855 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | ||||||||||||
3856 | int Idx = 0, Size = Offsets.Splits.size(); | ||||||||||||
3857 | for (;;) { | ||||||||||||
3858 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | ||||||||||||
3859 | auto AS = LI->getPointerAddressSpace(); | ||||||||||||
3860 | auto *PartPtrTy = PartTy->getPointerTo(AS); | ||||||||||||
3861 | LoadInst *PLoad = IRB.CreateAlignedLoad( | ||||||||||||
3862 | PartTy, | ||||||||||||
3863 | getAdjustedPtr(IRB, DL, BasePtr, | ||||||||||||
3864 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | ||||||||||||
3865 | PartPtrTy, BasePtr->getName() + "."), | ||||||||||||
3866 | getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false, | ||||||||||||
3867 | LI->getName()); | ||||||||||||
3868 | PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access, | ||||||||||||
3869 | LLVMContext::MD_access_group}); | ||||||||||||
3870 | |||||||||||||
3871 | // Append this load onto the list of split loads so we can find it later | ||||||||||||
3872 | // to rewrite the stores. | ||||||||||||
3873 | SplitLoads.push_back(PLoad); | ||||||||||||
3874 | |||||||||||||
3875 | // Now build a new slice for the alloca. | ||||||||||||
3876 | NewSlices.push_back( | ||||||||||||
3877 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | ||||||||||||
3878 | &PLoad->getOperandUse(PLoad->getPointerOperandIndex()), | ||||||||||||
3879 | /*IsSplittable*/ false)); | ||||||||||||
3880 | LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | ||||||||||||
3881 | << ", " << NewSlices.back().endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false) | ||||||||||||
3882 | << "): " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PLoad << "\n"; } } while (false); | ||||||||||||
3883 | |||||||||||||
3884 | // See if we've handled all the splits. | ||||||||||||
3885 | if (Idx >= Size) | ||||||||||||
3886 | break; | ||||||||||||
3887 | |||||||||||||
3888 | // Setup the next partition. | ||||||||||||
3889 | PartOffset = Offsets.Splits[Idx]; | ||||||||||||
3890 | ++Idx; | ||||||||||||
3891 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : LoadSize) - PartOffset; | ||||||||||||
3892 | } | ||||||||||||
3893 | |||||||||||||
3894 | // Now that we have the split loads, do the slow walk over all uses of the | ||||||||||||
3895 | // load and rewrite them as split stores, or save the split loads to use | ||||||||||||
3896 | // below if the store is going to be split there anyways. | ||||||||||||
3897 | bool DeferredStores = false; | ||||||||||||
3898 | for (User *LU : LI->users()) { | ||||||||||||
3899 | StoreInst *SI = cast<StoreInst>(LU); | ||||||||||||
3900 | if (!Stores.empty() && SplitOffsetsMap.count(SI)) { | ||||||||||||
3901 | DeferredStores = true; | ||||||||||||
3902 | LLVM_DEBUG(dbgs() << " Deferred splitting of store: " << *SIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false) | ||||||||||||
3903 | << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Deferred splitting of store: " << *SI << "\n"; } } while (false); | ||||||||||||
3904 | continue; | ||||||||||||
3905 | } | ||||||||||||
3906 | |||||||||||||
3907 | Value *StoreBasePtr = SI->getPointerOperand(); | ||||||||||||
3908 | IRB.SetInsertPoint(SI); | ||||||||||||
3909 | |||||||||||||
3910 | LLVM_DEBUG(dbgs() << " Splitting store of load: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store of load: " << *SI << "\n"; } } while (false); | ||||||||||||
3911 | |||||||||||||
3912 | for (int Idx = 0, Size = SplitLoads.size(); Idx < Size; ++Idx) { | ||||||||||||
3913 | LoadInst *PLoad = SplitLoads[Idx]; | ||||||||||||
3914 | uint64_t PartOffset = Idx == 0 ? 0 : Offsets.Splits[Idx - 1]; | ||||||||||||
3915 | auto *PartPtrTy = | ||||||||||||
3916 | PLoad->getType()->getPointerTo(SI->getPointerAddressSpace()); | ||||||||||||
3917 | |||||||||||||
3918 | auto AS = SI->getPointerAddressSpace(); | ||||||||||||
3919 | StoreInst *PStore = IRB.CreateAlignedStore( | ||||||||||||
3920 | PLoad, | ||||||||||||
3921 | getAdjustedPtr(IRB, DL, StoreBasePtr, | ||||||||||||
3922 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | ||||||||||||
3923 | PartPtrTy, StoreBasePtr->getName() + "."), | ||||||||||||
3924 | getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false); | ||||||||||||
3925 | PStore->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access, | ||||||||||||
3926 | LLVMContext::MD_access_group}); | ||||||||||||
3927 | LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " +" << PartOffset << ":" << *PStore << "\n"; } } while (false); | ||||||||||||
3928 | } | ||||||||||||
3929 | |||||||||||||
3930 | // We want to immediately iterate on any allocas impacted by splitting | ||||||||||||
3931 | // this store, and we have to track any promotable alloca (indicated by | ||||||||||||
3932 | // a direct store) as needing to be resplit because it is no longer | ||||||||||||
3933 | // promotable. | ||||||||||||
3934 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(StoreBasePtr)) { | ||||||||||||
3935 | ResplitPromotableAllocas.insert(OtherAI); | ||||||||||||
3936 | Worklist.insert(OtherAI); | ||||||||||||
3937 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | ||||||||||||
3938 | StoreBasePtr->stripInBoundsOffsets())) { | ||||||||||||
3939 | Worklist.insert(OtherAI); | ||||||||||||
3940 | } | ||||||||||||
3941 | |||||||||||||
3942 | // Mark the original store as dead. | ||||||||||||
3943 | DeadInsts.insert(SI); | ||||||||||||
3944 | } | ||||||||||||
3945 | |||||||||||||
3946 | // Save the split loads if there are deferred stores among the users. | ||||||||||||
3947 | if (DeferredStores) | ||||||||||||
3948 | SplitLoadsMap.insert(std::make_pair(LI, std::move(SplitLoads))); | ||||||||||||
3949 | |||||||||||||
3950 | // Mark the original load as dead and kill the original slice. | ||||||||||||
3951 | DeadInsts.insert(LI); | ||||||||||||
3952 | Offsets.S->kill(); | ||||||||||||
3953 | } | ||||||||||||
3954 | |||||||||||||
3955 | // Second, we rewrite all of the split stores. At this point, we know that | ||||||||||||
3956 | // all loads from this alloca have been split already. For stores of such | ||||||||||||
3957 | // loads, we can simply look up the pre-existing split loads. For stores of | ||||||||||||
3958 | // other loads, we split those loads first and then write split stores of | ||||||||||||
3959 | // them. | ||||||||||||
3960 | for (StoreInst *SI : Stores) { | ||||||||||||
3961 | auto *LI = cast<LoadInst>(SI->getValueOperand()); | ||||||||||||
3962 | IntegerType *Ty = cast<IntegerType>(LI->getType()); | ||||||||||||
3963 | uint64_t StoreSize = Ty->getBitWidth() / 8; | ||||||||||||
3964 | assert(StoreSize > 0 && "Cannot have a zero-sized integer store!")((StoreSize > 0 && "Cannot have a zero-sized integer store!" ) ? static_cast<void> (0) : __assert_fail ("StoreSize > 0 && \"Cannot have a zero-sized integer store!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3964, __PRETTY_FUNCTION__)); | ||||||||||||
3965 | |||||||||||||
3966 | auto &Offsets = SplitOffsetsMap[SI]; | ||||||||||||
3967 | assert(StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() &&((StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3968, __PRETTY_FUNCTION__)) | ||||||||||||
3968 | "Slice size should always match load size exactly!")((StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset () && "Slice size should always match load size exactly!" ) ? static_cast<void> (0) : __assert_fail ("StoreSize == Offsets.S->endOffset() - Offsets.S->beginOffset() && \"Slice size should always match load size exactly!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3968, __PRETTY_FUNCTION__)); | ||||||||||||
3969 | uint64_t BaseOffset = Offsets.S->beginOffset(); | ||||||||||||
3970 | assert(BaseOffset + StoreSize > BaseOffset &&((BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3971, __PRETTY_FUNCTION__)) | ||||||||||||
3971 | "Cannot represent alloca access size using 64-bit integers!")((BaseOffset + StoreSize > BaseOffset && "Cannot represent alloca access size using 64-bit integers!" ) ? static_cast<void> (0) : __assert_fail ("BaseOffset + StoreSize > BaseOffset && \"Cannot represent alloca access size using 64-bit integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3971, __PRETTY_FUNCTION__)); | ||||||||||||
3972 | |||||||||||||
3973 | Value *LoadBasePtr = LI->getPointerOperand(); | ||||||||||||
3974 | Instruction *StoreBasePtr = cast<Instruction>(SI->getPointerOperand()); | ||||||||||||
3975 | |||||||||||||
3976 | LLVM_DEBUG(dbgs() << " Splitting store: " << *SI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Splitting store: " << *SI << "\n"; } } while (false); | ||||||||||||
3977 | |||||||||||||
3978 | // Check whether we have an already split load. | ||||||||||||
3979 | auto SplitLoadsMapI = SplitLoadsMap.find(LI); | ||||||||||||
3980 | std::vector<LoadInst *> *SplitLoads = nullptr; | ||||||||||||
3981 | if (SplitLoadsMapI != SplitLoadsMap.end()) { | ||||||||||||
3982 | SplitLoads = &SplitLoadsMapI->second; | ||||||||||||
3983 | assert(SplitLoads->size() == Offsets.Splits.size() + 1 &&((SplitLoads->size() == Offsets.Splits.size() + 1 && "Too few split loads for the number of splits in the store!" ) ? static_cast<void> (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3984, __PRETTY_FUNCTION__)) | ||||||||||||
3984 | "Too few split loads for the number of splits in the store!")((SplitLoads->size() == Offsets.Splits.size() + 1 && "Too few split loads for the number of splits in the store!" ) ? static_cast<void> (0) : __assert_fail ("SplitLoads->size() == Offsets.Splits.size() + 1 && \"Too few split loads for the number of splits in the store!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 3984, __PRETTY_FUNCTION__)); | ||||||||||||
3985 | } else { | ||||||||||||
3986 | LLVM_DEBUG(dbgs() << " of load: " << *LI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of load: " << *LI << "\n"; } } while (false); | ||||||||||||
3987 | } | ||||||||||||
3988 | |||||||||||||
3989 | uint64_t PartOffset = 0, PartSize = Offsets.Splits.front(); | ||||||||||||
3990 | int Idx = 0, Size = Offsets.Splits.size(); | ||||||||||||
3991 | for (;;) { | ||||||||||||
3992 | auto *PartTy = Type::getIntNTy(Ty->getContext(), PartSize * 8); | ||||||||||||
3993 | auto *LoadPartPtrTy = PartTy->getPointerTo(LI->getPointerAddressSpace()); | ||||||||||||
3994 | auto *StorePartPtrTy = PartTy->getPointerTo(SI->getPointerAddressSpace()); | ||||||||||||
3995 | |||||||||||||
3996 | // Either lookup a split load or create one. | ||||||||||||
3997 | LoadInst *PLoad; | ||||||||||||
3998 | if (SplitLoads) { | ||||||||||||
3999 | PLoad = (*SplitLoads)[Idx]; | ||||||||||||
4000 | } else { | ||||||||||||
4001 | IRB.SetInsertPoint(LI); | ||||||||||||
4002 | auto AS = LI->getPointerAddressSpace(); | ||||||||||||
4003 | PLoad = IRB.CreateAlignedLoad( | ||||||||||||
4004 | PartTy, | ||||||||||||
4005 | getAdjustedPtr(IRB, DL, LoadBasePtr, | ||||||||||||
4006 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | ||||||||||||
4007 | LoadPartPtrTy, LoadBasePtr->getName() + "."), | ||||||||||||
4008 | getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false, | ||||||||||||
4009 | LI->getName()); | ||||||||||||
4010 | } | ||||||||||||
4011 | |||||||||||||
4012 | // And store this partition. | ||||||||||||
4013 | IRB.SetInsertPoint(SI); | ||||||||||||
4014 | auto AS = SI->getPointerAddressSpace(); | ||||||||||||
4015 | StoreInst *PStore = IRB.CreateAlignedStore( | ||||||||||||
4016 | PLoad, | ||||||||||||
4017 | getAdjustedPtr(IRB, DL, StoreBasePtr, | ||||||||||||
4018 | APInt(DL.getIndexSizeInBits(AS), PartOffset), | ||||||||||||
4019 | StorePartPtrTy, StoreBasePtr->getName() + "."), | ||||||||||||
4020 | getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false); | ||||||||||||
4021 | |||||||||||||
4022 | // Now build a new slice for the alloca. | ||||||||||||
4023 | NewSlices.push_back( | ||||||||||||
4024 | Slice(BaseOffset + PartOffset, BaseOffset + PartOffset + PartSize, | ||||||||||||
4025 | &PStore->getOperandUse(PStore->getPointerOperandIndex()), | ||||||||||||
4026 | /*IsSplittable*/ false)); | ||||||||||||
4027 | LLVM_DEBUG(dbgs() << " new slice [" << NewSlices.back().beginOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | ||||||||||||
4028 | << ", " << NewSlices.back().endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false) | ||||||||||||
4029 | << "): " << *PStore << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " new slice [" << NewSlices .back().beginOffset() << ", " << NewSlices.back() .endOffset() << "): " << *PStore << "\n"; } } while (false); | ||||||||||||
4030 | if (!SplitLoads) { | ||||||||||||
4031 | LLVM_DEBUG(dbgs() << " of split load: " << *PLoad << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " of split load: " << * PLoad << "\n"; } } while (false); | ||||||||||||
4032 | } | ||||||||||||
4033 | |||||||||||||
4034 | // See if we've finished all the splits. | ||||||||||||
4035 | if (Idx >= Size) | ||||||||||||
4036 | break; | ||||||||||||
4037 | |||||||||||||
4038 | // Setup the next partition. | ||||||||||||
4039 | PartOffset = Offsets.Splits[Idx]; | ||||||||||||
4040 | ++Idx; | ||||||||||||
4041 | PartSize = (Idx < Size ? Offsets.Splits[Idx] : StoreSize) - PartOffset; | ||||||||||||
4042 | } | ||||||||||||
4043 | |||||||||||||
4044 | // We want to immediately iterate on any allocas impacted by splitting | ||||||||||||
4045 | // this load, which is only relevant if it isn't a load of this alloca and | ||||||||||||
4046 | // thus we didn't already split the loads above. We also have to keep track | ||||||||||||
4047 | // of any promotable allocas we split loads on as they can no longer be | ||||||||||||
4048 | // promoted. | ||||||||||||
4049 | if (!SplitLoads) { | ||||||||||||
4050 | if (AllocaInst *OtherAI = dyn_cast<AllocaInst>(LoadBasePtr)) { | ||||||||||||
4051 | assert(OtherAI != &AI && "We can't re-split our own alloca!")((OtherAI != &AI && "We can't re-split our own alloca!" ) ? static_cast<void> (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 4051, __PRETTY_FUNCTION__)); | ||||||||||||
4052 | ResplitPromotableAllocas.insert(OtherAI); | ||||||||||||
4053 | Worklist.insert(OtherAI); | ||||||||||||
4054 | } else if (AllocaInst *OtherAI = dyn_cast<AllocaInst>( | ||||||||||||
4055 | LoadBasePtr->stripInBoundsOffsets())) { | ||||||||||||
4056 | assert(OtherAI != &AI && "We can't re-split our own alloca!")((OtherAI != &AI && "We can't re-split our own alloca!" ) ? static_cast<void> (0) : __assert_fail ("OtherAI != &AI && \"We can't re-split our own alloca!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 4056, __PRETTY_FUNCTION__)); | ||||||||||||
4057 | Worklist.insert(OtherAI); | ||||||||||||
4058 | } | ||||||||||||
4059 | } | ||||||||||||
4060 | |||||||||||||
4061 | // Mark the original store as dead now that we've split it up and kill its | ||||||||||||
4062 | // slice. Note that we leave the original load in place unless this store | ||||||||||||
4063 | // was its only use. It may in turn be split up if it is an alloca load | ||||||||||||
4064 | // for some other alloca, but it may be a normal load. This may introduce | ||||||||||||
4065 | // redundant loads, but where those can be merged the rest of the optimizer | ||||||||||||
4066 | // should handle the merging, and this uncovers SSA splits which is more | ||||||||||||
4067 | // important. In practice, the original loads will almost always be fully | ||||||||||||
4068 | // split and removed eventually, and the splits will be merged by any | ||||||||||||
4069 | // trivial CSE, including instcombine. | ||||||||||||
4070 | if (LI->hasOneUse()) { | ||||||||||||
4071 | assert(*LI->user_begin() == SI && "Single use isn't this store!")((*LI->user_begin() == SI && "Single use isn't this store!" ) ? static_cast<void> (0) : __assert_fail ("*LI->user_begin() == SI && \"Single use isn't this store!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 4071, __PRETTY_FUNCTION__)); | ||||||||||||
4072 | DeadInsts.insert(LI); | ||||||||||||
4073 | } | ||||||||||||
4074 | DeadInsts.insert(SI); | ||||||||||||
4075 | Offsets.S->kill(); | ||||||||||||
4076 | } | ||||||||||||
4077 | |||||||||||||
4078 | // Remove the killed slices that have ben pre-split. | ||||||||||||
4079 | AS.erase(llvm::remove_if(AS, [](const Slice &S) { return S.isDead(); }), | ||||||||||||
4080 | AS.end()); | ||||||||||||
4081 | |||||||||||||
4082 | // Insert our new slices. This will sort and merge them into the sorted | ||||||||||||
4083 | // sequence. | ||||||||||||
4084 | AS.insert(NewSlices); | ||||||||||||
4085 | |||||||||||||
4086 | LLVM_DEBUG(dbgs() << " Pre-split slices:\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Pre-split slices:\n"; } } while (false); | ||||||||||||
4087 | #ifndef NDEBUG | ||||||||||||
4088 | for (auto I = AS.begin(), E = AS.end(); I != E; ++I) | ||||||||||||
4089 | LLVM_DEBUG(AS.print(dbgs(), I, " "))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs(), I, " "); } } while (false); | ||||||||||||
4090 | #endif | ||||||||||||
4091 | |||||||||||||
4092 | // Finally, don't try to promote any allocas that new require re-splitting. | ||||||||||||
4093 | // They have already been added to the worklist above. | ||||||||||||
4094 | PromotableAllocas.erase( | ||||||||||||
4095 | llvm::remove_if( | ||||||||||||
4096 | PromotableAllocas, | ||||||||||||
4097 | [&](AllocaInst *AI) { return ResplitPromotableAllocas.count(AI); }), | ||||||||||||
4098 | PromotableAllocas.end()); | ||||||||||||
4099 | |||||||||||||
4100 | return true; | ||||||||||||
4101 | } | ||||||||||||
4102 | |||||||||||||
4103 | /// Rewrite an alloca partition's users. | ||||||||||||
4104 | /// | ||||||||||||
4105 | /// This routine drives both of the rewriting goals of the SROA pass. It tries | ||||||||||||
4106 | /// to rewrite uses of an alloca partition to be conducive for SSA value | ||||||||||||
4107 | /// promotion. If the partition needs a new, more refined alloca, this will | ||||||||||||
4108 | /// build that new alloca, preserving as much type information as possible, and | ||||||||||||
4109 | /// rewrite the uses of the old alloca to point at the new one and have the | ||||||||||||
4110 | /// appropriate new offsets. It also evaluates how successful the rewrite was | ||||||||||||
4111 | /// at enabling promotion and if it was successful queues the alloca to be | ||||||||||||
4112 | /// promoted. | ||||||||||||
4113 | AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS, | ||||||||||||
4114 | Partition &P) { | ||||||||||||
4115 | // Try to compute a friendly type for this partition of the alloca. This | ||||||||||||
4116 | // won't always succeed, in which case we fall back to a legal integer type | ||||||||||||
4117 | // or an i8 array of an appropriate size. | ||||||||||||
4118 | Type *SliceTy = nullptr; | ||||||||||||
4119 | const DataLayout &DL = AI.getModule()->getDataLayout(); | ||||||||||||
4120 | if (Type *CommonUseTy = findCommonType(P.begin(), P.end(), P.endOffset())) | ||||||||||||
4121 | if (DL.getTypeAllocSize(CommonUseTy) >= P.size()) | ||||||||||||
4122 | SliceTy = CommonUseTy; | ||||||||||||
4123 | if (!SliceTy) | ||||||||||||
4124 | if (Type *TypePartitionTy = getTypePartition(DL, AI.getAllocatedType(), | ||||||||||||
4125 | P.beginOffset(), P.size())) | ||||||||||||
4126 | SliceTy = TypePartitionTy; | ||||||||||||
4127 | if ((!SliceTy || (SliceTy->isArrayTy() && | ||||||||||||
4128 | SliceTy->getArrayElementType()->isIntegerTy())) && | ||||||||||||
4129 | DL.isLegalInteger(P.size() * 8)) | ||||||||||||
4130 | SliceTy = Type::getIntNTy(*C, P.size() * 8); | ||||||||||||
4131 | if (!SliceTy) | ||||||||||||
4132 | SliceTy = ArrayType::get(Type::getInt8Ty(*C), P.size()); | ||||||||||||
4133 | assert(DL.getTypeAllocSize(SliceTy) >= P.size())((DL.getTypeAllocSize(SliceTy) >= P.size()) ? static_cast< void> (0) : __assert_fail ("DL.getTypeAllocSize(SliceTy) >= P.size()" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 4133, __PRETTY_FUNCTION__)); | ||||||||||||
4134 | |||||||||||||
4135 | bool IsIntegerPromotable = isIntegerWideningViable(P, SliceTy, DL); | ||||||||||||
4136 | |||||||||||||
4137 | VectorType *VecTy = | ||||||||||||
4138 | IsIntegerPromotable ? nullptr : isVectorPromotionViable(P, DL); | ||||||||||||
4139 | if (VecTy) | ||||||||||||
4140 | SliceTy = VecTy; | ||||||||||||
4141 | |||||||||||||
4142 | // Check for the case where we're going to rewrite to a new alloca of the | ||||||||||||
4143 | // exact same type as the original, and with the same access offsets. In that | ||||||||||||
4144 | // case, re-use the existing alloca, but still run through the rewriter to | ||||||||||||
4145 | // perform phi and select speculation. | ||||||||||||
4146 | // P.beginOffset() can be non-zero even with the same type in a case with | ||||||||||||
4147 | // out-of-bounds access (e.g. @PR35657 function in SROA/basictest.ll). | ||||||||||||
4148 | AllocaInst *NewAI; | ||||||||||||
4149 | if (SliceTy == AI.getAllocatedType() && P.beginOffset() == 0) { | ||||||||||||
4150 | NewAI = &AI; | ||||||||||||
4151 | // FIXME: We should be able to bail at this point with "nothing changed". | ||||||||||||
4152 | // FIXME: We might want to defer PHI speculation until after here. | ||||||||||||
4153 | // FIXME: return nullptr; | ||||||||||||
4154 | } else { | ||||||||||||
4155 | // If alignment is unspecified we fallback on the one required by the ABI | ||||||||||||
4156 | // for this type. We also make sure the alignment is compatible with | ||||||||||||
4157 | // P.beginOffset(). | ||||||||||||
4158 | const Align Alignment = commonAlignment( | ||||||||||||
4159 | DL.getValueOrABITypeAlignment(MaybeAlign(AI.getAlignment()), | ||||||||||||
4160 | AI.getAllocatedType()), | ||||||||||||
4161 | P.beginOffset()); | ||||||||||||
4162 | // If we will get at least this much alignment from the type alone, leave | ||||||||||||
4163 | // the alloca's alignment unconstrained. | ||||||||||||
4164 | const bool IsUnconstrained = Alignment <= DL.getABITypeAlignment(SliceTy); | ||||||||||||
4165 | NewAI = new AllocaInst( | ||||||||||||
4166 | SliceTy, AI.getType()->getAddressSpace(), nullptr, | ||||||||||||
4167 | IsUnconstrained ? MaybeAlign() : Alignment, | ||||||||||||
4168 | AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI); | ||||||||||||
4169 | // Copy the old AI debug location over to the new one. | ||||||||||||
4170 | NewAI->setDebugLoc(AI.getDebugLoc()); | ||||||||||||
4171 | ++NumNewAllocas; | ||||||||||||
4172 | } | ||||||||||||
4173 | |||||||||||||
4174 | LLVM_DEBUG(dbgs() << "Rewriting alloca partition "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | ||||||||||||
4175 | << "[" << P.beginOffset() << "," << P.endOffset()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false) | ||||||||||||
4176 | << ") to: " << *NewAI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Rewriting alloca partition " << "[" << P.beginOffset() << "," << P.endOffset () << ") to: " << *NewAI << "\n"; } } while (false); | ||||||||||||
4177 | |||||||||||||
4178 | // Track the high watermark on the worklist as it is only relevant for | ||||||||||||
4179 | // promoted allocas. We will reset it to this point if the alloca is not in | ||||||||||||
4180 | // fact scheduled for promotion. | ||||||||||||
4181 | unsigned PPWOldSize = PostPromotionWorklist.size(); | ||||||||||||
4182 | unsigned NumUses = 0; | ||||||||||||
4183 | SmallSetVector<PHINode *, 8> PHIUsers; | ||||||||||||
4184 | SmallSetVector<SelectInst *, 8> SelectUsers; | ||||||||||||
4185 | |||||||||||||
4186 | AllocaSliceRewriter Rewriter(DL, AS, *this, AI, *NewAI, P.beginOffset(), | ||||||||||||
4187 | P.endOffset(), IsIntegerPromotable, VecTy, | ||||||||||||
4188 | PHIUsers, SelectUsers); | ||||||||||||
4189 | bool Promotable = true; | ||||||||||||
4190 | for (Slice *S : P.splitSliceTails()) { | ||||||||||||
4191 | Promotable &= Rewriter.visit(S); | ||||||||||||
4192 | ++NumUses; | ||||||||||||
4193 | } | ||||||||||||
4194 | for (Slice &S : P) { | ||||||||||||
4195 | Promotable &= Rewriter.visit(&S); | ||||||||||||
4196 | ++NumUses; | ||||||||||||
4197 | } | ||||||||||||
4198 | |||||||||||||
4199 | NumAllocaPartitionUses += NumUses; | ||||||||||||
4200 | MaxUsesPerAllocaPartition.updateMax(NumUses); | ||||||||||||
4201 | |||||||||||||
4202 | // Now that we've processed all the slices in the new partition, check if any | ||||||||||||
4203 | // PHIs or Selects would block promotion. | ||||||||||||
4204 | for (PHINode *PHI : PHIUsers) | ||||||||||||
4205 | if (!isSafePHIToSpeculate(*PHI)) { | ||||||||||||
4206 | Promotable = false; | ||||||||||||
4207 | PHIUsers.clear(); | ||||||||||||
4208 | SelectUsers.clear(); | ||||||||||||
4209 | break; | ||||||||||||
4210 | } | ||||||||||||
4211 | |||||||||||||
4212 | for (SelectInst *Sel : SelectUsers) | ||||||||||||
4213 | if (!isSafeSelectToSpeculate(*Sel)) { | ||||||||||||
4214 | Promotable = false; | ||||||||||||
4215 | PHIUsers.clear(); | ||||||||||||
4216 | SelectUsers.clear(); | ||||||||||||
4217 | break; | ||||||||||||
4218 | } | ||||||||||||
4219 | |||||||||||||
4220 | if (Promotable) { | ||||||||||||
4221 | if (PHIUsers.empty() && SelectUsers.empty()) { | ||||||||||||
4222 | // Promote the alloca. | ||||||||||||
4223 | PromotableAllocas.push_back(NewAI); | ||||||||||||
4224 | } else { | ||||||||||||
4225 | // If we have either PHIs or Selects to speculate, add them to those | ||||||||||||
4226 | // worklists and re-queue the new alloca so that we promote in on the | ||||||||||||
4227 | // next iteration. | ||||||||||||
4228 | for (PHINode *PHIUser : PHIUsers) | ||||||||||||
4229 | SpeculatablePHIs.insert(PHIUser); | ||||||||||||
4230 | for (SelectInst *SelectUser : SelectUsers) | ||||||||||||
4231 | SpeculatableSelects.insert(SelectUser); | ||||||||||||
4232 | Worklist.insert(NewAI); | ||||||||||||
4233 | } | ||||||||||||
4234 | } else { | ||||||||||||
4235 | // Drop any post-promotion work items if promotion didn't happen. | ||||||||||||
4236 | while (PostPromotionWorklist.size() > PPWOldSize) | ||||||||||||
4237 | PostPromotionWorklist.pop_back(); | ||||||||||||
4238 | |||||||||||||
4239 | // We couldn't promote and we didn't create a new partition, nothing | ||||||||||||
4240 | // happened. | ||||||||||||
4241 | if (NewAI == &AI) | ||||||||||||
4242 | return nullptr; | ||||||||||||
4243 | |||||||||||||
4244 | // If we can't promote the alloca, iterate on it to check for new | ||||||||||||
4245 | // refinements exposed by splitting the current alloca. Don't iterate on an | ||||||||||||
4246 | // alloca which didn't actually change and didn't get promoted. | ||||||||||||
4247 | Worklist.insert(NewAI); | ||||||||||||
4248 | } | ||||||||||||
4249 | |||||||||||||
4250 | return NewAI; | ||||||||||||
4251 | } | ||||||||||||
4252 | |||||||||||||
4253 | /// Walks the slices of an alloca and form partitions based on them, | ||||||||||||
4254 | /// rewriting each of their uses. | ||||||||||||
4255 | bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) { | ||||||||||||
4256 | if (AS.begin() == AS.end()) | ||||||||||||
4257 | return false; | ||||||||||||
4258 | |||||||||||||
4259 | unsigned NumPartitions = 0; | ||||||||||||
4260 | bool Changed = false; | ||||||||||||
4261 | const DataLayout &DL = AI.getModule()->getDataLayout(); | ||||||||||||
4262 | |||||||||||||
4263 | // First try to pre-split loads and stores. | ||||||||||||
4264 | Changed |= presplitLoadsAndStores(AI, AS); | ||||||||||||
4265 | |||||||||||||
4266 | // Now that we have identified any pre-splitting opportunities, | ||||||||||||
4267 | // mark loads and stores unsplittable except for the following case. | ||||||||||||
4268 | // We leave a slice splittable if all other slices are disjoint or fully | ||||||||||||
4269 | // included in the slice, such as whole-alloca loads and stores. | ||||||||||||
4270 | // If we fail to split these during pre-splitting, we want to force them | ||||||||||||
4271 | // to be rewritten into a partition. | ||||||||||||
4272 | bool IsSorted = true; | ||||||||||||
4273 | |||||||||||||
4274 | uint64_t AllocaSize = DL.getTypeAllocSize(AI.getAllocatedType()); | ||||||||||||
4275 | const uint64_t MaxBitVectorSize = 1024; | ||||||||||||
4276 | if (AllocaSize <= MaxBitVectorSize) { | ||||||||||||
4277 | // If a byte boundary is included in any load or store, a slice starting or | ||||||||||||
4278 | // ending at the boundary is not splittable. | ||||||||||||
4279 | SmallBitVector SplittableOffset(AllocaSize + 1, true); | ||||||||||||
4280 | for (Slice &S : AS) | ||||||||||||
4281 | for (unsigned O = S.beginOffset() + 1; | ||||||||||||
4282 | O < S.endOffset() && O < AllocaSize; O++) | ||||||||||||
4283 | SplittableOffset.reset(O); | ||||||||||||
4284 | |||||||||||||
4285 | for (Slice &S : AS) { | ||||||||||||
4286 | if (!S.isSplittable()) | ||||||||||||
4287 | continue; | ||||||||||||
4288 | |||||||||||||
4289 | if ((S.beginOffset() > AllocaSize || SplittableOffset[S.beginOffset()]) && | ||||||||||||
4290 | (S.endOffset() > AllocaSize || SplittableOffset[S.endOffset()])) | ||||||||||||
4291 | continue; | ||||||||||||
4292 | |||||||||||||
4293 | if (isa<LoadInst>(S.getUse()->getUser()) || | ||||||||||||
4294 | isa<StoreInst>(S.getUse()->getUser())) { | ||||||||||||
4295 | S.makeUnsplittable(); | ||||||||||||
4296 | IsSorted = false; | ||||||||||||
4297 | } | ||||||||||||
4298 | } | ||||||||||||
4299 | } | ||||||||||||
4300 | else { | ||||||||||||
4301 | // We only allow whole-alloca splittable loads and stores | ||||||||||||
4302 | // for a large alloca to avoid creating too large BitVector. | ||||||||||||
4303 | for (Slice &S : AS) { | ||||||||||||
4304 | if (!S.isSplittable()) | ||||||||||||
4305 | continue; | ||||||||||||
4306 | |||||||||||||
4307 | if (S.beginOffset() == 0 && S.endOffset() >= AllocaSize) | ||||||||||||
4308 | continue; | ||||||||||||
4309 | |||||||||||||
4310 | if (isa<LoadInst>(S.getUse()->getUser()) || | ||||||||||||
4311 | isa<StoreInst>(S.getUse()->getUser())) { | ||||||||||||
4312 | S.makeUnsplittable(); | ||||||||||||
4313 | IsSorted = false; | ||||||||||||
4314 | } | ||||||||||||
4315 | } | ||||||||||||
4316 | } | ||||||||||||
4317 | |||||||||||||
4318 | if (!IsSorted) | ||||||||||||
4319 | llvm::sort(AS); | ||||||||||||
4320 | |||||||||||||
4321 | /// Describes the allocas introduced by rewritePartition in order to migrate | ||||||||||||
4322 | /// the debug info. | ||||||||||||
4323 | struct Fragment { | ||||||||||||
4324 | AllocaInst *Alloca; | ||||||||||||
4325 | uint64_t Offset; | ||||||||||||
4326 | uint64_t Size; | ||||||||||||
4327 | Fragment(AllocaInst *AI, uint64_t O, uint64_t S) | ||||||||||||
4328 | : Alloca(AI), Offset(O), Size(S) {} | ||||||||||||
4329 | }; | ||||||||||||
4330 | SmallVector<Fragment, 4> Fragments; | ||||||||||||
4331 | |||||||||||||
4332 | // Rewrite each partition. | ||||||||||||
4333 | for (auto &P : AS.partitions()) { | ||||||||||||
4334 | if (AllocaInst *NewAI = rewritePartition(AI, AS, P)) { | ||||||||||||
4335 | Changed = true; | ||||||||||||
4336 | if (NewAI != &AI) { | ||||||||||||
4337 | uint64_t SizeOfByte = 8; | ||||||||||||
4338 | uint64_t AllocaSize = DL.getTypeSizeInBits(NewAI->getAllocatedType()); | ||||||||||||
4339 | // Don't include any padding. | ||||||||||||
4340 | uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte); | ||||||||||||
4341 | Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size)); | ||||||||||||
4342 | } | ||||||||||||
4343 | } | ||||||||||||
4344 | ++NumPartitions; | ||||||||||||
4345 | } | ||||||||||||
4346 | |||||||||||||
4347 | NumAllocaPartitions += NumPartitions; | ||||||||||||
4348 | MaxPartitionsPerAlloca.updateMax(NumPartitions); | ||||||||||||
4349 | |||||||||||||
4350 | // Migrate debug information from the old alloca to the new alloca(s) | ||||||||||||
4351 | // and the individual partitions. | ||||||||||||
4352 | TinyPtrVector<DbgVariableIntrinsic *> DbgDeclares = FindDbgAddrUses(&AI); | ||||||||||||
4353 | if (!DbgDeclares.empty()) { | ||||||||||||
4354 | auto *Var = DbgDeclares.front()->getVariable(); | ||||||||||||
4355 | auto *Expr = DbgDeclares.front()->getExpression(); | ||||||||||||
4356 | auto VarSize = Var->getSizeInBits(); | ||||||||||||
4357 | DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false); | ||||||||||||
4358 | uint64_t AllocaSize = DL.getTypeSizeInBits(AI.getAllocatedType()); | ||||||||||||
4359 | for (auto Fragment : Fragments) { | ||||||||||||
4360 | // Create a fragment expression describing the new partition or reuse AI's | ||||||||||||
4361 | // expression if there is only one partition. | ||||||||||||
4362 | auto *FragmentExpr = Expr; | ||||||||||||
4363 | if (Fragment.Size < AllocaSize || Expr->isFragment()) { | ||||||||||||
4364 | // If this alloca is already a scalar replacement of a larger aggregate, | ||||||||||||
4365 | // Fragment.Offset describes the offset inside the scalar. | ||||||||||||
4366 | auto ExprFragment = Expr->getFragmentInfo(); | ||||||||||||
4367 | uint64_t Offset = ExprFragment ? ExprFragment->OffsetInBits : 0; | ||||||||||||
4368 | uint64_t Start = Offset + Fragment.Offset; | ||||||||||||
4369 | uint64_t Size = Fragment.Size; | ||||||||||||
4370 | if (ExprFragment) { | ||||||||||||
4371 | uint64_t AbsEnd = | ||||||||||||
4372 | ExprFragment->OffsetInBits + ExprFragment->SizeInBits; | ||||||||||||
4373 | if (Start >= AbsEnd) | ||||||||||||
4374 | // No need to describe a SROAed padding. | ||||||||||||
4375 | continue; | ||||||||||||
4376 | Size = std::min(Size, AbsEnd - Start); | ||||||||||||
4377 | } | ||||||||||||
4378 | // The new, smaller fragment is stenciled out from the old fragment. | ||||||||||||
4379 | if (auto OrigFragment = FragmentExpr->getFragmentInfo()) { | ||||||||||||
4380 | assert(Start >= OrigFragment->OffsetInBits &&((Start >= OrigFragment->OffsetInBits && "new fragment is outside of original fragment" ) ? static_cast<void> (0) : __assert_fail ("Start >= OrigFragment->OffsetInBits && \"new fragment is outside of original fragment\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 4381, __PRETTY_FUNCTION__)) | ||||||||||||
4381 | "new fragment is outside of original fragment")((Start >= OrigFragment->OffsetInBits && "new fragment is outside of original fragment" ) ? static_cast<void> (0) : __assert_fail ("Start >= OrigFragment->OffsetInBits && \"new fragment is outside of original fragment\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Transforms/Scalar/SROA.cpp" , 4381, __PRETTY_FUNCTION__)); | ||||||||||||
4382 | Start -= OrigFragment->OffsetInBits; | ||||||||||||
4383 | } | ||||||||||||
4384 | |||||||||||||
4385 | // The alloca may be larger than the variable. | ||||||||||||
4386 | if (VarSize) { | ||||||||||||
4387 | if (Size > *VarSize) | ||||||||||||
4388 | Size = *VarSize; | ||||||||||||
4389 | if (Size == 0 || Start + Size > *VarSize) | ||||||||||||
4390 | continue; | ||||||||||||
4391 | } | ||||||||||||
4392 | |||||||||||||
4393 | // Avoid creating a fragment expression that covers the entire variable. | ||||||||||||
4394 | if (!VarSize || *VarSize != Size) { | ||||||||||||
4395 | if (auto E = | ||||||||||||
4396 | DIExpression::createFragmentExpression(Expr, Start, Size)) | ||||||||||||
4397 | FragmentExpr = *E; | ||||||||||||
4398 | else | ||||||||||||
4399 | continue; | ||||||||||||
4400 | } | ||||||||||||
4401 | } | ||||||||||||
4402 | |||||||||||||
4403 | // Remove any existing intrinsics describing the same alloca. | ||||||||||||
4404 | for (DbgVariableIntrinsic *OldDII : FindDbgAddrUses(Fragment.Alloca)) | ||||||||||||
4405 | OldDII->eraseFromParent(); | ||||||||||||
4406 | |||||||||||||
4407 | DIB.insertDeclare(Fragment.Alloca, Var, FragmentExpr, | ||||||||||||
4408 | DbgDeclares.front()->getDebugLoc(), &AI); | ||||||||||||
4409 | } | ||||||||||||
4410 | } | ||||||||||||
4411 | return Changed; | ||||||||||||
4412 | } | ||||||||||||
4413 | |||||||||||||
4414 | /// Clobber a use with undef, deleting the used value if it becomes dead. | ||||||||||||
4415 | void SROA::clobberUse(Use &U) { | ||||||||||||
4416 | Value *OldV = U; | ||||||||||||
4417 | // Replace the use with an undef value. | ||||||||||||
4418 | U = UndefValue::get(OldV->getType()); | ||||||||||||
4419 | |||||||||||||
4420 | // Check for this making an instruction dead. We have to garbage collect | ||||||||||||
4421 | // all the dead instructions to ensure the uses of any alloca end up being | ||||||||||||
4422 | // minimal. | ||||||||||||
4423 | if (Instruction *OldI = dyn_cast<Instruction>(OldV)) | ||||||||||||
4424 | if (isInstructionTriviallyDead(OldI)) { | ||||||||||||
4425 | DeadInsts.insert(OldI); | ||||||||||||
4426 | } | ||||||||||||
4427 | } | ||||||||||||
4428 | |||||||||||||
4429 | /// Analyze an alloca for SROA. | ||||||||||||
4430 | /// | ||||||||||||
4431 | /// This analyzes the alloca to ensure we can reason about it, builds | ||||||||||||
4432 | /// the slices of the alloca, and then hands it off to be split and | ||||||||||||
4433 | /// rewritten as needed. | ||||||||||||
4434 | bool SROA::runOnAlloca(AllocaInst &AI) { | ||||||||||||
4435 | LLVM_DEBUG(dbgs() << "SROA alloca: " << AI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA alloca: " << AI << "\n"; } } while (false); | ||||||||||||
4436 | ++NumAllocasAnalyzed; | ||||||||||||
4437 | |||||||||||||
4438 | // Special case dead allocas, as they're trivial. | ||||||||||||
4439 | if (AI.use_empty()) { | ||||||||||||
4440 | AI.eraseFromParent(); | ||||||||||||
4441 | return true; | ||||||||||||
4442 | } | ||||||||||||
4443 | const DataLayout &DL = AI.getModule()->getDataLayout(); | ||||||||||||
4444 | |||||||||||||
4445 | // Skip alloca forms that this analysis can't handle. | ||||||||||||
4446 | if (AI.isArrayAllocation() || !AI.getAllocatedType()->isSized() || | ||||||||||||
4447 | DL.getTypeAllocSize(AI.getAllocatedType()) == 0) | ||||||||||||
4448 | return false; | ||||||||||||
4449 | |||||||||||||
4450 | bool Changed = false; | ||||||||||||
4451 | |||||||||||||
4452 | // First, split any FCA loads and stores touching this alloca to promote | ||||||||||||
4453 | // better splitting and promotion opportunities. | ||||||||||||
4454 | AggLoadStoreRewriter AggRewriter(DL); | ||||||||||||
4455 | Changed |= AggRewriter.rewrite(AI); | ||||||||||||
4456 | |||||||||||||
4457 | // Build the slices using a recursive instruction-visiting builder. | ||||||||||||
4458 | AllocaSlices AS(DL, AI); | ||||||||||||
4459 | LLVM_DEBUG(AS.print(dbgs()))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { AS.print(dbgs()); } } while (false); | ||||||||||||
4460 | if (AS.isEscaped()) | ||||||||||||
4461 | return Changed; | ||||||||||||
4462 | |||||||||||||
4463 | // Delete all the dead users of this alloca before splitting and rewriting it. | ||||||||||||
4464 | for (Instruction *DeadUser : AS.getDeadUsers()) { | ||||||||||||
4465 | // Free up everything used by this instruction. | ||||||||||||
4466 | for (Use &DeadOp : DeadUser->operands()) | ||||||||||||
4467 | clobberUse(DeadOp); | ||||||||||||
4468 | |||||||||||||
4469 | // Now replace the uses of this instruction. | ||||||||||||
4470 | DeadUser->replaceAllUsesWith(UndefValue::get(DeadUser->getType())); | ||||||||||||
4471 | |||||||||||||
4472 | // And mark it for deletion. | ||||||||||||
4473 | DeadInsts.insert(DeadUser); | ||||||||||||
4474 | Changed = true; | ||||||||||||
4475 | } | ||||||||||||
4476 | for (Use *DeadOp : AS.getDeadOperands()) { | ||||||||||||
4477 | clobberUse(*DeadOp); | ||||||||||||
4478 | Changed = true; | ||||||||||||
4479 | } | ||||||||||||
4480 | |||||||||||||
4481 | // No slices to split. Leave the dead alloca for a later pass to clean up. | ||||||||||||
4482 | if (AS.begin() == AS.end()) | ||||||||||||
4483 | return Changed; | ||||||||||||
4484 | |||||||||||||
4485 | Changed |= splitAlloca(AI, AS); | ||||||||||||
4486 | |||||||||||||
4487 | LLVM_DEBUG(dbgs() << " Speculating PHIs\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating PHIs\n"; } } while (false); | ||||||||||||
4488 | while (!SpeculatablePHIs.empty()) | ||||||||||||
4489 | speculatePHINodeLoads(*SpeculatablePHIs.pop_back_val()); | ||||||||||||
4490 | |||||||||||||
4491 | LLVM_DEBUG(dbgs() << " Speculating Selects\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << " Speculating Selects\n"; } } while (false); | ||||||||||||
4492 | while (!SpeculatableSelects.empty()) | ||||||||||||
4493 | speculateSelectInstLoads(*SpeculatableSelects.pop_back_val()); | ||||||||||||
4494 | |||||||||||||
4495 | return Changed; | ||||||||||||
4496 | } | ||||||||||||
4497 | |||||||||||||
4498 | /// Delete the dead instructions accumulated in this run. | ||||||||||||
4499 | /// | ||||||||||||
4500 | /// Recursively deletes the dead instructions we've accumulated. This is done | ||||||||||||
4501 | /// at the very end to maximize locality of the recursive delete and to | ||||||||||||
4502 | /// minimize the problems of invalidated instruction pointers as such pointers | ||||||||||||
4503 | /// are used heavily in the intermediate stages of the algorithm. | ||||||||||||
4504 | /// | ||||||||||||
4505 | /// We also record the alloca instructions deleted here so that they aren't | ||||||||||||
4506 | /// subsequently handed to mem2reg to promote. | ||||||||||||
4507 | bool SROA::deleteDeadInstructions( | ||||||||||||
4508 | SmallPtrSetImpl<AllocaInst *> &DeletedAllocas) { | ||||||||||||
4509 | bool Changed = false; | ||||||||||||
4510 | while (!DeadInsts.empty()) { | ||||||||||||
4511 | Instruction *I = DeadInsts.pop_back_val(); | ||||||||||||
4512 | LLVM_DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Deleting dead instruction: " << *I << "\n"; } } while (false); | ||||||||||||
4513 | |||||||||||||
4514 | // If the instruction is an alloca, find the possible dbg.declare connected | ||||||||||||
4515 | // to it, and remove it too. We must do this before calling RAUW or we will | ||||||||||||
4516 | // not be able to find it. | ||||||||||||
4517 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) { | ||||||||||||
4518 | DeletedAllocas.insert(AI); | ||||||||||||
4519 | for (DbgVariableIntrinsic *OldDII : FindDbgAddrUses(AI)) | ||||||||||||
4520 | OldDII->eraseFromParent(); | ||||||||||||
4521 | } | ||||||||||||
4522 | |||||||||||||
4523 | I->replaceAllUsesWith(UndefValue::get(I->getType())); | ||||||||||||
4524 | |||||||||||||
4525 | for (Use &Operand : I->operands()) | ||||||||||||
4526 | if (Instruction *U = dyn_cast<Instruction>(Operand)) { | ||||||||||||
4527 | // Zero out the operand and see if it becomes trivially dead. | ||||||||||||
4528 | Operand = nullptr; | ||||||||||||
4529 | if (isInstructionTriviallyDead(U)) | ||||||||||||
4530 | DeadInsts.insert(U); | ||||||||||||
4531 | } | ||||||||||||
4532 | |||||||||||||
4533 | ++NumDeleted; | ||||||||||||
4534 | I->eraseFromParent(); | ||||||||||||
4535 | Changed = true; | ||||||||||||
4536 | } | ||||||||||||
4537 | return Changed; | ||||||||||||
4538 | } | ||||||||||||
4539 | |||||||||||||
4540 | /// Promote the allocas, using the best available technique. | ||||||||||||
4541 | /// | ||||||||||||
4542 | /// This attempts to promote whatever allocas have been identified as viable in | ||||||||||||
4543 | /// the PromotableAllocas list. If that list is empty, there is nothing to do. | ||||||||||||
4544 | /// This function returns whether any promotion occurred. | ||||||||||||
4545 | bool SROA::promoteAllocas(Function &F) { | ||||||||||||
4546 | if (PromotableAllocas.empty()) | ||||||||||||
4547 | return false; | ||||||||||||
4548 | |||||||||||||
4549 | NumPromoted += PromotableAllocas.size(); | ||||||||||||
4550 | |||||||||||||
4551 | LLVM_DEBUG(dbgs() << "Promoting allocas with mem2reg...\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "Promoting allocas with mem2reg...\n" ; } } while (false); | ||||||||||||
4552 | PromoteMemToReg(PromotableAllocas, *DT, AC); | ||||||||||||
4553 | PromotableAllocas.clear(); | ||||||||||||
4554 | return true; | ||||||||||||
4555 | } | ||||||||||||
4556 | |||||||||||||
4557 | PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT, | ||||||||||||
4558 | AssumptionCache &RunAC) { | ||||||||||||
4559 | LLVM_DEBUG(dbgs() << "SROA function: " << F.getName() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("sroa")) { dbgs() << "SROA function: " << F.getName () << "\n"; } } while (false); | ||||||||||||
4560 | C = &F.getContext(); | ||||||||||||
4561 | DT = &RunDT; | ||||||||||||
4562 | AC = &RunAC; | ||||||||||||
4563 | |||||||||||||
4564 | BasicBlock &EntryBB = F.getEntryBlock(); | ||||||||||||
4565 | for (BasicBlock::iterator I = EntryBB.begin(), E = std::prev(EntryBB.end()); | ||||||||||||
4566 | I != E; ++I) { | ||||||||||||
4567 | if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) | ||||||||||||
4568 | Worklist.insert(AI); | ||||||||||||
4569 | } | ||||||||||||
4570 | |||||||||||||
4571 | bool Changed = false; | ||||||||||||
4572 | // A set of deleted alloca instruction pointers which should be removed from | ||||||||||||
4573 | // the list of promotable allocas. | ||||||||||||
4574 | SmallPtrSet<AllocaInst *, 4> DeletedAllocas; | ||||||||||||
4575 | |||||||||||||
4576 | do { | ||||||||||||
4577 | while (!Worklist.empty()) { | ||||||||||||
4578 | Changed |= runOnAlloca(*Worklist.pop_back_val()); | ||||||||||||
4579 | Changed |= deleteDeadInstructions(DeletedAllocas); | ||||||||||||
4580 | |||||||||||||
4581 | // Remove the deleted allocas from various lists so that we don't try to | ||||||||||||
4582 | // continue processing them. | ||||||||||||
4583 | if (!DeletedAllocas.empty()) { | ||||||||||||
4584 | auto IsInSet = [&](AllocaInst *AI) { return DeletedAllocas.count(AI); }; | ||||||||||||
4585 | Worklist.remove_if(IsInSet); | ||||||||||||
4586 | PostPromotionWorklist.remove_if(IsInSet); | ||||||||||||
4587 | PromotableAllocas.erase(llvm::remove_if(PromotableAllocas, IsInSet), | ||||||||||||
4588 | PromotableAllocas.end()); | ||||||||||||
4589 | DeletedAllocas.clear(); | ||||||||||||
4590 | } | ||||||||||||
4591 | } | ||||||||||||
4592 | |||||||||||||
4593 | Changed |= promoteAllocas(F); | ||||||||||||
4594 | |||||||||||||
4595 | Worklist = PostPromotionWorklist; | ||||||||||||
4596 | PostPromotionWorklist.clear(); | ||||||||||||
4597 | } while (!Worklist.empty()); | ||||||||||||
4598 | |||||||||||||
4599 | if (!Changed) | ||||||||||||
4600 | return PreservedAnalyses::all(); | ||||||||||||
4601 | |||||||||||||
4602 | PreservedAnalyses PA; | ||||||||||||
4603 | PA.preserveSet<CFGAnalyses>(); | ||||||||||||
4604 | PA.preserve<GlobalsAA>(); | ||||||||||||
4605 | return PA; | ||||||||||||
4606 | } | ||||||||||||
4607 | |||||||||||||
4608 | PreservedAnalyses SROA::run(Function &F, FunctionAnalysisManager &AM) { | ||||||||||||
4609 | return runImpl(F, AM.getResult<DominatorTreeAnalysis>(F), | ||||||||||||
4610 | AM.getResult<AssumptionAnalysis>(F)); | ||||||||||||
4611 | } | ||||||||||||
4612 | |||||||||||||
4613 | /// A legacy pass for the legacy pass manager that wraps the \c SROA pass. | ||||||||||||
4614 | /// | ||||||||||||
4615 | /// This is in the llvm namespace purely to allow it to be a friend of the \c | ||||||||||||
4616 | /// SROA pass. | ||||||||||||
4617 | class llvm::sroa::SROALegacyPass : public FunctionPass { | ||||||||||||
4618 | /// The SROA implementation. | ||||||||||||
4619 | SROA Impl; | ||||||||||||
4620 | |||||||||||||
4621 | public: | ||||||||||||
4622 | static char ID; | ||||||||||||
4623 | |||||||||||||
4624 | SROALegacyPass() : FunctionPass(ID) { | ||||||||||||
4625 | initializeSROALegacyPassPass(*PassRegistry::getPassRegistry()); | ||||||||||||
4626 | } | ||||||||||||
4627 | |||||||||||||
4628 | bool runOnFunction(Function &F) override { | ||||||||||||
4629 | if (skipFunction(F)) | ||||||||||||
4630 | return false; | ||||||||||||
4631 | |||||||||||||
4632 | auto PA = Impl.runImpl( | ||||||||||||
4633 | F, getAnalysis<DominatorTreeWrapperPass>().getDomTree(), | ||||||||||||
4634 | getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F)); | ||||||||||||
4635 | return !PA.areAllPreserved(); | ||||||||||||
4636 | } | ||||||||||||
4637 | |||||||||||||
4638 | void getAnalysisUsage(AnalysisUsage &AU) const override { | ||||||||||||
4639 | AU.addRequired<AssumptionCacheTracker>(); | ||||||||||||
4640 | AU.addRequired<DominatorTreeWrapperPass>(); | ||||||||||||
4641 | AU.addPreserved<GlobalsAAWrapperPass>(); | ||||||||||||
4642 | AU.setPreservesCFG(); | ||||||||||||
4643 | } | ||||||||||||
4644 | |||||||||||||
4645 | StringRef getPassName() const override { return "SROA"; } | ||||||||||||
4646 | }; | ||||||||||||
4647 | |||||||||||||
4648 | char SROALegacyPass::ID = 0; | ||||||||||||
4649 | |||||||||||||
4650 | FunctionPass *llvm::createSROAPass() { return new SROALegacyPass(); } | ||||||||||||
4651 | |||||||||||||
4652 | INITIALIZE_PASS_BEGIN(SROALegacyPass, "sroa",static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | ||||||||||||
4653 | "Scalar Replacement Of Aggregates", false, false)static void *initializeSROALegacyPassPassOnce(PassRegistry & Registry) { | ||||||||||||
4654 | INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)initializeAssumptionCacheTrackerPass(Registry); | ||||||||||||
4655 | INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry); | ||||||||||||
4656 | INITIALIZE_PASS_END(SROALegacyPass, "sroa", "Scalar Replacement Of Aggregates",PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } | ||||||||||||
4657 | false, false)PassInfo *PI = new PassInfo( "Scalar Replacement Of Aggregates" , "sroa", &SROALegacyPass::ID, PassInfo::NormalCtor_t(callDefaultCtor <SROALegacyPass>), false, false); Registry.registerPass (*PI, true); return PI; } static llvm::once_flag InitializeSROALegacyPassPassFlag ; void llvm::initializeSROALegacyPassPass(PassRegistry &Registry ) { llvm::call_once(InitializeSROALegacyPassPassFlag, initializeSROALegacyPassPassOnce , std::ref(Registry)); } |
1 | //===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===// | ||||||
2 | // | ||||||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | ||||||
4 | // See https://llvm.org/LICENSE.txt for license information. | ||||||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | ||||||
6 | // | ||||||
7 | //===----------------------------------------------------------------------===// | ||||||
8 | // | ||||||
9 | // This file defines the IRBuilder class, which is used as a convenient way | ||||||
10 | // to create LLVM instructions with a consistent and simplified interface. | ||||||
11 | // | ||||||
12 | //===----------------------------------------------------------------------===// | ||||||
13 | |||||||
14 | #ifndef LLVM_IR_IRBUILDER_H | ||||||
15 | #define LLVM_IR_IRBUILDER_H | ||||||
16 | |||||||
17 | #include "llvm-c/Types.h" | ||||||
18 | #include "llvm/ADT/ArrayRef.h" | ||||||
19 | #include "llvm/ADT/None.h" | ||||||
20 | #include "llvm/ADT/StringRef.h" | ||||||
21 | #include "llvm/ADT/Twine.h" | ||||||
22 | #include "llvm/IR/BasicBlock.h" | ||||||
23 | #include "llvm/IR/Constant.h" | ||||||
24 | #include "llvm/IR/ConstantFolder.h" | ||||||
25 | #include "llvm/IR/Constants.h" | ||||||
26 | #include "llvm/IR/DataLayout.h" | ||||||
27 | #include "llvm/IR/DebugLoc.h" | ||||||
28 | #include "llvm/IR/DerivedTypes.h" | ||||||
29 | #include "llvm/IR/Function.h" | ||||||
30 | #include "llvm/IR/GlobalVariable.h" | ||||||
31 | #include "llvm/IR/InstrTypes.h" | ||||||
32 | #include "llvm/IR/Instruction.h" | ||||||
33 | #include "llvm/IR/Instructions.h" | ||||||
34 | #include "llvm/IR/IntrinsicInst.h" | ||||||
35 | #include "llvm/IR/LLVMContext.h" | ||||||
36 | #include "llvm/IR/Module.h" | ||||||
37 | #include "llvm/IR/Operator.h" | ||||||
38 | #include "llvm/IR/Type.h" | ||||||
39 | #include "llvm/IR/Value.h" | ||||||
40 | #include "llvm/IR/ValueHandle.h" | ||||||
41 | #include "llvm/Support/AtomicOrdering.h" | ||||||
42 | #include "llvm/Support/CBindingWrapping.h" | ||||||
43 | #include "llvm/Support/Casting.h" | ||||||
44 | #include <cassert> | ||||||
45 | #include <cstddef> | ||||||
46 | #include <cstdint> | ||||||
47 | #include <functional> | ||||||
48 | #include <utility> | ||||||
49 | |||||||
50 | namespace llvm { | ||||||
51 | |||||||
52 | class APInt; | ||||||
53 | class MDNode; | ||||||
54 | class Use; | ||||||
55 | |||||||
56 | /// This provides the default implementation of the IRBuilder | ||||||
57 | /// 'InsertHelper' method that is called whenever an instruction is created by | ||||||
58 | /// IRBuilder and needs to be inserted. | ||||||
59 | /// | ||||||
60 | /// By default, this inserts the instruction at the insertion point. | ||||||
61 | class IRBuilderDefaultInserter { | ||||||
62 | protected: | ||||||
63 | void InsertHelper(Instruction *I, const Twine &Name, | ||||||
64 | BasicBlock *BB, BasicBlock::iterator InsertPt) const { | ||||||
65 | if (BB) BB->getInstList().insert(InsertPt, I); | ||||||
66 | I->setName(Name); | ||||||
67 | } | ||||||
68 | }; | ||||||
69 | |||||||
70 | /// Provides an 'InsertHelper' that calls a user-provided callback after | ||||||
71 | /// performing the default insertion. | ||||||
72 | class IRBuilderCallbackInserter : IRBuilderDefaultInserter { | ||||||
73 | std::function<void(Instruction *)> Callback; | ||||||
74 | |||||||
75 | public: | ||||||
76 | IRBuilderCallbackInserter(std::function<void(Instruction *)> Callback) | ||||||
77 | : Callback(std::move(Callback)) {} | ||||||
78 | |||||||
79 | protected: | ||||||
80 | void InsertHelper(Instruction *I, const Twine &Name, | ||||||
81 | BasicBlock *BB, BasicBlock::iterator InsertPt) const { | ||||||
82 | IRBuilderDefaultInserter::InsertHelper(I, Name, BB, InsertPt); | ||||||
83 | Callback(I); | ||||||
84 | } | ||||||
85 | }; | ||||||
86 | |||||||
87 | /// Common base class shared among various IRBuilders. | ||||||
88 | class IRBuilderBase { | ||||||
89 | DebugLoc CurDbgLocation; | ||||||
90 | |||||||
91 | protected: | ||||||
92 | BasicBlock *BB; | ||||||
93 | BasicBlock::iterator InsertPt; | ||||||
94 | LLVMContext &Context; | ||||||
95 | |||||||
96 | MDNode *DefaultFPMathTag; | ||||||
97 | FastMathFlags FMF; | ||||||
98 | |||||||
99 | bool IsFPConstrained; | ||||||
100 | ConstrainedFPIntrinsic::ExceptionBehavior DefaultConstrainedExcept; | ||||||
101 | ConstrainedFPIntrinsic::RoundingMode DefaultConstrainedRounding; | ||||||
102 | |||||||
103 | ArrayRef<OperandBundleDef> DefaultOperandBundles; | ||||||
104 | |||||||
105 | public: | ||||||
106 | IRBuilderBase(LLVMContext &context, MDNode *FPMathTag = nullptr, | ||||||
107 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
108 | : Context(context), DefaultFPMathTag(FPMathTag), IsFPConstrained(false), | ||||||
109 | DefaultConstrainedExcept(ConstrainedFPIntrinsic::ebStrict), | ||||||
110 | DefaultConstrainedRounding(ConstrainedFPIntrinsic::rmDynamic), | ||||||
111 | DefaultOperandBundles(OpBundles) { | ||||||
112 | ClearInsertionPoint(); | ||||||
113 | } | ||||||
114 | |||||||
115 | //===--------------------------------------------------------------------===// | ||||||
116 | // Builder configuration methods | ||||||
117 | //===--------------------------------------------------------------------===// | ||||||
118 | |||||||
119 | /// Clear the insertion point: created instructions will not be | ||||||
120 | /// inserted into a block. | ||||||
121 | void ClearInsertionPoint() { | ||||||
122 | BB = nullptr; | ||||||
123 | InsertPt = BasicBlock::iterator(); | ||||||
124 | } | ||||||
125 | |||||||
126 | BasicBlock *GetInsertBlock() const { return BB; } | ||||||
127 | BasicBlock::iterator GetInsertPoint() const { return InsertPt; } | ||||||
128 | LLVMContext &getContext() const { return Context; } | ||||||
129 | |||||||
130 | /// This specifies that created instructions should be appended to the | ||||||
131 | /// end of the specified block. | ||||||
132 | void SetInsertPoint(BasicBlock *TheBB) { | ||||||
133 | BB = TheBB; | ||||||
134 | InsertPt = BB->end(); | ||||||
135 | } | ||||||
136 | |||||||
137 | /// This specifies that created instructions should be inserted before | ||||||
138 | /// the specified instruction. | ||||||
139 | void SetInsertPoint(Instruction *I) { | ||||||
140 | BB = I->getParent(); | ||||||
141 | InsertPt = I->getIterator(); | ||||||
142 | assert(InsertPt != BB->end() && "Can't read debug loc from end()")((InsertPt != BB->end() && "Can't read debug loc from end()" ) ? static_cast<void> (0) : __assert_fail ("InsertPt != BB->end() && \"Can't read debug loc from end()\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 142, __PRETTY_FUNCTION__)); | ||||||
143 | SetCurrentDebugLocation(I->getDebugLoc()); | ||||||
144 | } | ||||||
145 | |||||||
146 | /// This specifies that created instructions should be inserted at the | ||||||
147 | /// specified point. | ||||||
148 | void SetInsertPoint(BasicBlock *TheBB, BasicBlock::iterator IP) { | ||||||
149 | BB = TheBB; | ||||||
150 | InsertPt = IP; | ||||||
151 | if (IP != TheBB->end()) | ||||||
152 | SetCurrentDebugLocation(IP->getDebugLoc()); | ||||||
153 | } | ||||||
154 | |||||||
155 | /// Set location information used by debugging information. | ||||||
156 | void SetCurrentDebugLocation(DebugLoc L) { CurDbgLocation = std::move(L); } | ||||||
157 | |||||||
158 | /// Get location information used by debugging information. | ||||||
159 | const DebugLoc &getCurrentDebugLocation() const { return CurDbgLocation; } | ||||||
160 | |||||||
161 | /// If this builder has a current debug location, set it on the | ||||||
162 | /// specified instruction. | ||||||
163 | void SetInstDebugLocation(Instruction *I) const { | ||||||
164 | if (CurDbgLocation) | ||||||
165 | I->setDebugLoc(CurDbgLocation); | ||||||
166 | } | ||||||
167 | |||||||
168 | /// Get the return type of the current function that we're emitting | ||||||
169 | /// into. | ||||||
170 | Type *getCurrentFunctionReturnType() const; | ||||||
171 | |||||||
172 | /// InsertPoint - A saved insertion point. | ||||||
173 | class InsertPoint { | ||||||
174 | BasicBlock *Block = nullptr; | ||||||
175 | BasicBlock::iterator Point; | ||||||
176 | |||||||
177 | public: | ||||||
178 | /// Creates a new insertion point which doesn't point to anything. | ||||||
179 | InsertPoint() = default; | ||||||
180 | |||||||
181 | /// Creates a new insertion point at the given location. | ||||||
182 | InsertPoint(BasicBlock *InsertBlock, BasicBlock::iterator InsertPoint) | ||||||
183 | : Block(InsertBlock), Point(InsertPoint) {} | ||||||
184 | |||||||
185 | /// Returns true if this insert point is set. | ||||||
186 | bool isSet() const { return (Block != nullptr); } | ||||||
187 | |||||||
188 | BasicBlock *getBlock() const { return Block; } | ||||||
189 | BasicBlock::iterator getPoint() const { return Point; } | ||||||
190 | }; | ||||||
191 | |||||||
192 | /// Returns the current insert point. | ||||||
193 | InsertPoint saveIP() const { | ||||||
194 | return InsertPoint(GetInsertBlock(), GetInsertPoint()); | ||||||
195 | } | ||||||
196 | |||||||
197 | /// Returns the current insert point, clearing it in the process. | ||||||
198 | InsertPoint saveAndClearIP() { | ||||||
199 | InsertPoint IP(GetInsertBlock(), GetInsertPoint()); | ||||||
200 | ClearInsertionPoint(); | ||||||
201 | return IP; | ||||||
202 | } | ||||||
203 | |||||||
204 | /// Sets the current insert point to a previously-saved location. | ||||||
205 | void restoreIP(InsertPoint IP) { | ||||||
206 | if (IP.isSet()) | ||||||
207 | SetInsertPoint(IP.getBlock(), IP.getPoint()); | ||||||
208 | else | ||||||
209 | ClearInsertionPoint(); | ||||||
210 | } | ||||||
211 | |||||||
212 | /// Get the floating point math metadata being used. | ||||||
213 | MDNode *getDefaultFPMathTag() const { return DefaultFPMathTag; } | ||||||
214 | |||||||
215 | /// Get the flags to be applied to created floating point ops | ||||||
216 | FastMathFlags getFastMathFlags() const { return FMF; } | ||||||
217 | |||||||
218 | /// Clear the fast-math flags. | ||||||
219 | void clearFastMathFlags() { FMF.clear(); } | ||||||
220 | |||||||
221 | /// Set the floating point math metadata to be used. | ||||||
222 | void setDefaultFPMathTag(MDNode *FPMathTag) { DefaultFPMathTag = FPMathTag; } | ||||||
223 | |||||||
224 | /// Set the fast-math flags to be used with generated fp-math operators | ||||||
225 | void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; } | ||||||
226 | |||||||
227 | /// Enable/Disable use of constrained floating point math. When | ||||||
228 | /// enabled the CreateF<op>() calls instead create constrained | ||||||
229 | /// floating point intrinsic calls. Fast math flags are unaffected | ||||||
230 | /// by this setting. | ||||||
231 | void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; } | ||||||
232 | |||||||
233 | /// Query for the use of constrained floating point math | ||||||
234 | bool getIsFPConstrained() { return IsFPConstrained; } | ||||||
235 | |||||||
236 | /// Set the exception handling to be used with constrained floating point | ||||||
237 | void setDefaultConstrainedExcept( | ||||||
238 | ConstrainedFPIntrinsic::ExceptionBehavior NewExcept) { | ||||||
239 | DefaultConstrainedExcept = NewExcept; | ||||||
240 | } | ||||||
241 | |||||||
242 | /// Set the rounding mode handling to be used with constrained floating point | ||||||
243 | void setDefaultConstrainedRounding( | ||||||
244 | ConstrainedFPIntrinsic::RoundingMode NewRounding) { | ||||||
245 | DefaultConstrainedRounding = NewRounding; | ||||||
246 | } | ||||||
247 | |||||||
248 | /// Get the exception handling used with constrained floating point | ||||||
249 | ConstrainedFPIntrinsic::ExceptionBehavior getDefaultConstrainedExcept() { | ||||||
250 | return DefaultConstrainedExcept; | ||||||
251 | } | ||||||
252 | |||||||
253 | /// Get the rounding mode handling used with constrained floating point | ||||||
254 | ConstrainedFPIntrinsic::RoundingMode getDefaultConstrainedRounding() { | ||||||
255 | return DefaultConstrainedRounding; | ||||||
256 | } | ||||||
257 | |||||||
258 | void setConstrainedFPFunctionAttr() { | ||||||
259 | assert(BB && "Must have a basic block to set any function attributes!")((BB && "Must have a basic block to set any function attributes!" ) ? static_cast<void> (0) : __assert_fail ("BB && \"Must have a basic block to set any function attributes!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 259, __PRETTY_FUNCTION__)); | ||||||
260 | |||||||
261 | Function *F = BB->getParent(); | ||||||
262 | if (!F->hasFnAttribute(Attribute::StrictFP)) { | ||||||
263 | F->addFnAttr(Attribute::StrictFP); | ||||||
264 | } | ||||||
265 | } | ||||||
266 | |||||||
267 | void setConstrainedFPCallAttr(CallInst *I) { | ||||||
268 | if (!I->hasFnAttr(Attribute::StrictFP)) | ||||||
269 | I->addAttribute(AttributeList::FunctionIndex, Attribute::StrictFP); | ||||||
270 | setConstrainedFPFunctionAttr(); | ||||||
271 | } | ||||||
272 | |||||||
273 | //===--------------------------------------------------------------------===// | ||||||
274 | // RAII helpers. | ||||||
275 | //===--------------------------------------------------------------------===// | ||||||
276 | |||||||
277 | // RAII object that stores the current insertion point and restores it | ||||||
278 | // when the object is destroyed. This includes the debug location. | ||||||
279 | class InsertPointGuard { | ||||||
280 | IRBuilderBase &Builder; | ||||||
281 | AssertingVH<BasicBlock> Block; | ||||||
282 | BasicBlock::iterator Point; | ||||||
283 | DebugLoc DbgLoc; | ||||||
284 | |||||||
285 | public: | ||||||
286 | InsertPointGuard(IRBuilderBase &B) | ||||||
287 | : Builder(B), Block(B.GetInsertBlock()), Point(B.GetInsertPoint()), | ||||||
288 | DbgLoc(B.getCurrentDebugLocation()) {} | ||||||
289 | |||||||
290 | InsertPointGuard(const InsertPointGuard &) = delete; | ||||||
291 | InsertPointGuard &operator=(const InsertPointGuard &) = delete; | ||||||
292 | |||||||
293 | ~InsertPointGuard() { | ||||||
294 | Builder.restoreIP(InsertPoint(Block, Point)); | ||||||
295 | Builder.SetCurrentDebugLocation(DbgLoc); | ||||||
296 | } | ||||||
297 | }; | ||||||
298 | |||||||
299 | // RAII object that stores the current fast math settings and restores | ||||||
300 | // them when the object is destroyed. | ||||||
301 | class FastMathFlagGuard { | ||||||
302 | IRBuilderBase &Builder; | ||||||
303 | FastMathFlags FMF; | ||||||
304 | MDNode *FPMathTag; | ||||||
305 | |||||||
306 | public: | ||||||
307 | FastMathFlagGuard(IRBuilderBase &B) | ||||||
308 | : Builder(B), FMF(B.FMF), FPMathTag(B.DefaultFPMathTag) {} | ||||||
309 | |||||||
310 | FastMathFlagGuard(const FastMathFlagGuard &) = delete; | ||||||
311 | FastMathFlagGuard &operator=(const FastMathFlagGuard &) = delete; | ||||||
312 | |||||||
313 | ~FastMathFlagGuard() { | ||||||
314 | Builder.FMF = FMF; | ||||||
315 | Builder.DefaultFPMathTag = FPMathTag; | ||||||
316 | } | ||||||
317 | }; | ||||||
318 | |||||||
319 | //===--------------------------------------------------------------------===// | ||||||
320 | // Miscellaneous creation methods. | ||||||
321 | //===--------------------------------------------------------------------===// | ||||||
322 | |||||||
323 | /// Make a new global variable with initializer type i8* | ||||||
324 | /// | ||||||
325 | /// Make a new global variable with an initializer that has array of i8 type | ||||||
326 | /// filled in with the null terminated string value specified. The new global | ||||||
327 | /// variable will be marked mergable with any others of the same contents. If | ||||||
328 | /// Name is specified, it is the name of the global variable created. | ||||||
329 | GlobalVariable *CreateGlobalString(StringRef Str, const Twine &Name = "", | ||||||
330 | unsigned AddressSpace = 0); | ||||||
331 | |||||||
332 | /// Get a constant value representing either true or false. | ||||||
333 | ConstantInt *getInt1(bool V) { | ||||||
334 | return ConstantInt::get(getInt1Ty(), V); | ||||||
335 | } | ||||||
336 | |||||||
337 | /// Get the constant value for i1 true. | ||||||
338 | ConstantInt *getTrue() { | ||||||
339 | return ConstantInt::getTrue(Context); | ||||||
340 | } | ||||||
341 | |||||||
342 | /// Get the constant value for i1 false. | ||||||
343 | ConstantInt *getFalse() { | ||||||
344 | return ConstantInt::getFalse(Context); | ||||||
345 | } | ||||||
346 | |||||||
347 | /// Get a constant 8-bit value. | ||||||
348 | ConstantInt *getInt8(uint8_t C) { | ||||||
349 | return ConstantInt::get(getInt8Ty(), C); | ||||||
350 | } | ||||||
351 | |||||||
352 | /// Get a constant 16-bit value. | ||||||
353 | ConstantInt *getInt16(uint16_t C) { | ||||||
354 | return ConstantInt::get(getInt16Ty(), C); | ||||||
355 | } | ||||||
356 | |||||||
357 | /// Get a constant 32-bit value. | ||||||
358 | ConstantInt *getInt32(uint32_t C) { | ||||||
359 | return ConstantInt::get(getInt32Ty(), C); | ||||||
360 | } | ||||||
361 | |||||||
362 | /// Get a constant 64-bit value. | ||||||
363 | ConstantInt *getInt64(uint64_t C) { | ||||||
364 | return ConstantInt::get(getInt64Ty(), C); | ||||||
365 | } | ||||||
366 | |||||||
367 | /// Get a constant N-bit value, zero extended or truncated from | ||||||
368 | /// a 64-bit value. | ||||||
369 | ConstantInt *getIntN(unsigned N, uint64_t C) { | ||||||
370 | return ConstantInt::get(getIntNTy(N), C); | ||||||
371 | } | ||||||
372 | |||||||
373 | /// Get a constant integer value. | ||||||
374 | ConstantInt *getInt(const APInt &AI) { | ||||||
375 | return ConstantInt::get(Context, AI); | ||||||
376 | } | ||||||
377 | |||||||
378 | //===--------------------------------------------------------------------===// | ||||||
379 | // Type creation methods | ||||||
380 | //===--------------------------------------------------------------------===// | ||||||
381 | |||||||
382 | /// Fetch the type representing a single bit | ||||||
383 | IntegerType *getInt1Ty() { | ||||||
384 | return Type::getInt1Ty(Context); | ||||||
385 | } | ||||||
386 | |||||||
387 | /// Fetch the type representing an 8-bit integer. | ||||||
388 | IntegerType *getInt8Ty() { | ||||||
389 | return Type::getInt8Ty(Context); | ||||||
390 | } | ||||||
391 | |||||||
392 | /// Fetch the type representing a 16-bit integer. | ||||||
393 | IntegerType *getInt16Ty() { | ||||||
394 | return Type::getInt16Ty(Context); | ||||||
395 | } | ||||||
396 | |||||||
397 | /// Fetch the type representing a 32-bit integer. | ||||||
398 | IntegerType *getInt32Ty() { | ||||||
399 | return Type::getInt32Ty(Context); | ||||||
400 | } | ||||||
401 | |||||||
402 | /// Fetch the type representing a 64-bit integer. | ||||||
403 | IntegerType *getInt64Ty() { | ||||||
404 | return Type::getInt64Ty(Context); | ||||||
405 | } | ||||||
406 | |||||||
407 | /// Fetch the type representing a 128-bit integer. | ||||||
408 | IntegerType *getInt128Ty() { return Type::getInt128Ty(Context); } | ||||||
409 | |||||||
410 | /// Fetch the type representing an N-bit integer. | ||||||
411 | IntegerType *getIntNTy(unsigned N) { | ||||||
412 | return Type::getIntNTy(Context, N); | ||||||
413 | } | ||||||
414 | |||||||
415 | /// Fetch the type representing a 16-bit floating point value. | ||||||
416 | Type *getHalfTy() { | ||||||
417 | return Type::getHalfTy(Context); | ||||||
418 | } | ||||||
419 | |||||||
420 | /// Fetch the type representing a 32-bit floating point value. | ||||||
421 | Type *getFloatTy() { | ||||||
422 | return Type::getFloatTy(Context); | ||||||
423 | } | ||||||
424 | |||||||
425 | /// Fetch the type representing a 64-bit floating point value. | ||||||
426 | Type *getDoubleTy() { | ||||||
427 | return Type::getDoubleTy(Context); | ||||||
428 | } | ||||||
429 | |||||||
430 | /// Fetch the type representing void. | ||||||
431 | Type *getVoidTy() { | ||||||
432 | return Type::getVoidTy(Context); | ||||||
433 | } | ||||||
434 | |||||||
435 | /// Fetch the type representing a pointer to an 8-bit integer value. | ||||||
436 | PointerType *getInt8PtrTy(unsigned AddrSpace = 0) { | ||||||
437 | return Type::getInt8PtrTy(Context, AddrSpace); | ||||||
438 | } | ||||||
439 | |||||||
440 | /// Fetch the type representing a pointer to an integer value. | ||||||
441 | IntegerType *getIntPtrTy(const DataLayout &DL, unsigned AddrSpace = 0) { | ||||||
442 | return DL.getIntPtrType(Context, AddrSpace); | ||||||
443 | } | ||||||
444 | |||||||
445 | //===--------------------------------------------------------------------===// | ||||||
446 | // Intrinsic creation methods | ||||||
447 | //===--------------------------------------------------------------------===// | ||||||
448 | |||||||
449 | /// Create and insert a memset to the specified pointer and the | ||||||
450 | /// specified value. | ||||||
451 | /// | ||||||
452 | /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is | ||||||
453 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
454 | /// and noalias tags. | ||||||
455 | CallInst *CreateMemSet(Value *Ptr, Value *Val, uint64_t Size, unsigned Align, | ||||||
456 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
457 | MDNode *ScopeTag = nullptr, | ||||||
458 | MDNode *NoAliasTag = nullptr) { | ||||||
459 | return CreateMemSet(Ptr, Val, getInt64(Size), Align, isVolatile, | ||||||
460 | TBAATag, ScopeTag, NoAliasTag); | ||||||
461 | } | ||||||
462 | |||||||
463 | CallInst *CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align, | ||||||
464 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
465 | MDNode *ScopeTag = nullptr, | ||||||
466 | MDNode *NoAliasTag = nullptr); | ||||||
467 | |||||||
468 | /// Create and insert an element unordered-atomic memset of the region of | ||||||
469 | /// memory starting at the given pointer to the given value. | ||||||
470 | /// | ||||||
471 | /// If the pointer isn't an i8*, it will be converted. If a TBAA tag is | ||||||
472 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
473 | /// and noalias tags. | ||||||
474 | CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, | ||||||
475 | uint64_t Size, unsigned Align, | ||||||
476 | uint32_t ElementSize, | ||||||
477 | MDNode *TBAATag = nullptr, | ||||||
478 | MDNode *ScopeTag = nullptr, | ||||||
479 | MDNode *NoAliasTag = nullptr) { | ||||||
480 | return CreateElementUnorderedAtomicMemSet(Ptr, Val, getInt64(Size), Align, | ||||||
481 | ElementSize, TBAATag, ScopeTag, | ||||||
482 | NoAliasTag); | ||||||
483 | } | ||||||
484 | |||||||
485 | CallInst *CreateElementUnorderedAtomicMemSet(Value *Ptr, Value *Val, | ||||||
486 | Value *Size, unsigned Align, | ||||||
487 | uint32_t ElementSize, | ||||||
488 | MDNode *TBAATag = nullptr, | ||||||
489 | MDNode *ScopeTag = nullptr, | ||||||
490 | MDNode *NoAliasTag = nullptr); | ||||||
491 | |||||||
492 | /// Create and insert a memcpy between the specified pointers. | ||||||
493 | /// | ||||||
494 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
495 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
496 | /// and noalias tags. | ||||||
497 | CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, | ||||||
498 | unsigned SrcAlign, uint64_t Size, | ||||||
499 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
500 | MDNode *TBAAStructTag = nullptr, | ||||||
501 | MDNode *ScopeTag = nullptr, | ||||||
502 | MDNode *NoAliasTag = nullptr) { | ||||||
503 | return CreateMemCpy(Dst, DstAlign, Src, SrcAlign, getInt64(Size), | ||||||
504 | isVolatile, TBAATag, TBAAStructTag, ScopeTag, | ||||||
505 | NoAliasTag); | ||||||
506 | } | ||||||
507 | |||||||
508 | CallInst *CreateMemCpy(Value *Dst, unsigned DstAlign, Value *Src, | ||||||
509 | unsigned SrcAlign, Value *Size, | ||||||
510 | bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
511 | MDNode *TBAAStructTag = nullptr, | ||||||
512 | MDNode *ScopeTag = nullptr, | ||||||
513 | MDNode *NoAliasTag = nullptr); | ||||||
514 | |||||||
515 | /// Create and insert an element unordered-atomic memcpy between the | ||||||
516 | /// specified pointers. | ||||||
517 | /// | ||||||
518 | /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, respectively. | ||||||
519 | /// | ||||||
520 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
521 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
522 | /// and noalias tags. | ||||||
523 | CallInst *CreateElementUnorderedAtomicMemCpy( | ||||||
524 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, | ||||||
525 | uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
526 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
527 | MDNode *NoAliasTag = nullptr) { | ||||||
528 | return CreateElementUnorderedAtomicMemCpy( | ||||||
529 | Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag, | ||||||
530 | TBAAStructTag, ScopeTag, NoAliasTag); | ||||||
531 | } | ||||||
532 | |||||||
533 | CallInst *CreateElementUnorderedAtomicMemCpy( | ||||||
534 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, | ||||||
535 | uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
536 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
537 | MDNode *NoAliasTag = nullptr); | ||||||
538 | |||||||
539 | /// Create and insert a memmove between the specified | ||||||
540 | /// pointers. | ||||||
541 | /// | ||||||
542 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
543 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
544 | /// and noalias tags. | ||||||
545 | CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, | ||||||
546 | uint64_t Size, bool isVolatile = false, | ||||||
547 | MDNode *TBAATag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
548 | MDNode *NoAliasTag = nullptr) { | ||||||
549 | return CreateMemMove(Dst, DstAlign, Src, SrcAlign, getInt64(Size), isVolatile, | ||||||
550 | TBAATag, ScopeTag, NoAliasTag); | ||||||
551 | } | ||||||
552 | |||||||
553 | CallInst *CreateMemMove(Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, | ||||||
554 | Value *Size, bool isVolatile = false, MDNode *TBAATag = nullptr, | ||||||
555 | MDNode *ScopeTag = nullptr, | ||||||
556 | MDNode *NoAliasTag = nullptr); | ||||||
557 | |||||||
558 | /// \brief Create and insert an element unordered-atomic memmove between the | ||||||
559 | /// specified pointers. | ||||||
560 | /// | ||||||
561 | /// DstAlign/SrcAlign are the alignments of the Dst/Src pointers, | ||||||
562 | /// respectively. | ||||||
563 | /// | ||||||
564 | /// If the pointers aren't i8*, they will be converted. If a TBAA tag is | ||||||
565 | /// specified, it will be added to the instruction. Likewise with alias.scope | ||||||
566 | /// and noalias tags. | ||||||
567 | CallInst *CreateElementUnorderedAtomicMemMove( | ||||||
568 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, | ||||||
569 | uint64_t Size, uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
570 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
571 | MDNode *NoAliasTag = nullptr) { | ||||||
572 | return CreateElementUnorderedAtomicMemMove( | ||||||
573 | Dst, DstAlign, Src, SrcAlign, getInt64(Size), ElementSize, TBAATag, | ||||||
574 | TBAAStructTag, ScopeTag, NoAliasTag); | ||||||
575 | } | ||||||
576 | |||||||
577 | CallInst *CreateElementUnorderedAtomicMemMove( | ||||||
578 | Value *Dst, unsigned DstAlign, Value *Src, unsigned SrcAlign, Value *Size, | ||||||
579 | uint32_t ElementSize, MDNode *TBAATag = nullptr, | ||||||
580 | MDNode *TBAAStructTag = nullptr, MDNode *ScopeTag = nullptr, | ||||||
581 | MDNode *NoAliasTag = nullptr); | ||||||
582 | |||||||
583 | /// Create a vector fadd reduction intrinsic of the source vector. | ||||||
584 | /// The first parameter is a scalar accumulator value for ordered reductions. | ||||||
585 | CallInst *CreateFAddReduce(Value *Acc, Value *Src); | ||||||
586 | |||||||
587 | /// Create a vector fmul reduction intrinsic of the source vector. | ||||||
588 | /// The first parameter is a scalar accumulator value for ordered reductions. | ||||||
589 | CallInst *CreateFMulReduce(Value *Acc, Value *Src); | ||||||
590 | |||||||
591 | /// Create a vector int add reduction intrinsic of the source vector. | ||||||
592 | CallInst *CreateAddReduce(Value *Src); | ||||||
593 | |||||||
594 | /// Create a vector int mul reduction intrinsic of the source vector. | ||||||
595 | CallInst *CreateMulReduce(Value *Src); | ||||||
596 | |||||||
597 | /// Create a vector int AND reduction intrinsic of the source vector. | ||||||
598 | CallInst *CreateAndReduce(Value *Src); | ||||||
599 | |||||||
600 | /// Create a vector int OR reduction intrinsic of the source vector. | ||||||
601 | CallInst *CreateOrReduce(Value *Src); | ||||||
602 | |||||||
603 | /// Create a vector int XOR reduction intrinsic of the source vector. | ||||||
604 | CallInst *CreateXorReduce(Value *Src); | ||||||
605 | |||||||
606 | /// Create a vector integer max reduction intrinsic of the source | ||||||
607 | /// vector. | ||||||
608 | CallInst *CreateIntMaxReduce(Value *Src, bool IsSigned = false); | ||||||
609 | |||||||
610 | /// Create a vector integer min reduction intrinsic of the source | ||||||
611 | /// vector. | ||||||
612 | CallInst *CreateIntMinReduce(Value *Src, bool IsSigned = false); | ||||||
613 | |||||||
614 | /// Create a vector float max reduction intrinsic of the source | ||||||
615 | /// vector. | ||||||
616 | CallInst *CreateFPMaxReduce(Value *Src, bool NoNaN = false); | ||||||
617 | |||||||
618 | /// Create a vector float min reduction intrinsic of the source | ||||||
619 | /// vector. | ||||||
620 | CallInst *CreateFPMinReduce(Value *Src, bool NoNaN = false); | ||||||
621 | |||||||
622 | /// Create a lifetime.start intrinsic. | ||||||
623 | /// | ||||||
624 | /// If the pointer isn't i8* it will be converted. | ||||||
625 | CallInst *CreateLifetimeStart(Value *Ptr, ConstantInt *Size = nullptr); | ||||||
626 | |||||||
627 | /// Create a lifetime.end intrinsic. | ||||||
628 | /// | ||||||
629 | /// If the pointer isn't i8* it will be converted. | ||||||
630 | CallInst *CreateLifetimeEnd(Value *Ptr, ConstantInt *Size = nullptr); | ||||||
631 | |||||||
632 | /// Create a call to invariant.start intrinsic. | ||||||
633 | /// | ||||||
634 | /// If the pointer isn't i8* it will be converted. | ||||||
635 | CallInst *CreateInvariantStart(Value *Ptr, ConstantInt *Size = nullptr); | ||||||
636 | |||||||
637 | /// Create a call to Masked Load intrinsic | ||||||
638 | CallInst *CreateMaskedLoad(Value *Ptr, unsigned Align, Value *Mask, | ||||||
639 | Value *PassThru = nullptr, const Twine &Name = ""); | ||||||
640 | |||||||
641 | /// Create a call to Masked Store intrinsic | ||||||
642 | CallInst *CreateMaskedStore(Value *Val, Value *Ptr, unsigned Align, | ||||||
643 | Value *Mask); | ||||||
644 | |||||||
645 | /// Create a call to Masked Gather intrinsic | ||||||
646 | CallInst *CreateMaskedGather(Value *Ptrs, unsigned Align, | ||||||
647 | Value *Mask = nullptr, | ||||||
648 | Value *PassThru = nullptr, | ||||||
649 | const Twine& Name = ""); | ||||||
650 | |||||||
651 | /// Create a call to Masked Scatter intrinsic | ||||||
652 | CallInst *CreateMaskedScatter(Value *Val, Value *Ptrs, unsigned Align, | ||||||
653 | Value *Mask = nullptr); | ||||||
654 | |||||||
655 | /// Create an assume intrinsic call that allows the optimizer to | ||||||
656 | /// assume that the provided condition will be true. | ||||||
657 | CallInst *CreateAssumption(Value *Cond); | ||||||
658 | |||||||
659 | /// Create a call to the experimental.gc.statepoint intrinsic to | ||||||
660 | /// start a new statepoint sequence. | ||||||
661 | CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, | ||||||
662 | Value *ActualCallee, | ||||||
663 | ArrayRef<Value *> CallArgs, | ||||||
664 | ArrayRef<Value *> DeoptArgs, | ||||||
665 | ArrayRef<Value *> GCArgs, | ||||||
666 | const Twine &Name = ""); | ||||||
667 | |||||||
668 | /// Create a call to the experimental.gc.statepoint intrinsic to | ||||||
669 | /// start a new statepoint sequence. | ||||||
670 | CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, | ||||||
671 | Value *ActualCallee, uint32_t Flags, | ||||||
672 | ArrayRef<Use> CallArgs, | ||||||
673 | ArrayRef<Use> TransitionArgs, | ||||||
674 | ArrayRef<Use> DeoptArgs, | ||||||
675 | ArrayRef<Value *> GCArgs, | ||||||
676 | const Twine &Name = ""); | ||||||
677 | |||||||
678 | /// Conveninence function for the common case when CallArgs are filled | ||||||
679 | /// in using makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be | ||||||
680 | /// .get()'ed to get the Value pointer. | ||||||
681 | CallInst *CreateGCStatepointCall(uint64_t ID, uint32_t NumPatchBytes, | ||||||
682 | Value *ActualCallee, ArrayRef<Use> CallArgs, | ||||||
683 | ArrayRef<Value *> DeoptArgs, | ||||||
684 | ArrayRef<Value *> GCArgs, | ||||||
685 | const Twine &Name = ""); | ||||||
686 | |||||||
687 | /// Create an invoke to the experimental.gc.statepoint intrinsic to | ||||||
688 | /// start a new statepoint sequence. | ||||||
689 | InvokeInst * | ||||||
690 | CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, | ||||||
691 | Value *ActualInvokee, BasicBlock *NormalDest, | ||||||
692 | BasicBlock *UnwindDest, ArrayRef<Value *> InvokeArgs, | ||||||
693 | ArrayRef<Value *> DeoptArgs, | ||||||
694 | ArrayRef<Value *> GCArgs, const Twine &Name = ""); | ||||||
695 | |||||||
696 | /// Create an invoke to the experimental.gc.statepoint intrinsic to | ||||||
697 | /// start a new statepoint sequence. | ||||||
698 | InvokeInst *CreateGCStatepointInvoke( | ||||||
699 | uint64_t ID, uint32_t NumPatchBytes, Value *ActualInvokee, | ||||||
700 | BasicBlock *NormalDest, BasicBlock *UnwindDest, uint32_t Flags, | ||||||
701 | ArrayRef<Use> InvokeArgs, ArrayRef<Use> TransitionArgs, | ||||||
702 | ArrayRef<Use> DeoptArgs, ArrayRef<Value *> GCArgs, | ||||||
703 | const Twine &Name = ""); | ||||||
704 | |||||||
705 | // Convenience function for the common case when CallArgs are filled in using | ||||||
706 | // makeArrayRef(CS.arg_begin(), CS.arg_end()); Use needs to be .get()'ed to | ||||||
707 | // get the Value *. | ||||||
708 | InvokeInst * | ||||||
709 | CreateGCStatepointInvoke(uint64_t ID, uint32_t NumPatchBytes, | ||||||
710 | Value *ActualInvokee, BasicBlock *NormalDest, | ||||||
711 | BasicBlock *UnwindDest, ArrayRef<Use> InvokeArgs, | ||||||
712 | ArrayRef<Value *> DeoptArgs, | ||||||
713 | ArrayRef<Value *> GCArgs, const Twine &Name = ""); | ||||||
714 | |||||||
715 | /// Create a call to the experimental.gc.result intrinsic to extract | ||||||
716 | /// the result from a call wrapped in a statepoint. | ||||||
717 | CallInst *CreateGCResult(Instruction *Statepoint, | ||||||
718 | Type *ResultType, | ||||||
719 | const Twine &Name = ""); | ||||||
720 | |||||||
721 | /// Create a call to the experimental.gc.relocate intrinsics to | ||||||
722 | /// project the relocated value of one pointer from the statepoint. | ||||||
723 | CallInst *CreateGCRelocate(Instruction *Statepoint, | ||||||
724 | int BaseOffset, | ||||||
725 | int DerivedOffset, | ||||||
726 | Type *ResultType, | ||||||
727 | const Twine &Name = ""); | ||||||
728 | |||||||
729 | /// Create a call to intrinsic \p ID with 1 operand which is mangled on its | ||||||
730 | /// type. | ||||||
731 | CallInst *CreateUnaryIntrinsic(Intrinsic::ID ID, Value *V, | ||||||
732 | Instruction *FMFSource = nullptr, | ||||||
733 | const Twine &Name = ""); | ||||||
734 | |||||||
735 | /// Create a call to intrinsic \p ID with 2 operands which is mangled on the | ||||||
736 | /// first type. | ||||||
737 | CallInst *CreateBinaryIntrinsic(Intrinsic::ID ID, Value *LHS, Value *RHS, | ||||||
738 | Instruction *FMFSource = nullptr, | ||||||
739 | const Twine &Name = ""); | ||||||
740 | |||||||
741 | /// Create a call to intrinsic \p ID with \p args, mangled using \p Types. If | ||||||
742 | /// \p FMFSource is provided, copy fast-math-flags from that instruction to | ||||||
743 | /// the intrinsic. | ||||||
744 | CallInst *CreateIntrinsic(Intrinsic::ID ID, ArrayRef<Type *> Types, | ||||||
745 | ArrayRef<Value *> Args, | ||||||
746 | Instruction *FMFSource = nullptr, | ||||||
747 | const Twine &Name = ""); | ||||||
748 | |||||||
749 | /// Create call to the minnum intrinsic. | ||||||
750 | CallInst *CreateMinNum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
751 | return CreateBinaryIntrinsic(Intrinsic::minnum, LHS, RHS, nullptr, Name); | ||||||
752 | } | ||||||
753 | |||||||
754 | /// Create call to the maxnum intrinsic. | ||||||
755 | CallInst *CreateMaxNum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
756 | return CreateBinaryIntrinsic(Intrinsic::maxnum, LHS, RHS, nullptr, Name); | ||||||
757 | } | ||||||
758 | |||||||
759 | /// Create call to the minimum intrinsic. | ||||||
760 | CallInst *CreateMinimum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
761 | return CreateBinaryIntrinsic(Intrinsic::minimum, LHS, RHS, nullptr, Name); | ||||||
762 | } | ||||||
763 | |||||||
764 | /// Create call to the maximum intrinsic. | ||||||
765 | CallInst *CreateMaximum(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
766 | return CreateBinaryIntrinsic(Intrinsic::maximum, LHS, RHS, nullptr, Name); | ||||||
767 | } | ||||||
768 | |||||||
769 | private: | ||||||
770 | /// Create a call to a masked intrinsic with given Id. | ||||||
771 | CallInst *CreateMaskedIntrinsic(Intrinsic::ID Id, ArrayRef<Value *> Ops, | ||||||
772 | ArrayRef<Type *> OverloadedTypes, | ||||||
773 | const Twine &Name = ""); | ||||||
774 | |||||||
775 | Value *getCastedInt8PtrValue(Value *Ptr); | ||||||
776 | }; | ||||||
777 | |||||||
778 | /// This provides a uniform API for creating instructions and inserting | ||||||
779 | /// them into a basic block: either at the end of a BasicBlock, or at a specific | ||||||
780 | /// iterator location in a block. | ||||||
781 | /// | ||||||
782 | /// Note that the builder does not expose the full generality of LLVM | ||||||
783 | /// instructions. For access to extra instruction properties, use the mutators | ||||||
784 | /// (e.g. setVolatile) on the instructions after they have been | ||||||
785 | /// created. Convenience state exists to specify fast-math flags and fp-math | ||||||
786 | /// tags. | ||||||
787 | /// | ||||||
788 | /// The first template argument specifies a class to use for creating constants. | ||||||
789 | /// This defaults to creating minimally folded constants. The second template | ||||||
790 | /// argument allows clients to specify custom insertion hooks that are called on | ||||||
791 | /// every newly created insertion. | ||||||
792 | template <typename T = ConstantFolder, | ||||||
793 | typename Inserter = IRBuilderDefaultInserter> | ||||||
794 | class IRBuilder : public IRBuilderBase, public Inserter { | ||||||
795 | T Folder; | ||||||
796 | |||||||
797 | public: | ||||||
798 | IRBuilder(LLVMContext &C, const T &F, Inserter I = Inserter(), | ||||||
799 | MDNode *FPMathTag = nullptr, | ||||||
800 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
801 | : IRBuilderBase(C, FPMathTag, OpBundles), Inserter(std::move(I)), | ||||||
802 | Folder(F) {} | ||||||
803 | |||||||
804 | explicit IRBuilder(LLVMContext &C, MDNode *FPMathTag = nullptr, | ||||||
805 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
806 | : IRBuilderBase(C, FPMathTag, OpBundles) {} | ||||||
807 | |||||||
808 | explicit IRBuilder(BasicBlock *TheBB, const T &F, MDNode *FPMathTag = nullptr, | ||||||
809 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
810 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles), Folder(F) { | ||||||
811 | SetInsertPoint(TheBB); | ||||||
812 | } | ||||||
813 | |||||||
814 | explicit IRBuilder(BasicBlock *TheBB, MDNode *FPMathTag = nullptr, | ||||||
815 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
816 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles) { | ||||||
817 | SetInsertPoint(TheBB); | ||||||
818 | } | ||||||
819 | |||||||
820 | explicit IRBuilder(Instruction *IP, MDNode *FPMathTag = nullptr, | ||||||
821 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
822 | : IRBuilderBase(IP->getContext(), FPMathTag, OpBundles) { | ||||||
823 | SetInsertPoint(IP); | ||||||
824 | } | ||||||
825 | |||||||
826 | IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, const T &F, | ||||||
827 | MDNode *FPMathTag = nullptr, | ||||||
828 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
829 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles), Folder(F) { | ||||||
830 | SetInsertPoint(TheBB, IP); | ||||||
831 | } | ||||||
832 | |||||||
833 | IRBuilder(BasicBlock *TheBB, BasicBlock::iterator IP, | ||||||
834 | MDNode *FPMathTag = nullptr, | ||||||
835 | ArrayRef<OperandBundleDef> OpBundles = None) | ||||||
836 | : IRBuilderBase(TheBB->getContext(), FPMathTag, OpBundles) { | ||||||
837 | SetInsertPoint(TheBB, IP); | ||||||
838 | } | ||||||
839 | |||||||
840 | /// Get the constant folder being used. | ||||||
841 | const T &getFolder() { return Folder; } | ||||||
842 | |||||||
843 | /// Insert and return the specified instruction. | ||||||
844 | template<typename InstTy> | ||||||
845 | InstTy *Insert(InstTy *I, const Twine &Name = "") const { | ||||||
846 | this->InsertHelper(I, Name, BB, InsertPt); | ||||||
847 | this->SetInstDebugLocation(I); | ||||||
848 | return I; | ||||||
849 | } | ||||||
850 | |||||||
851 | /// No-op overload to handle constants. | ||||||
852 | Constant *Insert(Constant *C, const Twine& = "") const { | ||||||
853 | return C; | ||||||
854 | } | ||||||
855 | |||||||
856 | //===--------------------------------------------------------------------===// | ||||||
857 | // Instruction creation methods: Terminators | ||||||
858 | //===--------------------------------------------------------------------===// | ||||||
859 | |||||||
860 | private: | ||||||
861 | /// Helper to add branch weight and unpredictable metadata onto an | ||||||
862 | /// instruction. | ||||||
863 | /// \returns The annotated instruction. | ||||||
864 | template <typename InstTy> | ||||||
865 | InstTy *addBranchMetadata(InstTy *I, MDNode *Weights, MDNode *Unpredictable) { | ||||||
866 | if (Weights) | ||||||
867 | I->setMetadata(LLVMContext::MD_prof, Weights); | ||||||
868 | if (Unpredictable) | ||||||
869 | I->setMetadata(LLVMContext::MD_unpredictable, Unpredictable); | ||||||
870 | return I; | ||||||
871 | } | ||||||
872 | |||||||
873 | public: | ||||||
874 | /// Create a 'ret void' instruction. | ||||||
875 | ReturnInst *CreateRetVoid() { | ||||||
876 | return Insert(ReturnInst::Create(Context)); | ||||||
877 | } | ||||||
878 | |||||||
879 | /// Create a 'ret <val>' instruction. | ||||||
880 | ReturnInst *CreateRet(Value *V) { | ||||||
881 | return Insert(ReturnInst::Create(Context, V)); | ||||||
882 | } | ||||||
883 | |||||||
884 | /// Create a sequence of N insertvalue instructions, | ||||||
885 | /// with one Value from the retVals array each, that build a aggregate | ||||||
886 | /// return value one value at a time, and a ret instruction to return | ||||||
887 | /// the resulting aggregate value. | ||||||
888 | /// | ||||||
889 | /// This is a convenience function for code that uses aggregate return values | ||||||
890 | /// as a vehicle for having multiple return values. | ||||||
891 | ReturnInst *CreateAggregateRet(Value *const *retVals, unsigned N) { | ||||||
892 | Value *V = UndefValue::get(getCurrentFunctionReturnType()); | ||||||
893 | for (unsigned i = 0; i != N; ++i) | ||||||
894 | V = CreateInsertValue(V, retVals[i], i, "mrv"); | ||||||
895 | return Insert(ReturnInst::Create(Context, V)); | ||||||
896 | } | ||||||
897 | |||||||
898 | /// Create an unconditional 'br label X' instruction. | ||||||
899 | BranchInst *CreateBr(BasicBlock *Dest) { | ||||||
900 | return Insert(BranchInst::Create(Dest)); | ||||||
901 | } | ||||||
902 | |||||||
903 | /// Create a conditional 'br Cond, TrueDest, FalseDest' | ||||||
904 | /// instruction. | ||||||
905 | BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, | ||||||
906 | MDNode *BranchWeights = nullptr, | ||||||
907 | MDNode *Unpredictable = nullptr) { | ||||||
908 | return Insert(addBranchMetadata(BranchInst::Create(True, False, Cond), | ||||||
909 | BranchWeights, Unpredictable)); | ||||||
910 | } | ||||||
911 | |||||||
912 | /// Create a conditional 'br Cond, TrueDest, FalseDest' | ||||||
913 | /// instruction. Copy branch meta data if available. | ||||||
914 | BranchInst *CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, | ||||||
915 | Instruction *MDSrc) { | ||||||
916 | BranchInst *Br = BranchInst::Create(True, False, Cond); | ||||||
917 | if (MDSrc) { | ||||||
918 | unsigned WL[4] = {LLVMContext::MD_prof, LLVMContext::MD_unpredictable, | ||||||
919 | LLVMContext::MD_make_implicit, LLVMContext::MD_dbg}; | ||||||
920 | Br->copyMetadata(*MDSrc, makeArrayRef(&WL[0], 4)); | ||||||
921 | } | ||||||
922 | return Insert(Br); | ||||||
923 | } | ||||||
924 | |||||||
925 | /// Create a switch instruction with the specified value, default dest, | ||||||
926 | /// and with a hint for the number of cases that will be added (for efficient | ||||||
927 | /// allocation). | ||||||
928 | SwitchInst *CreateSwitch(Value *V, BasicBlock *Dest, unsigned NumCases = 10, | ||||||
929 | MDNode *BranchWeights = nullptr, | ||||||
930 | MDNode *Unpredictable = nullptr) { | ||||||
931 | return Insert(addBranchMetadata(SwitchInst::Create(V, Dest, NumCases), | ||||||
932 | BranchWeights, Unpredictable)); | ||||||
933 | } | ||||||
934 | |||||||
935 | /// Create an indirect branch instruction with the specified address | ||||||
936 | /// operand, with an optional hint for the number of destinations that will be | ||||||
937 | /// added (for efficient allocation). | ||||||
938 | IndirectBrInst *CreateIndirectBr(Value *Addr, unsigned NumDests = 10) { | ||||||
939 | return Insert(IndirectBrInst::Create(Addr, NumDests)); | ||||||
940 | } | ||||||
941 | |||||||
942 | /// Create an invoke instruction. | ||||||
943 | InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee, | ||||||
944 | BasicBlock *NormalDest, BasicBlock *UnwindDest, | ||||||
945 | ArrayRef<Value *> Args, | ||||||
946 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
947 | const Twine &Name = "") { | ||||||
948 | return Insert( | ||||||
949 | InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args, OpBundles), | ||||||
950 | Name); | ||||||
951 | } | ||||||
952 | InvokeInst *CreateInvoke(FunctionType *Ty, Value *Callee, | ||||||
953 | BasicBlock *NormalDest, BasicBlock *UnwindDest, | ||||||
954 | ArrayRef<Value *> Args = None, | ||||||
955 | const Twine &Name = "") { | ||||||
956 | return Insert(InvokeInst::Create(Ty, Callee, NormalDest, UnwindDest, Args), | ||||||
957 | Name); | ||||||
958 | } | ||||||
959 | |||||||
960 | InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest, | ||||||
961 | BasicBlock *UnwindDest, ArrayRef<Value *> Args, | ||||||
962 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
963 | const Twine &Name = "") { | ||||||
964 | return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(), | ||||||
965 | NormalDest, UnwindDest, Args, OpBundles, Name); | ||||||
966 | } | ||||||
967 | |||||||
968 | InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest, | ||||||
969 | BasicBlock *UnwindDest, | ||||||
970 | ArrayRef<Value *> Args = None, | ||||||
971 | const Twine &Name = "") { | ||||||
972 | return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(), | ||||||
973 | NormalDest, UnwindDest, Args, Name); | ||||||
974 | } | ||||||
975 | |||||||
976 | // Deprecated [opaque pointer types] | ||||||
977 | InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, | ||||||
978 | BasicBlock *UnwindDest, ArrayRef<Value *> Args, | ||||||
979 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
980 | const Twine &Name = "") { | ||||||
981 | return CreateInvoke( | ||||||
982 | cast<FunctionType>( | ||||||
983 | cast<PointerType>(Callee->getType())->getElementType()), | ||||||
984 | Callee, NormalDest, UnwindDest, Args, OpBundles, Name); | ||||||
985 | } | ||||||
986 | |||||||
987 | // Deprecated [opaque pointer types] | ||||||
988 | InvokeInst *CreateInvoke(Value *Callee, BasicBlock *NormalDest, | ||||||
989 | BasicBlock *UnwindDest, | ||||||
990 | ArrayRef<Value *> Args = None, | ||||||
991 | const Twine &Name = "") { | ||||||
992 | return CreateInvoke( | ||||||
993 | cast<FunctionType>( | ||||||
994 | cast<PointerType>(Callee->getType())->getElementType()), | ||||||
995 | Callee, NormalDest, UnwindDest, Args, Name); | ||||||
996 | } | ||||||
997 | |||||||
998 | /// \brief Create a callbr instruction. | ||||||
999 | CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, | ||||||
1000 | BasicBlock *DefaultDest, | ||||||
1001 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1002 | ArrayRef<Value *> Args = None, | ||||||
1003 | const Twine &Name = "") { | ||||||
1004 | return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, | ||||||
1005 | Args), Name); | ||||||
1006 | } | ||||||
1007 | CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, | ||||||
1008 | BasicBlock *DefaultDest, | ||||||
1009 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1010 | ArrayRef<Value *> Args, | ||||||
1011 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1012 | const Twine &Name = "") { | ||||||
1013 | return Insert( | ||||||
1014 | CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, | ||||||
1015 | OpBundles), Name); | ||||||
1016 | } | ||||||
1017 | |||||||
1018 | CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, | ||||||
1019 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1020 | ArrayRef<Value *> Args = None, | ||||||
1021 | const Twine &Name = "") { | ||||||
1022 | return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), | ||||||
1023 | DefaultDest, IndirectDests, Args, Name); | ||||||
1024 | } | ||||||
1025 | CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, | ||||||
1026 | ArrayRef<BasicBlock *> IndirectDests, | ||||||
1027 | ArrayRef<Value *> Args, | ||||||
1028 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
1029 | const Twine &Name = "") { | ||||||
1030 | return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), | ||||||
1031 | DefaultDest, IndirectDests, Args, Name); | ||||||
1032 | } | ||||||
1033 | |||||||
1034 | ResumeInst *CreateResume(Value *Exn) { | ||||||
1035 | return Insert(ResumeInst::Create(Exn)); | ||||||
1036 | } | ||||||
1037 | |||||||
1038 | CleanupReturnInst *CreateCleanupRet(CleanupPadInst *CleanupPad, | ||||||
1039 | BasicBlock *UnwindBB = nullptr) { | ||||||
1040 | return Insert(CleanupReturnInst::Create(CleanupPad, UnwindBB)); | ||||||
1041 | } | ||||||
1042 | |||||||
1043 | CatchSwitchInst *CreateCatchSwitch(Value *ParentPad, BasicBlock *UnwindBB, | ||||||
1044 | unsigned NumHandlers, | ||||||
1045 | const Twine &Name = "") { | ||||||
1046 | return Insert(CatchSwitchInst::Create(ParentPad, UnwindBB, NumHandlers), | ||||||
1047 | Name); | ||||||
1048 | } | ||||||
1049 | |||||||
1050 | CatchPadInst *CreateCatchPad(Value *ParentPad, ArrayRef<Value *> Args, | ||||||
1051 | const Twine &Name = "") { | ||||||
1052 | return Insert(CatchPadInst::Create(ParentPad, Args), Name); | ||||||
1053 | } | ||||||
1054 | |||||||
1055 | CleanupPadInst *CreateCleanupPad(Value *ParentPad, | ||||||
1056 | ArrayRef<Value *> Args = None, | ||||||
1057 | const Twine &Name = "") { | ||||||
1058 | return Insert(CleanupPadInst::Create(ParentPad, Args), Name); | ||||||
1059 | } | ||||||
1060 | |||||||
1061 | CatchReturnInst *CreateCatchRet(CatchPadInst *CatchPad, BasicBlock *BB) { | ||||||
1062 | return Insert(CatchReturnInst::Create(CatchPad, BB)); | ||||||
1063 | } | ||||||
1064 | |||||||
1065 | UnreachableInst *CreateUnreachable() { | ||||||
1066 | return Insert(new UnreachableInst(Context)); | ||||||
1067 | } | ||||||
1068 | |||||||
1069 | //===--------------------------------------------------------------------===// | ||||||
1070 | // Instruction creation methods: Binary Operators | ||||||
1071 | //===--------------------------------------------------------------------===// | ||||||
1072 | private: | ||||||
1073 | BinaryOperator *CreateInsertNUWNSWBinOp(BinaryOperator::BinaryOps Opc, | ||||||
1074 | Value *LHS, Value *RHS, | ||||||
1075 | const Twine &Name, | ||||||
1076 | bool HasNUW, bool HasNSW) { | ||||||
1077 | BinaryOperator *BO = Insert(BinaryOperator::Create(Opc, LHS, RHS), Name); | ||||||
1078 | if (HasNUW) BO->setHasNoUnsignedWrap(); | ||||||
1079 | if (HasNSW) BO->setHasNoSignedWrap(); | ||||||
1080 | return BO; | ||||||
1081 | } | ||||||
1082 | |||||||
1083 | Instruction *setFPAttrs(Instruction *I, MDNode *FPMD, | ||||||
1084 | FastMathFlags FMF) const { | ||||||
1085 | if (!FPMD) | ||||||
1086 | FPMD = DefaultFPMathTag; | ||||||
1087 | if (FPMD) | ||||||
1088 | I->setMetadata(LLVMContext::MD_fpmath, FPMD); | ||||||
1089 | I->setFastMathFlags(FMF); | ||||||
1090 | return I; | ||||||
1091 | } | ||||||
1092 | |||||||
1093 | Value *foldConstant(Instruction::BinaryOps Opc, Value *L, | ||||||
1094 | Value *R, const Twine &Name) const { | ||||||
1095 | auto *LC = dyn_cast<Constant>(L); | ||||||
1096 | auto *RC = dyn_cast<Constant>(R); | ||||||
1097 | return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr; | ||||||
1098 | } | ||||||
1099 | |||||||
1100 | Value *getConstrainedFPRounding( | ||||||
1101 | Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding) { | ||||||
1102 | ConstrainedFPIntrinsic::RoundingMode UseRounding = | ||||||
1103 | DefaultConstrainedRounding; | ||||||
1104 | |||||||
1105 | if (Rounding.hasValue()) | ||||||
1106 | UseRounding = Rounding.getValue(); | ||||||
1107 | |||||||
1108 | Optional<StringRef> RoundingStr = | ||||||
1109 | ConstrainedFPIntrinsic::RoundingModeToStr(UseRounding); | ||||||
1110 | assert(RoundingStr.hasValue() && "Garbage strict rounding mode!")((RoundingStr.hasValue() && "Garbage strict rounding mode!" ) ? static_cast<void> (0) : __assert_fail ("RoundingStr.hasValue() && \"Garbage strict rounding mode!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1110, __PRETTY_FUNCTION__)); | ||||||
1111 | auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue()); | ||||||
1112 | |||||||
1113 | return MetadataAsValue::get(Context, RoundingMDS); | ||||||
1114 | } | ||||||
1115 | |||||||
1116 | Value *getConstrainedFPExcept( | ||||||
1117 | Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except) { | ||||||
1118 | ConstrainedFPIntrinsic::ExceptionBehavior UseExcept = | ||||||
1119 | DefaultConstrainedExcept; | ||||||
1120 | |||||||
1121 | if (Except.hasValue()) | ||||||
1122 | UseExcept = Except.getValue(); | ||||||
1123 | |||||||
1124 | Optional<StringRef> ExceptStr = | ||||||
1125 | ConstrainedFPIntrinsic::ExceptionBehaviorToStr(UseExcept); | ||||||
1126 | assert(ExceptStr.hasValue() && "Garbage strict exception behavior!")((ExceptStr.hasValue() && "Garbage strict exception behavior!" ) ? static_cast<void> (0) : __assert_fail ("ExceptStr.hasValue() && \"Garbage strict exception behavior!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1126, __PRETTY_FUNCTION__)); | ||||||
1127 | auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue()); | ||||||
1128 | |||||||
1129 | return MetadataAsValue::get(Context, ExceptMDS); | ||||||
1130 | } | ||||||
1131 | |||||||
1132 | public: | ||||||
1133 | Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1134 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1135 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1136 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1137 | return Insert(Folder.CreateAdd(LC, RC, HasNUW, HasNSW), Name); | ||||||
1138 | return CreateInsertNUWNSWBinOp(Instruction::Add, LHS, RHS, Name, | ||||||
1139 | HasNUW, HasNSW); | ||||||
1140 | } | ||||||
1141 | |||||||
1142 | Value *CreateNSWAdd(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1143 | return CreateAdd(LHS, RHS, Name, false, true); | ||||||
1144 | } | ||||||
1145 | |||||||
1146 | Value *CreateNUWAdd(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1147 | return CreateAdd(LHS, RHS, Name, true, false); | ||||||
1148 | } | ||||||
1149 | |||||||
1150 | Value *CreateSub(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1151 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1152 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1153 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1154 | return Insert(Folder.CreateSub(LC, RC, HasNUW, HasNSW), Name); | ||||||
1155 | return CreateInsertNUWNSWBinOp(Instruction::Sub, LHS, RHS, Name, | ||||||
1156 | HasNUW, HasNSW); | ||||||
1157 | } | ||||||
1158 | |||||||
1159 | Value *CreateNSWSub(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1160 | return CreateSub(LHS, RHS, Name, false, true); | ||||||
1161 | } | ||||||
1162 | |||||||
1163 | Value *CreateNUWSub(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1164 | return CreateSub(LHS, RHS, Name, true, false); | ||||||
1165 | } | ||||||
1166 | |||||||
1167 | Value *CreateMul(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1168 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1169 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1170 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1171 | return Insert(Folder.CreateMul(LC, RC, HasNUW, HasNSW), Name); | ||||||
1172 | return CreateInsertNUWNSWBinOp(Instruction::Mul, LHS, RHS, Name, | ||||||
1173 | HasNUW, HasNSW); | ||||||
1174 | } | ||||||
1175 | |||||||
1176 | Value *CreateNSWMul(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1177 | return CreateMul(LHS, RHS, Name, false, true); | ||||||
1178 | } | ||||||
1179 | |||||||
1180 | Value *CreateNUWMul(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1181 | return CreateMul(LHS, RHS, Name, true, false); | ||||||
1182 | } | ||||||
1183 | |||||||
1184 | Value *CreateUDiv(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1185 | bool isExact = false) { | ||||||
1186 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1187 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1188 | return Insert(Folder.CreateUDiv(LC, RC, isExact), Name); | ||||||
1189 | if (!isExact) | ||||||
1190 | return Insert(BinaryOperator::CreateUDiv(LHS, RHS), Name); | ||||||
1191 | return Insert(BinaryOperator::CreateExactUDiv(LHS, RHS), Name); | ||||||
1192 | } | ||||||
1193 | |||||||
1194 | Value *CreateExactUDiv(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1195 | return CreateUDiv(LHS, RHS, Name, true); | ||||||
1196 | } | ||||||
1197 | |||||||
1198 | Value *CreateSDiv(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1199 | bool isExact = false) { | ||||||
1200 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1201 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1202 | return Insert(Folder.CreateSDiv(LC, RC, isExact), Name); | ||||||
1203 | if (!isExact) | ||||||
1204 | return Insert(BinaryOperator::CreateSDiv(LHS, RHS), Name); | ||||||
1205 | return Insert(BinaryOperator::CreateExactSDiv(LHS, RHS), Name); | ||||||
1206 | } | ||||||
1207 | |||||||
1208 | Value *CreateExactSDiv(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1209 | return CreateSDiv(LHS, RHS, Name, true); | ||||||
1210 | } | ||||||
1211 | |||||||
1212 | Value *CreateURem(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1213 | if (Value *V = foldConstant(Instruction::URem, LHS, RHS, Name)) return V; | ||||||
1214 | return Insert(BinaryOperator::CreateURem(LHS, RHS), Name); | ||||||
1215 | } | ||||||
1216 | |||||||
1217 | Value *CreateSRem(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1218 | if (Value *V = foldConstant(Instruction::SRem, LHS, RHS, Name)) return V; | ||||||
1219 | return Insert(BinaryOperator::CreateSRem(LHS, RHS), Name); | ||||||
1220 | } | ||||||
1221 | |||||||
1222 | Value *CreateShl(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1223 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1224 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1225 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1226 | return Insert(Folder.CreateShl(LC, RC, HasNUW, HasNSW), Name); | ||||||
1227 | return CreateInsertNUWNSWBinOp(Instruction::Shl, LHS, RHS, Name, | ||||||
1228 | HasNUW, HasNSW); | ||||||
1229 | } | ||||||
1230 | |||||||
1231 | Value *CreateShl(Value *LHS, const APInt &RHS, const Twine &Name = "", | ||||||
1232 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1233 | return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name, | ||||||
1234 | HasNUW, HasNSW); | ||||||
1235 | } | ||||||
1236 | |||||||
1237 | Value *CreateShl(Value *LHS, uint64_t RHS, const Twine &Name = "", | ||||||
1238 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1239 | return CreateShl(LHS, ConstantInt::get(LHS->getType(), RHS), Name, | ||||||
1240 | HasNUW, HasNSW); | ||||||
1241 | } | ||||||
1242 | |||||||
1243 | Value *CreateLShr(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1244 | bool isExact = false) { | ||||||
1245 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1246 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1247 | return Insert(Folder.CreateLShr(LC, RC, isExact), Name); | ||||||
1248 | if (!isExact) | ||||||
1249 | return Insert(BinaryOperator::CreateLShr(LHS, RHS), Name); | ||||||
1250 | return Insert(BinaryOperator::CreateExactLShr(LHS, RHS), Name); | ||||||
1251 | } | ||||||
1252 | |||||||
1253 | Value *CreateLShr(Value *LHS, const APInt &RHS, const Twine &Name = "", | ||||||
1254 | bool isExact = false) { | ||||||
1255 | return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1256 | } | ||||||
1257 | |||||||
1258 | Value *CreateLShr(Value *LHS, uint64_t RHS, const Twine &Name = "", | ||||||
1259 | bool isExact = false) { | ||||||
1260 | return CreateLShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1261 | } | ||||||
1262 | |||||||
1263 | Value *CreateAShr(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1264 | bool isExact = false) { | ||||||
1265 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1266 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
1267 | return Insert(Folder.CreateAShr(LC, RC, isExact), Name); | ||||||
1268 | if (!isExact) | ||||||
1269 | return Insert(BinaryOperator::CreateAShr(LHS, RHS), Name); | ||||||
1270 | return Insert(BinaryOperator::CreateExactAShr(LHS, RHS), Name); | ||||||
1271 | } | ||||||
1272 | |||||||
1273 | Value *CreateAShr(Value *LHS, const APInt &RHS, const Twine &Name = "", | ||||||
1274 | bool isExact = false) { | ||||||
1275 | return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1276 | } | ||||||
1277 | |||||||
1278 | Value *CreateAShr(Value *LHS, uint64_t RHS, const Twine &Name = "", | ||||||
1279 | bool isExact = false) { | ||||||
1280 | return CreateAShr(LHS, ConstantInt::get(LHS->getType(), RHS), Name,isExact); | ||||||
1281 | } | ||||||
1282 | |||||||
1283 | Value *CreateAnd(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1284 | if (auto *RC = dyn_cast<Constant>(RHS)) { | ||||||
1285 | if (isa<ConstantInt>(RC) && cast<ConstantInt>(RC)->isMinusOne()) | ||||||
1286 | return LHS; // LHS & -1 -> LHS | ||||||
1287 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1288 | return Insert(Folder.CreateAnd(LC, RC), Name); | ||||||
1289 | } | ||||||
1290 | return Insert(BinaryOperator::CreateAnd(LHS, RHS), Name); | ||||||
1291 | } | ||||||
1292 | |||||||
1293 | Value *CreateAnd(Value *LHS, const APInt &RHS, const Twine &Name = "") { | ||||||
1294 | return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1295 | } | ||||||
1296 | |||||||
1297 | Value *CreateAnd(Value *LHS, uint64_t RHS, const Twine &Name = "") { | ||||||
1298 | return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1299 | } | ||||||
1300 | |||||||
1301 | Value *CreateAnd(ArrayRef<Value*> Ops) { | ||||||
1302 | assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail ("!Ops.empty()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1302, __PRETTY_FUNCTION__)); | ||||||
1303 | Value *Accum = Ops[0]; | ||||||
1304 | for (unsigned i = 1; i < Ops.size(); i++) | ||||||
1305 | Accum = CreateAnd(Accum, Ops[i]); | ||||||
1306 | return Accum; | ||||||
1307 | } | ||||||
1308 | |||||||
1309 | Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1310 | if (auto *RC = dyn_cast<Constant>(RHS)) { | ||||||
1311 | if (RC->isNullValue()) | ||||||
1312 | return LHS; // LHS | 0 -> LHS | ||||||
1313 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
1314 | return Insert(Folder.CreateOr(LC, RC), Name); | ||||||
1315 | } | ||||||
1316 | return Insert(BinaryOperator::CreateOr(LHS, RHS), Name); | ||||||
1317 | } | ||||||
1318 | |||||||
1319 | Value *CreateOr(Value *LHS, const APInt &RHS, const Twine &Name = "") { | ||||||
1320 | return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1321 | } | ||||||
1322 | |||||||
1323 | Value *CreateOr(Value *LHS, uint64_t RHS, const Twine &Name = "") { | ||||||
1324 | return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1325 | } | ||||||
1326 | |||||||
1327 | Value *CreateOr(ArrayRef<Value*> Ops) { | ||||||
1328 | assert(!Ops.empty())((!Ops.empty()) ? static_cast<void> (0) : __assert_fail ("!Ops.empty()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1328, __PRETTY_FUNCTION__)); | ||||||
1329 | Value *Accum = Ops[0]; | ||||||
1330 | for (unsigned i = 1; i < Ops.size(); i++) | ||||||
1331 | Accum = CreateOr(Accum, Ops[i]); | ||||||
1332 | return Accum; | ||||||
1333 | } | ||||||
1334 | |||||||
1335 | Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
1336 | if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V; | ||||||
1337 | return Insert(BinaryOperator::CreateXor(LHS, RHS), Name); | ||||||
1338 | } | ||||||
1339 | |||||||
1340 | Value *CreateXor(Value *LHS, const APInt &RHS, const Twine &Name = "") { | ||||||
1341 | return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1342 | } | ||||||
1343 | |||||||
1344 | Value *CreateXor(Value *LHS, uint64_t RHS, const Twine &Name = "") { | ||||||
1345 | return CreateXor(LHS, ConstantInt::get(LHS->getType(), RHS), Name); | ||||||
1346 | } | ||||||
1347 | |||||||
1348 | Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "", | ||||||
1349 | MDNode *FPMD = nullptr) { | ||||||
1350 | if (IsFPConstrained) | ||||||
1351 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, | ||||||
1352 | L, R, nullptr, Name, FPMD); | ||||||
1353 | |||||||
1354 | if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; | ||||||
1355 | Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF); | ||||||
1356 | return Insert(I, Name); | ||||||
1357 | } | ||||||
1358 | |||||||
1359 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1360 | /// default FMF. | ||||||
1361 | Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1362 | const Twine &Name = "") { | ||||||
1363 | if (IsFPConstrained) | ||||||
1364 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, | ||||||
1365 | L, R, FMFSource, Name); | ||||||
1366 | |||||||
1367 | if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; | ||||||
1368 | Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, | ||||||
1369 | FMFSource->getFastMathFlags()); | ||||||
1370 | return Insert(I, Name); | ||||||
1371 | } | ||||||
1372 | |||||||
1373 | Value *CreateFSub(Value *L, Value *R, const Twine &Name = "", | ||||||
1374 | MDNode *FPMD = nullptr) { | ||||||
1375 | if (IsFPConstrained) | ||||||
1376 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, | ||||||
1377 | L, R, nullptr, Name, FPMD); | ||||||
1378 | |||||||
1379 | if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; | ||||||
1380 | Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF); | ||||||
1381 | return Insert(I, Name); | ||||||
1382 | } | ||||||
1383 | |||||||
1384 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1385 | /// default FMF. | ||||||
1386 | Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1387 | const Twine &Name = "") { | ||||||
1388 | if (IsFPConstrained) | ||||||
1389 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, | ||||||
1390 | L, R, FMFSource, Name); | ||||||
1391 | |||||||
1392 | if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; | ||||||
1393 | Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, | ||||||
1394 | FMFSource->getFastMathFlags()); | ||||||
1395 | return Insert(I, Name); | ||||||
1396 | } | ||||||
1397 | |||||||
1398 | Value *CreateFMul(Value *L, Value *R, const Twine &Name = "", | ||||||
1399 | MDNode *FPMD = nullptr) { | ||||||
1400 | if (IsFPConstrained) | ||||||
1401 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, | ||||||
1402 | L, R, nullptr, Name, FPMD); | ||||||
1403 | |||||||
1404 | if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; | ||||||
1405 | Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF); | ||||||
1406 | return Insert(I, Name); | ||||||
1407 | } | ||||||
1408 | |||||||
1409 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1410 | /// default FMF. | ||||||
1411 | Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1412 | const Twine &Name = "") { | ||||||
1413 | if (IsFPConstrained) | ||||||
1414 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, | ||||||
1415 | L, R, FMFSource, Name); | ||||||
1416 | |||||||
1417 | if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; | ||||||
1418 | Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, | ||||||
1419 | FMFSource->getFastMathFlags()); | ||||||
1420 | return Insert(I, Name); | ||||||
1421 | } | ||||||
1422 | |||||||
1423 | Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "", | ||||||
1424 | MDNode *FPMD = nullptr) { | ||||||
1425 | if (IsFPConstrained) | ||||||
1426 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, | ||||||
1427 | L, R, nullptr, Name, FPMD); | ||||||
1428 | |||||||
1429 | if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; | ||||||
1430 | Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF); | ||||||
1431 | return Insert(I, Name); | ||||||
1432 | } | ||||||
1433 | |||||||
1434 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1435 | /// default FMF. | ||||||
1436 | Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1437 | const Twine &Name = "") { | ||||||
1438 | if (IsFPConstrained) | ||||||
1439 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, | ||||||
1440 | L, R, FMFSource, Name); | ||||||
1441 | |||||||
1442 | if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; | ||||||
1443 | Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, | ||||||
1444 | FMFSource->getFastMathFlags()); | ||||||
1445 | return Insert(I, Name); | ||||||
1446 | } | ||||||
1447 | |||||||
1448 | Value *CreateFRem(Value *L, Value *R, const Twine &Name = "", | ||||||
1449 | MDNode *FPMD = nullptr) { | ||||||
1450 | if (IsFPConstrained) | ||||||
1451 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, | ||||||
1452 | L, R, nullptr, Name, FPMD); | ||||||
1453 | |||||||
1454 | if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; | ||||||
1455 | Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF); | ||||||
1456 | return Insert(I, Name); | ||||||
1457 | } | ||||||
1458 | |||||||
1459 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1460 | /// default FMF. | ||||||
1461 | Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource, | ||||||
1462 | const Twine &Name = "") { | ||||||
1463 | if (IsFPConstrained) | ||||||
1464 | return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, | ||||||
1465 | L, R, FMFSource, Name); | ||||||
1466 | |||||||
1467 | if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; | ||||||
1468 | Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, | ||||||
1469 | FMFSource->getFastMathFlags()); | ||||||
1470 | return Insert(I, Name); | ||||||
1471 | } | ||||||
1472 | |||||||
1473 | Value *CreateBinOp(Instruction::BinaryOps Opc, | ||||||
1474 | Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
1475 | MDNode *FPMathTag = nullptr) { | ||||||
1476 | if (Value *V = foldConstant(Opc, LHS, RHS, Name)) return V; | ||||||
1477 | Instruction *BinOp = BinaryOperator::Create(Opc, LHS, RHS); | ||||||
1478 | if (isa<FPMathOperator>(BinOp)) | ||||||
1479 | setFPAttrs(BinOp, FPMathTag, FMF); | ||||||
1480 | return Insert(BinOp, Name); | ||||||
1481 | } | ||||||
1482 | |||||||
1483 | CallInst *CreateConstrainedFPBinOp( | ||||||
1484 | Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr, | ||||||
1485 | const Twine &Name = "", MDNode *FPMathTag = nullptr, | ||||||
1486 | Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding = None, | ||||||
1487 | Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except = None) { | ||||||
1488 | Value *RoundingV = getConstrainedFPRounding(Rounding); | ||||||
1489 | Value *ExceptV = getConstrainedFPExcept(Except); | ||||||
1490 | |||||||
1491 | FastMathFlags UseFMF = FMF; | ||||||
1492 | if (FMFSource) | ||||||
1493 | UseFMF = FMFSource->getFastMathFlags(); | ||||||
1494 | |||||||
1495 | CallInst *C = CreateIntrinsic(ID, {L->getType()}, | ||||||
1496 | {L, R, RoundingV, ExceptV}, nullptr, Name); | ||||||
1497 | setConstrainedFPCallAttr(C); | ||||||
1498 | setFPAttrs(C, FPMathTag, UseFMF); | ||||||
1499 | return C; | ||||||
1500 | } | ||||||
1501 | |||||||
1502 | Value *CreateNeg(Value *V, const Twine &Name = "", | ||||||
1503 | bool HasNUW = false, bool HasNSW = false) { | ||||||
1504 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1505 | return Insert(Folder.CreateNeg(VC, HasNUW, HasNSW), Name); | ||||||
1506 | BinaryOperator *BO = Insert(BinaryOperator::CreateNeg(V), Name); | ||||||
1507 | if (HasNUW) BO->setHasNoUnsignedWrap(); | ||||||
1508 | if (HasNSW) BO->setHasNoSignedWrap(); | ||||||
1509 | return BO; | ||||||
1510 | } | ||||||
1511 | |||||||
1512 | Value *CreateNSWNeg(Value *V, const Twine &Name = "") { | ||||||
1513 | return CreateNeg(V, Name, false, true); | ||||||
1514 | } | ||||||
1515 | |||||||
1516 | Value *CreateNUWNeg(Value *V, const Twine &Name = "") { | ||||||
1517 | return CreateNeg(V, Name, true, false); | ||||||
1518 | } | ||||||
1519 | |||||||
1520 | Value *CreateFNeg(Value *V, const Twine &Name = "", | ||||||
1521 | MDNode *FPMathTag = nullptr) { | ||||||
1522 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1523 | return Insert(Folder.CreateFNeg(VC), Name); | ||||||
1524 | return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), FPMathTag, FMF), | ||||||
1525 | Name); | ||||||
1526 | } | ||||||
1527 | |||||||
1528 | /// Copy fast-math-flags from an instruction rather than using the builder's | ||||||
1529 | /// default FMF. | ||||||
1530 | Value *CreateFNegFMF(Value *V, Instruction *FMFSource, | ||||||
1531 | const Twine &Name = "") { | ||||||
1532 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1533 | return Insert(Folder.CreateFNeg(VC), Name); | ||||||
1534 | return Insert(setFPAttrs(UnaryOperator::CreateFNeg(V), nullptr, | ||||||
1535 | FMFSource->getFastMathFlags()), | ||||||
1536 | Name); | ||||||
1537 | } | ||||||
1538 | |||||||
1539 | Value *CreateNot(Value *V, const Twine &Name = "") { | ||||||
1540 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1541 | return Insert(Folder.CreateNot(VC), Name); | ||||||
1542 | return Insert(BinaryOperator::CreateNot(V), Name); | ||||||
1543 | } | ||||||
1544 | |||||||
1545 | Value *CreateUnOp(Instruction::UnaryOps Opc, | ||||||
1546 | Value *V, const Twine &Name = "", | ||||||
1547 | MDNode *FPMathTag = nullptr) { | ||||||
1548 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1549 | return Insert(Folder.CreateUnOp(Opc, VC), Name); | ||||||
1550 | Instruction *UnOp = UnaryOperator::Create(Opc, V); | ||||||
1551 | if (isa<FPMathOperator>(UnOp)) | ||||||
1552 | setFPAttrs(UnOp, FPMathTag, FMF); | ||||||
1553 | return Insert(UnOp, Name); | ||||||
1554 | } | ||||||
1555 | |||||||
1556 | /// Create either a UnaryOperator or BinaryOperator depending on \p Opc. | ||||||
1557 | /// Correct number of operands must be passed accordingly. | ||||||
1558 | Value *CreateNAryOp(unsigned Opc, ArrayRef<Value *> Ops, | ||||||
1559 | const Twine &Name = "", | ||||||
1560 | MDNode *FPMathTag = nullptr) { | ||||||
1561 | if (Instruction::isBinaryOp(Opc)) { | ||||||
1562 | assert(Ops.size() == 2 && "Invalid number of operands!")((Ops.size() == 2 && "Invalid number of operands!") ? static_cast<void> (0) : __assert_fail ("Ops.size() == 2 && \"Invalid number of operands!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1562, __PRETTY_FUNCTION__)); | ||||||
1563 | return CreateBinOp(static_cast<Instruction::BinaryOps>(Opc), | ||||||
1564 | Ops[0], Ops[1], Name, FPMathTag); | ||||||
1565 | } | ||||||
1566 | if (Instruction::isUnaryOp(Opc)) { | ||||||
1567 | assert(Ops.size() == 1 && "Invalid number of operands!")((Ops.size() == 1 && "Invalid number of operands!") ? static_cast<void> (0) : __assert_fail ("Ops.size() == 1 && \"Invalid number of operands!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1567, __PRETTY_FUNCTION__)); | ||||||
1568 | return CreateUnOp(static_cast<Instruction::UnaryOps>(Opc), | ||||||
1569 | Ops[0], Name, FPMathTag); | ||||||
1570 | } | ||||||
1571 | llvm_unreachable("Unexpected opcode!")::llvm::llvm_unreachable_internal("Unexpected opcode!", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1571); | ||||||
1572 | } | ||||||
1573 | |||||||
1574 | //===--------------------------------------------------------------------===// | ||||||
1575 | // Instruction creation methods: Memory Instructions | ||||||
1576 | //===--------------------------------------------------------------------===// | ||||||
1577 | |||||||
1578 | AllocaInst *CreateAlloca(Type *Ty, unsigned AddrSpace, | ||||||
1579 | Value *ArraySize = nullptr, const Twine &Name = "") { | ||||||
1580 | return Insert(new AllocaInst(Ty, AddrSpace, ArraySize), Name); | ||||||
1581 | } | ||||||
1582 | |||||||
1583 | AllocaInst *CreateAlloca(Type *Ty, Value *ArraySize = nullptr, | ||||||
1584 | const Twine &Name = "") { | ||||||
1585 | const DataLayout &DL = BB->getParent()->getParent()->getDataLayout(); | ||||||
1586 | return Insert(new AllocaInst(Ty, DL.getAllocaAddrSpace(), ArraySize), Name); | ||||||
1587 | } | ||||||
1588 | |||||||
1589 | /// Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of | ||||||
1590 | /// converting the string to 'bool' for the isVolatile parameter. | ||||||
1591 | LoadInst *CreateLoad(Type *Ty, Value *Ptr, const char *Name) { | ||||||
1592 | return Insert(new LoadInst(Ty, Ptr), Name); | ||||||
1593 | } | ||||||
1594 | |||||||
1595 | LoadInst *CreateLoad(Type *Ty, Value *Ptr, const Twine &Name = "") { | ||||||
1596 | return Insert(new LoadInst(Ty, Ptr), Name); | ||||||
1597 | } | ||||||
1598 | |||||||
1599 | LoadInst *CreateLoad(Type *Ty, Value *Ptr, bool isVolatile, | ||||||
1600 | const Twine &Name = "") { | ||||||
1601 | return Insert(new LoadInst(Ty, Ptr, Twine(), isVolatile), Name); | ||||||
1602 | } | ||||||
1603 | |||||||
1604 | // Deprecated [opaque pointer types] | ||||||
1605 | LoadInst *CreateLoad(Value *Ptr, const char *Name) { | ||||||
1606 | return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name); | ||||||
1607 | } | ||||||
1608 | |||||||
1609 | // Deprecated [opaque pointer types] | ||||||
1610 | LoadInst *CreateLoad(Value *Ptr, const Twine &Name = "") { | ||||||
1611 | return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, Name); | ||||||
1612 | } | ||||||
1613 | |||||||
1614 | // Deprecated [opaque pointer types] | ||||||
1615 | LoadInst *CreateLoad(Value *Ptr, bool isVolatile, const Twine &Name = "") { | ||||||
1616 | return CreateLoad(Ptr->getType()->getPointerElementType(), Ptr, isVolatile, | ||||||
1617 | Name); | ||||||
1618 | } | ||||||
1619 | |||||||
1620 | StoreInst *CreateStore(Value *Val, Value *Ptr, bool isVolatile = false) { | ||||||
1621 | return Insert(new StoreInst(Val, Ptr, isVolatile)); | ||||||
1622 | } | ||||||
1623 | |||||||
1624 | /// Provided to resolve 'CreateAlignedLoad(Ptr, Align, "...")' | ||||||
1625 | /// correctly, instead of converting the string to 'bool' for the isVolatile | ||||||
1626 | /// parameter. | ||||||
1627 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, | ||||||
1628 | const char *Name) { | ||||||
1629 | LoadInst *LI = CreateLoad(Ty, Ptr, Name); | ||||||
1630 | LI->setAlignment(MaybeAlign(Align)); | ||||||
1631 | return LI; | ||||||
1632 | } | ||||||
1633 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, | ||||||
1634 | const Twine &Name = "") { | ||||||
1635 | LoadInst *LI = CreateLoad(Ty, Ptr, Name); | ||||||
1636 | LI->setAlignment(MaybeAlign(Align)); | ||||||
1637 | return LI; | ||||||
1638 | } | ||||||
1639 | LoadInst *CreateAlignedLoad(Type *Ty, Value *Ptr, unsigned Align, | ||||||
1640 | bool isVolatile, const Twine &Name = "") { | ||||||
1641 | LoadInst *LI = CreateLoad(Ty, Ptr, isVolatile, Name); | ||||||
1642 | LI->setAlignment(MaybeAlign(Align)); | ||||||
1643 | return LI; | ||||||
1644 | } | ||||||
1645 | |||||||
1646 | // Deprecated [opaque pointer types] | ||||||
1647 | LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, const char *Name) { | ||||||
1648 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1649 | Align, Name); | ||||||
1650 | } | ||||||
1651 | // Deprecated [opaque pointer types] | ||||||
1652 | LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, | ||||||
1653 | const Twine &Name = "") { | ||||||
1654 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1655 | Align, Name); | ||||||
1656 | } | ||||||
1657 | // Deprecated [opaque pointer types] | ||||||
1658 | LoadInst *CreateAlignedLoad(Value *Ptr, unsigned Align, bool isVolatile, | ||||||
1659 | const Twine &Name = "") { | ||||||
1660 | return CreateAlignedLoad(Ptr->getType()->getPointerElementType(), Ptr, | ||||||
1661 | Align, isVolatile, Name); | ||||||
1662 | } | ||||||
1663 | |||||||
1664 | StoreInst *CreateAlignedStore(Value *Val, Value *Ptr, unsigned Align, | ||||||
1665 | bool isVolatile = false) { | ||||||
1666 | StoreInst *SI = CreateStore(Val, Ptr, isVolatile); | ||||||
1667 | SI->setAlignment(MaybeAlign(Align)); | ||||||
1668 | return SI; | ||||||
1669 | } | ||||||
1670 | |||||||
1671 | FenceInst *CreateFence(AtomicOrdering Ordering, | ||||||
1672 | SyncScope::ID SSID = SyncScope::System, | ||||||
1673 | const Twine &Name = "") { | ||||||
1674 | return Insert(new FenceInst(Context, Ordering, SSID), Name); | ||||||
1675 | } | ||||||
1676 | |||||||
1677 | AtomicCmpXchgInst * | ||||||
1678 | CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New, | ||||||
1679 | AtomicOrdering SuccessOrdering, | ||||||
1680 | AtomicOrdering FailureOrdering, | ||||||
1681 | SyncScope::ID SSID = SyncScope::System) { | ||||||
1682 | return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, | ||||||
1683 | FailureOrdering, SSID)); | ||||||
1684 | } | ||||||
1685 | |||||||
1686 | AtomicRMWInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, | ||||||
1687 | AtomicOrdering Ordering, | ||||||
1688 | SyncScope::ID SSID = SyncScope::System) { | ||||||
1689 | return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SSID)); | ||||||
1690 | } | ||||||
1691 | |||||||
1692 | Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1693 | const Twine &Name = "") { | ||||||
1694 | return CreateGEP(nullptr, Ptr, IdxList, Name); | ||||||
1695 | } | ||||||
1696 | |||||||
1697 | Value *CreateGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1698 | const Twine &Name = "") { | ||||||
1699 | if (auto *PC = dyn_cast<Constant>(Ptr)) { | ||||||
1700 | // Every index must be constant. | ||||||
1701 | size_t i, e; | ||||||
1702 | for (i = 0, e = IdxList.size(); i != e; ++i) | ||||||
1703 | if (!isa<Constant>(IdxList[i])) | ||||||
1704 | break; | ||||||
1705 | if (i == e) | ||||||
1706 | return Insert(Folder.CreateGetElementPtr(Ty, PC, IdxList), Name); | ||||||
1707 | } | ||||||
1708 | return Insert(GetElementPtrInst::Create(Ty, Ptr, IdxList), Name); | ||||||
1709 | } | ||||||
1710 | |||||||
1711 | Value *CreateInBoundsGEP(Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1712 | const Twine &Name = "") { | ||||||
1713 | return CreateInBoundsGEP(nullptr, Ptr, IdxList, Name); | ||||||
1714 | } | ||||||
1715 | |||||||
1716 | Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, ArrayRef<Value *> IdxList, | ||||||
1717 | const Twine &Name = "") { | ||||||
1718 | if (auto *PC = dyn_cast<Constant>(Ptr)) { | ||||||
1719 | // Every index must be constant. | ||||||
1720 | size_t i, e; | ||||||
1721 | for (i = 0, e = IdxList.size(); i != e; ++i) | ||||||
1722 | if (!isa<Constant>(IdxList[i])) | ||||||
1723 | break; | ||||||
1724 | if (i == e) | ||||||
1725 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IdxList), | ||||||
1726 | Name); | ||||||
1727 | } | ||||||
1728 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, IdxList), Name); | ||||||
1729 | } | ||||||
1730 | |||||||
1731 | Value *CreateGEP(Value *Ptr, Value *Idx, const Twine &Name = "") { | ||||||
1732 | return CreateGEP(nullptr, Ptr, Idx, Name); | ||||||
1733 | } | ||||||
1734 | |||||||
1735 | Value *CreateGEP(Type *Ty, Value *Ptr, Value *Idx, const Twine &Name = "") { | ||||||
1736 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1737 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
1738 | return Insert(Folder.CreateGetElementPtr(Ty, PC, IC), Name); | ||||||
1739 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); | ||||||
1740 | } | ||||||
1741 | |||||||
1742 | Value *CreateInBoundsGEP(Type *Ty, Value *Ptr, Value *Idx, | ||||||
1743 | const Twine &Name = "") { | ||||||
1744 | if (auto *PC
| ||||||
1745 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
1746 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, IC), Name); | ||||||
1747 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); | ||||||
1748 | } | ||||||
1749 | |||||||
1750 | Value *CreateConstGEP1_32(Value *Ptr, unsigned Idx0, const Twine &Name = "") { | ||||||
1751 | return CreateConstGEP1_32(nullptr, Ptr, Idx0, Name); | ||||||
1752 | } | ||||||
1753 | |||||||
1754 | Value *CreateConstGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, | ||||||
1755 | const Twine &Name = "") { | ||||||
1756 | Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0); | ||||||
1757 | |||||||
1758 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1759 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name); | ||||||
1760 | |||||||
1761 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); | ||||||
1762 | } | ||||||
1763 | |||||||
1764 | Value *CreateConstInBoundsGEP1_32(Type *Ty, Value *Ptr, unsigned Idx0, | ||||||
1765 | const Twine &Name = "") { | ||||||
1766 | Value *Idx = ConstantInt::get(Type::getInt32Ty(Context), Idx0); | ||||||
1767 | |||||||
1768 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1769 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name); | ||||||
1770 | |||||||
1771 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); | ||||||
1772 | } | ||||||
1773 | |||||||
1774 | Value *CreateConstGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, | ||||||
1775 | const Twine &Name = "") { | ||||||
1776 | Value *Idxs[] = { | ||||||
1777 | ConstantInt::get(Type::getInt32Ty(Context), Idx0), | ||||||
1778 | ConstantInt::get(Type::getInt32Ty(Context), Idx1) | ||||||
1779 | }; | ||||||
1780 | |||||||
1781 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1782 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1783 | |||||||
1784 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name); | ||||||
1785 | } | ||||||
1786 | |||||||
1787 | Value *CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, | ||||||
1788 | unsigned Idx1, const Twine &Name = "") { | ||||||
1789 | Value *Idxs[] = { | ||||||
1790 | ConstantInt::get(Type::getInt32Ty(Context), Idx0), | ||||||
1791 | ConstantInt::get(Type::getInt32Ty(Context), Idx1) | ||||||
1792 | }; | ||||||
1793 | |||||||
1794 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1795 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1796 | |||||||
1797 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name); | ||||||
1798 | } | ||||||
1799 | |||||||
1800 | Value *CreateConstGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0, | ||||||
1801 | const Twine &Name = "") { | ||||||
1802 | Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0); | ||||||
1803 | |||||||
1804 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1805 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idx), Name); | ||||||
1806 | |||||||
1807 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idx), Name); | ||||||
1808 | } | ||||||
1809 | |||||||
1810 | Value *CreateConstGEP1_64(Value *Ptr, uint64_t Idx0, const Twine &Name = "") { | ||||||
1811 | return CreateConstGEP1_64(nullptr, Ptr, Idx0, Name); | ||||||
1812 | } | ||||||
1813 | |||||||
1814 | Value *CreateConstInBoundsGEP1_64(Type *Ty, Value *Ptr, uint64_t Idx0, | ||||||
1815 | const Twine &Name = "") { | ||||||
1816 | Value *Idx = ConstantInt::get(Type::getInt64Ty(Context), Idx0); | ||||||
1817 | |||||||
1818 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1819 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idx), Name); | ||||||
1820 | |||||||
1821 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idx), Name); | ||||||
1822 | } | ||||||
1823 | |||||||
1824 | Value *CreateConstInBoundsGEP1_64(Value *Ptr, uint64_t Idx0, | ||||||
1825 | const Twine &Name = "") { | ||||||
1826 | return CreateConstInBoundsGEP1_64(nullptr, Ptr, Idx0, Name); | ||||||
1827 | } | ||||||
1828 | |||||||
1829 | Value *CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, | ||||||
1830 | const Twine &Name = "") { | ||||||
1831 | Value *Idxs[] = { | ||||||
1832 | ConstantInt::get(Type::getInt64Ty(Context), Idx0), | ||||||
1833 | ConstantInt::get(Type::getInt64Ty(Context), Idx1) | ||||||
1834 | }; | ||||||
1835 | |||||||
1836 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1837 | return Insert(Folder.CreateGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1838 | |||||||
1839 | return Insert(GetElementPtrInst::Create(Ty, Ptr, Idxs), Name); | ||||||
1840 | } | ||||||
1841 | |||||||
1842 | Value *CreateConstGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1, | ||||||
1843 | const Twine &Name = "") { | ||||||
1844 | return CreateConstGEP2_64(nullptr, Ptr, Idx0, Idx1, Name); | ||||||
1845 | } | ||||||
1846 | |||||||
1847 | Value *CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, | ||||||
1848 | uint64_t Idx1, const Twine &Name = "") { | ||||||
1849 | Value *Idxs[] = { | ||||||
1850 | ConstantInt::get(Type::getInt64Ty(Context), Idx0), | ||||||
1851 | ConstantInt::get(Type::getInt64Ty(Context), Idx1) | ||||||
1852 | }; | ||||||
1853 | |||||||
1854 | if (auto *PC = dyn_cast<Constant>(Ptr)) | ||||||
1855 | return Insert(Folder.CreateInBoundsGetElementPtr(Ty, PC, Idxs), Name); | ||||||
1856 | |||||||
1857 | return Insert(GetElementPtrInst::CreateInBounds(Ty, Ptr, Idxs), Name); | ||||||
1858 | } | ||||||
1859 | |||||||
1860 | Value *CreateConstInBoundsGEP2_64(Value *Ptr, uint64_t Idx0, uint64_t Idx1, | ||||||
1861 | const Twine &Name = "") { | ||||||
1862 | return CreateConstInBoundsGEP2_64(nullptr, Ptr, Idx0, Idx1, Name); | ||||||
1863 | } | ||||||
1864 | |||||||
1865 | Value *CreateStructGEP(Type *Ty, Value *Ptr, unsigned Idx, | ||||||
1866 | const Twine &Name = "") { | ||||||
1867 | return CreateConstInBoundsGEP2_32(Ty, Ptr, 0, Idx, Name); | ||||||
1868 | } | ||||||
1869 | |||||||
1870 | Value *CreateStructGEP(Value *Ptr, unsigned Idx, const Twine &Name = "") { | ||||||
1871 | return CreateConstInBoundsGEP2_32(nullptr, Ptr, 0, Idx, Name); | ||||||
1872 | } | ||||||
1873 | |||||||
1874 | /// Same as CreateGlobalString, but return a pointer with "i8*" type | ||||||
1875 | /// instead of a pointer to array of i8. | ||||||
1876 | Constant *CreateGlobalStringPtr(StringRef Str, const Twine &Name = "", | ||||||
1877 | unsigned AddressSpace = 0) { | ||||||
1878 | GlobalVariable *GV = CreateGlobalString(Str, Name, AddressSpace); | ||||||
1879 | Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); | ||||||
1880 | Constant *Indices[] = {Zero, Zero}; | ||||||
1881 | return ConstantExpr::getInBoundsGetElementPtr(GV->getValueType(), GV, | ||||||
1882 | Indices); | ||||||
1883 | } | ||||||
1884 | |||||||
1885 | //===--------------------------------------------------------------------===// | ||||||
1886 | // Instruction creation methods: Cast/Conversion Operators | ||||||
1887 | //===--------------------------------------------------------------------===// | ||||||
1888 | |||||||
1889 | Value *CreateTrunc(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1890 | return CreateCast(Instruction::Trunc, V, DestTy, Name); | ||||||
1891 | } | ||||||
1892 | |||||||
1893 | Value *CreateZExt(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1894 | return CreateCast(Instruction::ZExt, V, DestTy, Name); | ||||||
1895 | } | ||||||
1896 | |||||||
1897 | Value *CreateSExt(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1898 | return CreateCast(Instruction::SExt, V, DestTy, Name); | ||||||
1899 | } | ||||||
1900 | |||||||
1901 | /// Create a ZExt or Trunc from the integer value V to DestTy. Return | ||||||
1902 | /// the value untouched if the type of V is already DestTy. | ||||||
1903 | Value *CreateZExtOrTrunc(Value *V, Type *DestTy, | ||||||
1904 | const Twine &Name = "") { | ||||||
1905 | assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only zero extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1907, __PRETTY_FUNCTION__)) | ||||||
1906 | DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only zero extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1907, __PRETTY_FUNCTION__)) | ||||||
1907 | "Can only zero extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only zero extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only zero extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1907, __PRETTY_FUNCTION__)); | ||||||
1908 | Type *VTy = V->getType(); | ||||||
1909 | if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits()) | ||||||
1910 | return CreateZExt(V, DestTy, Name); | ||||||
1911 | if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits()) | ||||||
1912 | return CreateTrunc(V, DestTy, Name); | ||||||
1913 | return V; | ||||||
1914 | } | ||||||
1915 | |||||||
1916 | /// Create a SExt or Trunc from the integer value V to DestTy. Return | ||||||
1917 | /// the value untouched if the type of V is already DestTy. | ||||||
1918 | Value *CreateSExtOrTrunc(Value *V, Type *DestTy, | ||||||
1919 | const Twine &Name = "") { | ||||||
1920 | assert(V->getType()->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only sign extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1922, __PRETTY_FUNCTION__)) | ||||||
1921 | DestTy->isIntOrIntVectorTy() &&((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only sign extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1922, __PRETTY_FUNCTION__)) | ||||||
1922 | "Can only sign extend/truncate integers!")((V->getType()->isIntOrIntVectorTy() && DestTy-> isIntOrIntVectorTy() && "Can only sign extend/truncate integers!" ) ? static_cast<void> (0) : __assert_fail ("V->getType()->isIntOrIntVectorTy() && DestTy->isIntOrIntVectorTy() && \"Can only sign extend/truncate integers!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 1922, __PRETTY_FUNCTION__)); | ||||||
1923 | Type *VTy = V->getType(); | ||||||
1924 | if (VTy->getScalarSizeInBits() < DestTy->getScalarSizeInBits()) | ||||||
1925 | return CreateSExt(V, DestTy, Name); | ||||||
1926 | if (VTy->getScalarSizeInBits() > DestTy->getScalarSizeInBits()) | ||||||
1927 | return CreateTrunc(V, DestTy, Name); | ||||||
1928 | return V; | ||||||
1929 | } | ||||||
1930 | |||||||
1931 | Value *CreateFPToUI(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1932 | if (IsFPConstrained) | ||||||
1933 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptoui, | ||||||
1934 | V, DestTy, nullptr, Name); | ||||||
1935 | return CreateCast(Instruction::FPToUI, V, DestTy, Name); | ||||||
1936 | } | ||||||
1937 | |||||||
1938 | Value *CreateFPToSI(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1939 | if (IsFPConstrained) | ||||||
1940 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fptosi, | ||||||
1941 | V, DestTy, nullptr, Name); | ||||||
1942 | return CreateCast(Instruction::FPToSI, V, DestTy, Name); | ||||||
1943 | } | ||||||
1944 | |||||||
1945 | Value *CreateUIToFP(Value *V, Type *DestTy, const Twine &Name = ""){ | ||||||
1946 | return CreateCast(Instruction::UIToFP, V, DestTy, Name); | ||||||
1947 | } | ||||||
1948 | |||||||
1949 | Value *CreateSIToFP(Value *V, Type *DestTy, const Twine &Name = ""){ | ||||||
1950 | return CreateCast(Instruction::SIToFP, V, DestTy, Name); | ||||||
1951 | } | ||||||
1952 | |||||||
1953 | Value *CreateFPTrunc(Value *V, Type *DestTy, | ||||||
1954 | const Twine &Name = "") { | ||||||
1955 | if (IsFPConstrained) | ||||||
1956 | return CreateConstrainedFPCast( | ||||||
1957 | Intrinsic::experimental_constrained_fptrunc, V, DestTy, nullptr, | ||||||
1958 | Name); | ||||||
1959 | return CreateCast(Instruction::FPTrunc, V, DestTy, Name); | ||||||
1960 | } | ||||||
1961 | |||||||
1962 | Value *CreateFPExt(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
1963 | if (IsFPConstrained) | ||||||
1964 | return CreateConstrainedFPCast(Intrinsic::experimental_constrained_fpext, | ||||||
1965 | V, DestTy, nullptr, Name); | ||||||
1966 | return CreateCast(Instruction::FPExt, V, DestTy, Name); | ||||||
1967 | } | ||||||
1968 | |||||||
1969 | Value *CreatePtrToInt(Value *V, Type *DestTy, | ||||||
1970 | const Twine &Name = "") { | ||||||
1971 | return CreateCast(Instruction::PtrToInt, V, DestTy, Name); | ||||||
1972 | } | ||||||
1973 | |||||||
1974 | Value *CreateIntToPtr(Value *V, Type *DestTy, | ||||||
1975 | const Twine &Name = "") { | ||||||
1976 | return CreateCast(Instruction::IntToPtr, V, DestTy, Name); | ||||||
1977 | } | ||||||
1978 | |||||||
1979 | Value *CreateBitCast(Value *V, Type *DestTy, | ||||||
1980 | const Twine &Name = "") { | ||||||
1981 | return CreateCast(Instruction::BitCast, V, DestTy, Name); | ||||||
1982 | } | ||||||
1983 | |||||||
1984 | Value *CreateAddrSpaceCast(Value *V, Type *DestTy, | ||||||
1985 | const Twine &Name = "") { | ||||||
1986 | return CreateCast(Instruction::AddrSpaceCast, V, DestTy, Name); | ||||||
1987 | } | ||||||
1988 | |||||||
1989 | Value *CreateZExtOrBitCast(Value *V, Type *DestTy, | ||||||
1990 | const Twine &Name = "") { | ||||||
1991 | if (V->getType() == DestTy) | ||||||
1992 | return V; | ||||||
1993 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
1994 | return Insert(Folder.CreateZExtOrBitCast(VC, DestTy), Name); | ||||||
1995 | return Insert(CastInst::CreateZExtOrBitCast(V, DestTy), Name); | ||||||
1996 | } | ||||||
1997 | |||||||
1998 | Value *CreateSExtOrBitCast(Value *V, Type *DestTy, | ||||||
1999 | const Twine &Name = "") { | ||||||
2000 | if (V->getType() == DestTy) | ||||||
2001 | return V; | ||||||
2002 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2003 | return Insert(Folder.CreateSExtOrBitCast(VC, DestTy), Name); | ||||||
2004 | return Insert(CastInst::CreateSExtOrBitCast(V, DestTy), Name); | ||||||
2005 | } | ||||||
2006 | |||||||
2007 | Value *CreateTruncOrBitCast(Value *V, Type *DestTy, | ||||||
2008 | const Twine &Name = "") { | ||||||
2009 | if (V->getType() == DestTy) | ||||||
2010 | return V; | ||||||
2011 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2012 | return Insert(Folder.CreateTruncOrBitCast(VC, DestTy), Name); | ||||||
2013 | return Insert(CastInst::CreateTruncOrBitCast(V, DestTy), Name); | ||||||
2014 | } | ||||||
2015 | |||||||
2016 | Value *CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, | ||||||
2017 | const Twine &Name = "") { | ||||||
2018 | if (V->getType() == DestTy) | ||||||
2019 | return V; | ||||||
2020 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2021 | return Insert(Folder.CreateCast(Op, VC, DestTy), Name); | ||||||
2022 | return Insert(CastInst::Create(Op, V, DestTy), Name); | ||||||
2023 | } | ||||||
2024 | |||||||
2025 | Value *CreatePointerCast(Value *V, Type *DestTy, | ||||||
2026 | const Twine &Name = "") { | ||||||
2027 | if (V->getType() == DestTy) | ||||||
2028 | return V; | ||||||
2029 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2030 | return Insert(Folder.CreatePointerCast(VC, DestTy), Name); | ||||||
2031 | return Insert(CastInst::CreatePointerCast(V, DestTy), Name); | ||||||
2032 | } | ||||||
2033 | |||||||
2034 | Value *CreatePointerBitCastOrAddrSpaceCast(Value *V, Type *DestTy, | ||||||
2035 | const Twine &Name = "") { | ||||||
2036 | if (V->getType() == DestTy) | ||||||
2037 | return V; | ||||||
2038 | |||||||
2039 | if (auto *VC = dyn_cast<Constant>(V)) { | ||||||
2040 | return Insert(Folder.CreatePointerBitCastOrAddrSpaceCast(VC, DestTy), | ||||||
2041 | Name); | ||||||
2042 | } | ||||||
2043 | |||||||
2044 | return Insert(CastInst::CreatePointerBitCastOrAddrSpaceCast(V, DestTy), | ||||||
2045 | Name); | ||||||
2046 | } | ||||||
2047 | |||||||
2048 | Value *CreateIntCast(Value *V, Type *DestTy, bool isSigned, | ||||||
2049 | const Twine &Name = "") { | ||||||
2050 | if (V->getType() == DestTy) | ||||||
2051 | return V; | ||||||
2052 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2053 | return Insert(Folder.CreateIntCast(VC, DestTy, isSigned), Name); | ||||||
2054 | return Insert(CastInst::CreateIntegerCast(V, DestTy, isSigned), Name); | ||||||
2055 | } | ||||||
2056 | |||||||
2057 | Value *CreateBitOrPointerCast(Value *V, Type *DestTy, | ||||||
2058 | const Twine &Name = "") { | ||||||
2059 | if (V->getType() == DestTy) | ||||||
2060 | return V; | ||||||
2061 | if (V->getType()->isPtrOrPtrVectorTy() && DestTy->isIntOrIntVectorTy()) | ||||||
2062 | return CreatePtrToInt(V, DestTy, Name); | ||||||
2063 | if (V->getType()->isIntOrIntVectorTy() && DestTy->isPtrOrPtrVectorTy()) | ||||||
2064 | return CreateIntToPtr(V, DestTy, Name); | ||||||
2065 | |||||||
2066 | return CreateBitCast(V, DestTy, Name); | ||||||
2067 | } | ||||||
2068 | |||||||
2069 | Value *CreateFPCast(Value *V, Type *DestTy, const Twine &Name = "") { | ||||||
2070 | if (V->getType() == DestTy) | ||||||
2071 | return V; | ||||||
2072 | if (auto *VC = dyn_cast<Constant>(V)) | ||||||
2073 | return Insert(Folder.CreateFPCast(VC, DestTy), Name); | ||||||
2074 | return Insert(CastInst::CreateFPCast(V, DestTy), Name); | ||||||
2075 | } | ||||||
2076 | |||||||
2077 | CallInst *CreateConstrainedFPCast( | ||||||
2078 | Intrinsic::ID ID, Value *V, Type *DestTy, | ||||||
2079 | Instruction *FMFSource = nullptr, const Twine &Name = "", | ||||||
2080 | MDNode *FPMathTag = nullptr, | ||||||
2081 | Optional<ConstrainedFPIntrinsic::RoundingMode> Rounding = None, | ||||||
2082 | Optional<ConstrainedFPIntrinsic::ExceptionBehavior> Except = None) { | ||||||
2083 | Value *ExceptV = getConstrainedFPExcept(Except); | ||||||
2084 | |||||||
2085 | FastMathFlags UseFMF = FMF; | ||||||
2086 | if (FMFSource) | ||||||
2087 | UseFMF = FMFSource->getFastMathFlags(); | ||||||
2088 | |||||||
2089 | CallInst *C; | ||||||
2090 | switch (ID) { | ||||||
2091 | default: { | ||||||
2092 | Value *RoundingV = getConstrainedFPRounding(Rounding); | ||||||
2093 | C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, RoundingV, ExceptV}, | ||||||
2094 | nullptr, Name); | ||||||
2095 | } break; | ||||||
2096 | case Intrinsic::experimental_constrained_fpext: | ||||||
2097 | case Intrinsic::experimental_constrained_fptoui: | ||||||
2098 | case Intrinsic::experimental_constrained_fptosi: | ||||||
2099 | C = CreateIntrinsic(ID, {DestTy, V->getType()}, {V, ExceptV}, nullptr, | ||||||
2100 | Name); | ||||||
2101 | break; | ||||||
2102 | } | ||||||
2103 | setConstrainedFPCallAttr(C); | ||||||
2104 | |||||||
2105 | if (isa<FPMathOperator>(C)) | ||||||
2106 | setFPAttrs(C, FPMathTag, UseFMF); | ||||||
2107 | return C; | ||||||
2108 | } | ||||||
2109 | |||||||
2110 | // Provided to resolve 'CreateIntCast(Ptr, Ptr, "...")', giving a | ||||||
2111 | // compile time error, instead of converting the string to bool for the | ||||||
2112 | // isSigned parameter. | ||||||
2113 | Value *CreateIntCast(Value *, Type *, const char *) = delete; | ||||||
2114 | |||||||
2115 | //===--------------------------------------------------------------------===// | ||||||
2116 | // Instruction creation methods: Compare Instructions | ||||||
2117 | //===--------------------------------------------------------------------===// | ||||||
2118 | |||||||
2119 | Value *CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2120 | return CreateICmp(ICmpInst::ICMP_EQ, LHS, RHS, Name); | ||||||
2121 | } | ||||||
2122 | |||||||
2123 | Value *CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2124 | return CreateICmp(ICmpInst::ICMP_NE, LHS, RHS, Name); | ||||||
2125 | } | ||||||
2126 | |||||||
2127 | Value *CreateICmpUGT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2128 | return CreateICmp(ICmpInst::ICMP_UGT, LHS, RHS, Name); | ||||||
2129 | } | ||||||
2130 | |||||||
2131 | Value *CreateICmpUGE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2132 | return CreateICmp(ICmpInst::ICMP_UGE, LHS, RHS, Name); | ||||||
2133 | } | ||||||
2134 | |||||||
2135 | Value *CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2136 | return CreateICmp(ICmpInst::ICMP_ULT, LHS, RHS, Name); | ||||||
2137 | } | ||||||
2138 | |||||||
2139 | Value *CreateICmpULE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2140 | return CreateICmp(ICmpInst::ICMP_ULE, LHS, RHS, Name); | ||||||
2141 | } | ||||||
2142 | |||||||
2143 | Value *CreateICmpSGT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2144 | return CreateICmp(ICmpInst::ICMP_SGT, LHS, RHS, Name); | ||||||
2145 | } | ||||||
2146 | |||||||
2147 | Value *CreateICmpSGE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2148 | return CreateICmp(ICmpInst::ICMP_SGE, LHS, RHS, Name); | ||||||
2149 | } | ||||||
2150 | |||||||
2151 | Value *CreateICmpSLT(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2152 | return CreateICmp(ICmpInst::ICMP_SLT, LHS, RHS, Name); | ||||||
2153 | } | ||||||
2154 | |||||||
2155 | Value *CreateICmpSLE(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2156 | return CreateICmp(ICmpInst::ICMP_SLE, LHS, RHS, Name); | ||||||
2157 | } | ||||||
2158 | |||||||
2159 | Value *CreateFCmpOEQ(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2160 | MDNode *FPMathTag = nullptr) { | ||||||
2161 | return CreateFCmp(FCmpInst::FCMP_OEQ, LHS, RHS, Name, FPMathTag); | ||||||
2162 | } | ||||||
2163 | |||||||
2164 | Value *CreateFCmpOGT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2165 | MDNode *FPMathTag = nullptr) { | ||||||
2166 | return CreateFCmp(FCmpInst::FCMP_OGT, LHS, RHS, Name, FPMathTag); | ||||||
2167 | } | ||||||
2168 | |||||||
2169 | Value *CreateFCmpOGE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2170 | MDNode *FPMathTag = nullptr) { | ||||||
2171 | return CreateFCmp(FCmpInst::FCMP_OGE, LHS, RHS, Name, FPMathTag); | ||||||
2172 | } | ||||||
2173 | |||||||
2174 | Value *CreateFCmpOLT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2175 | MDNode *FPMathTag = nullptr) { | ||||||
2176 | return CreateFCmp(FCmpInst::FCMP_OLT, LHS, RHS, Name, FPMathTag); | ||||||
2177 | } | ||||||
2178 | |||||||
2179 | Value *CreateFCmpOLE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2180 | MDNode *FPMathTag = nullptr) { | ||||||
2181 | return CreateFCmp(FCmpInst::FCMP_OLE, LHS, RHS, Name, FPMathTag); | ||||||
2182 | } | ||||||
2183 | |||||||
2184 | Value *CreateFCmpONE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2185 | MDNode *FPMathTag = nullptr) { | ||||||
2186 | return CreateFCmp(FCmpInst::FCMP_ONE, LHS, RHS, Name, FPMathTag); | ||||||
2187 | } | ||||||
2188 | |||||||
2189 | Value *CreateFCmpORD(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2190 | MDNode *FPMathTag = nullptr) { | ||||||
2191 | return CreateFCmp(FCmpInst::FCMP_ORD, LHS, RHS, Name, FPMathTag); | ||||||
2192 | } | ||||||
2193 | |||||||
2194 | Value *CreateFCmpUNO(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2195 | MDNode *FPMathTag = nullptr) { | ||||||
2196 | return CreateFCmp(FCmpInst::FCMP_UNO, LHS, RHS, Name, FPMathTag); | ||||||
2197 | } | ||||||
2198 | |||||||
2199 | Value *CreateFCmpUEQ(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2200 | MDNode *FPMathTag = nullptr) { | ||||||
2201 | return CreateFCmp(FCmpInst::FCMP_UEQ, LHS, RHS, Name, FPMathTag); | ||||||
2202 | } | ||||||
2203 | |||||||
2204 | Value *CreateFCmpUGT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2205 | MDNode *FPMathTag = nullptr) { | ||||||
2206 | return CreateFCmp(FCmpInst::FCMP_UGT, LHS, RHS, Name, FPMathTag); | ||||||
2207 | } | ||||||
2208 | |||||||
2209 | Value *CreateFCmpUGE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2210 | MDNode *FPMathTag = nullptr) { | ||||||
2211 | return CreateFCmp(FCmpInst::FCMP_UGE, LHS, RHS, Name, FPMathTag); | ||||||
2212 | } | ||||||
2213 | |||||||
2214 | Value *CreateFCmpULT(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2215 | MDNode *FPMathTag = nullptr) { | ||||||
2216 | return CreateFCmp(FCmpInst::FCMP_ULT, LHS, RHS, Name, FPMathTag); | ||||||
2217 | } | ||||||
2218 | |||||||
2219 | Value *CreateFCmpULE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2220 | MDNode *FPMathTag = nullptr) { | ||||||
2221 | return CreateFCmp(FCmpInst::FCMP_ULE, LHS, RHS, Name, FPMathTag); | ||||||
2222 | } | ||||||
2223 | |||||||
2224 | Value *CreateFCmpUNE(Value *LHS, Value *RHS, const Twine &Name = "", | ||||||
2225 | MDNode *FPMathTag = nullptr) { | ||||||
2226 | return CreateFCmp(FCmpInst::FCMP_UNE, LHS, RHS, Name, FPMathTag); | ||||||
2227 | } | ||||||
2228 | |||||||
2229 | Value *CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, | ||||||
2230 | const Twine &Name = "") { | ||||||
2231 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
2232 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
2233 | return Insert(Folder.CreateICmp(P, LC, RC), Name); | ||||||
2234 | return Insert(new ICmpInst(P, LHS, RHS), Name); | ||||||
2235 | } | ||||||
2236 | |||||||
2237 | Value *CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, | ||||||
2238 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2239 | if (auto *LC = dyn_cast<Constant>(LHS)) | ||||||
2240 | if (auto *RC = dyn_cast<Constant>(RHS)) | ||||||
2241 | return Insert(Folder.CreateFCmp(P, LC, RC), Name); | ||||||
2242 | return Insert(setFPAttrs(new FCmpInst(P, LHS, RHS), FPMathTag, FMF), Name); | ||||||
2243 | } | ||||||
2244 | |||||||
2245 | //===--------------------------------------------------------------------===// | ||||||
2246 | // Instruction creation methods: Other Instructions | ||||||
2247 | //===--------------------------------------------------------------------===// | ||||||
2248 | |||||||
2249 | PHINode *CreatePHI(Type *Ty, unsigned NumReservedValues, | ||||||
2250 | const Twine &Name = "") { | ||||||
2251 | PHINode *Phi = PHINode::Create(Ty, NumReservedValues); | ||||||
2252 | if (isa<FPMathOperator>(Phi)) | ||||||
2253 | setFPAttrs(Phi, nullptr /* MDNode* */, FMF); | ||||||
2254 | return Insert(Phi, Name); | ||||||
2255 | } | ||||||
2256 | |||||||
2257 | CallInst *CreateCall(FunctionType *FTy, Value *Callee, | ||||||
2258 | ArrayRef<Value *> Args = None, const Twine &Name = "", | ||||||
2259 | MDNode *FPMathTag = nullptr) { | ||||||
2260 | CallInst *CI = CallInst::Create(FTy, Callee, Args, DefaultOperandBundles); | ||||||
2261 | if (IsFPConstrained) | ||||||
2262 | setConstrainedFPCallAttr(CI); | ||||||
2263 | if (isa<FPMathOperator>(CI)) | ||||||
2264 | setFPAttrs(CI, FPMathTag, FMF); | ||||||
2265 | return Insert(CI, Name); | ||||||
2266 | } | ||||||
2267 | |||||||
2268 | CallInst *CreateCall(FunctionType *FTy, Value *Callee, ArrayRef<Value *> Args, | ||||||
2269 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
2270 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2271 | CallInst *CI = CallInst::Create(FTy, Callee, Args, OpBundles); | ||||||
2272 | if (IsFPConstrained) | ||||||
2273 | setConstrainedFPCallAttr(CI); | ||||||
2274 | if (isa<FPMathOperator>(CI)) | ||||||
2275 | setFPAttrs(CI, FPMathTag, FMF); | ||||||
2276 | return Insert(CI, Name); | ||||||
2277 | } | ||||||
2278 | |||||||
2279 | CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args = None, | ||||||
2280 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2281 | return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name, | ||||||
2282 | FPMathTag); | ||||||
2283 | } | ||||||
2284 | |||||||
2285 | CallInst *CreateCall(FunctionCallee Callee, ArrayRef<Value *> Args, | ||||||
2286 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
2287 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2288 | return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, | ||||||
2289 | OpBundles, Name, FPMathTag); | ||||||
2290 | } | ||||||
2291 | |||||||
2292 | // Deprecated [opaque pointer types] | ||||||
2293 | CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args = None, | ||||||
2294 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2295 | return CreateCall( | ||||||
2296 | cast<FunctionType>(Callee->getType()->getPointerElementType()), Callee, | ||||||
2297 | Args, Name, FPMathTag); | ||||||
2298 | } | ||||||
2299 | |||||||
2300 | // Deprecated [opaque pointer types] | ||||||
2301 | CallInst *CreateCall(Value *Callee, ArrayRef<Value *> Args, | ||||||
2302 | ArrayRef<OperandBundleDef> OpBundles, | ||||||
2303 | const Twine &Name = "", MDNode *FPMathTag = nullptr) { | ||||||
2304 | return CreateCall( | ||||||
2305 | cast<FunctionType>(Callee->getType()->getPointerElementType()), Callee, | ||||||
2306 | Args, OpBundles, Name, FPMathTag); | ||||||
2307 | } | ||||||
2308 | |||||||
2309 | Value *CreateSelect(Value *C, Value *True, Value *False, | ||||||
2310 | const Twine &Name = "", Instruction *MDFrom = nullptr) { | ||||||
2311 | if (auto *CC = dyn_cast<Constant>(C)) | ||||||
2312 | if (auto *TC = dyn_cast<Constant>(True)) | ||||||
2313 | if (auto *FC = dyn_cast<Constant>(False)) | ||||||
2314 | return Insert(Folder.CreateSelect(CC, TC, FC), Name); | ||||||
2315 | |||||||
2316 | SelectInst *Sel = SelectInst::Create(C, True, False); | ||||||
2317 | if (MDFrom) { | ||||||
2318 | MDNode *Prof = MDFrom->getMetadata(LLVMContext::MD_prof); | ||||||
2319 | MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); | ||||||
2320 | Sel = addBranchMetadata(Sel, Prof, Unpred); | ||||||
2321 | } | ||||||
2322 | if (isa<FPMathOperator>(Sel)) | ||||||
2323 | setFPAttrs(Sel, nullptr /* MDNode* */, FMF); | ||||||
2324 | return Insert(Sel, Name); | ||||||
2325 | } | ||||||
2326 | |||||||
2327 | VAArgInst *CreateVAArg(Value *List, Type *Ty, const Twine &Name = "") { | ||||||
2328 | return Insert(new VAArgInst(List, Ty), Name); | ||||||
2329 | } | ||||||
2330 | |||||||
2331 | Value *CreateExtractElement(Value *Vec, Value *Idx, | ||||||
2332 | const Twine &Name = "") { | ||||||
2333 | if (auto *VC = dyn_cast<Constant>(Vec)) | ||||||
2334 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
2335 | return Insert(Folder.CreateExtractElement(VC, IC), Name); | ||||||
2336 | return Insert(ExtractElementInst::Create(Vec, Idx), Name); | ||||||
2337 | } | ||||||
2338 | |||||||
2339 | Value *CreateExtractElement(Value *Vec, uint64_t Idx, | ||||||
2340 | const Twine &Name = "") { | ||||||
2341 | return CreateExtractElement(Vec, getInt64(Idx), Name); | ||||||
2342 | } | ||||||
2343 | |||||||
2344 | Value *CreateInsertElement(Value *Vec, Value *NewElt, Value *Idx, | ||||||
2345 | const Twine &Name = "") { | ||||||
2346 | if (auto *VC = dyn_cast<Constant>(Vec)) | ||||||
2347 | if (auto *NC = dyn_cast<Constant>(NewElt)) | ||||||
2348 | if (auto *IC = dyn_cast<Constant>(Idx)) | ||||||
2349 | return Insert(Folder.CreateInsertElement(VC, NC, IC), Name); | ||||||
2350 | return Insert(InsertElementInst::Create(Vec, NewElt, Idx), Name); | ||||||
2351 | } | ||||||
2352 | |||||||
2353 | Value *CreateInsertElement(Value *Vec, Value *NewElt, uint64_t Idx, | ||||||
2354 | const Twine &Name = "") { | ||||||
2355 | return CreateInsertElement(Vec, NewElt, getInt64(Idx), Name); | ||||||
2356 | } | ||||||
2357 | |||||||
2358 | Value *CreateShuffleVector(Value *V1, Value *V2, Value *Mask, | ||||||
2359 | const Twine &Name = "") { | ||||||
2360 | if (auto *V1C = dyn_cast<Constant>(V1)) | ||||||
2361 | if (auto *V2C = dyn_cast<Constant>(V2)) | ||||||
2362 | if (auto *MC = dyn_cast<Constant>(Mask)) | ||||||
2363 | return Insert(Folder.CreateShuffleVector(V1C, V2C, MC), Name); | ||||||
2364 | return Insert(new ShuffleVectorInst(V1, V2, Mask), Name); | ||||||
2365 | } | ||||||
2366 | |||||||
2367 | Value *CreateShuffleVector(Value *V1, Value *V2, ArrayRef<uint32_t> IntMask, | ||||||
2368 | const Twine &Name = "") { | ||||||
2369 | Value *Mask = ConstantDataVector::get(Context, IntMask); | ||||||
2370 | return CreateShuffleVector(V1, V2, Mask, Name); | ||||||
2371 | } | ||||||
2372 | |||||||
2373 | Value *CreateExtractValue(Value *Agg, | ||||||
2374 | ArrayRef<unsigned> Idxs, | ||||||
2375 | const Twine &Name = "") { | ||||||
2376 | if (auto *AggC = dyn_cast<Constant>(Agg)) | ||||||
2377 | return Insert(Folder.CreateExtractValue(AggC, Idxs), Name); | ||||||
2378 | return Insert(ExtractValueInst::Create(Agg, Idxs), Name); | ||||||
2379 | } | ||||||
2380 | |||||||
2381 | Value *CreateInsertValue(Value *Agg, Value *Val, | ||||||
2382 | ArrayRef<unsigned> Idxs, | ||||||
2383 | const Twine &Name = "") { | ||||||
2384 | if (auto *AggC = dyn_cast<Constant>(Agg)) | ||||||
2385 | if (auto *ValC = dyn_cast<Constant>(Val)) | ||||||
2386 | return Insert(Folder.CreateInsertValue(AggC, ValC, Idxs), Name); | ||||||
2387 | return Insert(InsertValueInst::Create(Agg, Val, Idxs), Name); | ||||||
2388 | } | ||||||
2389 | |||||||
2390 | LandingPadInst *CreateLandingPad(Type *Ty, unsigned NumClauses, | ||||||
2391 | const Twine &Name = "") { | ||||||
2392 | return Insert(LandingPadInst::Create(Ty, NumClauses), Name); | ||||||
2393 | } | ||||||
2394 | |||||||
2395 | Value *CreateFreeze(Value *V, const Twine &Name = "") { | ||||||
2396 | return Insert(UnaryOperator::CreateFreeze(V, Name)); | ||||||
2397 | } | ||||||
2398 | |||||||
2399 | //===--------------------------------------------------------------------===// | ||||||
2400 | // Utility creation methods | ||||||
2401 | //===--------------------------------------------------------------------===// | ||||||
2402 | |||||||
2403 | /// Return an i1 value testing if \p Arg is null. | ||||||
2404 | Value *CreateIsNull(Value *Arg, const Twine &Name = "") { | ||||||
2405 | return CreateICmpEQ(Arg, Constant::getNullValue(Arg->getType()), | ||||||
2406 | Name); | ||||||
2407 | } | ||||||
2408 | |||||||
2409 | /// Return an i1 value testing if \p Arg is not null. | ||||||
2410 | Value *CreateIsNotNull(Value *Arg, const Twine &Name = "") { | ||||||
2411 | return CreateICmpNE(Arg, Constant::getNullValue(Arg->getType()), | ||||||
2412 | Name); | ||||||
2413 | } | ||||||
2414 | |||||||
2415 | /// Return the i64 difference between two pointer values, dividing out | ||||||
2416 | /// the size of the pointed-to objects. | ||||||
2417 | /// | ||||||
2418 | /// This is intended to implement C-style pointer subtraction. As such, the | ||||||
2419 | /// pointers must be appropriately aligned for their element types and | ||||||
2420 | /// pointing into the same object. | ||||||
2421 | Value *CreatePtrDiff(Value *LHS, Value *RHS, const Twine &Name = "") { | ||||||
2422 | assert(LHS->getType() == RHS->getType() &&((LHS->getType() == RHS->getType() && "Pointer subtraction operand types must match!" ) ? static_cast<void> (0) : __assert_fail ("LHS->getType() == RHS->getType() && \"Pointer subtraction operand types must match!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2423, __PRETTY_FUNCTION__)) | ||||||
2423 | "Pointer subtraction operand types must match!")((LHS->getType() == RHS->getType() && "Pointer subtraction operand types must match!" ) ? static_cast<void> (0) : __assert_fail ("LHS->getType() == RHS->getType() && \"Pointer subtraction operand types must match!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2423, __PRETTY_FUNCTION__)); | ||||||
2424 | auto *ArgType = cast<PointerType>(LHS->getType()); | ||||||
2425 | Value *LHS_int = CreatePtrToInt(LHS, Type::getInt64Ty(Context)); | ||||||
2426 | Value *RHS_int = CreatePtrToInt(RHS, Type::getInt64Ty(Context)); | ||||||
2427 | Value *Difference = CreateSub(LHS_int, RHS_int); | ||||||
2428 | return CreateExactSDiv(Difference, | ||||||
2429 | ConstantExpr::getSizeOf(ArgType->getElementType()), | ||||||
2430 | Name); | ||||||
2431 | } | ||||||
2432 | |||||||
2433 | /// Create a launder.invariant.group intrinsic call. If Ptr type is | ||||||
2434 | /// different from pointer to i8, it's casted to pointer to i8 in the same | ||||||
2435 | /// address space before call and casted back to Ptr type after call. | ||||||
2436 | Value *CreateLaunderInvariantGroup(Value *Ptr) { | ||||||
2437 | assert(isa<PointerType>(Ptr->getType()) &&((isa<PointerType>(Ptr->getType()) && "launder.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"launder.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2438, __PRETTY_FUNCTION__)) | ||||||
2438 | "launder.invariant.group only applies to pointers.")((isa<PointerType>(Ptr->getType()) && "launder.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"launder.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2438, __PRETTY_FUNCTION__)); | ||||||
2439 | // FIXME: we could potentially avoid casts to/from i8*. | ||||||
2440 | auto *PtrType = Ptr->getType(); | ||||||
2441 | auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); | ||||||
2442 | if (PtrType != Int8PtrTy) | ||||||
2443 | Ptr = CreateBitCast(Ptr, Int8PtrTy); | ||||||
2444 | Module *M = BB->getParent()->getParent(); | ||||||
2445 | Function *FnLaunderInvariantGroup = Intrinsic::getDeclaration( | ||||||
2446 | M, Intrinsic::launder_invariant_group, {Int8PtrTy}); | ||||||
2447 | |||||||
2448 | assert(FnLaunderInvariantGroup->getReturnType() == Int8PtrTy &&((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2451, __PRETTY_FUNCTION__)) | ||||||
2449 | FnLaunderInvariantGroup->getFunctionType()->getParamType(0) ==((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2451, __PRETTY_FUNCTION__)) | ||||||
2450 | Int8PtrTy &&((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2451, __PRETTY_FUNCTION__)) | ||||||
2451 | "LaunderInvariantGroup should take and return the same type")((FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "LaunderInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnLaunderInvariantGroup->getReturnType() == Int8PtrTy && FnLaunderInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"LaunderInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2451, __PRETTY_FUNCTION__)); | ||||||
2452 | |||||||
2453 | CallInst *Fn = CreateCall(FnLaunderInvariantGroup, {Ptr}); | ||||||
2454 | |||||||
2455 | if (PtrType != Int8PtrTy) | ||||||
2456 | return CreateBitCast(Fn, PtrType); | ||||||
2457 | return Fn; | ||||||
2458 | } | ||||||
2459 | |||||||
2460 | /// \brief Create a strip.invariant.group intrinsic call. If Ptr type is | ||||||
2461 | /// different from pointer to i8, it's casted to pointer to i8 in the same | ||||||
2462 | /// address space before call and casted back to Ptr type after call. | ||||||
2463 | Value *CreateStripInvariantGroup(Value *Ptr) { | ||||||
2464 | assert(isa<PointerType>(Ptr->getType()) &&((isa<PointerType>(Ptr->getType()) && "strip.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"strip.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2465, __PRETTY_FUNCTION__)) | ||||||
2465 | "strip.invariant.group only applies to pointers.")((isa<PointerType>(Ptr->getType()) && "strip.invariant.group only applies to pointers." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Ptr->getType()) && \"strip.invariant.group only applies to pointers.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2465, __PRETTY_FUNCTION__)); | ||||||
2466 | |||||||
2467 | // FIXME: we could potentially avoid casts to/from i8*. | ||||||
2468 | auto *PtrType = Ptr->getType(); | ||||||
2469 | auto *Int8PtrTy = getInt8PtrTy(PtrType->getPointerAddressSpace()); | ||||||
2470 | if (PtrType != Int8PtrTy) | ||||||
2471 | Ptr = CreateBitCast(Ptr, Int8PtrTy); | ||||||
2472 | Module *M = BB->getParent()->getParent(); | ||||||
2473 | Function *FnStripInvariantGroup = Intrinsic::getDeclaration( | ||||||
2474 | M, Intrinsic::strip_invariant_group, {Int8PtrTy}); | ||||||
2475 | |||||||
2476 | assert(FnStripInvariantGroup->getReturnType() == Int8PtrTy &&((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2479, __PRETTY_FUNCTION__)) | ||||||
2477 | FnStripInvariantGroup->getFunctionType()->getParamType(0) ==((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2479, __PRETTY_FUNCTION__)) | ||||||
2478 | Int8PtrTy &&((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2479, __PRETTY_FUNCTION__)) | ||||||
2479 | "StripInvariantGroup should take and return the same type")((FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType (0) == Int8PtrTy && "StripInvariantGroup should take and return the same type" ) ? static_cast<void> (0) : __assert_fail ("FnStripInvariantGroup->getReturnType() == Int8PtrTy && FnStripInvariantGroup->getFunctionType()->getParamType(0) == Int8PtrTy && \"StripInvariantGroup should take and return the same type\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2479, __PRETTY_FUNCTION__)); | ||||||
2480 | |||||||
2481 | CallInst *Fn = CreateCall(FnStripInvariantGroup, {Ptr}); | ||||||
2482 | |||||||
2483 | if (PtrType != Int8PtrTy) | ||||||
2484 | return CreateBitCast(Fn, PtrType); | ||||||
2485 | return Fn; | ||||||
2486 | } | ||||||
2487 | |||||||
2488 | /// Return a vector value that contains \arg V broadcasted to \p | ||||||
2489 | /// NumElts elements. | ||||||
2490 | Value *CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name = "") { | ||||||
2491 | assert(NumElts > 0 && "Cannot splat to an empty vector!")((NumElts > 0 && "Cannot splat to an empty vector!" ) ? static_cast<void> (0) : __assert_fail ("NumElts > 0 && \"Cannot splat to an empty vector!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2491, __PRETTY_FUNCTION__)); | ||||||
2492 | |||||||
2493 | // First insert it into an undef vector so we can shuffle it. | ||||||
2494 | Type *I32Ty = getInt32Ty(); | ||||||
2495 | Value *Undef = UndefValue::get(VectorType::get(V->getType(), NumElts)); | ||||||
2496 | V = CreateInsertElement(Undef, V, ConstantInt::get(I32Ty, 0), | ||||||
2497 | Name + ".splatinsert"); | ||||||
2498 | |||||||
2499 | // Shuffle the value across the desired number of elements. | ||||||
2500 | Value *Zeros = ConstantAggregateZero::get(VectorType::get(I32Ty, NumElts)); | ||||||
2501 | return CreateShuffleVector(V, Undef, Zeros, Name + ".splat"); | ||||||
2502 | } | ||||||
2503 | |||||||
2504 | /// Return a value that has been extracted from a larger integer type. | ||||||
2505 | Value *CreateExtractInteger(const DataLayout &DL, Value *From, | ||||||
2506 | IntegerType *ExtractedTy, uint64_t Offset, | ||||||
2507 | const Twine &Name) { | ||||||
2508 | auto *IntTy = cast<IntegerType>(From->getType()); | ||||||
2509 | assert(DL.getTypeStoreSize(ExtractedTy) + Offset <=((DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize (IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2511, __PRETTY_FUNCTION__)) | ||||||
2510 | DL.getTypeStoreSize(IntTy) &&((DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize (IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2511, __PRETTY_FUNCTION__)) | ||||||
2511 | "Element extends past full value")((DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize (IntTy) && "Element extends past full value") ? static_cast <void> (0) : __assert_fail ("DL.getTypeStoreSize(ExtractedTy) + Offset <= DL.getTypeStoreSize(IntTy) && \"Element extends past full value\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2511, __PRETTY_FUNCTION__)); | ||||||
2512 | uint64_t ShAmt = 8 * Offset; | ||||||
2513 | Value *V = From; | ||||||
2514 | if (DL.isBigEndian()) | ||||||
2515 | ShAmt = 8 * (DL.getTypeStoreSize(IntTy) - | ||||||
2516 | DL.getTypeStoreSize(ExtractedTy) - Offset); | ||||||
2517 | if (ShAmt) { | ||||||
2518 | V = CreateLShr(V, ShAmt, Name + ".shift"); | ||||||
2519 | } | ||||||
2520 | assert(ExtractedTy->getBitWidth() <= IntTy->getBitWidth() &&((ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast <void> (0) : __assert_fail ("ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2521, __PRETTY_FUNCTION__)) | ||||||
2521 | "Cannot extract to a larger integer!")((ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && "Cannot extract to a larger integer!") ? static_cast <void> (0) : __assert_fail ("ExtractedTy->getBitWidth() <= IntTy->getBitWidth() && \"Cannot extract to a larger integer!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2521, __PRETTY_FUNCTION__)); | ||||||
2522 | if (ExtractedTy != IntTy) { | ||||||
2523 | V = CreateTrunc(V, ExtractedTy, Name + ".trunc"); | ||||||
2524 | } | ||||||
2525 | return V; | ||||||
2526 | } | ||||||
2527 | |||||||
2528 | Value *CreatePreserveArrayAccessIndex(Type *ElTy, Value *Base, | ||||||
2529 | unsigned Dimension, unsigned LastIndex, | ||||||
2530 | MDNode *DbgInfo) { | ||||||
2531 | assert(isa<PointerType>(Base->getType()) &&((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.array.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.array.access.index.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2532, __PRETTY_FUNCTION__)) | ||||||
2532 | "Invalid Base ptr type for preserve.array.access.index.")((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.array.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.array.access.index.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2532, __PRETTY_FUNCTION__)); | ||||||
2533 | auto *BaseType = Base->getType(); | ||||||
2534 | |||||||
2535 | Value *LastIndexV = getInt32(LastIndex); | ||||||
2536 | Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); | ||||||
2537 | SmallVector<Value *, 4> IdxList; | ||||||
2538 | for (unsigned I = 0; I < Dimension; ++I) | ||||||
2539 | IdxList.push_back(Zero); | ||||||
2540 | IdxList.push_back(LastIndexV); | ||||||
2541 | |||||||
2542 | Type *ResultType = | ||||||
2543 | GetElementPtrInst::getGEPReturnType(ElTy, Base, IdxList); | ||||||
2544 | |||||||
2545 | Module *M = BB->getParent()->getParent(); | ||||||
2546 | Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( | ||||||
2547 | M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); | ||||||
2548 | |||||||
2549 | Value *DimV = getInt32(Dimension); | ||||||
2550 | CallInst *Fn = | ||||||
2551 | CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); | ||||||
2552 | if (DbgInfo) | ||||||
2553 | Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); | ||||||
2554 | |||||||
2555 | return Fn; | ||||||
2556 | } | ||||||
2557 | |||||||
2558 | Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex, | ||||||
2559 | MDNode *DbgInfo) { | ||||||
2560 | assert(isa<PointerType>(Base->getType()) &&((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.union.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.union.access.index.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2561, __PRETTY_FUNCTION__)) | ||||||
2561 | "Invalid Base ptr type for preserve.union.access.index.")((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.union.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.union.access.index.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2561, __PRETTY_FUNCTION__)); | ||||||
2562 | auto *BaseType = Base->getType(); | ||||||
2563 | |||||||
2564 | Module *M = BB->getParent()->getParent(); | ||||||
2565 | Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( | ||||||
2566 | M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); | ||||||
2567 | |||||||
2568 | Value *DIIndex = getInt32(FieldIndex); | ||||||
2569 | CallInst *Fn = | ||||||
2570 | CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); | ||||||
2571 | if (DbgInfo) | ||||||
2572 | Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); | ||||||
2573 | |||||||
2574 | return Fn; | ||||||
2575 | } | ||||||
2576 | |||||||
2577 | Value *CreatePreserveStructAccessIndex(Type *ElTy, Value *Base, | ||||||
2578 | unsigned Index, unsigned FieldIndex, | ||||||
2579 | MDNode *DbgInfo) { | ||||||
2580 | assert(isa<PointerType>(Base->getType()) &&((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.struct.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.struct.access.index.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2581, __PRETTY_FUNCTION__)) | ||||||
2581 | "Invalid Base ptr type for preserve.struct.access.index.")((isa<PointerType>(Base->getType()) && "Invalid Base ptr type for preserve.struct.access.index." ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(Base->getType()) && \"Invalid Base ptr type for preserve.struct.access.index.\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2581, __PRETTY_FUNCTION__)); | ||||||
2582 | auto *BaseType = Base->getType(); | ||||||
2583 | |||||||
2584 | Value *GEPIndex = getInt32(Index); | ||||||
2585 | Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); | ||||||
2586 | Type *ResultType = | ||||||
2587 | GetElementPtrInst::getGEPReturnType(ElTy, Base, {Zero, GEPIndex}); | ||||||
2588 | |||||||
2589 | Module *M = BB->getParent()->getParent(); | ||||||
2590 | Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( | ||||||
2591 | M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); | ||||||
2592 | |||||||
2593 | Value *DIIndex = getInt32(FieldIndex); | ||||||
2594 | CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, | ||||||
2595 | {Base, GEPIndex, DIIndex}); | ||||||
2596 | if (DbgInfo) | ||||||
2597 | Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); | ||||||
2598 | |||||||
2599 | return Fn; | ||||||
2600 | } | ||||||
2601 | |||||||
2602 | private: | ||||||
2603 | /// Helper function that creates an assume intrinsic call that | ||||||
2604 | /// represents an alignment assumption on the provided Ptr, Mask, Type | ||||||
2605 | /// and Offset. It may be sometimes useful to do some other logic | ||||||
2606 | /// based on this alignment check, thus it can be stored into 'TheCheck'. | ||||||
2607 | CallInst *CreateAlignmentAssumptionHelper(const DataLayout &DL, | ||||||
2608 | Value *PtrValue, Value *Mask, | ||||||
2609 | Type *IntPtrTy, Value *OffsetValue, | ||||||
2610 | Value **TheCheck) { | ||||||
2611 | Value *PtrIntValue = CreatePtrToInt(PtrValue, IntPtrTy, "ptrint"); | ||||||
2612 | |||||||
2613 | if (OffsetValue) { | ||||||
2614 | bool IsOffsetZero = false; | ||||||
2615 | if (const auto *CI = dyn_cast<ConstantInt>(OffsetValue)) | ||||||
2616 | IsOffsetZero = CI->isZero(); | ||||||
2617 | |||||||
2618 | if (!IsOffsetZero) { | ||||||
2619 | if (OffsetValue->getType() != IntPtrTy) | ||||||
2620 | OffsetValue = CreateIntCast(OffsetValue, IntPtrTy, /*isSigned*/ true, | ||||||
2621 | "offsetcast"); | ||||||
2622 | PtrIntValue = CreateSub(PtrIntValue, OffsetValue, "offsetptr"); | ||||||
2623 | } | ||||||
2624 | } | ||||||
2625 | |||||||
2626 | Value *Zero = ConstantInt::get(IntPtrTy, 0); | ||||||
2627 | Value *MaskedPtr = CreateAnd(PtrIntValue, Mask, "maskedptr"); | ||||||
2628 | Value *InvCond = CreateICmpEQ(MaskedPtr, Zero, "maskcond"); | ||||||
2629 | if (TheCheck) | ||||||
2630 | *TheCheck = InvCond; | ||||||
2631 | |||||||
2632 | return CreateAssumption(InvCond); | ||||||
2633 | } | ||||||
2634 | |||||||
2635 | public: | ||||||
2636 | /// Create an assume intrinsic call that represents an alignment | ||||||
2637 | /// assumption on the provided pointer. | ||||||
2638 | /// | ||||||
2639 | /// An optional offset can be provided, and if it is provided, the offset | ||||||
2640 | /// must be subtracted from the provided pointer to get the pointer with the | ||||||
2641 | /// specified alignment. | ||||||
2642 | /// | ||||||
2643 | /// It may be sometimes useful to do some other logic | ||||||
2644 | /// based on this alignment check, thus it can be stored into 'TheCheck'. | ||||||
2645 | CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, | ||||||
2646 | unsigned Alignment, | ||||||
2647 | Value *OffsetValue = nullptr, | ||||||
2648 | Value **TheCheck = nullptr) { | ||||||
2649 | assert(isa<PointerType>(PtrValue->getType()) &&((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2650, __PRETTY_FUNCTION__)) | ||||||
2650 | "trying to create an alignment assumption on a non-pointer?")((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2650, __PRETTY_FUNCTION__)); | ||||||
2651 | assert(Alignment != 0 && "Invalid Alignment")((Alignment != 0 && "Invalid Alignment") ? static_cast <void> (0) : __assert_fail ("Alignment != 0 && \"Invalid Alignment\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2651, __PRETTY_FUNCTION__)); | ||||||
2652 | auto *PtrTy = cast<PointerType>(PtrValue->getType()); | ||||||
2653 | Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); | ||||||
2654 | |||||||
2655 | Value *Mask = ConstantInt::get(IntPtrTy, Alignment - 1); | ||||||
2656 | return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, | ||||||
2657 | OffsetValue, TheCheck); | ||||||
2658 | } | ||||||
2659 | |||||||
2660 | /// Create an assume intrinsic call that represents an alignment | ||||||
2661 | /// assumption on the provided pointer. | ||||||
2662 | /// | ||||||
2663 | /// An optional offset can be provided, and if it is provided, the offset | ||||||
2664 | /// must be subtracted from the provided pointer to get the pointer with the | ||||||
2665 | /// specified alignment. | ||||||
2666 | /// | ||||||
2667 | /// It may be sometimes useful to do some other logic | ||||||
2668 | /// based on this alignment check, thus it can be stored into 'TheCheck'. | ||||||
2669 | /// | ||||||
2670 | /// This overload handles the condition where the Alignment is dependent | ||||||
2671 | /// on an existing value rather than a static value. | ||||||
2672 | CallInst *CreateAlignmentAssumption(const DataLayout &DL, Value *PtrValue, | ||||||
2673 | Value *Alignment, | ||||||
2674 | Value *OffsetValue = nullptr, | ||||||
2675 | Value **TheCheck = nullptr) { | ||||||
2676 | assert(isa<PointerType>(PtrValue->getType()) &&((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2677, __PRETTY_FUNCTION__)) | ||||||
2677 | "trying to create an alignment assumption on a non-pointer?")((isa<PointerType>(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?" ) ? static_cast<void> (0) : __assert_fail ("isa<PointerType>(PtrValue->getType()) && \"trying to create an alignment assumption on a non-pointer?\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/IR/IRBuilder.h" , 2677, __PRETTY_FUNCTION__)); | ||||||
2678 | auto *PtrTy = cast<PointerType>(PtrValue->getType()); | ||||||
2679 | Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); | ||||||
2680 | |||||||
2681 | if (Alignment->getType() != IntPtrTy) | ||||||
2682 | Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ false, | ||||||
2683 | "alignmentcast"); | ||||||
2684 | |||||||
2685 | Value *Mask = CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "mask"); | ||||||
2686 | |||||||
2687 | return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, | ||||||
2688 | OffsetValue, TheCheck); | ||||||
2689 | } | ||||||
2690 | }; | ||||||
2691 | |||||||
2692 | // Create wrappers for C Binding types (see CBindingWrapping.h). | ||||||
2693 | DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRBuilder<>, LLVMBuilderRef)inline IRBuilder<> *unwrap(LLVMBuilderRef P) { return reinterpret_cast <IRBuilder<>*>(P); } inline LLVMBuilderRef wrap(const IRBuilder<> *P) { return reinterpret_cast<LLVMBuilderRef >(const_cast<IRBuilder<>*>(P)); } | ||||||
2694 | |||||||
2695 | } // end namespace llvm | ||||||
2696 | |||||||
2697 | #endif // LLVM_IR_IRBUILDER_H |
1 | //===- Twine.h - Fast Temporary String Concatenation ------------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | |
9 | #ifndef LLVM_ADT_TWINE_H |
10 | #define LLVM_ADT_TWINE_H |
11 | |
12 | #include "llvm/ADT/SmallVector.h" |
13 | #include "llvm/ADT/StringRef.h" |
14 | #include "llvm/Support/ErrorHandling.h" |
15 | #include <cassert> |
16 | #include <cstdint> |
17 | #include <string> |
18 | |
19 | namespace llvm { |
20 | |
21 | class formatv_object_base; |
22 | class raw_ostream; |
23 | |
24 | /// Twine - A lightweight data structure for efficiently representing the |
25 | /// concatenation of temporary values as strings. |
26 | /// |
27 | /// A Twine is a kind of rope, it represents a concatenated string using a |
28 | /// binary-tree, where the string is the preorder of the nodes. Since the |
29 | /// Twine can be efficiently rendered into a buffer when its result is used, |
30 | /// it avoids the cost of generating temporary values for intermediate string |
31 | /// results -- particularly in cases when the Twine result is never |
32 | /// required. By explicitly tracking the type of leaf nodes, we can also avoid |
33 | /// the creation of temporary strings for conversions operations (such as |
34 | /// appending an integer to a string). |
35 | /// |
36 | /// A Twine is not intended for use directly and should not be stored, its |
37 | /// implementation relies on the ability to store pointers to temporary stack |
38 | /// objects which may be deallocated at the end of a statement. Twines should |
39 | /// only be used accepted as const references in arguments, when an API wishes |
40 | /// to accept possibly-concatenated strings. |
41 | /// |
42 | /// Twines support a special 'null' value, which always concatenates to form |
43 | /// itself, and renders as an empty string. This can be returned from APIs to |
44 | /// effectively nullify any concatenations performed on the result. |
45 | /// |
46 | /// \b Implementation |
47 | /// |
48 | /// Given the nature of a Twine, it is not possible for the Twine's |
49 | /// concatenation method to construct interior nodes; the result must be |
50 | /// represented inside the returned value. For this reason a Twine object |
51 | /// actually holds two values, the left- and right-hand sides of a |
52 | /// concatenation. We also have nullary Twine objects, which are effectively |
53 | /// sentinel values that represent empty strings. |
54 | /// |
55 | /// Thus, a Twine can effectively have zero, one, or two children. The \see |
56 | /// isNullary(), \see isUnary(), and \see isBinary() predicates exist for |
57 | /// testing the number of children. |
58 | /// |
59 | /// We maintain a number of invariants on Twine objects (FIXME: Why): |
60 | /// - Nullary twines are always represented with their Kind on the left-hand |
61 | /// side, and the Empty kind on the right-hand side. |
62 | /// - Unary twines are always represented with the value on the left-hand |
63 | /// side, and the Empty kind on the right-hand side. |
64 | /// - If a Twine has another Twine as a child, that child should always be |
65 | /// binary (otherwise it could have been folded into the parent). |
66 | /// |
67 | /// These invariants are check by \see isValid(). |
68 | /// |
69 | /// \b Efficiency Considerations |
70 | /// |
71 | /// The Twine is designed to yield efficient and small code for common |
72 | /// situations. For this reason, the concat() method is inlined so that |
73 | /// concatenations of leaf nodes can be optimized into stores directly into a |
74 | /// single stack allocated object. |
75 | /// |
76 | /// In practice, not all compilers can be trusted to optimize concat() fully, |
77 | /// so we provide two additional methods (and accompanying operator+ |
78 | /// overloads) to guarantee that particularly important cases (cstring plus |
79 | /// StringRef) codegen as desired. |
80 | class Twine { |
81 | /// NodeKind - Represent the type of an argument. |
82 | enum NodeKind : unsigned char { |
83 | /// An empty string; the result of concatenating anything with it is also |
84 | /// empty. |
85 | NullKind, |
86 | |
87 | /// The empty string. |
88 | EmptyKind, |
89 | |
90 | /// A pointer to a Twine instance. |
91 | TwineKind, |
92 | |
93 | /// A pointer to a C string instance. |
94 | CStringKind, |
95 | |
96 | /// A pointer to an std::string instance. |
97 | StdStringKind, |
98 | |
99 | /// A pointer to a StringRef instance. |
100 | StringRefKind, |
101 | |
102 | /// A pointer to a SmallString instance. |
103 | SmallStringKind, |
104 | |
105 | /// A pointer to a formatv_object_base instance. |
106 | FormatvObjectKind, |
107 | |
108 | /// A char value, to render as a character. |
109 | CharKind, |
110 | |
111 | /// An unsigned int value, to render as an unsigned decimal integer. |
112 | DecUIKind, |
113 | |
114 | /// An int value, to render as a signed decimal integer. |
115 | DecIKind, |
116 | |
117 | /// A pointer to an unsigned long value, to render as an unsigned decimal |
118 | /// integer. |
119 | DecULKind, |
120 | |
121 | /// A pointer to a long value, to render as a signed decimal integer. |
122 | DecLKind, |
123 | |
124 | /// A pointer to an unsigned long long value, to render as an unsigned |
125 | /// decimal integer. |
126 | DecULLKind, |
127 | |
128 | /// A pointer to a long long value, to render as a signed decimal integer. |
129 | DecLLKind, |
130 | |
131 | /// A pointer to a uint64_t value, to render as an unsigned hexadecimal |
132 | /// integer. |
133 | UHexKind |
134 | }; |
135 | |
136 | union Child |
137 | { |
138 | const Twine *twine; |
139 | const char *cString; |
140 | const std::string *stdString; |
141 | const StringRef *stringRef; |
142 | const SmallVectorImpl<char> *smallString; |
143 | const formatv_object_base *formatvObject; |
144 | char character; |
145 | unsigned int decUI; |
146 | int decI; |
147 | const unsigned long *decUL; |
148 | const long *decL; |
149 | const unsigned long long *decULL; |
150 | const long long *decLL; |
151 | const uint64_t *uHex; |
152 | }; |
153 | |
154 | /// LHS - The prefix in the concatenation, which may be uninitialized for |
155 | /// Null or Empty kinds. |
156 | Child LHS = {0}; |
157 | |
158 | /// RHS - The suffix in the concatenation, which may be uninitialized for |
159 | /// Null or Empty kinds. |
160 | Child RHS = {0}; |
161 | |
162 | /// LHSKind - The NodeKind of the left hand side, \see getLHSKind(). |
163 | NodeKind LHSKind = EmptyKind; |
164 | |
165 | /// RHSKind - The NodeKind of the right hand side, \see getRHSKind(). |
166 | NodeKind RHSKind = EmptyKind; |
167 | |
168 | /// Construct a nullary twine; the kind must be NullKind or EmptyKind. |
169 | explicit Twine(NodeKind Kind) : LHSKind(Kind) { |
170 | assert(isNullary() && "Invalid kind!")((isNullary() && "Invalid kind!") ? static_cast<void > (0) : __assert_fail ("isNullary() && \"Invalid kind!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 170, __PRETTY_FUNCTION__)); |
171 | } |
172 | |
173 | /// Construct a binary twine. |
174 | explicit Twine(const Twine &LHS, const Twine &RHS) |
175 | : LHSKind(TwineKind), RHSKind(TwineKind) { |
176 | this->LHS.twine = &LHS; |
177 | this->RHS.twine = &RHS; |
178 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 178, __PRETTY_FUNCTION__)); |
179 | } |
180 | |
181 | /// Construct a twine from explicit values. |
182 | explicit Twine(Child LHS, NodeKind LHSKind, Child RHS, NodeKind RHSKind) |
183 | : LHS(LHS), RHS(RHS), LHSKind(LHSKind), RHSKind(RHSKind) { |
184 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 184, __PRETTY_FUNCTION__)); |
185 | } |
186 | |
187 | /// Check for the null twine. |
188 | bool isNull() const { |
189 | return getLHSKind() == NullKind; |
190 | } |
191 | |
192 | /// Check for the empty twine. |
193 | bool isEmpty() const { |
194 | return getLHSKind() == EmptyKind; |
195 | } |
196 | |
197 | /// Check if this is a nullary twine (null or empty). |
198 | bool isNullary() const { |
199 | return isNull() || isEmpty(); |
200 | } |
201 | |
202 | /// Check if this is a unary twine. |
203 | bool isUnary() const { |
204 | return getRHSKind() == EmptyKind && !isNullary(); |
205 | } |
206 | |
207 | /// Check if this is a binary twine. |
208 | bool isBinary() const { |
209 | return getLHSKind() != NullKind && getRHSKind() != EmptyKind; |
210 | } |
211 | |
212 | /// Check if this is a valid twine (satisfying the invariants on |
213 | /// order and number of arguments). |
214 | bool isValid() const { |
215 | // Nullary twines always have Empty on the RHS. |
216 | if (isNullary() && getRHSKind() != EmptyKind) |
217 | return false; |
218 | |
219 | // Null should never appear on the RHS. |
220 | if (getRHSKind() == NullKind) |
221 | return false; |
222 | |
223 | // The RHS cannot be non-empty if the LHS is empty. |
224 | if (getRHSKind() != EmptyKind && getLHSKind() == EmptyKind) |
225 | return false; |
226 | |
227 | // A twine child should always be binary. |
228 | if (getLHSKind() == TwineKind && |
229 | !LHS.twine->isBinary()) |
230 | return false; |
231 | if (getRHSKind() == TwineKind && |
232 | !RHS.twine->isBinary()) |
233 | return false; |
234 | |
235 | return true; |
236 | } |
237 | |
238 | /// Get the NodeKind of the left-hand side. |
239 | NodeKind getLHSKind() const { return LHSKind; } |
240 | |
241 | /// Get the NodeKind of the right-hand side. |
242 | NodeKind getRHSKind() const { return RHSKind; } |
243 | |
244 | /// Print one child from a twine. |
245 | void printOneChild(raw_ostream &OS, Child Ptr, NodeKind Kind) const; |
246 | |
247 | /// Print the representation of one child from a twine. |
248 | void printOneChildRepr(raw_ostream &OS, Child Ptr, |
249 | NodeKind Kind) const; |
250 | |
251 | public: |
252 | /// @name Constructors |
253 | /// @{ |
254 | |
255 | /// Construct from an empty string. |
256 | /*implicit*/ Twine() { |
257 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 257, __PRETTY_FUNCTION__)); |
258 | } |
259 | |
260 | Twine(const Twine &) = default; |
261 | |
262 | /// Construct from a C string. |
263 | /// |
264 | /// We take care here to optimize "" into the empty twine -- this will be |
265 | /// optimized out for string constants. This allows Twine arguments have |
266 | /// default "" values, without introducing unnecessary string constants. |
267 | /*implicit*/ Twine(const char *Str) { |
268 | if (Str[0] != '\0') { |
269 | LHS.cString = Str; |
270 | LHSKind = CStringKind; |
271 | } else |
272 | LHSKind = EmptyKind; |
273 | |
274 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 274, __PRETTY_FUNCTION__)); |
275 | } |
276 | /// Delete the implicit conversion from nullptr as Twine(const char *) |
277 | /// cannot take nullptr. |
278 | /*implicit*/ Twine(std::nullptr_t) = delete; |
279 | |
280 | /// Construct from an std::string. |
281 | /*implicit*/ Twine(const std::string &Str) : LHSKind(StdStringKind) { |
282 | LHS.stdString = &Str; |
283 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 283, __PRETTY_FUNCTION__)); |
284 | } |
285 | |
286 | /// Construct from a StringRef. |
287 | /*implicit*/ Twine(const StringRef &Str) : LHSKind(StringRefKind) { |
288 | LHS.stringRef = &Str; |
289 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 289, __PRETTY_FUNCTION__)); |
290 | } |
291 | |
292 | /// Construct from a SmallString. |
293 | /*implicit*/ Twine(const SmallVectorImpl<char> &Str) |
294 | : LHSKind(SmallStringKind) { |
295 | LHS.smallString = &Str; |
296 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 296, __PRETTY_FUNCTION__)); |
297 | } |
298 | |
299 | /// Construct from a formatv_object_base. |
300 | /*implicit*/ Twine(const formatv_object_base &Fmt) |
301 | : LHSKind(FormatvObjectKind) { |
302 | LHS.formatvObject = &Fmt; |
303 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 303, __PRETTY_FUNCTION__)); |
304 | } |
305 | |
306 | /// Construct from a char. |
307 | explicit Twine(char Val) : LHSKind(CharKind) { |
308 | LHS.character = Val; |
309 | } |
310 | |
311 | /// Construct from a signed char. |
312 | explicit Twine(signed char Val) : LHSKind(CharKind) { |
313 | LHS.character = static_cast<char>(Val); |
314 | } |
315 | |
316 | /// Construct from an unsigned char. |
317 | explicit Twine(unsigned char Val) : LHSKind(CharKind) { |
318 | LHS.character = static_cast<char>(Val); |
319 | } |
320 | |
321 | /// Construct a twine to print \p Val as an unsigned decimal integer. |
322 | explicit Twine(unsigned Val) : LHSKind(DecUIKind) { |
323 | LHS.decUI = Val; |
324 | } |
325 | |
326 | /// Construct a twine to print \p Val as a signed decimal integer. |
327 | explicit Twine(int Val) : LHSKind(DecIKind) { |
328 | LHS.decI = Val; |
329 | } |
330 | |
331 | /// Construct a twine to print \p Val as an unsigned decimal integer. |
332 | explicit Twine(const unsigned long &Val) : LHSKind(DecULKind) { |
333 | LHS.decUL = &Val; |
334 | } |
335 | |
336 | /// Construct a twine to print \p Val as a signed decimal integer. |
337 | explicit Twine(const long &Val) : LHSKind(DecLKind) { |
338 | LHS.decL = &Val; |
339 | } |
340 | |
341 | /// Construct a twine to print \p Val as an unsigned decimal integer. |
342 | explicit Twine(const unsigned long long &Val) : LHSKind(DecULLKind) { |
343 | LHS.decULL = &Val; |
344 | } |
345 | |
346 | /// Construct a twine to print \p Val as a signed decimal integer. |
347 | explicit Twine(const long long &Val) : LHSKind(DecLLKind) { |
348 | LHS.decLL = &Val; |
349 | } |
350 | |
351 | // FIXME: Unfortunately, to make sure this is as efficient as possible we |
352 | // need extra binary constructors from particular types. We can't rely on |
353 | // the compiler to be smart enough to fold operator+()/concat() down to the |
354 | // right thing. Yet. |
355 | |
356 | /// Construct as the concatenation of a C string and a StringRef. |
357 | /*implicit*/ Twine(const char *LHS, const StringRef &RHS) |
358 | : LHSKind(CStringKind), RHSKind(StringRefKind) { |
359 | this->LHS.cString = LHS; |
360 | this->RHS.stringRef = &RHS; |
361 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 361, __PRETTY_FUNCTION__)); |
362 | } |
363 | |
364 | /// Construct as the concatenation of a StringRef and a C string. |
365 | /*implicit*/ Twine(const StringRef &LHS, const char *RHS) |
366 | : LHSKind(StringRefKind), RHSKind(CStringKind) { |
367 | this->LHS.stringRef = &LHS; |
368 | this->RHS.cString = RHS; |
369 | assert(isValid() && "Invalid twine!")((isValid() && "Invalid twine!") ? static_cast<void > (0) : __assert_fail ("isValid() && \"Invalid twine!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 369, __PRETTY_FUNCTION__)); |
370 | } |
371 | |
372 | /// Since the intended use of twines is as temporary objects, assignments |
373 | /// when concatenating might cause undefined behavior or stack corruptions |
374 | Twine &operator=(const Twine &) = delete; |
375 | |
376 | /// Create a 'null' string, which is an empty string that always |
377 | /// concatenates to form another empty string. |
378 | static Twine createNull() { |
379 | return Twine(NullKind); |
380 | } |
381 | |
382 | /// @} |
383 | /// @name Numeric Conversions |
384 | /// @{ |
385 | |
386 | // Construct a twine to print \p Val as an unsigned hexadecimal integer. |
387 | static Twine utohexstr(const uint64_t &Val) { |
388 | Child LHS, RHS; |
389 | LHS.uHex = &Val; |
390 | RHS.twine = nullptr; |
391 | return Twine(LHS, UHexKind, RHS, EmptyKind); |
392 | } |
393 | |
394 | /// @} |
395 | /// @name Predicate Operations |
396 | /// @{ |
397 | |
398 | /// Check if this twine is trivially empty; a false return value does not |
399 | /// necessarily mean the twine is empty. |
400 | bool isTriviallyEmpty() const { |
401 | return isNullary(); |
402 | } |
403 | |
404 | /// Return true if this twine can be dynamically accessed as a single |
405 | /// StringRef value with getSingleStringRef(). |
406 | bool isSingleStringRef() const { |
407 | if (getRHSKind() != EmptyKind) return false; |
408 | |
409 | switch (getLHSKind()) { |
410 | case EmptyKind: |
411 | case CStringKind: |
412 | case StdStringKind: |
413 | case StringRefKind: |
414 | case SmallStringKind: |
415 | return true; |
416 | default: |
417 | return false; |
418 | } |
419 | } |
420 | |
421 | /// @} |
422 | /// @name String Operations |
423 | /// @{ |
424 | |
425 | Twine concat(const Twine &Suffix) const; |
426 | |
427 | /// @} |
428 | /// @name Output & Conversion. |
429 | /// @{ |
430 | |
431 | /// Return the twine contents as a std::string. |
432 | std::string str() const; |
433 | |
434 | /// Append the concatenated string into the given SmallString or SmallVector. |
435 | void toVector(SmallVectorImpl<char> &Out) const; |
436 | |
437 | /// This returns the twine as a single StringRef. This method is only valid |
438 | /// if isSingleStringRef() is true. |
439 | StringRef getSingleStringRef() const { |
440 | assert(isSingleStringRef() &&"This cannot be had as a single stringref!")((isSingleStringRef() &&"This cannot be had as a single stringref!" ) ? static_cast<void> (0) : __assert_fail ("isSingleStringRef() &&\"This cannot be had as a single stringref!\"" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 440, __PRETTY_FUNCTION__)); |
441 | switch (getLHSKind()) { |
442 | default: llvm_unreachable("Out of sync with isSingleStringRef")::llvm::llvm_unreachable_internal("Out of sync with isSingleStringRef" , "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/ADT/Twine.h" , 442); |
443 | case EmptyKind: return StringRef(); |
444 | case CStringKind: return StringRef(LHS.cString); |
445 | case StdStringKind: return StringRef(*LHS.stdString); |
446 | case StringRefKind: return *LHS.stringRef; |
447 | case SmallStringKind: |
448 | return StringRef(LHS.smallString->data(), LHS.smallString->size()); |
449 | } |
450 | } |
451 | |
452 | /// This returns the twine as a single StringRef if it can be |
453 | /// represented as such. Otherwise the twine is written into the given |
454 | /// SmallVector and a StringRef to the SmallVector's data is returned. |
455 | StringRef toStringRef(SmallVectorImpl<char> &Out) const { |
456 | if (isSingleStringRef()) |
457 | return getSingleStringRef(); |
458 | toVector(Out); |
459 | return StringRef(Out.data(), Out.size()); |
460 | } |
461 | |
462 | /// This returns the twine as a single null terminated StringRef if it |
463 | /// can be represented as such. Otherwise the twine is written into the |
464 | /// given SmallVector and a StringRef to the SmallVector's data is returned. |
465 | /// |
466 | /// The returned StringRef's size does not include the null terminator. |
467 | StringRef toNullTerminatedStringRef(SmallVectorImpl<char> &Out) const; |
468 | |
469 | /// Write the concatenated string represented by this twine to the |
470 | /// stream \p OS. |
471 | void print(raw_ostream &OS) const; |
472 | |
473 | /// Dump the concatenated string represented by this twine to stderr. |
474 | void dump() const; |
475 | |
476 | /// Write the representation of this twine to the stream \p OS. |
477 | void printRepr(raw_ostream &OS) const; |
478 | |
479 | /// Dump the representation of this twine to stderr. |
480 | void dumpRepr() const; |
481 | |
482 | /// @} |
483 | }; |
484 | |
485 | /// @name Twine Inline Implementations |
486 | /// @{ |
487 | |
488 | inline Twine Twine::concat(const Twine &Suffix) const { |
489 | // Concatenation with null is null. |
490 | if (isNull() || Suffix.isNull()) |
491 | return Twine(NullKind); |
492 | |
493 | // Concatenation with empty yields the other side. |
494 | if (isEmpty()) |
495 | return Suffix; |
496 | if (Suffix.isEmpty()) |
497 | return *this; |
498 | |
499 | // Otherwise we need to create a new node, taking care to fold in unary |
500 | // twines. |
501 | Child NewLHS, NewRHS; |
502 | NewLHS.twine = this; |
503 | NewRHS.twine = &Suffix; |
504 | NodeKind NewLHSKind = TwineKind, NewRHSKind = TwineKind; |
505 | if (isUnary()) { |
506 | NewLHS = LHS; |
507 | NewLHSKind = getLHSKind(); |
508 | } |
509 | if (Suffix.isUnary()) { |
510 | NewRHS = Suffix.LHS; |
511 | NewRHSKind = Suffix.getLHSKind(); |
512 | } |
513 | |
514 | return Twine(NewLHS, NewLHSKind, NewRHS, NewRHSKind); |
515 | } |
516 | |
517 | inline Twine operator+(const Twine &LHS, const Twine &RHS) { |
518 | return LHS.concat(RHS); |
519 | } |
520 | |
521 | /// Additional overload to guarantee simplified codegen; this is equivalent to |
522 | /// concat(). |
523 | |
524 | inline Twine operator+(const char *LHS, const StringRef &RHS) { |
525 | return Twine(LHS, RHS); |
526 | } |
527 | |
528 | /// Additional overload to guarantee simplified codegen; this is equivalent to |
529 | /// concat(). |
530 | |
531 | inline Twine operator+(const StringRef &LHS, const char *RHS) { |
532 | return Twine(LHS, RHS); |
533 | } |
534 | |
535 | inline raw_ostream &operator<<(raw_ostream &OS, const Twine &RHS) { |
536 | RHS.print(OS); |
537 | return OS; |
538 | } |
539 | |
540 | /// @} |
541 | |
542 | } // end namespace llvm |
543 | |
544 | #endif // LLVM_ADT_TWINE_H |