Bug Summary

File:lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
Warning:line 7270, column 14
Forming reference to null pointer

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name SelectionDAGBuilder.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-eagerly-assume -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-7/lib/clang/7.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/include -I /build/llvm-toolchain-snapshot-7~svn338205/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/x86_64-linux-gnu/c++/8 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/8/../../../../include/c++/8/backward -internal-isystem /usr/include/clang/7.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-7/lib/clang/7.0.0/include -internal-externc-isystem /usr/lib/gcc/x86_64-linux-gnu/8/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-7~svn338205/build-llvm/lib/CodeGen/SelectionDAG -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-07-29-043837-17923-1 -x c++ /build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp -faddrsig
1//===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This implements routines for translating from LLVM IR into SelectionDAG IR.
11//
12//===----------------------------------------------------------------------===//
13
14#include "SelectionDAGBuilder.h"
15#include "SDNodeDbgValue.h"
16#include "llvm/ADT/APFloat.h"
17#include "llvm/ADT/APInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/None.h"
22#include "llvm/ADT/Optional.h"
23#include "llvm/ADT/STLExtras.h"
24#include "llvm/ADT/SmallPtrSet.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/SmallVector.h"
27#include "llvm/ADT/StringRef.h"
28#include "llvm/ADT/Triple.h"
29#include "llvm/ADT/Twine.h"
30#include "llvm/Analysis/AliasAnalysis.h"
31#include "llvm/Analysis/BranchProbabilityInfo.h"
32#include "llvm/Analysis/ConstantFolding.h"
33#include "llvm/Analysis/EHPersonalities.h"
34#include "llvm/Analysis/Loads.h"
35#include "llvm/Analysis/MemoryLocation.h"
36#include "llvm/Analysis/TargetLibraryInfo.h"
37#include "llvm/Analysis/ValueTracking.h"
38#include "llvm/Analysis/VectorUtils.h"
39#include "llvm/CodeGen/Analysis.h"
40#include "llvm/CodeGen/FunctionLoweringInfo.h"
41#include "llvm/CodeGen/GCMetadata.h"
42#include "llvm/CodeGen/ISDOpcodes.h"
43#include "llvm/CodeGen/MachineBasicBlock.h"
44#include "llvm/CodeGen/MachineFrameInfo.h"
45#include "llvm/CodeGen/MachineFunction.h"
46#include "llvm/CodeGen/MachineInstr.h"
47#include "llvm/CodeGen/MachineInstrBuilder.h"
48#include "llvm/CodeGen/MachineJumpTableInfo.h"
49#include "llvm/CodeGen/MachineMemOperand.h"
50#include "llvm/CodeGen/MachineModuleInfo.h"
51#include "llvm/CodeGen/MachineOperand.h"
52#include "llvm/CodeGen/MachineRegisterInfo.h"
53#include "llvm/CodeGen/RuntimeLibcalls.h"
54#include "llvm/CodeGen/SelectionDAG.h"
55#include "llvm/CodeGen/SelectionDAGNodes.h"
56#include "llvm/CodeGen/SelectionDAGTargetInfo.h"
57#include "llvm/CodeGen/StackMaps.h"
58#include "llvm/CodeGen/TargetFrameLowering.h"
59#include "llvm/CodeGen/TargetInstrInfo.h"
60#include "llvm/CodeGen/TargetLowering.h"
61#include "llvm/CodeGen/TargetOpcodes.h"
62#include "llvm/CodeGen/TargetRegisterInfo.h"
63#include "llvm/CodeGen/TargetSubtargetInfo.h"
64#include "llvm/CodeGen/ValueTypes.h"
65#include "llvm/CodeGen/WinEHFuncInfo.h"
66#include "llvm/IR/Argument.h"
67#include "llvm/IR/Attributes.h"
68#include "llvm/IR/BasicBlock.h"
69#include "llvm/IR/CFG.h"
70#include "llvm/IR/CallSite.h"
71#include "llvm/IR/CallingConv.h"
72#include "llvm/IR/Constant.h"
73#include "llvm/IR/ConstantRange.h"
74#include "llvm/IR/Constants.h"
75#include "llvm/IR/DataLayout.h"
76#include "llvm/IR/DebugInfoMetadata.h"
77#include "llvm/IR/DebugLoc.h"
78#include "llvm/IR/DerivedTypes.h"
79#include "llvm/IR/Function.h"
80#include "llvm/IR/GetElementPtrTypeIterator.h"
81#include "llvm/IR/InlineAsm.h"
82#include "llvm/IR/InstrTypes.h"
83#include "llvm/IR/Instruction.h"
84#include "llvm/IR/Instructions.h"
85#include "llvm/IR/IntrinsicInst.h"
86#include "llvm/IR/Intrinsics.h"
87#include "llvm/IR/LLVMContext.h"
88#include "llvm/IR/Metadata.h"
89#include "llvm/IR/Module.h"
90#include "llvm/IR/Operator.h"
91#include "llvm/IR/Statepoint.h"
92#include "llvm/IR/Type.h"
93#include "llvm/IR/User.h"
94#include "llvm/IR/Value.h"
95#include "llvm/MC/MCContext.h"
96#include "llvm/MC/MCSymbol.h"
97#include "llvm/Support/AtomicOrdering.h"
98#include "llvm/Support/BranchProbability.h"
99#include "llvm/Support/Casting.h"
100#include "llvm/Support/CodeGen.h"
101#include "llvm/Support/CommandLine.h"
102#include "llvm/Support/Compiler.h"
103#include "llvm/Support/Debug.h"
104#include "llvm/Support/ErrorHandling.h"
105#include "llvm/Support/MachineValueType.h"
106#include "llvm/Support/MathExtras.h"
107#include "llvm/Support/raw_ostream.h"
108#include "llvm/Target/TargetIntrinsicInfo.h"
109#include "llvm/Target/TargetMachine.h"
110#include "llvm/Target/TargetOptions.h"
111#include <algorithm>
112#include <cassert>
113#include <cstddef>
114#include <cstdint>
115#include <cstring>
116#include <iterator>
117#include <limits>
118#include <numeric>
119#include <tuple>
120#include <utility>
121#include <vector>
122
123using namespace llvm;
124
125#define DEBUG_TYPE"isel" "isel"
126
127/// LimitFloatPrecision - Generate low-precision inline sequences for
128/// some float libcalls (6, 8 or 12 bits).
129static unsigned LimitFloatPrecision;
130
131static cl::opt<unsigned, true>
132 LimitFPPrecision("limit-float-precision",
133 cl::desc("Generate low-precision inline sequences "
134 "for some float libcalls"),
135 cl::location(LimitFloatPrecision), cl::Hidden,
136 cl::init(0));
137
138static cl::opt<unsigned> SwitchPeelThreshold(
139 "switch-peel-threshold", cl::Hidden, cl::init(66),
140 cl::desc("Set the case probability threshold for peeling the case from a "
141 "switch statement. A value greater than 100 will void this "
142 "optimization"));
143
144// Limit the width of DAG chains. This is important in general to prevent
145// DAG-based analysis from blowing up. For example, alias analysis and
146// load clustering may not complete in reasonable time. It is difficult to
147// recognize and avoid this situation within each individual analysis, and
148// future analyses are likely to have the same behavior. Limiting DAG width is
149// the safe approach and will be especially important with global DAGs.
150//
151// MaxParallelChains default is arbitrarily high to avoid affecting
152// optimization, but could be lowered to improve compile time. Any ld-ld-st-st
153// sequence over this should have been converted to llvm.memcpy by the
154// frontend. It is easy to induce this behavior with .ll code such as:
155// %buffer = alloca [4096 x i8]
156// %data = load [4096 x i8]* %argPtr
157// store [4096 x i8] %data, [4096 x i8]* %buffer
158static const unsigned MaxParallelChains = 64;
159
160// Return the calling convention if the Value passed requires ABI mangling as it
161// is a parameter to a function or a return value from a function which is not
162// an intrinsic.
163static Optional<CallingConv::ID> getABIRegCopyCC(const Value *V) {
164 if (auto *R = dyn_cast<ReturnInst>(V))
165 return R->getParent()->getParent()->getCallingConv();
166
167 if (auto *CI = dyn_cast<CallInst>(V)) {
168 const bool IsInlineAsm = CI->isInlineAsm();
169 const bool IsIndirectFunctionCall =
170 !IsInlineAsm && !CI->getCalledFunction();
171
172 // It is possible that the call instruction is an inline asm statement or an
173 // indirect function call in which case the return value of
174 // getCalledFunction() would be nullptr.
175 const bool IsInstrinsicCall =
176 !IsInlineAsm && !IsIndirectFunctionCall &&
177 CI->getCalledFunction()->getIntrinsicID() != Intrinsic::not_intrinsic;
178
179 if (!IsInlineAsm && !IsInstrinsicCall)
180 return CI->getCallingConv();
181 }
182
183 return None;
184}
185
186static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
187 const SDValue *Parts, unsigned NumParts,
188 MVT PartVT, EVT ValueVT, const Value *V,
189 Optional<CallingConv::ID> CC);
190
191/// getCopyFromParts - Create a value that contains the specified legal parts
192/// combined into the value they represent. If the parts combine to a type
193/// larger than ValueVT then AssertOp can be used to specify whether the extra
194/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
195/// (ISD::AssertSext).
196static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL,
197 const SDValue *Parts, unsigned NumParts,
198 MVT PartVT, EVT ValueVT, const Value *V,
199 Optional<CallingConv::ID> CC = None,
200 Optional<ISD::NodeType> AssertOp = None) {
201 if (ValueVT.isVector())
202 return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT, V,
203 CC);
204
205 assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 205, __extension__ __PRETTY_FUNCTION__));
206 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
207 SDValue Val = Parts[0];
208
209 if (NumParts > 1) {
210 // Assemble the value from multiple parts.
211 if (ValueVT.isInteger()) {
212 unsigned PartBits = PartVT.getSizeInBits();
213 unsigned ValueBits = ValueVT.getSizeInBits();
214
215 // Assemble the power of 2 part.
216 unsigned RoundParts = NumParts & (NumParts - 1) ?
217 1 << Log2_32(NumParts) : NumParts;
218 unsigned RoundBits = PartBits * RoundParts;
219 EVT RoundVT = RoundBits == ValueBits ?
220 ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits);
221 SDValue Lo, Hi;
222
223 EVT HalfVT = EVT::getIntegerVT(*DAG.getContext(), RoundBits/2);
224
225 if (RoundParts > 2) {
226 Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
227 PartVT, HalfVT, V);
228 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
229 RoundParts / 2, PartVT, HalfVT, V);
230 } else {
231 Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
232 Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
233 }
234
235 if (DAG.getDataLayout().isBigEndian())
236 std::swap(Lo, Hi);
237
238 Val = DAG.getNode(ISD::BUILD_PAIR, DL, RoundVT, Lo, Hi);
239
240 if (RoundParts < NumParts) {
241 // Assemble the trailing non-power-of-2 part.
242 unsigned OddParts = NumParts - RoundParts;
243 EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
244 Hi = getCopyFromParts(DAG, DL, Parts + RoundParts, OddParts, PartVT,
245 OddVT, V, CC);
246
247 // Combine the round and odd parts.
248 Lo = Val;
249 if (DAG.getDataLayout().isBigEndian())
250 std::swap(Lo, Hi);
251 EVT TotalVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
252 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, TotalVT, Hi);
253 Hi =
254 DAG.getNode(ISD::SHL, DL, TotalVT, Hi,
255 DAG.getConstant(Lo.getValueSizeInBits(), DL,
256 TLI.getPointerTy(DAG.getDataLayout())));
257 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, TotalVT, Lo);
258 Val = DAG.getNode(ISD::OR, DL, TotalVT, Lo, Hi);
259 }
260 } else if (PartVT.isFloatingPoint()) {
261 // FP split into multiple FP parts (for ppcf128)
262 assert(ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 &&(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 263, __extension__ __PRETTY_FUNCTION__))
263 "Unexpected split")(static_cast <bool> (ValueVT == EVT(MVT::ppcf128) &&
PartVT == MVT::f64 && "Unexpected split") ? void (0)
: __assert_fail ("ValueVT == EVT(MVT::ppcf128) && PartVT == MVT::f64 && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 263, __extension__ __PRETTY_FUNCTION__));
264 SDValue Lo, Hi;
265 Lo = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[0]);
266 Hi = DAG.getNode(ISD::BITCAST, DL, EVT(MVT::f64), Parts[1]);
267 if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
268 std::swap(Lo, Hi);
269 Val = DAG.getNode(ISD::BUILD_PAIR, DL, ValueVT, Lo, Hi);
270 } else {
271 // FP split into integer parts (soft fp)
272 assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 273, __extension__ __PRETTY_FUNCTION__))
273 !PartVT.isVector() && "Unexpected split")(static_cast <bool> (ValueVT.isFloatingPoint() &&
PartVT.isInteger() && !PartVT.isVector() && "Unexpected split"
) ? void (0) : __assert_fail ("ValueVT.isFloatingPoint() && PartVT.isInteger() && !PartVT.isVector() && \"Unexpected split\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 273, __extension__ __PRETTY_FUNCTION__));
274 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
275 Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V, CC);
276 }
277 }
278
279 // There is now one part, held in Val. Correct it to match ValueVT.
280 // PartEVT is the type of the register class that holds the value.
281 // ValueVT is the type of the inline asm operation.
282 EVT PartEVT = Val.getValueType();
283
284 if (PartEVT == ValueVT)
285 return Val;
286
287 if (PartEVT.isInteger() && ValueVT.isFloatingPoint() &&
288 ValueVT.bitsLT(PartEVT)) {
289 // For an FP value in an integer part, we need to truncate to the right
290 // width first.
291 PartEVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
292 Val = DAG.getNode(ISD::TRUNCATE, DL, PartEVT, Val);
293 }
294
295 // Handle types that have the same size.
296 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits())
297 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
298
299 // Handle types with different sizes.
300 if (PartEVT.isInteger() && ValueVT.isInteger()) {
301 if (ValueVT.bitsLT(PartEVT)) {
302 // For a truncate, see if we have any information to
303 // indicate whether the truncated bits will always be
304 // zero or sign-extension.
305 if (AssertOp.hasValue())
306 Val = DAG.getNode(*AssertOp, DL, PartEVT, Val,
307 DAG.getValueType(ValueVT));
308 return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
309 }
310 return DAG.getNode(ISD::ANY_EXTEND, DL, ValueVT, Val);
311 }
312
313 if (PartEVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
314 // FP_ROUND's are always exact here.
315 if (ValueVT.bitsLT(Val.getValueType()))
316 return DAG.getNode(
317 ISD::FP_ROUND, DL, ValueVT, Val,
318 DAG.getTargetConstant(1, DL, TLI.getPointerTy(DAG.getDataLayout())));
319
320 return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val);
321 }
322
323 llvm_unreachable("Unknown mismatch!")::llvm::llvm_unreachable_internal("Unknown mismatch!", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 323);
324}
325
326static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V,
327 const Twine &ErrMsg) {
328 const Instruction *I = dyn_cast_or_null<Instruction>(V);
329 if (!V)
330 return Ctx.emitError(ErrMsg);
331
332 const char *AsmError = ", possible invalid constraint for vector type";
333 if (const CallInst *CI = dyn_cast<CallInst>(I))
334 if (isa<InlineAsm>(CI->getCalledValue()))
335 return Ctx.emitError(I, ErrMsg + AsmError);
336
337 return Ctx.emitError(I, ErrMsg);
338}
339
340/// getCopyFromPartsVector - Create a value that contains the specified legal
341/// parts combined into the value they represent. If the parts combine to a
342/// type larger than ValueVT then AssertOp can be used to specify whether the
343/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
344/// ValueVT (ISD::AssertSext).
345static SDValue getCopyFromPartsVector(SelectionDAG &DAG, const SDLoc &DL,
346 const SDValue *Parts, unsigned NumParts,
347 MVT PartVT, EVT ValueVT, const Value *V,
348 Optional<CallingConv::ID> CallConv) {
349 assert(ValueVT.isVector() && "Not a vector value")(static_cast <bool> (ValueVT.isVector() && "Not a vector value"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector value\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 349, __extension__ __PRETTY_FUNCTION__));
350 assert(NumParts > 0 && "No parts to assemble!")(static_cast <bool> (NumParts > 0 && "No parts to assemble!"
) ? void (0) : __assert_fail ("NumParts > 0 && \"No parts to assemble!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 350, __extension__ __PRETTY_FUNCTION__));
351 const bool IsABIRegCopy = CallConv.hasValue();
352
353 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
354 SDValue Val = Parts[0];
355
356 // Handle a multi-element vector.
357 if (NumParts > 1) {
358 EVT IntermediateVT;
359 MVT RegisterVT;
360 unsigned NumIntermediates;
361 unsigned NumRegs;
362
363 if (IsABIRegCopy) {
364 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
365 *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
366 NumIntermediates, RegisterVT);
367 } else {
368 NumRegs =
369 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
370 NumIntermediates, RegisterVT);
371 }
372
373 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 373, __extension__ __PRETTY_FUNCTION__));
374 NumParts = NumRegs; // Silence a compiler warning.
375 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 375, __extension__ __PRETTY_FUNCTION__));
376 assert(RegisterVT.getSizeInBits() ==(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 378, __extension__ __PRETTY_FUNCTION__))
377 Parts[0].getSimpleValueType().getSizeInBits() &&(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 378, __extension__ __PRETTY_FUNCTION__))
378 "Part type sizes don't match!")(static_cast <bool> (RegisterVT.getSizeInBits() == Parts
[0].getSimpleValueType().getSizeInBits() && "Part type sizes don't match!"
) ? void (0) : __assert_fail ("RegisterVT.getSizeInBits() == Parts[0].getSimpleValueType().getSizeInBits() && \"Part type sizes don't match!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 378, __extension__ __PRETTY_FUNCTION__));
379
380 // Assemble the parts into intermediate operands.
381 SmallVector<SDValue, 8> Ops(NumIntermediates);
382 if (NumIntermediates == NumParts) {
383 // If the register was not expanded, truncate or copy the value,
384 // as appropriate.
385 for (unsigned i = 0; i != NumParts; ++i)
386 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
387 PartVT, IntermediateVT, V);
388 } else if (NumParts > 0) {
389 // If the intermediate type was expanded, build the intermediate
390 // operands from the parts.
391 assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 392, __extension__ __PRETTY_FUNCTION__))
392 "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 392, __extension__ __PRETTY_FUNCTION__));
393 unsigned Factor = NumParts / NumIntermediates;
394 for (unsigned i = 0; i != NumIntermediates; ++i)
395 Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
396 PartVT, IntermediateVT, V);
397 }
398
399 // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
400 // intermediate operands.
401 EVT BuiltVectorTy =
402 EVT::getVectorVT(*DAG.getContext(), IntermediateVT.getScalarType(),
403 (IntermediateVT.isVector()
404 ? IntermediateVT.getVectorNumElements() * NumParts
405 : NumIntermediates));
406 Val = DAG.getNode(IntermediateVT.isVector() ? ISD::CONCAT_VECTORS
407 : ISD::BUILD_VECTOR,
408 DL, BuiltVectorTy, Ops);
409 }
410
411 // There is now one part, held in Val. Correct it to match ValueVT.
412 EVT PartEVT = Val.getValueType();
413
414 if (PartEVT == ValueVT)
415 return Val;
416
417 if (PartEVT.isVector()) {
418 // If the element type of the source/dest vectors are the same, but the
419 // parts vector has more elements than the value vector, then we have a
420 // vector widening case (e.g. <2 x float> -> <4 x float>). Extract the
421 // elements we want.
422 if (PartEVT.getVectorElementType() == ValueVT.getVectorElementType()) {
423 assert(PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() &&(static_cast <bool> (PartEVT.getVectorNumElements() >
ValueVT.getVectorNumElements() && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 424, __extension__ __PRETTY_FUNCTION__))
424 "Cannot narrow, it would be a lossy transformation")(static_cast <bool> (PartEVT.getVectorNumElements() >
ValueVT.getVectorNumElements() && "Cannot narrow, it would be a lossy transformation"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements() && \"Cannot narrow, it would be a lossy transformation\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 424, __extension__ __PRETTY_FUNCTION__));
425 return DAG.getNode(
426 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
427 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
428 }
429
430 // Vector/Vector bitcast.
431 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits())
432 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
433
434 assert(PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() &&(static_cast <bool> (PartEVT.getVectorNumElements() == ValueVT
.getVectorNumElements() && "Cannot handle this kind of promotion"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 435, __extension__ __PRETTY_FUNCTION__))
435 "Cannot handle this kind of promotion")(static_cast <bool> (PartEVT.getVectorNumElements() == ValueVT
.getVectorNumElements() && "Cannot handle this kind of promotion"
) ? void (0) : __assert_fail ("PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements() && \"Cannot handle this kind of promotion\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 435, __extension__ __PRETTY_FUNCTION__));
436 // Promoted vector extract
437 return DAG.getAnyExtOrTrunc(Val, DL, ValueVT);
438
439 }
440
441 // Trivial bitcast if the types are the same size and the destination
442 // vector type is legal.
443 if (PartEVT.getSizeInBits() == ValueVT.getSizeInBits() &&
444 TLI.isTypeLegal(ValueVT))
445 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
446
447 if (ValueVT.getVectorNumElements() != 1) {
448 // Certain ABIs require that vectors are passed as integers. For vectors
449 // are the same size, this is an obvious bitcast.
450 if (ValueVT.getSizeInBits() == PartEVT.getSizeInBits()) {
451 return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
452 } else if (ValueVT.getSizeInBits() < PartEVT.getSizeInBits()) {
453 // Bitcast Val back the original type and extract the corresponding
454 // vector we want.
455 unsigned Elts = PartEVT.getSizeInBits() / ValueVT.getScalarSizeInBits();
456 EVT WiderVecType = EVT::getVectorVT(*DAG.getContext(),
457 ValueVT.getVectorElementType(), Elts);
458 Val = DAG.getBitcast(WiderVecType, Val);
459 return DAG.getNode(
460 ISD::EXTRACT_SUBVECTOR, DL, ValueVT, Val,
461 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
462 }
463
464 diagnosePossiblyInvalidConstraint(
465 *DAG.getContext(), V, "non-trivial scalar-to-vector conversion");
466 return DAG.getUNDEF(ValueVT);
467 }
468
469 // Handle cases such as i8 -> <1 x i1>
470 EVT ValueSVT = ValueVT.getVectorElementType();
471 if (ValueVT.getVectorNumElements() == 1 && ValueSVT != PartEVT)
472 Val = ValueVT.isFloatingPoint() ? DAG.getFPExtendOrRound(Val, DL, ValueSVT)
473 : DAG.getAnyExtOrTrunc(Val, DL, ValueSVT);
474
475 return DAG.getBuildVector(ValueVT, DL, Val);
476}
477
478static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &dl,
479 SDValue Val, SDValue *Parts, unsigned NumParts,
480 MVT PartVT, const Value *V,
481 Optional<CallingConv::ID> CallConv);
482
483/// getCopyToParts - Create a series of nodes that contain the specified value
484/// split into legal parts. If the parts contain more bits than Val, then, for
485/// integers, ExtendKind can be used to specify how to generate the extra bits.
486static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
487 SDValue *Parts, unsigned NumParts, MVT PartVT,
488 const Value *V,
489 Optional<CallingConv::ID> CallConv = None,
490 ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
491 EVT ValueVT = Val.getValueType();
492
493 // Handle the vector case separately.
494 if (ValueVT.isVector())
495 return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V,
496 CallConv);
497
498 unsigned PartBits = PartVT.getSizeInBits();
499 unsigned OrigNumParts = NumParts;
500 assert(DAG.getTargetLoweringInfo().isTypeLegal(PartVT) &&(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __extension__ __PRETTY_FUNCTION__))
501 "Copying to an illegal type!")(static_cast <bool> (DAG.getTargetLoweringInfo().isTypeLegal
(PartVT) && "Copying to an illegal type!") ? void (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(PartVT) && \"Copying to an illegal type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 501, __extension__ __PRETTY_FUNCTION__));
502
503 if (NumParts == 0)
504 return;
505
506 assert(!ValueVT.isVector() && "Vector case handled elsewhere")(static_cast <bool> (!ValueVT.isVector() && "Vector case handled elsewhere"
) ? void (0) : __assert_fail ("!ValueVT.isVector() && \"Vector case handled elsewhere\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 506, __extension__ __PRETTY_FUNCTION__));
507 EVT PartEVT = PartVT;
508 if (PartEVT == ValueVT) {
509 assert(NumParts == 1 && "No-op copy with multiple parts!")(static_cast <bool> (NumParts == 1 && "No-op copy with multiple parts!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"No-op copy with multiple parts!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 509, __extension__ __PRETTY_FUNCTION__));
510 Parts[0] = Val;
511 return;
512 }
513
514 if (NumParts * PartBits > ValueVT.getSizeInBits()) {
515 // If the parts cover more bits than the value has, promote the value.
516 if (PartVT.isFloatingPoint() && ValueVT.isFloatingPoint()) {
517 assert(NumParts == 1 && "Do not know what to promote to!")(static_cast <bool> (NumParts == 1 && "Do not know what to promote to!"
) ? void (0) : __assert_fail ("NumParts == 1 && \"Do not know what to promote to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 517, __extension__ __PRETTY_FUNCTION__));
518 Val = DAG.getNode(ISD::FP_EXTEND, DL, PartVT, Val);
519 } else {
520 if (ValueVT.isFloatingPoint()) {
521 // FP values need to be bitcast, then extended if they are being put
522 // into a larger container.
523 ValueVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
524 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
525 }
526 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 528, __extension__ __PRETTY_FUNCTION__))
527 ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 528, __extension__ __PRETTY_FUNCTION__))
528 "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 528, __extension__ __PRETTY_FUNCTION__));
529 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
530 Val = DAG.getNode(ExtendKind, DL, ValueVT, Val);
531 if (PartVT == MVT::x86mmx)
532 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
533 }
534 } else if (PartBits == ValueVT.getSizeInBits()) {
535 // Different types of the same size.
536 assert(NumParts == 1 && PartEVT != ValueVT)(static_cast <bool> (NumParts == 1 && PartEVT !=
ValueVT) ? void (0) : __assert_fail ("NumParts == 1 && PartEVT != ValueVT"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 536, __extension__ __PRETTY_FUNCTION__));
537 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
538 } else if (NumParts * PartBits < ValueVT.getSizeInBits()) {
539 // If the parts cover less bits than value has, truncate the value.
540 assert((PartVT.isInteger() || PartVT == MVT::x86mmx) &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 542, __extension__ __PRETTY_FUNCTION__))
541 ValueVT.isInteger() &&(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 542, __extension__ __PRETTY_FUNCTION__))
542 "Unknown mismatch!")(static_cast <bool> ((PartVT.isInteger() || PartVT == MVT
::x86mmx) && ValueVT.isInteger() && "Unknown mismatch!"
) ? void (0) : __assert_fail ("(PartVT.isInteger() || PartVT == MVT::x86mmx) && ValueVT.isInteger() && \"Unknown mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 542, __extension__ __PRETTY_FUNCTION__));
543 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
544 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
545 if (PartVT == MVT::x86mmx)
546 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
547 }
548
549 // The value may have changed - recompute ValueVT.
550 ValueVT = Val.getValueType();
551 assert(NumParts * PartBits == ValueVT.getSizeInBits() &&(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 552, __extension__ __PRETTY_FUNCTION__))
552 "Failed to tile the value with PartVT!")(static_cast <bool> (NumParts * PartBits == ValueVT.getSizeInBits
() && "Failed to tile the value with PartVT!") ? void
(0) : __assert_fail ("NumParts * PartBits == ValueVT.getSizeInBits() && \"Failed to tile the value with PartVT!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 552, __extension__ __PRETTY_FUNCTION__));
553
554 if (NumParts == 1) {
555 if (PartEVT != ValueVT) {
556 diagnosePossiblyInvalidConstraint(*DAG.getContext(), V,
557 "scalar-to-vector conversion failed");
558 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
559 }
560
561 Parts[0] = Val;
562 return;
563 }
564
565 // Expand the value into multiple parts.
566 if (NumParts & (NumParts - 1)) {
567 // The number of parts is not a power of 2. Split off and copy the tail.
568 assert(PartVT.isInteger() && ValueVT.isInteger() &&(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 569, __extension__ __PRETTY_FUNCTION__))
569 "Do not know what to expand to!")(static_cast <bool> (PartVT.isInteger() && ValueVT
.isInteger() && "Do not know what to expand to!") ? void
(0) : __assert_fail ("PartVT.isInteger() && ValueVT.isInteger() && \"Do not know what to expand to!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 569, __extension__ __PRETTY_FUNCTION__));
570 unsigned RoundParts = 1 << Log2_32(NumParts);
571 unsigned RoundBits = RoundParts * PartBits;
572 unsigned OddParts = NumParts - RoundParts;
573 SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
574 DAG.getIntPtrConstant(RoundBits, DL));
575 getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V,
576 CallConv);
577
578 if (DAG.getDataLayout().isBigEndian())
579 // The odd parts were reversed by getCopyToParts - unreverse them.
580 std::reverse(Parts + RoundParts, Parts + NumParts);
581
582 NumParts = RoundParts;
583 ValueVT = EVT::getIntegerVT(*DAG.getContext(), NumParts * PartBits);
584 Val = DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val);
585 }
586
587 // The number of parts is a power of 2. Repeatedly bisect the value using
588 // EXTRACT_ELEMENT.
589 Parts[0] = DAG.getNode(ISD::BITCAST, DL,
590 EVT::getIntegerVT(*DAG.getContext(),
591 ValueVT.getSizeInBits()),
592 Val);
593
594 for (unsigned StepSize = NumParts; StepSize > 1; StepSize /= 2) {
595 for (unsigned i = 0; i < NumParts; i += StepSize) {
596 unsigned ThisBits = StepSize * PartBits / 2;
597 EVT ThisVT = EVT::getIntegerVT(*DAG.getContext(), ThisBits);
598 SDValue &Part0 = Parts[i];
599 SDValue &Part1 = Parts[i+StepSize/2];
600
601 Part1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
602 ThisVT, Part0, DAG.getIntPtrConstant(1, DL));
603 Part0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL,
604 ThisVT, Part0, DAG.getIntPtrConstant(0, DL));
605
606 if (ThisBits == PartBits && ThisVT != PartVT) {
607 Part0 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part0);
608 Part1 = DAG.getNode(ISD::BITCAST, DL, PartVT, Part1);
609 }
610 }
611 }
612
613 if (DAG.getDataLayout().isBigEndian())
614 std::reverse(Parts, Parts + OrigNumParts);
615}
616
617/// getCopyToPartsVector - Create a series of nodes that contain the specified
618/// value split into legal parts.
619static void getCopyToPartsVector(SelectionDAG &DAG, const SDLoc &DL,
620 SDValue Val, SDValue *Parts, unsigned NumParts,
621 MVT PartVT, const Value *V,
622 Optional<CallingConv::ID> CallConv) {
623 EVT ValueVT = Val.getValueType();
624 assert(ValueVT.isVector() && "Not a vector")(static_cast <bool> (ValueVT.isVector() && "Not a vector"
) ? void (0) : __assert_fail ("ValueVT.isVector() && \"Not a vector\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 624, __extension__ __PRETTY_FUNCTION__));
625 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
626 const bool IsABIRegCopy = CallConv.hasValue();
627
628 if (NumParts == 1) {
629 EVT PartEVT = PartVT;
630 if (PartEVT == ValueVT) {
631 // Nothing to do.
632 } else if (PartVT.getSizeInBits() == ValueVT.getSizeInBits()) {
633 // Bitconvert vector->vector case.
634 Val = DAG.getNode(ISD::BITCAST, DL, PartVT, Val);
635 } else if (PartVT.isVector() &&
636 PartEVT.getVectorElementType() == ValueVT.getVectorElementType() &&
637 PartEVT.getVectorNumElements() > ValueVT.getVectorNumElements()) {
638 EVT ElementVT = PartVT.getVectorElementType();
639 // Vector widening case, e.g. <2 x float> -> <4 x float>. Shuffle in
640 // undef elements.
641 SmallVector<SDValue, 16> Ops;
642 for (unsigned i = 0, e = ValueVT.getVectorNumElements(); i != e; ++i)
643 Ops.push_back(DAG.getNode(
644 ISD::EXTRACT_VECTOR_ELT, DL, ElementVT, Val,
645 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))));
646
647 for (unsigned i = ValueVT.getVectorNumElements(),
648 e = PartVT.getVectorNumElements(); i != e; ++i)
649 Ops.push_back(DAG.getUNDEF(ElementVT));
650
651 Val = DAG.getBuildVector(PartVT, DL, Ops);
652
653 // FIXME: Use CONCAT for 2x -> 4x.
654
655 //SDValue UndefElts = DAG.getUNDEF(VectorTy);
656 //Val = DAG.getNode(ISD::CONCAT_VECTORS, DL, PartVT, Val, UndefElts);
657 } else if (PartVT.isVector() &&
658 PartEVT.getVectorElementType().bitsGE(
659 ValueVT.getVectorElementType()) &&
660 PartEVT.getVectorNumElements() == ValueVT.getVectorNumElements()) {
661
662 // Promoted vector extract
663 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
664 } else {
665 if (ValueVT.getVectorNumElements() == 1) {
666 Val = DAG.getNode(
667 ISD::EXTRACT_VECTOR_ELT, DL, PartVT, Val,
668 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
669 } else {
670 assert(PartVT.getSizeInBits() > ValueVT.getSizeInBits() &&(static_cast <bool> (PartVT.getSizeInBits() > ValueVT
.getSizeInBits() && "lossy conversion of vector to scalar type"
) ? void (0) : __assert_fail ("PartVT.getSizeInBits() > ValueVT.getSizeInBits() && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 671, __extension__ __PRETTY_FUNCTION__))
671 "lossy conversion of vector to scalar type")(static_cast <bool> (PartVT.getSizeInBits() > ValueVT
.getSizeInBits() && "lossy conversion of vector to scalar type"
) ? void (0) : __assert_fail ("PartVT.getSizeInBits() > ValueVT.getSizeInBits() && \"lossy conversion of vector to scalar type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 671, __extension__ __PRETTY_FUNCTION__));
672 EVT IntermediateType =
673 EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
674 Val = DAG.getBitcast(IntermediateType, Val);
675 Val = DAG.getAnyExtOrTrunc(Val, DL, PartVT);
676 }
677 }
678
679 assert(Val.getValueType() == PartVT && "Unexpected vector part value type")(static_cast <bool> (Val.getValueType() == PartVT &&
"Unexpected vector part value type") ? void (0) : __assert_fail
("Val.getValueType() == PartVT && \"Unexpected vector part value type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 679, __extension__ __PRETTY_FUNCTION__));
680 Parts[0] = Val;
681 return;
682 }
683
684 // Handle a multi-element vector.
685 EVT IntermediateVT;
686 MVT RegisterVT;
687 unsigned NumIntermediates;
688 unsigned NumRegs;
689 if (IsABIRegCopy) {
690 NumRegs = TLI.getVectorTypeBreakdownForCallingConv(
691 *DAG.getContext(), CallConv.getValue(), ValueVT, IntermediateVT,
692 NumIntermediates, RegisterVT);
693 } else {
694 NumRegs =
695 TLI.getVectorTypeBreakdown(*DAG.getContext(), ValueVT, IntermediateVT,
696 NumIntermediates, RegisterVT);
697 }
698 unsigned NumElements = ValueVT.getVectorNumElements();
699
700 assert(NumRegs == NumParts && "Part count doesn't match vector breakdown!")(static_cast <bool> (NumRegs == NumParts && "Part count doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("NumRegs == NumParts && \"Part count doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 700, __extension__ __PRETTY_FUNCTION__));
701 NumParts = NumRegs; // Silence a compiler warning.
702 assert(RegisterVT == PartVT && "Part type doesn't match vector breakdown!")(static_cast <bool> (RegisterVT == PartVT && "Part type doesn't match vector breakdown!"
) ? void (0) : __assert_fail ("RegisterVT == PartVT && \"Part type doesn't match vector breakdown!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 702, __extension__ __PRETTY_FUNCTION__));
703
704 // Convert the vector to the appropiate type if necessary.
705 unsigned DestVectorNoElts =
706 NumIntermediates *
707 (IntermediateVT.isVector() ? IntermediateVT.getVectorNumElements() : 1);
708 EVT BuiltVectorTy = EVT::getVectorVT(
709 *DAG.getContext(), IntermediateVT.getScalarType(), DestVectorNoElts);
710 if (Val.getValueType() != BuiltVectorTy)
711 Val = DAG.getNode(ISD::BITCAST, DL, BuiltVectorTy, Val);
712
713 // Split the vector into intermediate operands.
714 SmallVector<SDValue, 8> Ops(NumIntermediates);
715 for (unsigned i = 0; i != NumIntermediates; ++i) {
716 if (IntermediateVT.isVector())
717 Ops[i] =
718 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, IntermediateVT, Val,
719 DAG.getConstant(i * (NumElements / NumIntermediates), DL,
720 TLI.getVectorIdxTy(DAG.getDataLayout())));
721 else
722 Ops[i] = DAG.getNode(
723 ISD::EXTRACT_VECTOR_ELT, DL, IntermediateVT, Val,
724 DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
725 }
726
727 // Split the intermediate operands into legal parts.
728 if (NumParts == NumIntermediates) {
729 // If the register was not expanded, promote or copy the value,
730 // as appropriate.
731 for (unsigned i = 0; i != NumParts; ++i)
732 getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V, CallConv);
733 } else if (NumParts > 0) {
734 // If the intermediate type was expanded, split each the value into
735 // legal parts.
736 assert(NumIntermediates != 0 && "division by zero")(static_cast <bool> (NumIntermediates != 0 && "division by zero"
) ? void (0) : __assert_fail ("NumIntermediates != 0 && \"division by zero\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 736, __extension__ __PRETTY_FUNCTION__));
737 assert(NumParts % NumIntermediates == 0 &&(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 738, __extension__ __PRETTY_FUNCTION__))
738 "Must expand into a divisible number of parts!")(static_cast <bool> (NumParts % NumIntermediates == 0 &&
"Must expand into a divisible number of parts!") ? void (0) :
__assert_fail ("NumParts % NumIntermediates == 0 && \"Must expand into a divisible number of parts!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 738, __extension__ __PRETTY_FUNCTION__));
739 unsigned Factor = NumParts / NumIntermediates;
740 for (unsigned i = 0; i != NumIntermediates; ++i)
741 getCopyToParts(DAG, DL, Ops[i], &Parts[i * Factor], Factor, PartVT, V,
742 CallConv);
743 }
744}
745
746RegsForValue::RegsForValue(const SmallVector<unsigned, 4> &regs, MVT regvt,
747 EVT valuevt, Optional<CallingConv::ID> CC)
748 : ValueVTs(1, valuevt), RegVTs(1, regvt), Regs(regs),
749 RegCount(1, regs.size()), CallConv(CC) {}
750
751RegsForValue::RegsForValue(LLVMContext &Context, const TargetLowering &TLI,
752 const DataLayout &DL, unsigned Reg, Type *Ty,
753 Optional<CallingConv::ID> CC) {
754 ComputeValueVTs(TLI, DL, Ty, ValueVTs);
755
756 CallConv = CC;
757
758 for (EVT ValueVT : ValueVTs) {
759 unsigned NumRegs =
760 isABIMangled()
761 ? TLI.getNumRegistersForCallingConv(Context, CC.getValue(), ValueVT)
762 : TLI.getNumRegisters(Context, ValueVT);
763 MVT RegisterVT =
764 isABIMangled()
765 ? TLI.getRegisterTypeForCallingConv(Context, CC.getValue(), ValueVT)
766 : TLI.getRegisterType(Context, ValueVT);
767 for (unsigned i = 0; i != NumRegs; ++i)
768 Regs.push_back(Reg + i);
769 RegVTs.push_back(RegisterVT);
770 RegCount.push_back(NumRegs);
771 Reg += NumRegs;
772 }
773}
774
775SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
776 FunctionLoweringInfo &FuncInfo,
777 const SDLoc &dl, SDValue &Chain,
778 SDValue *Flag, const Value *V) const {
779 // A Value with type {} or [0 x %t] needs no registers.
780 if (ValueVTs.empty())
781 return SDValue();
782
783 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
784
785 // Assemble the legal parts into the final values.
786 SmallVector<SDValue, 4> Values(ValueVTs.size());
787 SmallVector<SDValue, 8> Parts;
788 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
789 // Copy the legal parts from the registers.
790 EVT ValueVT = ValueVTs[Value];
791 unsigned NumRegs = RegCount[Value];
792 MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
793 *DAG.getContext(),
794 CallConv.getValue(), RegVTs[Value])
795 : RegVTs[Value];
796
797 Parts.resize(NumRegs);
798 for (unsigned i = 0; i != NumRegs; ++i) {
799 SDValue P;
800 if (!Flag) {
801 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT);
802 } else {
803 P = DAG.getCopyFromReg(Chain, dl, Regs[Part+i], RegisterVT, *Flag);
804 *Flag = P.getValue(2);
805 }
806
807 Chain = P.getValue(1);
808 Parts[i] = P;
809
810 // If the source register was virtual and if we know something about it,
811 // add an assert node.
812 if (!TargetRegisterInfo::isVirtualRegister(Regs[Part+i]) ||
813 !RegisterVT.isInteger() || RegisterVT.isVector())
814 continue;
815
816 const FunctionLoweringInfo::LiveOutInfo *LOI =
817 FuncInfo.GetLiveOutRegInfo(Regs[Part+i]);
818 if (!LOI)
819 continue;
820
821 unsigned RegSize = RegisterVT.getSizeInBits();
822 unsigned NumSignBits = LOI->NumSignBits;
823 unsigned NumZeroBits = LOI->Known.countMinLeadingZeros();
824
825 if (NumZeroBits == RegSize) {
826 // The current value is a zero.
827 // Explicitly express that as it would be easier for
828 // optimizations to kick in.
829 Parts[i] = DAG.getConstant(0, dl, RegisterVT);
830 continue;
831 }
832
833 // FIXME: We capture more information than the dag can represent. For
834 // now, just use the tightest assertzext/assertsext possible.
835 bool isSExt;
836 EVT FromVT(MVT::Other);
837 if (NumZeroBits) {
838 FromVT = EVT::getIntegerVT(*DAG.getContext(), RegSize - NumZeroBits);
839 isSExt = false;
840 } else if (NumSignBits > 1) {
841 FromVT =
842 EVT::getIntegerVT(*DAG.getContext(), RegSize - NumSignBits + 1);
843 isSExt = true;
844 } else {
845 continue;
846 }
847 // Add an assertion node.
848 assert(FromVT != MVT::Other)(static_cast <bool> (FromVT != MVT::Other) ? void (0) :
__assert_fail ("FromVT != MVT::Other", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 848, __extension__ __PRETTY_FUNCTION__));
849 Parts[i] = DAG.getNode(isSExt ? ISD::AssertSext : ISD::AssertZext, dl,
850 RegisterVT, P, DAG.getValueType(FromVT));
851 }
852
853 Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(), NumRegs,
854 RegisterVT, ValueVT, V, CallConv);
855 Part += NumRegs;
856 Parts.clear();
857 }
858
859 return DAG.getNode(ISD::MERGE_VALUES, dl, DAG.getVTList(ValueVTs), Values);
860}
861
862void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG,
863 const SDLoc &dl, SDValue &Chain, SDValue *Flag,
864 const Value *V,
865 ISD::NodeType PreferredExtendType) const {
866 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
867 ISD::NodeType ExtendKind = PreferredExtendType;
868
869 // Get the list of the values's legal parts.
870 unsigned NumRegs = Regs.size();
871 SmallVector<SDValue, 8> Parts(NumRegs);
872 for (unsigned Value = 0, Part = 0, e = ValueVTs.size(); Value != e; ++Value) {
873 unsigned NumParts = RegCount[Value];
874
875 MVT RegisterVT = isABIMangled() ? TLI.getRegisterTypeForCallingConv(
876 *DAG.getContext(),
877 CallConv.getValue(), RegVTs[Value])
878 : RegVTs[Value];
879
880 if (ExtendKind == ISD::ANY_EXTEND && TLI.isZExtFree(Val, RegisterVT))
881 ExtendKind = ISD::ZERO_EXTEND;
882
883 getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value), &Parts[Part],
884 NumParts, RegisterVT, V, CallConv, ExtendKind);
885 Part += NumParts;
886 }
887
888 // Copy the parts into the registers.
889 SmallVector<SDValue, 8> Chains(NumRegs);
890 for (unsigned i = 0; i != NumRegs; ++i) {
891 SDValue Part;
892 if (!Flag) {
893 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i]);
894 } else {
895 Part = DAG.getCopyToReg(Chain, dl, Regs[i], Parts[i], *Flag);
896 *Flag = Part.getValue(1);
897 }
898
899 Chains[i] = Part.getValue(0);
900 }
901
902 if (NumRegs == 1 || Flag)
903 // If NumRegs > 1 && Flag is used then the use of the last CopyToReg is
904 // flagged to it. That is the CopyToReg nodes and the user are considered
905 // a single scheduling unit. If we create a TokenFactor and return it as
906 // chain, then the TokenFactor is both a predecessor (operand) of the
907 // user as well as a successor (the TF operands are flagged to the user).
908 // c1, f1 = CopyToReg
909 // c2, f2 = CopyToReg
910 // c3 = TokenFactor c1, c2
911 // ...
912 // = op c3, ..., f2
913 Chain = Chains[NumRegs-1];
914 else
915 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
916}
917
918void RegsForValue::AddInlineAsmOperands(unsigned Code, bool HasMatching,
919 unsigned MatchingIdx, const SDLoc &dl,
920 SelectionDAG &DAG,
921 std::vector<SDValue> &Ops) const {
922 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
923
924 unsigned Flag = InlineAsm::getFlagWord(Code, Regs.size());
925 if (HasMatching)
926 Flag = InlineAsm::getFlagWordForMatchingOp(Flag, MatchingIdx);
927 else if (!Regs.empty() &&
928 TargetRegisterInfo::isVirtualRegister(Regs.front())) {
929 // Put the register class of the virtual registers in the flag word. That
930 // way, later passes can recompute register class constraints for inline
931 // assembly as well as normal instructions.
932 // Don't do this for tied operands that can use the regclass information
933 // from the def.
934 const MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
935 const TargetRegisterClass *RC = MRI.getRegClass(Regs.front());
936 Flag = InlineAsm::getFlagWordForRegClass(Flag, RC->getID());
937 }
938
939 SDValue Res = DAG.getTargetConstant(Flag, dl, MVT::i32);
940 Ops.push_back(Res);
941
942 if (Code == InlineAsm::Kind_Clobber) {
943 // Clobbers should always have a 1:1 mapping with registers, and may
944 // reference registers that have illegal (e.g. vector) types. Hence, we
945 // shouldn't try to apply any sort of splitting logic to them.
946 assert(Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() &&(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 947, __extension__ __PRETTY_FUNCTION__))
947 "No 1:1 mapping from clobbers to regs?")(static_cast <bool> (Regs.size() == RegVTs.size() &&
Regs.size() == ValueVTs.size() && "No 1:1 mapping from clobbers to regs?"
) ? void (0) : __assert_fail ("Regs.size() == RegVTs.size() && Regs.size() == ValueVTs.size() && \"No 1:1 mapping from clobbers to regs?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 947, __extension__ __PRETTY_FUNCTION__));
948 unsigned SP = TLI.getStackPointerRegisterToSaveRestore();
949 (void)SP;
950 for (unsigned I = 0, E = ValueVTs.size(); I != E; ++I) {
951 Ops.push_back(DAG.getRegister(Regs[I], RegVTs[I]));
952 assert((static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
953 (Regs[I] != SP ||(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
954 DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) &&(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 955, __extension__ __PRETTY_FUNCTION__))
955 "If we clobbered the stack pointer, MFI should know about it.")(static_cast <bool> ((Regs[I] != SP || DAG.getMachineFunction
().getFrameInfo().hasOpaqueSPAdjustment()) && "If we clobbered the stack pointer, MFI should know about it."
) ? void (0) : __assert_fail ("(Regs[I] != SP || DAG.getMachineFunction().getFrameInfo().hasOpaqueSPAdjustment()) && \"If we clobbered the stack pointer, MFI should know about it.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 955, __extension__ __PRETTY_FUNCTION__));
956 }
957 return;
958 }
959
960 for (unsigned Value = 0, Reg = 0, e = ValueVTs.size(); Value != e; ++Value) {
961 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), ValueVTs[Value]);
962 MVT RegisterVT = RegVTs[Value];
963 for (unsigned i = 0; i != NumRegs; ++i) {
964 assert(Reg < Regs.size() && "Mismatch in # registers expected")(static_cast <bool> (Reg < Regs.size() && "Mismatch in # registers expected"
) ? void (0) : __assert_fail ("Reg < Regs.size() && \"Mismatch in # registers expected\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 964, __extension__ __PRETTY_FUNCTION__));
965 unsigned TheReg = Regs[Reg++];
966 Ops.push_back(DAG.getRegister(TheReg, RegisterVT));
967 }
968 }
969}
970
971SmallVector<std::pair<unsigned, unsigned>, 4>
972RegsForValue::getRegsAndSizes() const {
973 SmallVector<std::pair<unsigned, unsigned>, 4> OutVec;
974 unsigned I = 0;
975 for (auto CountAndVT : zip_first(RegCount, RegVTs)) {
976 unsigned RegCount = std::get<0>(CountAndVT);
977 MVT RegisterVT = std::get<1>(CountAndVT);
978 unsigned RegisterSize = RegisterVT.getSizeInBits();
979 for (unsigned E = I + RegCount; I != E; ++I)
980 OutVec.push_back(std::make_pair(Regs[I], RegisterSize));
981 }
982 return OutVec;
983}
984
985void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa,
986 const TargetLibraryInfo *li) {
987 AA = aa;
988 GFI = gfi;
989 LibInfo = li;
990 DL = &DAG.getDataLayout();
991 Context = DAG.getContext();
992 LPadToCallSiteMap.clear();
993}
994
995void SelectionDAGBuilder::clear() {
996 NodeMap.clear();
997 UnusedArgNodeMap.clear();
998 PendingLoads.clear();
999 PendingExports.clear();
1000 CurInst = nullptr;
1001 HasTailCall = false;
1002 SDNodeOrder = LowestSDNodeOrder;
1003 StatepointLowering.clear();
1004}
1005
1006void SelectionDAGBuilder::clearDanglingDebugInfo() {
1007 DanglingDebugInfoMap.clear();
1008}
1009
1010SDValue SelectionDAGBuilder::getRoot() {
1011 if (PendingLoads.empty())
1012 return DAG.getRoot();
1013
1014 if (PendingLoads.size() == 1) {
1015 SDValue Root = PendingLoads[0];
1016 DAG.setRoot(Root);
1017 PendingLoads.clear();
1018 return Root;
1019 }
1020
1021 // Otherwise, we have to make a token factor node.
1022 SDValue Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1023 PendingLoads);
1024 PendingLoads.clear();
1025 DAG.setRoot(Root);
1026 return Root;
1027}
1028
1029SDValue SelectionDAGBuilder::getControlRoot() {
1030 SDValue Root = DAG.getRoot();
1031
1032 if (PendingExports.empty())
1033 return Root;
1034
1035 // Turn all of the CopyToReg chains into one factored node.
1036 if (Root.getOpcode() != ISD::EntryToken) {
1037 unsigned i = 0, e = PendingExports.size();
1038 for (; i != e; ++i) {
1039 assert(PendingExports[i].getNode()->getNumOperands() > 1)(static_cast <bool> (PendingExports[i].getNode()->getNumOperands
() > 1) ? void (0) : __assert_fail ("PendingExports[i].getNode()->getNumOperands() > 1"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1039, __extension__ __PRETTY_FUNCTION__));
1040 if (PendingExports[i].getNode()->getOperand(0) == Root)
1041 break; // Don't add the root if we already indirectly depend on it.
1042 }
1043
1044 if (i == e)
1045 PendingExports.push_back(Root);
1046 }
1047
1048 Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other,
1049 PendingExports);
1050 PendingExports.clear();
1051 DAG.setRoot(Root);
1052 return Root;
1053}
1054
1055void SelectionDAGBuilder::visit(const Instruction &I) {
1056 // Set up outgoing PHI node register values before emitting the terminator.
1057 if (isa<TerminatorInst>(&I)) {
1058 HandlePHINodesInSuccessorBlocks(I.getParent());
1059 }
1060
1061 // Increase the SDNodeOrder if dealing with a non-debug instruction.
1062 if (!isa<DbgInfoIntrinsic>(I))
1063 ++SDNodeOrder;
1064
1065 CurInst = &I;
1066
1067 visit(I.getOpcode(), I);
1068
1069 if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) {
1070 // Propagate the fast-math-flags of this IR instruction to the DAG node that
1071 // maps to this instruction.
1072 // TODO: We could handle all flags (nsw, etc) here.
1073 // TODO: If an IR instruction maps to >1 node, only the final node will have
1074 // flags set.
1075 if (SDNode *Node = getNodeForIRValue(&I)) {
1076 SDNodeFlags IncomingFlags;
1077 IncomingFlags.copyFMF(*FPMO);
1078 if (!Node->getFlags().isDefined())
1079 Node->setFlags(IncomingFlags);
1080 else
1081 Node->intersectFlagsWith(IncomingFlags);
1082 }
1083 }
1084
1085 if (!isa<TerminatorInst>(&I) && !HasTailCall &&
1086 !isStatepoint(&I)) // statepoints handle their exports internally
1087 CopyToExportRegsIfNeeded(&I);
1088
1089 CurInst = nullptr;
1090}
1091
1092void SelectionDAGBuilder::visitPHI(const PHINode &) {
1093 llvm_unreachable("SelectionDAGBuilder shouldn't visit PHI nodes!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit PHI nodes!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1093);
1094}
1095
1096void SelectionDAGBuilder::visit(unsigned Opcode, const User &I) {
1097 // Note: this doesn't use InstVisitor, because it has to work with
1098 // ConstantExpr's in addition to instructions.
1099 switch (Opcode) {
1100 default: llvm_unreachable("Unknown instruction type encountered!")::llvm::llvm_unreachable_internal("Unknown instruction type encountered!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1100);
1101 // Build the switch statement using the Instruction.def file.
1102#define HANDLE_INST(NUM, OPCODE, CLASS) \
1103 case Instruction::OPCODE: visit##OPCODE((const CLASS&)I); break;
1104#include "llvm/IR/Instruction.def"
1105 }
1106}
1107
1108void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable,
1109 const DIExpression *Expr) {
1110 auto isMatchingDbgValue = [&](DanglingDebugInfo &DDI) {
1111 const DbgValueInst *DI = DDI.getDI();
1112 DIVariable *DanglingVariable = DI->getVariable();
1113 DIExpression *DanglingExpr = DI->getExpression();
1114 if (DanglingVariable == Variable && Expr->fragmentsOverlap(DanglingExpr)) {
1115 LLVM_DEBUG(dbgs() << "Dropping dangling debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping dangling debug info for "
<< *DI << "\n"; } } while (false);
1116 return true;
1117 }
1118 return false;
1119 };
1120
1121 for (auto &DDIMI : DanglingDebugInfoMap) {
1122 DanglingDebugInfoVector &DDIV = DDIMI.second;
1123 DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end());
1124 }
1125}
1126
1127// resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V,
1128// generate the debug data structures now that we've seen its definition.
1129void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V,
1130 SDValue Val) {
1131 auto DanglingDbgInfoIt = DanglingDebugInfoMap.find(V);
1132 if (DanglingDbgInfoIt == DanglingDebugInfoMap.end())
1133 return;
1134
1135 DanglingDebugInfoVector &DDIV = DanglingDbgInfoIt->second;
1136 for (auto &DDI : DDIV) {
1137 const DbgValueInst *DI = DDI.getDI();
1138 assert(DI && "Ill-formed DanglingDebugInfo")(static_cast <bool> (DI && "Ill-formed DanglingDebugInfo"
) ? void (0) : __assert_fail ("DI && \"Ill-formed DanglingDebugInfo\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1138, __extension__ __PRETTY_FUNCTION__));
1139 DebugLoc dl = DDI.getdl();
1140 unsigned ValSDNodeOrder = Val.getNode()->getIROrder();
1141 unsigned DbgSDNodeOrder = DDI.getSDNodeOrder();
1142 DILocalVariable *Variable = DI->getVariable();
1143 DIExpression *Expr = DI->getExpression();
1144 assert(Variable->isValidLocationForIntrinsic(dl) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1145, __extension__ __PRETTY_FUNCTION__))
1145 "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(dl) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(dl) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1145, __extension__ __PRETTY_FUNCTION__));
1146 SDDbgValue *SDV;
1147 if (Val.getNode()) {
1148 if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) {
1149 LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false)
1150 << DbgSDNodeOrder << "] for:\n " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolve dangling debug info [order="
<< DbgSDNodeOrder << "] for:\n " << *DI <<
"\n"; } } while (false);
1151 LLVM_DEBUG(dbgs() << " By mapping to:\n "; Val.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " By mapping to:\n "; Val.dump
(); } } while (false);
1152 // Increase the SDNodeOrder for the DbgValue here to make sure it is
1153 // inserted after the definition of Val when emitting the instructions
1154 // after ISel. An alternative could be to teach
1155 // ScheduleDAGSDNodes::EmitSchedule to delay the insertion properly.
1156 LLVM_DEBUG(if (ValSDNodeOrder > DbgSDNodeOrder) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)
1157 << "changing SDNodeOrder from " << DbgSDNodeOrder << " to "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
)
1158 << ValSDNodeOrder << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { if (ValSDNodeOrder > DbgSDNodeOrder) dbgs() <<
"changing SDNodeOrder from " << DbgSDNodeOrder <<
" to " << ValSDNodeOrder << "\n"; } } while (false
);
1159 SDV = getDbgValue(Val, Variable, Expr, dl,
1160 std::max(DbgSDNodeOrder, ValSDNodeOrder));
1161 DAG.AddDbgValue(SDV, Val.getNode(), false);
1162 } else
1163 LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false)
1164 << "in EmitFuncArgumentDbgValue\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Resolved dangling debug info for "
<< *DI << "in EmitFuncArgumentDbgValue\n"; } } while
(false);
1165 } else
1166 LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
*DI << "\n"; } } while (false);
1167 }
1168 DDIV.clear();
1169}
1170
1171/// getCopyFromRegs - If there was virtual register allocated for the value V
1172/// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise.
1173SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) {
1174 DenseMap<const Value *, unsigned>::iterator It = FuncInfo.ValueMap.find(V);
1175 SDValue Result;
1176
1177 if (It != FuncInfo.ValueMap.end()) {
1178 unsigned InReg = It->second;
1179
1180 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
1181 DAG.getDataLayout(), InReg, Ty, getABIRegCopyCC(V));
1182 SDValue Chain = DAG.getEntryNode();
1183 Result = RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr,
1184 V);
1185 resolveDanglingDebugInfo(V, Result);
1186 }
1187
1188 return Result;
1189}
1190
1191/// getValue - Return an SDValue for the given Value.
1192SDValue SelectionDAGBuilder::getValue(const Value *V) {
1193 // If we already have an SDValue for this value, use it. It's important
1194 // to do this first, so that we don't create a CopyFromReg if we already
1195 // have a regular SDValue.
1196 SDValue &N = NodeMap[V];
1197 if (N.getNode()) return N;
1198
1199 // If there's a virtual register allocated and initialized for this
1200 // value, use it.
1201 if (SDValue copyFromReg = getCopyFromRegs(V, V->getType()))
1202 return copyFromReg;
1203
1204 // Otherwise create a new SDValue and remember it.
1205 SDValue Val = getValueImpl(V);
1206 NodeMap[V] = Val;
1207 resolveDanglingDebugInfo(V, Val);
1208 return Val;
1209}
1210
1211// Return true if SDValue exists for the given Value
1212bool SelectionDAGBuilder::findValue(const Value *V) const {
1213 return (NodeMap.find(V) != NodeMap.end()) ||
1214 (FuncInfo.ValueMap.find(V) != FuncInfo.ValueMap.end());
1215}
1216
1217/// getNonRegisterValue - Return an SDValue for the given Value, but
1218/// don't look in FuncInfo.ValueMap for a virtual register.
1219SDValue SelectionDAGBuilder::getNonRegisterValue(const Value *V) {
1220 // If we already have an SDValue for this value, use it.
1221 SDValue &N = NodeMap[V];
1222 if (N.getNode()) {
1223 if (isa<ConstantSDNode>(N) || isa<ConstantFPSDNode>(N)) {
1224 // Remove the debug location from the node as the node is about to be used
1225 // in a location which may differ from the original debug location. This
1226 // is relevant to Constant and ConstantFP nodes because they can appear
1227 // as constant expressions inside PHI nodes.
1228 N->setDebugLoc(DebugLoc());
1229 }
1230 return N;
1231 }
1232
1233 // Otherwise create a new SDValue and remember it.
1234 SDValue Val = getValueImpl(V);
1235 NodeMap[V] = Val;
1236 resolveDanglingDebugInfo(V, Val);
1237 return Val;
1238}
1239
1240/// getValueImpl - Helper function for getValue and getNonRegisterValue.
1241/// Create an SDValue for the given value.
1242SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
1243 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1244
1245 if (const Constant *C = dyn_cast<Constant>(V)) {
1246 EVT VT = TLI.getValueType(DAG.getDataLayout(), V->getType(), true);
1247
1248 if (const ConstantInt *CI = dyn_cast<ConstantInt>(C))
1249 return DAG.getConstant(*CI, getCurSDLoc(), VT);
1250
1251 if (const GlobalValue *GV = dyn_cast<GlobalValue>(C))
1252 return DAG.getGlobalAddress(GV, getCurSDLoc(), VT);
1253
1254 if (isa<ConstantPointerNull>(C)) {
1255 unsigned AS = V->getType()->getPointerAddressSpace();
1256 return DAG.getConstant(0, getCurSDLoc(),
1257 TLI.getPointerTy(DAG.getDataLayout(), AS));
1258 }
1259
1260 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(C))
1261 return DAG.getConstantFP(*CFP, getCurSDLoc(), VT);
1262
1263 if (isa<UndefValue>(C) && !V->getType()->isAggregateType())
1264 return DAG.getUNDEF(VT);
1265
1266 if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
1267 visit(CE->getOpcode(), *CE);
1268 SDValue N1 = NodeMap[V];
1269 assert(N1.getNode() && "visit didn't populate the NodeMap!")(static_cast <bool> (N1.getNode() && "visit didn't populate the NodeMap!"
) ? void (0) : __assert_fail ("N1.getNode() && \"visit didn't populate the NodeMap!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1269, __extension__ __PRETTY_FUNCTION__));
1270 return N1;
1271 }
1272
1273 if (isa<ConstantStruct>(C) || isa<ConstantArray>(C)) {
1274 SmallVector<SDValue, 4> Constants;
1275 for (User::const_op_iterator OI = C->op_begin(), OE = C->op_end();
1276 OI != OE; ++OI) {
1277 SDNode *Val = getValue(*OI).getNode();
1278 // If the operand is an empty aggregate, there are no values.
1279 if (!Val) continue;
1280 // Add each leaf value from the operand to the Constants list
1281 // to form a flattened list of all the values.
1282 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1283 Constants.push_back(SDValue(Val, i));
1284 }
1285
1286 return DAG.getMergeValues(Constants, getCurSDLoc());
1287 }
1288
1289 if (const ConstantDataSequential *CDS =
1290 dyn_cast<ConstantDataSequential>(C)) {
1291 SmallVector<SDValue, 4> Ops;
1292 for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
1293 SDNode *Val = getValue(CDS->getElementAsConstant(i)).getNode();
1294 // Add each leaf value from the operand to the Constants list
1295 // to form a flattened list of all the values.
1296 for (unsigned i = 0, e = Val->getNumValues(); i != e; ++i)
1297 Ops.push_back(SDValue(Val, i));
1298 }
1299
1300 if (isa<ArrayType>(CDS->getType()))
1301 return DAG.getMergeValues(Ops, getCurSDLoc());
1302 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1303 }
1304
1305 if (C->getType()->isStructTy() || C->getType()->isArrayTy()) {
1306 assert((isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) &&(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1307, __extension__ __PRETTY_FUNCTION__))
1307 "Unknown struct or array constant!")(static_cast <bool> ((isa<ConstantAggregateZero>(
C) || isa<UndefValue>(C)) && "Unknown struct or array constant!"
) ? void (0) : __assert_fail ("(isa<ConstantAggregateZero>(C) || isa<UndefValue>(C)) && \"Unknown struct or array constant!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1307, __extension__ __PRETTY_FUNCTION__));
1308
1309 SmallVector<EVT, 4> ValueVTs;
1310 ComputeValueVTs(TLI, DAG.getDataLayout(), C->getType(), ValueVTs);
1311 unsigned NumElts = ValueVTs.size();
1312 if (NumElts == 0)
1313 return SDValue(); // empty struct
1314 SmallVector<SDValue, 4> Constants(NumElts);
1315 for (unsigned i = 0; i != NumElts; ++i) {
1316 EVT EltVT = ValueVTs[i];
1317 if (isa<UndefValue>(C))
1318 Constants[i] = DAG.getUNDEF(EltVT);
1319 else if (EltVT.isFloatingPoint())
1320 Constants[i] = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1321 else
1322 Constants[i] = DAG.getConstant(0, getCurSDLoc(), EltVT);
1323 }
1324
1325 return DAG.getMergeValues(Constants, getCurSDLoc());
1326 }
1327
1328 if (const BlockAddress *BA = dyn_cast<BlockAddress>(C))
1329 return DAG.getBlockAddress(BA, VT);
1330
1331 VectorType *VecTy = cast<VectorType>(V->getType());
1332 unsigned NumElements = VecTy->getNumElements();
1333
1334 // Now that we know the number and type of the elements, get that number of
1335 // elements into the Ops array based on what kind of constant it is.
1336 SmallVector<SDValue, 16> Ops;
1337 if (const ConstantVector *CV = dyn_cast<ConstantVector>(C)) {
1338 for (unsigned i = 0; i != NumElements; ++i)
1339 Ops.push_back(getValue(CV->getOperand(i)));
1340 } else {
1341 assert(isa<ConstantAggregateZero>(C) && "Unknown vector constant!")(static_cast <bool> (isa<ConstantAggregateZero>(C
) && "Unknown vector constant!") ? void (0) : __assert_fail
("isa<ConstantAggregateZero>(C) && \"Unknown vector constant!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1341, __extension__ __PRETTY_FUNCTION__));
1342 EVT EltVT =
1343 TLI.getValueType(DAG.getDataLayout(), VecTy->getElementType());
1344
1345 SDValue Op;
1346 if (EltVT.isFloatingPoint())
1347 Op = DAG.getConstantFP(0, getCurSDLoc(), EltVT);
1348 else
1349 Op = DAG.getConstant(0, getCurSDLoc(), EltVT);
1350 Ops.assign(NumElements, Op);
1351 }
1352
1353 // Create a BUILD_VECTOR node.
1354 return NodeMap[V] = DAG.getBuildVector(VT, getCurSDLoc(), Ops);
1355 }
1356
1357 // If this is a static alloca, generate it as the frameindex instead of
1358 // computation.
1359 if (const AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
1360 DenseMap<const AllocaInst*, int>::iterator SI =
1361 FuncInfo.StaticAllocaMap.find(AI);
1362 if (SI != FuncInfo.StaticAllocaMap.end())
1363 return DAG.getFrameIndex(SI->second,
1364 TLI.getFrameIndexTy(DAG.getDataLayout()));
1365 }
1366
1367 // If this is an instruction which fast-isel has deferred, select it now.
1368 if (const Instruction *Inst = dyn_cast<Instruction>(V)) {
1369 unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
1370
1371 RegsForValue RFV(*DAG.getContext(), TLI, DAG.getDataLayout(), InReg,
1372 Inst->getType(), getABIRegCopyCC(V));
1373 SDValue Chain = DAG.getEntryNode();
1374 return RFV.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), Chain, nullptr, V);
1375 }
1376
1377 llvm_unreachable("Can't get register for value!")::llvm::llvm_unreachable_internal("Can't get register for value!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1377);
1378}
1379
1380void SelectionDAGBuilder::visitCatchPad(const CatchPadInst &I) {
1381 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1382 bool IsMSVCCXX = Pers == EHPersonality::MSVC_CXX;
1383 bool IsCoreCLR = Pers == EHPersonality::CoreCLR;
1384 bool IsSEH = isAsynchronousEHPersonality(Pers);
1385 bool IsWasmCXX = Pers == EHPersonality::Wasm_CXX;
1386 MachineBasicBlock *CatchPadMBB = FuncInfo.MBB;
1387 if (!IsSEH)
1388 CatchPadMBB->setIsEHScopeEntry();
1389 // In MSVC C++ and CoreCLR, catchblocks are funclets and need prologues.
1390 if (IsMSVCCXX || IsCoreCLR)
1391 CatchPadMBB->setIsEHFuncletEntry();
1392 // Wasm does not need catchpads anymore
1393 if (!IsWasmCXX)
1394 DAG.setRoot(DAG.getNode(ISD::CATCHPAD, getCurSDLoc(), MVT::Other,
1395 getControlRoot()));
1396}
1397
1398void SelectionDAGBuilder::visitCatchRet(const CatchReturnInst &I) {
1399 // Update machine-CFG edge.
1400 MachineBasicBlock *TargetMBB = FuncInfo.MBBMap[I.getSuccessor()];
1401 FuncInfo.MBB->addSuccessor(TargetMBB);
1402
1403 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1404 bool IsSEH = isAsynchronousEHPersonality(Pers);
1405 if (IsSEH) {
1406 // If this is not a fall-through branch or optimizations are switched off,
1407 // emit the branch.
1408 if (TargetMBB != NextBlock(FuncInfo.MBB) ||
1409 TM.getOptLevel() == CodeGenOpt::None)
1410 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
1411 getControlRoot(), DAG.getBasicBlock(TargetMBB)));
1412 return;
1413 }
1414
1415 // Figure out the funclet membership for the catchret's successor.
1416 // This will be used by the FuncletLayout pass to determine how to order the
1417 // BB's.
1418 // A 'catchret' returns to the outer scope's color.
1419 Value *ParentPad = I.getCatchSwitchParentPad();
1420 const BasicBlock *SuccessorColor;
1421 if (isa<ConstantTokenNone>(ParentPad))
1422 SuccessorColor = &FuncInfo.Fn->getEntryBlock();
1423 else
1424 SuccessorColor = cast<Instruction>(ParentPad)->getParent();
1425 assert(SuccessorColor && "No parent funclet for catchret!")(static_cast <bool> (SuccessorColor && "No parent funclet for catchret!"
) ? void (0) : __assert_fail ("SuccessorColor && \"No parent funclet for catchret!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1425, __extension__ __PRETTY_FUNCTION__));
1426 MachineBasicBlock *SuccessorColorMBB = FuncInfo.MBBMap[SuccessorColor];
1427 assert(SuccessorColorMBB && "No MBB for SuccessorColor!")(static_cast <bool> (SuccessorColorMBB && "No MBB for SuccessorColor!"
) ? void (0) : __assert_fail ("SuccessorColorMBB && \"No MBB for SuccessorColor!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1427, __extension__ __PRETTY_FUNCTION__));
1428
1429 // Create the terminator node.
1430 SDValue Ret = DAG.getNode(ISD::CATCHRET, getCurSDLoc(), MVT::Other,
1431 getControlRoot(), DAG.getBasicBlock(TargetMBB),
1432 DAG.getBasicBlock(SuccessorColorMBB));
1433 DAG.setRoot(Ret);
1434}
1435
1436void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) {
1437 // Don't emit any special code for the cleanuppad instruction. It just marks
1438 // the start of an EH scope/funclet.
1439 FuncInfo.MBB->setIsEHScopeEntry();
1440 FuncInfo.MBB->setIsEHFuncletEntry();
1441 FuncInfo.MBB->setIsCleanupFuncletEntry();
1442}
1443
1444/// When an invoke or a cleanupret unwinds to the next EH pad, there are
1445/// many places it could ultimately go. In the IR, we have a single unwind
1446/// destination, but in the machine CFG, we enumerate all the possible blocks.
1447/// This function skips over imaginary basic blocks that hold catchswitch
1448/// instructions, and finds all the "real" machine
1449/// basic block destinations. As those destinations may not be successors of
1450/// EHPadBB, here we also calculate the edge probability to those destinations.
1451/// The passed-in Prob is the edge probability to EHPadBB.
1452static void findUnwindDestinations(
1453 FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB,
1454 BranchProbability Prob,
1455 SmallVectorImpl<std::pair<MachineBasicBlock *, BranchProbability>>
1456 &UnwindDests) {
1457 EHPersonality Personality =
1458 classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
1459 bool IsMSVCCXX = Personality == EHPersonality::MSVC_CXX;
1460 bool IsCoreCLR = Personality == EHPersonality::CoreCLR;
1461 bool IsSEH = isAsynchronousEHPersonality(Personality);
1462
1463 while (EHPadBB) {
1464 const Instruction *Pad = EHPadBB->getFirstNonPHI();
1465 BasicBlock *NewEHPadBB = nullptr;
1466 if (isa<LandingPadInst>(Pad)) {
1467 // Stop on landingpads. They are not funclets.
1468 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1469 break;
1470 } else if (isa<CleanupPadInst>(Pad)) {
1471 // Stop on cleanup pads. Cleanups are always funclet entries for all known
1472 // personalities.
1473 UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob);
1474 UnwindDests.back().first->setIsEHScopeEntry();
1475 UnwindDests.back().first->setIsEHFuncletEntry();
1476 break;
1477 } else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(Pad)) {
1478 // Add the catchpad handlers to the possible destinations.
1479 for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) {
1480 UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob);
1481 // For MSVC++ and the CLR, catchblocks are funclets and need prologues.
1482 if (IsMSVCCXX || IsCoreCLR)
1483 UnwindDests.back().first->setIsEHFuncletEntry();
1484 if (!IsSEH)
1485 UnwindDests.back().first->setIsEHScopeEntry();
1486 }
1487 NewEHPadBB = CatchSwitch->getUnwindDest();
1488 } else {
1489 continue;
1490 }
1491
1492 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1493 if (BPI && NewEHPadBB)
1494 Prob *= BPI->getEdgeProbability(EHPadBB, NewEHPadBB);
1495 EHPadBB = NewEHPadBB;
1496 }
1497}
1498
1499void SelectionDAGBuilder::visitCleanupRet(const CleanupReturnInst &I) {
1500 // Update successor info.
1501 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
1502 auto UnwindDest = I.getUnwindDest();
1503 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1504 BranchProbability UnwindDestProb =
1505 (BPI && UnwindDest)
1506 ? BPI->getEdgeProbability(FuncInfo.MBB->getBasicBlock(), UnwindDest)
1507 : BranchProbability::getZero();
1508 findUnwindDestinations(FuncInfo, UnwindDest, UnwindDestProb, UnwindDests);
1509 for (auto &UnwindDest : UnwindDests) {
1510 UnwindDest.first->setIsEHPad();
1511 addSuccessorWithProb(FuncInfo.MBB, UnwindDest.first, UnwindDest.second);
1512 }
1513 FuncInfo.MBB->normalizeSuccProbs();
1514
1515 // Create the terminator node.
1516 SDValue Ret =
1517 DAG.getNode(ISD::CLEANUPRET, getCurSDLoc(), MVT::Other, getControlRoot());
1518 DAG.setRoot(Ret);
1519}
1520
1521void SelectionDAGBuilder::visitCatchSwitch(const CatchSwitchInst &CSI) {
1522 report_fatal_error("visitCatchSwitch not yet implemented!");
1523}
1524
1525void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
1526 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1527 auto &DL = DAG.getDataLayout();
1528 SDValue Chain = getControlRoot();
1529 SmallVector<ISD::OutputArg, 8> Outs;
1530 SmallVector<SDValue, 8> OutVals;
1531
1532 // Calls to @llvm.experimental.deoptimize don't generate a return value, so
1533 // lower
1534 //
1535 // %val = call <ty> @llvm.experimental.deoptimize()
1536 // ret <ty> %val
1537 //
1538 // differently.
1539 if (I.getParent()->getTerminatingDeoptimizeCall()) {
1540 LowerDeoptimizingReturn();
1541 return;
1542 }
1543
1544 if (!FuncInfo.CanLowerReturn) {
1545 unsigned DemoteReg = FuncInfo.DemoteRegister;
1546 const Function *F = I.getParent()->getParent();
1547
1548 // Emit a store of the return value through the virtual register.
1549 // Leave Outs empty so that LowerReturn won't try to load return
1550 // registers the usual way.
1551 SmallVector<EVT, 1> PtrValueVTs;
1552 ComputeValueVTs(TLI, DL,
1553 F->getReturnType()->getPointerTo(
1554 DAG.getDataLayout().getAllocaAddrSpace()),
1555 PtrValueVTs);
1556
1557 SDValue RetPtr = DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(),
1558 DemoteReg, PtrValueVTs[0]);
1559 SDValue RetOp = getValue(I.getOperand(0));
1560
1561 SmallVector<EVT, 4> ValueVTs;
1562 SmallVector<uint64_t, 4> Offsets;
1563 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets);
1564 unsigned NumValues = ValueVTs.size();
1565
1566 SmallVector<SDValue, 4> Chains(NumValues);
1567 for (unsigned i = 0; i != NumValues; ++i) {
1568 // An aggregate return value cannot wrap around the address space, so
1569 // offsets to its parts don't wrap either.
1570 SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]);
1571 Chains[i] = DAG.getStore(
1572 Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i),
1573 // FIXME: better loc info would be nice.
1574 Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1575 }
1576
1577 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(),
1578 MVT::Other, Chains);
1579 } else if (I.getNumOperands() != 0) {
1580 SmallVector<EVT, 4> ValueVTs;
1581 ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs);
1582 unsigned NumValues = ValueVTs.size();
1583 if (NumValues) {
1584 SDValue RetOp = getValue(I.getOperand(0));
1585
1586 const Function *F = I.getParent()->getParent();
1587
1588 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
1589 if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1590 Attribute::SExt))
1591 ExtendKind = ISD::SIGN_EXTEND;
1592 else if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex,
1593 Attribute::ZExt))
1594 ExtendKind = ISD::ZERO_EXTEND;
1595
1596 LLVMContext &Context = F->getContext();
1597 bool RetInReg = F->getAttributes().hasAttribute(
1598 AttributeList::ReturnIndex, Attribute::InReg);
1599
1600 for (unsigned j = 0; j != NumValues; ++j) {
1601 EVT VT = ValueVTs[j];
1602
1603 if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
1604 VT = TLI.getTypeForExtReturn(Context, VT, ExtendKind);
1605
1606 CallingConv::ID CC = F->getCallingConv();
1607
1608 unsigned NumParts = TLI.getNumRegistersForCallingConv(Context, CC, VT);
1609 MVT PartVT = TLI.getRegisterTypeForCallingConv(Context, CC, VT);
1610 SmallVector<SDValue, 4> Parts(NumParts);
1611 getCopyToParts(DAG, getCurSDLoc(),
1612 SDValue(RetOp.getNode(), RetOp.getResNo() + j),
1613 &Parts[0], NumParts, PartVT, &I, CC, ExtendKind);
1614
1615 // 'inreg' on function refers to return value
1616 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1617 if (RetInReg)
1618 Flags.setInReg();
1619
1620 // Propagate extension type if any
1621 if (ExtendKind == ISD::SIGN_EXTEND)
1622 Flags.setSExt();
1623 else if (ExtendKind == ISD::ZERO_EXTEND)
1624 Flags.setZExt();
1625
1626 for (unsigned i = 0; i < NumParts; ++i) {
1627 Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
1628 VT, /*isfixed=*/true, 0, 0));
1629 OutVals.push_back(Parts[i]);
1630 }
1631 }
1632 }
1633 }
1634
1635 // Push in swifterror virtual register as the last element of Outs. This makes
1636 // sure swifterror virtual register will be returned in the swifterror
1637 // physical register.
1638 const Function *F = I.getParent()->getParent();
1639 if (TLI.supportSwiftError() &&
1640 F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) {
1641 assert(FuncInfo.SwiftErrorArg && "Need a swift error argument")(static_cast <bool> (FuncInfo.SwiftErrorArg && "Need a swift error argument"
) ? void (0) : __assert_fail ("FuncInfo.SwiftErrorArg && \"Need a swift error argument\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1641, __extension__ __PRETTY_FUNCTION__));
1642 ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
1643 Flags.setSwiftError();
1644 Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/,
1645 EVT(TLI.getPointerTy(DL)) /*argvt*/,
1646 true /*isfixed*/, 1 /*origidx*/,
1647 0 /*partOffs*/));
1648 // Create SDNode for the swifterror virtual register.
1649 OutVals.push_back(
1650 DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt(
1651 &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first,
1652 EVT(TLI.getPointerTy(DL))));
1653 }
1654
1655 bool isVarArg = DAG.getMachineFunction().getFunction().isVarArg();
1656 CallingConv::ID CallConv =
1657 DAG.getMachineFunction().getFunction().getCallingConv();
1658 Chain = DAG.getTargetLoweringInfo().LowerReturn(
1659 Chain, CallConv, isVarArg, Outs, OutVals, getCurSDLoc(), DAG);
1660
1661 // Verify that the target's LowerReturn behaved as expected.
1662 assert(Chain.getNode() && Chain.getValueType() == MVT::Other &&(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1663, __extension__ __PRETTY_FUNCTION__))
1663 "LowerReturn didn't return a valid chain!")(static_cast <bool> (Chain.getNode() && Chain.getValueType
() == MVT::Other && "LowerReturn didn't return a valid chain!"
) ? void (0) : __assert_fail ("Chain.getNode() && Chain.getValueType() == MVT::Other && \"LowerReturn didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1663, __extension__ __PRETTY_FUNCTION__));
1664
1665 // Update the DAG with the new chain value resulting from return lowering.
1666 DAG.setRoot(Chain);
1667}
1668
1669/// CopyToExportRegsIfNeeded - If the given value has virtual registers
1670/// created for it, emit nodes to copy the value into the virtual
1671/// registers.
1672void SelectionDAGBuilder::CopyToExportRegsIfNeeded(const Value *V) {
1673 // Skip empty types
1674 if (V->getType()->isEmptyTy())
1675 return;
1676
1677 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
1678 if (VMI != FuncInfo.ValueMap.end()) {
1679 assert(!V->use_empty() && "Unused value assigned virtual registers!")(static_cast <bool> (!V->use_empty() && "Unused value assigned virtual registers!"
) ? void (0) : __assert_fail ("!V->use_empty() && \"Unused value assigned virtual registers!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1679, __extension__ __PRETTY_FUNCTION__));
1680 CopyValueToVirtualRegister(V, VMI->second);
1681 }
1682}
1683
1684/// ExportFromCurrentBlock - If this condition isn't known to be exported from
1685/// the current basic block, add it to ValueMap now so that we'll get a
1686/// CopyTo/FromReg.
1687void SelectionDAGBuilder::ExportFromCurrentBlock(const Value *V) {
1688 // No need to export constants.
1689 if (!isa<Instruction>(V) && !isa<Argument>(V)) return;
1690
1691 // Already exported?
1692 if (FuncInfo.isExportedInst(V)) return;
1693
1694 unsigned Reg = FuncInfo.InitializeRegForValue(V);
1695 CopyValueToVirtualRegister(V, Reg);
1696}
1697
1698bool SelectionDAGBuilder::isExportableFromCurrentBlock(const Value *V,
1699 const BasicBlock *FromBB) {
1700 // The operands of the setcc have to be in this block. We don't know
1701 // how to export them from some other block.
1702 if (const Instruction *VI = dyn_cast<Instruction>(V)) {
1703 // Can export from current BB.
1704 if (VI->getParent() == FromBB)
1705 return true;
1706
1707 // Is already exported, noop.
1708 return FuncInfo.isExportedInst(V);
1709 }
1710
1711 // If this is an argument, we can export it if the BB is the entry block or
1712 // if it is already exported.
1713 if (isa<Argument>(V)) {
1714 if (FromBB == &FromBB->getParent()->getEntryBlock())
1715 return true;
1716
1717 // Otherwise, can only export this if it is already exported.
1718 return FuncInfo.isExportedInst(V);
1719 }
1720
1721 // Otherwise, constants can always be exported.
1722 return true;
1723}
1724
1725/// Return branch probability calculated by BranchProbabilityInfo for IR blocks.
1726BranchProbability
1727SelectionDAGBuilder::getEdgeProbability(const MachineBasicBlock *Src,
1728 const MachineBasicBlock *Dst) const {
1729 BranchProbabilityInfo *BPI = FuncInfo.BPI;
1730 const BasicBlock *SrcBB = Src->getBasicBlock();
1731 const BasicBlock *DstBB = Dst->getBasicBlock();
1732 if (!BPI) {
1733 // If BPI is not available, set the default probability as 1 / N, where N is
1734 // the number of successors.
1735 auto SuccSize = std::max<uint32_t>(succ_size(SrcBB), 1);
1736 return BranchProbability(1, SuccSize);
1737 }
1738 return BPI->getEdgeProbability(SrcBB, DstBB);
1739}
1740
1741void SelectionDAGBuilder::addSuccessorWithProb(MachineBasicBlock *Src,
1742 MachineBasicBlock *Dst,
1743 BranchProbability Prob) {
1744 if (!FuncInfo.BPI)
1745 Src->addSuccessorWithoutProb(Dst);
1746 else {
1747 if (Prob.isUnknown())
1748 Prob = getEdgeProbability(Src, Dst);
1749 Src->addSuccessor(Dst, Prob);
1750 }
1751}
1752
1753static bool InBlock(const Value *V, const BasicBlock *BB) {
1754 if (const Instruction *I = dyn_cast<Instruction>(V))
1755 return I->getParent() == BB;
1756 return true;
1757}
1758
1759/// EmitBranchForMergedCondition - Helper method for FindMergedConditions.
1760/// This function emits a branch and is used at the leaves of an OR or an
1761/// AND operator tree.
1762void
1763SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond,
1764 MachineBasicBlock *TBB,
1765 MachineBasicBlock *FBB,
1766 MachineBasicBlock *CurBB,
1767 MachineBasicBlock *SwitchBB,
1768 BranchProbability TProb,
1769 BranchProbability FProb,
1770 bool InvertCond) {
1771 const BasicBlock *BB = CurBB->getBasicBlock();
1772
1773 // If the leaf of the tree is a comparison, merge the condition into
1774 // the caseblock.
1775 if (const CmpInst *BOp = dyn_cast<CmpInst>(Cond)) {
1776 // The operands of the cmp have to be in this block. We don't know
1777 // how to export them from some other block. If this is the first block
1778 // of the sequence, no exporting is needed.
1779 if (CurBB == SwitchBB ||
1780 (isExportableFromCurrentBlock(BOp->getOperand(0), BB) &&
1781 isExportableFromCurrentBlock(BOp->getOperand(1), BB))) {
1782 ISD::CondCode Condition;
1783 if (const ICmpInst *IC = dyn_cast<ICmpInst>(Cond)) {
1784 ICmpInst::Predicate Pred =
1785 InvertCond ? IC->getInversePredicate() : IC->getPredicate();
1786 Condition = getICmpCondCode(Pred);
1787 } else {
1788 const FCmpInst *FC = cast<FCmpInst>(Cond);
1789 FCmpInst::Predicate Pred =
1790 InvertCond ? FC->getInversePredicate() : FC->getPredicate();
1791 Condition = getFCmpCondCode(Pred);
1792 if (TM.Options.NoNaNsFPMath)
1793 Condition = getFCmpCodeWithoutNaN(Condition);
1794 }
1795
1796 CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr,
1797 TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1798 SwitchCases.push_back(CB);
1799 return;
1800 }
1801 }
1802
1803 // Create a CaseBlock record representing this branch.
1804 ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ;
1805 CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()),
1806 nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb);
1807 SwitchCases.push_back(CB);
1808}
1809
1810/// FindMergedConditions - If Cond is an expression like
1811void SelectionDAGBuilder::FindMergedConditions(const Value *Cond,
1812 MachineBasicBlock *TBB,
1813 MachineBasicBlock *FBB,
1814 MachineBasicBlock *CurBB,
1815 MachineBasicBlock *SwitchBB,
1816 Instruction::BinaryOps Opc,
1817 BranchProbability TProb,
1818 BranchProbability FProb,
1819 bool InvertCond) {
1820 // Skip over not part of the tree and remember to invert op and operands at
1821 // next level.
1822 if (BinaryOperator::isNot(Cond) && Cond->hasOneUse()) {
1823 const Value *CondOp = BinaryOperator::getNotArgument(Cond);
1824 if (InBlock(CondOp, CurBB->getBasicBlock())) {
1825 FindMergedConditions(CondOp, TBB, FBB, CurBB, SwitchBB, Opc, TProb, FProb,
1826 !InvertCond);
1827 return;
1828 }
1829 }
1830
1831 const Instruction *BOp = dyn_cast<Instruction>(Cond);
1832 // Compute the effective opcode for Cond, taking into account whether it needs
1833 // to be inverted, e.g.
1834 // and (not (or A, B)), C
1835 // gets lowered as
1836 // and (and (not A, not B), C)
1837 unsigned BOpc = 0;
1838 if (BOp) {
1839 BOpc = BOp->getOpcode();
1840 if (InvertCond) {
1841 if (BOpc == Instruction::And)
1842 BOpc = Instruction::Or;
1843 else if (BOpc == Instruction::Or)
1844 BOpc = Instruction::And;
1845 }
1846 }
1847
1848 // If this node is not part of the or/and tree, emit it as a branch.
1849 if (!BOp || !(isa<BinaryOperator>(BOp) || isa<CmpInst>(BOp)) ||
1850 BOpc != unsigned(Opc) || !BOp->hasOneUse() ||
1851 BOp->getParent() != CurBB->getBasicBlock() ||
1852 !InBlock(BOp->getOperand(0), CurBB->getBasicBlock()) ||
1853 !InBlock(BOp->getOperand(1), CurBB->getBasicBlock())) {
1854 EmitBranchForMergedCondition(Cond, TBB, FBB, CurBB, SwitchBB,
1855 TProb, FProb, InvertCond);
1856 return;
1857 }
1858
1859 // Create TmpBB after CurBB.
1860 MachineFunction::iterator BBI(CurBB);
1861 MachineFunction &MF = DAG.getMachineFunction();
1862 MachineBasicBlock *TmpBB = MF.CreateMachineBasicBlock(CurBB->getBasicBlock());
1863 CurBB->getParent()->insert(++BBI, TmpBB);
1864
1865 if (Opc == Instruction::Or) {
1866 // Codegen X | Y as:
1867 // BB1:
1868 // jmp_if_X TBB
1869 // jmp TmpBB
1870 // TmpBB:
1871 // jmp_if_Y TBB
1872 // jmp FBB
1873 //
1874
1875 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1876 // The requirement is that
1877 // TrueProb for BB1 + (FalseProb for BB1 * TrueProb for TmpBB)
1878 // = TrueProb for original BB.
1879 // Assuming the original probabilities are A and B, one choice is to set
1880 // BB1's probabilities to A/2 and A/2+B, and set TmpBB's probabilities to
1881 // A/(1+B) and 2B/(1+B). This choice assumes that
1882 // TrueProb for BB1 == FalseProb for BB1 * TrueProb for TmpBB.
1883 // Another choice is to assume TrueProb for BB1 equals to TrueProb for
1884 // TmpBB, but the math is more complicated.
1885
1886 auto NewTrueProb = TProb / 2;
1887 auto NewFalseProb = TProb / 2 + FProb;
1888 // Emit the LHS condition.
1889 FindMergedConditions(BOp->getOperand(0), TBB, TmpBB, CurBB, SwitchBB, Opc,
1890 NewTrueProb, NewFalseProb, InvertCond);
1891
1892 // Normalize A/2 and B to get A/(1+B) and 2B/(1+B).
1893 SmallVector<BranchProbability, 2> Probs{TProb / 2, FProb};
1894 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1895 // Emit the RHS condition into TmpBB.
1896 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1897 Probs[0], Probs[1], InvertCond);
1898 } else {
1899 assert(Opc == Instruction::And && "Unknown merge op!")(static_cast <bool> (Opc == Instruction::And &&
"Unknown merge op!") ? void (0) : __assert_fail ("Opc == Instruction::And && \"Unknown merge op!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 1899, __extension__ __PRETTY_FUNCTION__));
1900 // Codegen X & Y as:
1901 // BB1:
1902 // jmp_if_X TmpBB
1903 // jmp FBB
1904 // TmpBB:
1905 // jmp_if_Y TBB
1906 // jmp FBB
1907 //
1908 // This requires creation of TmpBB after CurBB.
1909
1910 // We have flexibility in setting Prob for BB1 and Prob for TmpBB.
1911 // The requirement is that
1912 // FalseProb for BB1 + (TrueProb for BB1 * FalseProb for TmpBB)
1913 // = FalseProb for original BB.
1914 // Assuming the original probabilities are A and B, one choice is to set
1915 // BB1's probabilities to A+B/2 and B/2, and set TmpBB's probabilities to
1916 // 2A/(1+A) and B/(1+A). This choice assumes that FalseProb for BB1 ==
1917 // TrueProb for BB1 * FalseProb for TmpBB.
1918
1919 auto NewTrueProb = TProb + FProb / 2;
1920 auto NewFalseProb = FProb / 2;
1921 // Emit the LHS condition.
1922 FindMergedConditions(BOp->getOperand(0), TmpBB, FBB, CurBB, SwitchBB, Opc,
1923 NewTrueProb, NewFalseProb, InvertCond);
1924
1925 // Normalize A and B/2 to get 2A/(1+A) and B/(1+A).
1926 SmallVector<BranchProbability, 2> Probs{TProb, FProb / 2};
1927 BranchProbability::normalizeProbabilities(Probs.begin(), Probs.end());
1928 // Emit the RHS condition into TmpBB.
1929 FindMergedConditions(BOp->getOperand(1), TBB, FBB, TmpBB, SwitchBB, Opc,
1930 Probs[0], Probs[1], InvertCond);
1931 }
1932}
1933
1934/// If the set of cases should be emitted as a series of branches, return true.
1935/// If we should emit this as a bunch of and/or'd together conditions, return
1936/// false.
1937bool
1938SelectionDAGBuilder::ShouldEmitAsBranches(const std::vector<CaseBlock> &Cases) {
1939 if (Cases.size() != 2) return true;
1940
1941 // If this is two comparisons of the same values or'd or and'd together, they
1942 // will get folded into a single comparison, so don't emit two blocks.
1943 if ((Cases[0].CmpLHS == Cases[1].CmpLHS &&
1944 Cases[0].CmpRHS == Cases[1].CmpRHS) ||
1945 (Cases[0].CmpRHS == Cases[1].CmpLHS &&
1946 Cases[0].CmpLHS == Cases[1].CmpRHS)) {
1947 return false;
1948 }
1949
1950 // Handle: (X != null) | (Y != null) --> (X|Y) != 0
1951 // Handle: (X == null) & (Y == null) --> (X|Y) == 0
1952 if (Cases[0].CmpRHS == Cases[1].CmpRHS &&
1953 Cases[0].CC == Cases[1].CC &&
1954 isa<Constant>(Cases[0].CmpRHS) &&
1955 cast<Constant>(Cases[0].CmpRHS)->isNullValue()) {
1956 if (Cases[0].CC == ISD::SETEQ && Cases[0].TrueBB == Cases[1].ThisBB)
1957 return false;
1958 if (Cases[0].CC == ISD::SETNE && Cases[0].FalseBB == Cases[1].ThisBB)
1959 return false;
1960 }
1961
1962 return true;
1963}
1964
1965void SelectionDAGBuilder::visitBr(const BranchInst &I) {
1966 MachineBasicBlock *BrMBB = FuncInfo.MBB;
1967
1968 // Update machine-CFG edges.
1969 MachineBasicBlock *Succ0MBB = FuncInfo.MBBMap[I.getSuccessor(0)];
1970
1971 if (I.isUnconditional()) {
1972 // Update machine-CFG edges.
1973 BrMBB->addSuccessor(Succ0MBB);
1974
1975 // If this is not a fall-through branch or optimizations are switched off,
1976 // emit the branch.
1977 if (Succ0MBB != NextBlock(BrMBB) || TM.getOptLevel() == CodeGenOpt::None)
1978 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
1979 MVT::Other, getControlRoot(),
1980 DAG.getBasicBlock(Succ0MBB)));
1981
1982 return;
1983 }
1984
1985 // If this condition is one of the special cases we handle, do special stuff
1986 // now.
1987 const Value *CondVal = I.getCondition();
1988 MachineBasicBlock *Succ1MBB = FuncInfo.MBBMap[I.getSuccessor(1)];
1989
1990 // If this is a series of conditions that are or'd or and'd together, emit
1991 // this as a sequence of branches instead of setcc's with and/or operations.
1992 // As long as jumps are not expensive, this should improve performance.
1993 // For example, instead of something like:
1994 // cmp A, B
1995 // C = seteq
1996 // cmp D, E
1997 // F = setle
1998 // or C, F
1999 // jnz foo
2000 // Emit:
2001 // cmp A, B
2002 // je foo
2003 // cmp D, E
2004 // jle foo
2005 if (const BinaryOperator *BOp = dyn_cast<BinaryOperator>(CondVal)) {
2006 Instruction::BinaryOps Opcode = BOp->getOpcode();
2007 if (!DAG.getTargetLoweringInfo().isJumpExpensive() && BOp->hasOneUse() &&
2008 !I.getMetadata(LLVMContext::MD_unpredictable) &&
2009 (Opcode == Instruction::And || Opcode == Instruction::Or)) {
2010 FindMergedConditions(BOp, Succ0MBB, Succ1MBB, BrMBB, BrMBB,
2011 Opcode,
2012 getEdgeProbability(BrMBB, Succ0MBB),
2013 getEdgeProbability(BrMBB, Succ1MBB),
2014 /*InvertCond=*/false);
2015 // If the compares in later blocks need to use values not currently
2016 // exported from this block, export them now. This block should always
2017 // be the first entry.
2018 assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!")(static_cast <bool> (SwitchCases[0].ThisBB == BrMBB &&
"Unexpected lowering!") ? void (0) : __assert_fail ("SwitchCases[0].ThisBB == BrMBB && \"Unexpected lowering!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2018, __extension__ __PRETTY_FUNCTION__));
2019
2020 // Allow some cases to be rejected.
2021 if (ShouldEmitAsBranches(SwitchCases)) {
2022 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) {
2023 ExportFromCurrentBlock(SwitchCases[i].CmpLHS);
2024 ExportFromCurrentBlock(SwitchCases[i].CmpRHS);
2025 }
2026
2027 // Emit the branch for this block.
2028 visitSwitchCase(SwitchCases[0], BrMBB);
2029 SwitchCases.erase(SwitchCases.begin());
2030 return;
2031 }
2032
2033 // Okay, we decided not to do this, remove any inserted MBB's and clear
2034 // SwitchCases.
2035 for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i)
2036 FuncInfo.MF->erase(SwitchCases[i].ThisBB);
2037
2038 SwitchCases.clear();
2039 }
2040 }
2041
2042 // Create a CaseBlock record representing this branch.
2043 CaseBlock CB(ISD::SETEQ, CondVal, ConstantInt::getTrue(*DAG.getContext()),
2044 nullptr, Succ0MBB, Succ1MBB, BrMBB, getCurSDLoc());
2045
2046 // Use visitSwitchCase to actually insert the fast branch sequence for this
2047 // cond branch.
2048 visitSwitchCase(CB, BrMBB);
2049}
2050
2051/// visitSwitchCase - Emits the necessary code to represent a single node in
2052/// the binary search tree resulting from lowering a switch instruction.
2053void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
2054 MachineBasicBlock *SwitchBB) {
2055 SDValue Cond;
2056 SDValue CondLHS = getValue(CB.CmpLHS);
2057 SDLoc dl = CB.DL;
2058
2059 // Build the setcc now.
2060 if (!CB.CmpMHS) {
2061 // Fold "(X == true)" to X and "(X == false)" to !X to
2062 // handle common cases produced by branch lowering.
2063 if (CB.CmpRHS == ConstantInt::getTrue(*DAG.getContext()) &&
2064 CB.CC == ISD::SETEQ)
2065 Cond = CondLHS;
2066 else if (CB.CmpRHS == ConstantInt::getFalse(*DAG.getContext()) &&
2067 CB.CC == ISD::SETEQ) {
2068 SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType());
2069 Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True);
2070 } else
2071 Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC);
2072 } else {
2073 assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now")(static_cast <bool> (CB.CC == ISD::SETLE && "Can handle only LE ranges now"
) ? void (0) : __assert_fail ("CB.CC == ISD::SETLE && \"Can handle only LE ranges now\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2073, __extension__ __PRETTY_FUNCTION__));
2074
2075 const APInt& Low = cast<ConstantInt>(CB.CmpLHS)->getValue();
2076 const APInt& High = cast<ConstantInt>(CB.CmpRHS)->getValue();
2077
2078 SDValue CmpOp = getValue(CB.CmpMHS);
2079 EVT VT = CmpOp.getValueType();
2080
2081 if (cast<ConstantInt>(CB.CmpLHS)->isMinValue(true)) {
2082 Cond = DAG.getSetCC(dl, MVT::i1, CmpOp, DAG.getConstant(High, dl, VT),
2083 ISD::SETLE);
2084 } else {
2085 SDValue SUB = DAG.getNode(ISD::SUB, dl,
2086 VT, CmpOp, DAG.getConstant(Low, dl, VT));
2087 Cond = DAG.getSetCC(dl, MVT::i1, SUB,
2088 DAG.getConstant(High-Low, dl, VT), ISD::SETULE);
2089 }
2090 }
2091
2092 // Update successor info
2093 addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb);
2094 // TrueBB and FalseBB are always different unless the incoming IR is
2095 // degenerate. This only happens when running llc on weird IR.
2096 if (CB.TrueBB != CB.FalseBB)
2097 addSuccessorWithProb(SwitchBB, CB.FalseBB, CB.FalseProb);
2098 SwitchBB->normalizeSuccProbs();
2099
2100 // If the lhs block is the next block, invert the condition so that we can
2101 // fall through to the lhs instead of the rhs block.
2102 if (CB.TrueBB == NextBlock(SwitchBB)) {
2103 std::swap(CB.TrueBB, CB.FalseBB);
2104 SDValue True = DAG.getConstant(1, dl, Cond.getValueType());
2105 Cond = DAG.getNode(ISD::XOR, dl, Cond.getValueType(), Cond, True);
2106 }
2107
2108 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2109 MVT::Other, getControlRoot(), Cond,
2110 DAG.getBasicBlock(CB.TrueBB));
2111
2112 // Insert the false branch. Do this even if it's a fall through branch,
2113 // this makes it easier to do DAG optimizations which require inverting
2114 // the branch condition.
2115 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2116 DAG.getBasicBlock(CB.FalseBB));
2117
2118 DAG.setRoot(BrCond);
2119}
2120
2121/// visitJumpTable - Emit JumpTable node in the current MBB
2122void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) {
2123 // Emit the code for the jump table
2124 assert(JT.Reg != -1U && "Should lower JT Header first!")(static_cast <bool> (JT.Reg != -1U && "Should lower JT Header first!"
) ? void (0) : __assert_fail ("JT.Reg != -1U && \"Should lower JT Header first!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2124, __extension__ __PRETTY_FUNCTION__));
2125 EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
2126 SDValue Index = DAG.getCopyFromReg(getControlRoot(), getCurSDLoc(),
2127 JT.Reg, PTy);
2128 SDValue Table = DAG.getJumpTable(JT.JTI, PTy);
2129 SDValue BrJumpTable = DAG.getNode(ISD::BR_JT, getCurSDLoc(),
2130 MVT::Other, Index.getValue(1),
2131 Table, Index);
2132 DAG.setRoot(BrJumpTable);
2133}
2134
2135/// visitJumpTableHeader - This function emits necessary code to produce index
2136/// in the JumpTable from switch case.
2137void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT,
2138 JumpTableHeader &JTH,
2139 MachineBasicBlock *SwitchBB) {
2140 SDLoc dl = getCurSDLoc();
2141
2142 // Subtract the lowest switch case value from the value being switched on and
2143 // conditional branch to default mbb if the result is greater than the
2144 // difference between smallest and largest cases.
2145 SDValue SwitchOp = getValue(JTH.SValue);
2146 EVT VT = SwitchOp.getValueType();
2147 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2148 DAG.getConstant(JTH.First, dl, VT));
2149
2150 // The SDNode we just created, which holds the value being switched on minus
2151 // the smallest case value, needs to be copied to a virtual register so it
2152 // can be used as an index into the jump table in a subsequent basic block.
2153 // This value may be smaller or larger than the target's pointer type, and
2154 // therefore require extension or truncating.
2155 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2156 SwitchOp = DAG.getZExtOrTrunc(Sub, dl, TLI.getPointerTy(DAG.getDataLayout()));
2157
2158 unsigned JumpTableReg =
2159 FuncInfo.CreateReg(TLI.getPointerTy(DAG.getDataLayout()));
2160 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl,
2161 JumpTableReg, SwitchOp);
2162 JT.Reg = JumpTableReg;
2163
2164 // Emit the range check for the jump table, and branch to the default block
2165 // for the switch statement if the value being switched on exceeds the largest
2166 // case in the switch.
2167 SDValue CMP = DAG.getSetCC(
2168 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2169 Sub.getValueType()),
2170 Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT);
2171
2172 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2173 MVT::Other, CopyTo, CMP,
2174 DAG.getBasicBlock(JT.Default));
2175
2176 // Avoid emitting unnecessary branches to the next block.
2177 if (JT.MBB != NextBlock(SwitchBB))
2178 BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond,
2179 DAG.getBasicBlock(JT.MBB));
2180
2181 DAG.setRoot(BrCond);
2182}
2183
2184/// Create a LOAD_STACK_GUARD node, and let it carry the target specific global
2185/// variable if there exists one.
2186static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL,
2187 SDValue &Chain) {
2188 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2189 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2190 MachineFunction &MF = DAG.getMachineFunction();
2191 Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent());
2192 MachineSDNode *Node =
2193 DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain);
2194 if (Global) {
2195 MachinePointerInfo MPInfo(Global);
2196 MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1);
2197 auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant |
2198 MachineMemOperand::MODereferenceable;
2199 *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8,
2200 DAG.getEVTAlignment(PtrTy));
2201 Node->setMemRefs(MemRefs, MemRefs + 1);
2202 }
2203 return SDValue(Node, 0);
2204}
2205
2206/// Codegen a new tail for a stack protector check ParentMBB which has had its
2207/// tail spliced into a stack protector check success bb.
2208///
2209/// For a high level explanation of how this fits into the stack protector
2210/// generation see the comment on the declaration of class
2211/// StackProtectorDescriptor.
2212void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD,
2213 MachineBasicBlock *ParentBB) {
2214
2215 // First create the loads to the guard/stack slot for the comparison.
2216 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2217 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
2218
2219 MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo();
2220 int FI = MFI.getStackProtectorIndex();
2221
2222 SDValue Guard;
2223 SDLoc dl = getCurSDLoc();
2224 SDValue StackSlotPtr = DAG.getFrameIndex(FI, PtrTy);
2225 const Module &M = *ParentBB->getParent()->getFunction().getParent();
2226 unsigned Align = DL->getPrefTypeAlignment(Type::getInt8PtrTy(M.getContext()));
2227
2228 // Generate code to load the content of the guard slot.
2229 SDValue GuardVal = DAG.getLoad(
2230 PtrTy, dl, DAG.getEntryNode(), StackSlotPtr,
2231 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align,
2232 MachineMemOperand::MOVolatile);
2233
2234 if (TLI.useStackGuardXorFP())
2235 GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl);
2236
2237 // Retrieve guard check function, nullptr if instrumentation is inlined.
2238 if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) {
2239 // The target provides a guard check function to validate the guard value.
2240 // Generate a call to that function with the content of the guard slot as
2241 // argument.
2242 auto *Fn = cast<Function>(GuardCheck);
2243 FunctionType *FnTy = Fn->getFunctionType();
2244 assert(FnTy->getNumParams() == 1 && "Invalid function signature")(static_cast <bool> (FnTy->getNumParams() == 1 &&
"Invalid function signature") ? void (0) : __assert_fail ("FnTy->getNumParams() == 1 && \"Invalid function signature\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2244, __extension__ __PRETTY_FUNCTION__));
2245
2246 TargetLowering::ArgListTy Args;
2247 TargetLowering::ArgListEntry Entry;
2248 Entry.Node = GuardVal;
2249 Entry.Ty = FnTy->getParamType(0);
2250 if (Fn->hasAttribute(1, Attribute::AttrKind::InReg))
2251 Entry.IsInReg = true;
2252 Args.push_back(Entry);
2253
2254 TargetLowering::CallLoweringInfo CLI(DAG);
2255 CLI.setDebugLoc(getCurSDLoc())
2256 .setChain(DAG.getEntryNode())
2257 .setCallee(Fn->getCallingConv(), FnTy->getReturnType(),
2258 getValue(GuardCheck), std::move(Args));
2259
2260 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
2261 DAG.setRoot(Result.second);
2262 return;
2263 }
2264
2265 // If useLoadStackGuardNode returns true, generate LOAD_STACK_GUARD.
2266 // Otherwise, emit a volatile load to retrieve the stack guard value.
2267 SDValue Chain = DAG.getEntryNode();
2268 if (TLI.useLoadStackGuardNode()) {
2269 Guard = getLoadStackGuard(DAG, dl, Chain);
2270 } else {
2271 const Value *IRGuard = TLI.getSDagStackGuard(M);
2272 SDValue GuardPtr = getValue(IRGuard);
2273
2274 Guard =
2275 DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0),
2276 Align, MachineMemOperand::MOVolatile);
2277 }
2278
2279 // Perform the comparison via a subtract/getsetcc.
2280 EVT VT = Guard.getValueType();
2281 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, Guard, GuardVal);
2282
2283 SDValue Cmp = DAG.getSetCC(dl, TLI.getSetCCResultType(DAG.getDataLayout(),
2284 *DAG.getContext(),
2285 Sub.getValueType()),
2286 Sub, DAG.getConstant(0, dl, VT), ISD::SETNE);
2287
2288 // If the sub is not 0, then we know the guard/stackslot do not equal, so
2289 // branch to failure MBB.
2290 SDValue BrCond = DAG.getNode(ISD::BRCOND, dl,
2291 MVT::Other, GuardVal.getOperand(0),
2292 Cmp, DAG.getBasicBlock(SPD.getFailureMBB()));
2293 // Otherwise branch to success MBB.
2294 SDValue Br = DAG.getNode(ISD::BR, dl,
2295 MVT::Other, BrCond,
2296 DAG.getBasicBlock(SPD.getSuccessMBB()));
2297
2298 DAG.setRoot(Br);
2299}
2300
2301/// Codegen the failure basic block for a stack protector check.
2302///
2303/// A failure stack protector machine basic block consists simply of a call to
2304/// __stack_chk_fail().
2305///
2306/// For a high level explanation of how this fits into the stack protector
2307/// generation see the comment on the declaration of class
2308/// StackProtectorDescriptor.
2309void
2310SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) {
2311 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2312 SDValue Chain =
2313 TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid,
2314 None, false, getCurSDLoc(), false, false).second;
2315 DAG.setRoot(Chain);
2316}
2317
2318/// visitBitTestHeader - This function emits necessary code to produce value
2319/// suitable for "bit tests"
2320void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
2321 MachineBasicBlock *SwitchBB) {
2322 SDLoc dl = getCurSDLoc();
2323
2324 // Subtract the minimum value
2325 SDValue SwitchOp = getValue(B.SValue);
2326 EVT VT = SwitchOp.getValueType();
2327 SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp,
2328 DAG.getConstant(B.First, dl, VT));
2329
2330 // Check range
2331 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2332 SDValue RangeCmp = DAG.getSetCC(
2333 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
2334 Sub.getValueType()),
2335 Sub, DAG.getConstant(B.Range, dl, VT), ISD::SETUGT);
2336
2337 // Determine the type of the test operands.
2338 bool UsePtrType = false;
2339 if (!TLI.isTypeLegal(VT))
2340 UsePtrType = true;
2341 else {
2342 for (unsigned i = 0, e = B.Cases.size(); i != e; ++i)
2343 if (!isUIntN(VT.getSizeInBits(), B.Cases[i].Mask)) {
2344 // Switch table case range are encoded into series of masks.
2345 // Just use pointer type, it's guaranteed to fit.
2346 UsePtrType = true;
2347 break;
2348 }
2349 }
2350 if (UsePtrType) {
2351 VT = TLI.getPointerTy(DAG.getDataLayout());
2352 Sub = DAG.getZExtOrTrunc(Sub, dl, VT);
2353 }
2354
2355 B.RegVT = VT.getSimpleVT();
2356 B.Reg = FuncInfo.CreateReg(B.RegVT);
2357 SDValue CopyTo = DAG.getCopyToReg(getControlRoot(), dl, B.Reg, Sub);
2358
2359 MachineBasicBlock* MBB = B.Cases[0].ThisBB;
2360
2361 addSuccessorWithProb(SwitchBB, B.Default, B.DefaultProb);
2362 addSuccessorWithProb(SwitchBB, MBB, B.Prob);
2363 SwitchBB->normalizeSuccProbs();
2364
2365 SDValue BrRange = DAG.getNode(ISD::BRCOND, dl,
2366 MVT::Other, CopyTo, RangeCmp,
2367 DAG.getBasicBlock(B.Default));
2368
2369 // Avoid emitting unnecessary branches to the next block.
2370 if (MBB != NextBlock(SwitchBB))
2371 BrRange = DAG.getNode(ISD::BR, dl, MVT::Other, BrRange,
2372 DAG.getBasicBlock(MBB));
2373
2374 DAG.setRoot(BrRange);
2375}
2376
2377/// visitBitTestCase - this function produces one "bit test"
2378void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
2379 MachineBasicBlock* NextMBB,
2380 BranchProbability BranchProbToNext,
2381 unsigned Reg,
2382 BitTestCase &B,
2383 MachineBasicBlock *SwitchBB) {
2384 SDLoc dl = getCurSDLoc();
2385 MVT VT = BB.RegVT;
2386 SDValue ShiftOp = DAG.getCopyFromReg(getControlRoot(), dl, Reg, VT);
2387 SDValue Cmp;
2388 unsigned PopCount = countPopulation(B.Mask);
2389 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2390 if (PopCount == 1) {
2391 // Testing for a single bit; just compare the shift count with what it
2392 // would need to be to shift a 1 bit in that position.
2393 Cmp = DAG.getSetCC(
2394 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2395 ShiftOp, DAG.getConstant(countTrailingZeros(B.Mask), dl, VT),
2396 ISD::SETEQ);
2397 } else if (PopCount == BB.Range) {
2398 // There is only one zero bit in the range, test for it directly.
2399 Cmp = DAG.getSetCC(
2400 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2401 ShiftOp, DAG.getConstant(countTrailingOnes(B.Mask), dl, VT),
2402 ISD::SETNE);
2403 } else {
2404 // Make desired shift
2405 SDValue SwitchVal = DAG.getNode(ISD::SHL, dl, VT,
2406 DAG.getConstant(1, dl, VT), ShiftOp);
2407
2408 // Emit bit tests and jumps
2409 SDValue AndOp = DAG.getNode(ISD::AND, dl,
2410 VT, SwitchVal, DAG.getConstant(B.Mask, dl, VT));
2411 Cmp = DAG.getSetCC(
2412 dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT),
2413 AndOp, DAG.getConstant(0, dl, VT), ISD::SETNE);
2414 }
2415
2416 // The branch probability from SwitchBB to B.TargetBB is B.ExtraProb.
2417 addSuccessorWithProb(SwitchBB, B.TargetBB, B.ExtraProb);
2418 // The branch probability from SwitchBB to NextMBB is BranchProbToNext.
2419 addSuccessorWithProb(SwitchBB, NextMBB, BranchProbToNext);
2420 // It is not guaranteed that the sum of B.ExtraProb and BranchProbToNext is
2421 // one as they are relative probabilities (and thus work more like weights),
2422 // and hence we need to normalize them to let the sum of them become one.
2423 SwitchBB->normalizeSuccProbs();
2424
2425 SDValue BrAnd = DAG.getNode(ISD::BRCOND, dl,
2426 MVT::Other, getControlRoot(),
2427 Cmp, DAG.getBasicBlock(B.TargetBB));
2428
2429 // Avoid emitting unnecessary branches to the next block.
2430 if (NextMBB != NextBlock(SwitchBB))
2431 BrAnd = DAG.getNode(ISD::BR, dl, MVT::Other, BrAnd,
2432 DAG.getBasicBlock(NextMBB));
2433
2434 DAG.setRoot(BrAnd);
2435}
2436
2437void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) {
2438 MachineBasicBlock *InvokeMBB = FuncInfo.MBB;
2439
2440 // Retrieve successors. Look through artificial IR level blocks like
2441 // catchswitch for successors.
2442 MachineBasicBlock *Return = FuncInfo.MBBMap[I.getSuccessor(0)];
2443 const BasicBlock *EHPadBB = I.getSuccessor(1);
2444
2445 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
2446 // have to do anything here to lower funclet bundles.
2447 assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2449, __extension__ __PRETTY_FUNCTION__))
2448 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2449, __extension__ __PRETTY_FUNCTION__))
2449 "Cannot lower invokes with arbitrary operand bundles yet!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower invokes with arbitrary operand bundles yet!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower invokes with arbitrary operand bundles yet!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2449, __extension__ __PRETTY_FUNCTION__));
2450
2451 const Value *Callee(I.getCalledValue());
2452 const Function *Fn = dyn_cast<Function>(Callee);
2453 if (isa<InlineAsm>(Callee))
2454 visitInlineAsm(&I);
2455 else if (Fn && Fn->isIntrinsic()) {
2456 switch (Fn->getIntrinsicID()) {
2457 default:
2458 llvm_unreachable("Cannot invoke this intrinsic")::llvm::llvm_unreachable_internal("Cannot invoke this intrinsic"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2458);
2459 case Intrinsic::donothing:
2460 // Ignore invokes to @llvm.donothing: jump directly to the next BB.
2461 break;
2462 case Intrinsic::experimental_patchpoint_void:
2463 case Intrinsic::experimental_patchpoint_i64:
2464 visitPatchpoint(&I, EHPadBB);
2465 break;
2466 case Intrinsic::experimental_gc_statepoint:
2467 LowerStatepoint(ImmutableStatepoint(&I), EHPadBB);
2468 break;
2469 }
2470 } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) {
2471 // Currently we do not lower any intrinsic calls with deopt operand bundles.
2472 // Eventually we will support lowering the @llvm.experimental.deoptimize
2473 // intrinsic, and right now there are no plans to support other intrinsics
2474 // with deopt state.
2475 LowerCallSiteWithDeoptBundle(&I, getValue(Callee), EHPadBB);
2476 } else {
2477 LowerCallTo(&I, getValue(Callee), false, EHPadBB);
2478 }
2479
2480 // If the value of the invoke is used outside of its defining block, make it
2481 // available as a virtual register.
2482 // We already took care of the exported value for the statepoint instruction
2483 // during call to the LowerStatepoint.
2484 if (!isStatepoint(I)) {
2485 CopyToExportRegsIfNeeded(&I);
2486 }
2487
2488 SmallVector<std::pair<MachineBasicBlock *, BranchProbability>, 1> UnwindDests;
2489 BranchProbabilityInfo *BPI = FuncInfo.BPI;
2490 BranchProbability EHPadBBProb =
2491 BPI ? BPI->getEdgeProbability(InvokeMBB->getBasicBlock(), EHPadBB)
2492 : BranchProbability::getZero();
2493 findUnwindDestinations(FuncInfo, EHPadBB, EHPadBBProb, UnwindDests);
2494
2495 // Update successor info.
2496 addSuccessorWithProb(InvokeMBB, Return);
2497 for (auto &UnwindDest : UnwindDests) {
2498 UnwindDest.first->setIsEHPad();
2499 addSuccessorWithProb(InvokeMBB, UnwindDest.first, UnwindDest.second);
2500 }
2501 InvokeMBB->normalizeSuccProbs();
2502
2503 // Drop into normal successor.
2504 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(),
2505 MVT::Other, getControlRoot(),
2506 DAG.getBasicBlock(Return)));
2507}
2508
2509void SelectionDAGBuilder::visitResume(const ResumeInst &RI) {
2510 llvm_unreachable("SelectionDAGBuilder shouldn't visit resume instructions!")::llvm::llvm_unreachable_internal("SelectionDAGBuilder shouldn't visit resume instructions!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2510);
2511}
2512
2513void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) {
2514 assert(FuncInfo.MBB->isEHPad() &&(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2515, __extension__ __PRETTY_FUNCTION__))
2515 "Call to landingpad not in landing pad!")(static_cast <bool> (FuncInfo.MBB->isEHPad() &&
"Call to landingpad not in landing pad!") ? void (0) : __assert_fail
("FuncInfo.MBB->isEHPad() && \"Call to landingpad not in landing pad!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2515, __extension__ __PRETTY_FUNCTION__));
2516
2517 MachineBasicBlock *MBB = FuncInfo.MBB;
2518 addLandingPadInfo(LP, *MBB);
2519
2520 // If there aren't registers to copy the values into (e.g., during SjLj
2521 // exceptions), then don't bother to create these DAG nodes.
2522 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
2523 const Constant *PersonalityFn = FuncInfo.Fn->getPersonalityFn();
2524 if (TLI.getExceptionPointerRegister(PersonalityFn) == 0 &&
2525 TLI.getExceptionSelectorRegister(PersonalityFn) == 0)
2526 return;
2527
2528 // If landingpad's return type is token type, we don't create DAG nodes
2529 // for its exception pointer and selector value. The extraction of exception
2530 // pointer or selector value from token type landingpads is not currently
2531 // supported.
2532 if (LP.getType()->isTokenTy())
2533 return;
2534
2535 SmallVector<EVT, 2> ValueVTs;
2536 SDLoc dl = getCurSDLoc();
2537 ComputeValueVTs(TLI, DAG.getDataLayout(), LP.getType(), ValueVTs);
2538 assert(ValueVTs.size() == 2 && "Only two-valued landingpads are supported")(static_cast <bool> (ValueVTs.size() == 2 && "Only two-valued landingpads are supported"
) ? void (0) : __assert_fail ("ValueVTs.size() == 2 && \"Only two-valued landingpads are supported\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2538, __extension__ __PRETTY_FUNCTION__));
2539
2540 // Get the two live-in registers as SDValues. The physregs have already been
2541 // copied into virtual registers.
2542 SDValue Ops[2];
2543 if (FuncInfo.ExceptionPointerVirtReg) {
2544 Ops[0] = DAG.getZExtOrTrunc(
2545 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2546 FuncInfo.ExceptionPointerVirtReg,
2547 TLI.getPointerTy(DAG.getDataLayout())),
2548 dl, ValueVTs[0]);
2549 } else {
2550 Ops[0] = DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()));
2551 }
2552 Ops[1] = DAG.getZExtOrTrunc(
2553 DAG.getCopyFromReg(DAG.getEntryNode(), dl,
2554 FuncInfo.ExceptionSelectorVirtReg,
2555 TLI.getPointerTy(DAG.getDataLayout())),
2556 dl, ValueVTs[1]);
2557
2558 // Merge into one.
2559 SDValue Res = DAG.getNode(ISD::MERGE_VALUES, dl,
2560 DAG.getVTList(ValueVTs), Ops);
2561 setValue(&LP, Res);
2562}
2563
2564void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) {
2565#ifndef NDEBUG
2566 for (const CaseCluster &CC : Clusters)
2567 assert(CC.Low == CC.High && "Input clusters must be single-case")(static_cast <bool> (CC.Low == CC.High && "Input clusters must be single-case"
) ? void (0) : __assert_fail ("CC.Low == CC.High && \"Input clusters must be single-case\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2567, __extension__ __PRETTY_FUNCTION__));
2568#endif
2569
2570 llvm::sort(Clusters.begin(), Clusters.end(),
2571 [](const CaseCluster &a, const CaseCluster &b) {
2572 return a.Low->getValue().slt(b.Low->getValue());
2573 });
2574
2575 // Merge adjacent clusters with the same destination.
2576 const unsigned N = Clusters.size();
2577 unsigned DstIndex = 0;
2578 for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) {
2579 CaseCluster &CC = Clusters[SrcIndex];
2580 const ConstantInt *CaseVal = CC.Low;
2581 MachineBasicBlock *Succ = CC.MBB;
2582
2583 if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ &&
2584 (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) {
2585 // If this case has the same successor and is a neighbour, merge it into
2586 // the previous cluster.
2587 Clusters[DstIndex - 1].High = CaseVal;
2588 Clusters[DstIndex - 1].Prob += CC.Prob;
2589 } else {
2590 std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex],
2591 sizeof(Clusters[SrcIndex]));
2592 }
2593 }
2594 Clusters.resize(DstIndex);
2595}
2596
2597void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First,
2598 MachineBasicBlock *Last) {
2599 // Update JTCases.
2600 for (unsigned i = 0, e = JTCases.size(); i != e; ++i)
2601 if (JTCases[i].first.HeaderBB == First)
2602 JTCases[i].first.HeaderBB = Last;
2603
2604 // Update BitTestCases.
2605 for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i)
2606 if (BitTestCases[i].Parent == First)
2607 BitTestCases[i].Parent = Last;
2608}
2609
2610void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
2611 MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
2612
2613 // Update machine-CFG edges with unique successors.
2614 SmallSet<BasicBlock*, 32> Done;
2615 for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
2616 BasicBlock *BB = I.getSuccessor(i);
2617 bool Inserted = Done.insert(BB).second;
2618 if (!Inserted)
2619 continue;
2620
2621 MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
2622 addSuccessorWithProb(IndirectBrMBB, Succ);
2623 }
2624 IndirectBrMBB->normalizeSuccProbs();
2625
2626 DAG.setRoot(DAG.getNode(ISD::BRIND, getCurSDLoc(),
2627 MVT::Other, getControlRoot(),
2628 getValue(I.getAddress())));
2629}
2630
2631void SelectionDAGBuilder::visitUnreachable(const UnreachableInst &I) {
2632 if (!DAG.getTarget().Options.TrapUnreachable)
2633 return;
2634
2635 // We may be able to ignore unreachable behind a noreturn call.
2636 if (DAG.getTarget().Options.NoTrapAfterNoreturn) {
2637 const BasicBlock &BB = *I.getParent();
2638 if (&I != &BB.front()) {
2639 BasicBlock::const_iterator PredI =
2640 std::prev(BasicBlock::const_iterator(&I));
2641 if (const CallInst *Call = dyn_cast<CallInst>(&*PredI)) {
2642 if (Call->doesNotReturn())
2643 return;
2644 }
2645 }
2646 }
2647
2648 DAG.setRoot(DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));
2649}
2650
2651void SelectionDAGBuilder::visitFSub(const User &I) {
2652 // -0.0 - X --> fneg
2653 Type *Ty = I.getType();
2654 if (isa<Constant>(I.getOperand(0)) &&
2655 I.getOperand(0) == ConstantFP::getZeroValueForNegation(Ty)) {
2656 SDValue Op2 = getValue(I.getOperand(1));
2657 setValue(&I, DAG.getNode(ISD::FNEG, getCurSDLoc(),
2658 Op2.getValueType(), Op2));
2659 return;
2660 }
2661
2662 visitBinary(I, ISD::FSUB);
2663}
2664
2665/// Checks if the given instruction performs a vector reduction, in which case
2666/// we have the freedom to alter the elements in the result as long as the
2667/// reduction of them stays unchanged.
2668static bool isVectorReductionOp(const User *I) {
2669 const Instruction *Inst = dyn_cast<Instruction>(I);
2670 if (!Inst || !Inst->getType()->isVectorTy())
2671 return false;
2672
2673 auto OpCode = Inst->getOpcode();
2674 switch (OpCode) {
2675 case Instruction::Add:
2676 case Instruction::Mul:
2677 case Instruction::And:
2678 case Instruction::Or:
2679 case Instruction::Xor:
2680 break;
2681 case Instruction::FAdd:
2682 case Instruction::FMul:
2683 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2684 if (FPOp->getFastMathFlags().isFast())
2685 break;
2686 LLVM_FALLTHROUGH[[clang::fallthrough]];
2687 default:
2688 return false;
2689 }
2690
2691 unsigned ElemNum = Inst->getType()->getVectorNumElements();
2692 // Ensure the reduction size is a power of 2.
2693 if (!isPowerOf2_32(ElemNum))
2694 return false;
2695
2696 unsigned ElemNumToReduce = ElemNum;
2697
2698 // Do DFS search on the def-use chain from the given instruction. We only
2699 // allow four kinds of operations during the search until we reach the
2700 // instruction that extracts the first element from the vector:
2701 //
2702 // 1. The reduction operation of the same opcode as the given instruction.
2703 //
2704 // 2. PHI node.
2705 //
2706 // 3. ShuffleVector instruction together with a reduction operation that
2707 // does a partial reduction.
2708 //
2709 // 4. ExtractElement that extracts the first element from the vector, and we
2710 // stop searching the def-use chain here.
2711 //
2712 // 3 & 4 above perform a reduction on all elements of the vector. We push defs
2713 // from 1-3 to the stack to continue the DFS. The given instruction is not
2714 // a reduction operation if we meet any other instructions other than those
2715 // listed above.
2716
2717 SmallVector<const User *, 16> UsersToVisit{Inst};
2718 SmallPtrSet<const User *, 16> Visited;
2719 bool ReduxExtracted = false;
2720
2721 while (!UsersToVisit.empty()) {
2722 auto User = UsersToVisit.back();
2723 UsersToVisit.pop_back();
2724 if (!Visited.insert(User).second)
2725 continue;
2726
2727 for (const auto &U : User->users()) {
2728 auto Inst = dyn_cast<Instruction>(U);
2729 if (!Inst)
2730 return false;
2731
2732 if (Inst->getOpcode() == OpCode || isa<PHINode>(U)) {
2733 if (const FPMathOperator *FPOp = dyn_cast<const FPMathOperator>(Inst))
2734 if (!isa<PHINode>(FPOp) && !FPOp->getFastMathFlags().isFast())
2735 return false;
2736 UsersToVisit.push_back(U);
2737 } else if (const ShuffleVectorInst *ShufInst =
2738 dyn_cast<ShuffleVectorInst>(U)) {
2739 // Detect the following pattern: A ShuffleVector instruction together
2740 // with a reduction that do partial reduction on the first and second
2741 // ElemNumToReduce / 2 elements, and store the result in
2742 // ElemNumToReduce / 2 elements in another vector.
2743
2744 unsigned ResultElements = ShufInst->getType()->getVectorNumElements();
2745 if (ResultElements < ElemNum)
2746 return false;
2747
2748 if (ElemNumToReduce == 1)
2749 return false;
2750 if (!isa<UndefValue>(U->getOperand(1)))
2751 return false;
2752 for (unsigned i = 0; i < ElemNumToReduce / 2; ++i)
2753 if (ShufInst->getMaskValue(i) != int(i + ElemNumToReduce / 2))
2754 return false;
2755 for (unsigned i = ElemNumToReduce / 2; i < ElemNum; ++i)
2756 if (ShufInst->getMaskValue(i) != -1)
2757 return false;
2758
2759 // There is only one user of this ShuffleVector instruction, which
2760 // must be a reduction operation.
2761 if (!U->hasOneUse())
2762 return false;
2763
2764 auto U2 = dyn_cast<Instruction>(*U->user_begin());
2765 if (!U2 || U2->getOpcode() != OpCode)
2766 return false;
2767
2768 // Check operands of the reduction operation.
2769 if ((U2->getOperand(0) == U->getOperand(0) && U2->getOperand(1) == U) ||
2770 (U2->getOperand(1) == U->getOperand(0) && U2->getOperand(0) == U)) {
2771 UsersToVisit.push_back(U2);
2772 ElemNumToReduce /= 2;
2773 } else
2774 return false;
2775 } else if (isa<ExtractElementInst>(U)) {
2776 // At this moment we should have reduced all elements in the vector.
2777 if (ElemNumToReduce != 1)
2778 return false;
2779
2780 const ConstantInt *Val = dyn_cast<ConstantInt>(U->getOperand(1));
2781 if (!Val || !Val->isZero())
2782 return false;
2783
2784 ReduxExtracted = true;
2785 } else
2786 return false;
2787 }
2788 }
2789 return ReduxExtracted;
2790}
2791
2792void SelectionDAGBuilder::visitBinary(const User &I, unsigned Opcode) {
2793 SDNodeFlags Flags;
2794 if (auto *OFBinOp = dyn_cast<OverflowingBinaryOperator>(&I)) {
2795 Flags.setNoSignedWrap(OFBinOp->hasNoSignedWrap());
2796 Flags.setNoUnsignedWrap(OFBinOp->hasNoUnsignedWrap());
2797 }
2798 if (auto *ExactOp = dyn_cast<PossiblyExactOperator>(&I)) {
2799 Flags.setExact(ExactOp->isExact());
2800 }
2801 if (isVectorReductionOp(&I)) {
2802 Flags.setVectorReduction(true);
2803 LLVM_DEBUG(dbgs() << "Detected a reduction operation:" << I << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Detected a reduction operation:"
<< I << "\n"; } } while (false);
2804 }
2805
2806 SDValue Op1 = getValue(I.getOperand(0));
2807 SDValue Op2 = getValue(I.getOperand(1));
2808 SDValue BinNodeValue = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(),
2809 Op1, Op2, Flags);
2810 setValue(&I, BinNodeValue);
2811}
2812
2813void SelectionDAGBuilder::visitShift(const User &I, unsigned Opcode) {
2814 SDValue Op1 = getValue(I.getOperand(0));
2815 SDValue Op2 = getValue(I.getOperand(1));
2816
2817 EVT ShiftTy = DAG.getTargetLoweringInfo().getShiftAmountTy(
2818 Op2.getValueType(), DAG.getDataLayout());
2819
2820 // Coerce the shift amount to the right type if we can.
2821 if (!I.getType()->isVectorTy() && Op2.getValueType() != ShiftTy) {
2822 unsigned ShiftSize = ShiftTy.getSizeInBits();
2823 unsigned Op2Size = Op2.getValueSizeInBits();
2824 SDLoc DL = getCurSDLoc();
2825
2826 // If the operand is smaller than the shift count type, promote it.
2827 if (ShiftSize > Op2Size)
2828 Op2 = DAG.getNode(ISD::ZERO_EXTEND, DL, ShiftTy, Op2);
2829
2830 // If the operand is larger than the shift count type but the shift
2831 // count type has enough bits to represent any shift value, truncate
2832 // it now. This is a common case and it exposes the truncate to
2833 // optimization early.
2834 else if (ShiftSize >= Log2_32_Ceil(Op2.getValueSizeInBits()))
2835 Op2 = DAG.getNode(ISD::TRUNCATE, DL, ShiftTy, Op2);
2836 // Otherwise we'll need to temporarily settle for some other convenient
2837 // type. Type legalization will make adjustments once the shiftee is split.
2838 else
2839 Op2 = DAG.getZExtOrTrunc(Op2, DL, MVT::i32);
2840 }
2841
2842 bool nuw = false;
2843 bool nsw = false;
2844 bool exact = false;
2845
2846 if (Opcode == ISD::SRL || Opcode == ISD::SRA || Opcode == ISD::SHL) {
2847
2848 if (const OverflowingBinaryOperator *OFBinOp =
2849 dyn_cast<const OverflowingBinaryOperator>(&I)) {
2850 nuw = OFBinOp->hasNoUnsignedWrap();
2851 nsw = OFBinOp->hasNoSignedWrap();
2852 }
2853 if (const PossiblyExactOperator *ExactOp =
2854 dyn_cast<const PossiblyExactOperator>(&I))
2855 exact = ExactOp->isExact();
2856 }
2857 SDNodeFlags Flags;
2858 Flags.setExact(exact);
2859 Flags.setNoSignedWrap(nsw);
2860 Flags.setNoUnsignedWrap(nuw);
2861 SDValue Res = DAG.getNode(Opcode, getCurSDLoc(), Op1.getValueType(), Op1, Op2,
2862 Flags);
2863 setValue(&I, Res);
2864}
2865
2866void SelectionDAGBuilder::visitSDiv(const User &I) {
2867 SDValue Op1 = getValue(I.getOperand(0));
2868 SDValue Op2 = getValue(I.getOperand(1));
2869
2870 SDNodeFlags Flags;
2871 Flags.setExact(isa<PossiblyExactOperator>(&I) &&
2872 cast<PossiblyExactOperator>(&I)->isExact());
2873 setValue(&I, DAG.getNode(ISD::SDIV, getCurSDLoc(), Op1.getValueType(), Op1,
2874 Op2, Flags));
2875}
2876
2877void SelectionDAGBuilder::visitICmp(const User &I) {
2878 ICmpInst::Predicate predicate = ICmpInst::BAD_ICMP_PREDICATE;
2879 if (const ICmpInst *IC = dyn_cast<ICmpInst>(&I))
2880 predicate = IC->getPredicate();
2881 else if (const ConstantExpr *IC = dyn_cast<ConstantExpr>(&I))
2882 predicate = ICmpInst::Predicate(IC->getPredicate());
2883 SDValue Op1 = getValue(I.getOperand(0));
2884 SDValue Op2 = getValue(I.getOperand(1));
2885 ISD::CondCode Opcode = getICmpCondCode(predicate);
2886
2887 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2888 I.getType());
2889 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode));
2890}
2891
2892void SelectionDAGBuilder::visitFCmp(const User &I) {
2893 FCmpInst::Predicate predicate = FCmpInst::BAD_FCMP_PREDICATE;
2894 if (const FCmpInst *FC = dyn_cast<FCmpInst>(&I))
2895 predicate = FC->getPredicate();
2896 else if (const ConstantExpr *FC = dyn_cast<ConstantExpr>(&I))
2897 predicate = FCmpInst::Predicate(FC->getPredicate());
2898 SDValue Op1 = getValue(I.getOperand(0));
2899 SDValue Op2 = getValue(I.getOperand(1));
2900
2901 ISD::CondCode Condition = getFCmpCondCode(predicate);
2902 auto *FPMO = dyn_cast<FPMathOperator>(&I);
2903 if ((FPMO && FPMO->hasNoNaNs()) || TM.Options.NoNaNsFPMath)
2904 Condition = getFCmpCodeWithoutNaN(Condition);
2905
2906 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
2907 I.getType());
2908 setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Condition));
2909}
2910
2911// Check if the condition of the select has one use or two users that are both
2912// selects with the same condition.
2913static bool hasOnlySelectUsers(const Value *Cond) {
2914 return llvm::all_of(Cond->users(), [](const Value *V) {
2915 return isa<SelectInst>(V);
2916 });
2917}
2918
2919void SelectionDAGBuilder::visitSelect(const User &I) {
2920 SmallVector<EVT, 4> ValueVTs;
2921 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), I.getType(),
2922 ValueVTs);
2923 unsigned NumValues = ValueVTs.size();
2924 if (NumValues == 0) return;
2925
2926 SmallVector<SDValue, 4> Values(NumValues);
2927 SDValue Cond = getValue(I.getOperand(0));
2928 SDValue LHSVal = getValue(I.getOperand(1));
2929 SDValue RHSVal = getValue(I.getOperand(2));
2930 auto BaseOps = {Cond};
2931 ISD::NodeType OpCode = Cond.getValueType().isVector() ?
2932 ISD::VSELECT : ISD::SELECT;
2933
2934 // Min/max matching is only viable if all output VTs are the same.
2935 if (std::equal(ValueVTs.begin(), ValueVTs.end(), ValueVTs.begin())) {
2936 EVT VT = ValueVTs[0];
2937 LLVMContext &Ctx = *DAG.getContext();
2938 auto &TLI = DAG.getTargetLoweringInfo();
2939
2940 // We care about the legality of the operation after it has been type
2941 // legalized.
2942 while (TLI.getTypeAction(Ctx, VT) != TargetLoweringBase::TypeLegal &&
2943 VT != TLI.getTypeToTransformTo(Ctx, VT))
2944 VT = TLI.getTypeToTransformTo(Ctx, VT);
2945
2946 // If the vselect is legal, assume we want to leave this as a vector setcc +
2947 // vselect. Otherwise, if this is going to be scalarized, we want to see if
2948 // min/max is legal on the scalar type.
2949 bool UseScalarMinMax = VT.isVector() &&
2950 !TLI.isOperationLegalOrCustom(ISD::VSELECT, VT);
2951
2952 Value *LHS, *RHS;
2953 auto SPR = matchSelectPattern(const_cast<User*>(&I), LHS, RHS);
2954 ISD::NodeType Opc = ISD::DELETED_NODE;
2955 switch (SPR.Flavor) {
2956 case SPF_UMAX: Opc = ISD::UMAX; break;
2957 case SPF_UMIN: Opc = ISD::UMIN; break;
2958 case SPF_SMAX: Opc = ISD::SMAX; break;
2959 case SPF_SMIN: Opc = ISD::SMIN; break;
2960 case SPF_FMINNUM:
2961 switch (SPR.NaNBehavior) {
2962 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2962);
2963 case SPNB_RETURNS_NAN: Opc = ISD::FMINNAN; break;
2964 case SPNB_RETURNS_OTHER: Opc = ISD::FMINNUM; break;
2965 case SPNB_RETURNS_ANY: {
2966 if (TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT))
2967 Opc = ISD::FMINNUM;
2968 else if (TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT))
2969 Opc = ISD::FMINNAN;
2970 else if (UseScalarMinMax)
2971 Opc = TLI.isOperationLegalOrCustom(ISD::FMINNUM, VT.getScalarType()) ?
2972 ISD::FMINNUM : ISD::FMINNAN;
2973 break;
2974 }
2975 }
2976 break;
2977 case SPF_FMAXNUM:
2978 switch (SPR.NaNBehavior) {
2979 case SPNB_NA: llvm_unreachable("No NaN behavior for FP op?")::llvm::llvm_unreachable_internal("No NaN behavior for FP op?"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 2979);
2980 case SPNB_RETURNS_NAN: Opc = ISD::FMAXNAN; break;
2981 case SPNB_RETURNS_OTHER: Opc = ISD::FMAXNUM; break;
2982 case SPNB_RETURNS_ANY:
2983
2984 if (TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT))
2985 Opc = ISD::FMAXNUM;
2986 else if (TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT))
2987 Opc = ISD::FMAXNAN;
2988 else if (UseScalarMinMax)
2989 Opc = TLI.isOperationLegalOrCustom(ISD::FMAXNUM, VT.getScalarType()) ?
2990 ISD::FMAXNUM : ISD::FMAXNAN;
2991 break;
2992 }
2993 break;
2994 default: break;
2995 }
2996
2997 if (Opc != ISD::DELETED_NODE &&
2998 (TLI.isOperationLegalOrCustom(Opc, VT) ||
2999 (UseScalarMinMax &&
3000 TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) &&
3001 // If the underlying comparison instruction is used by any other
3002 // instruction, the consumed instructions won't be destroyed, so it is
3003 // not profitable to convert to a min/max.
3004 hasOnlySelectUsers(cast<SelectInst>(I).getCondition())) {
3005 OpCode = Opc;
3006 LHSVal = getValue(LHS);
3007 RHSVal = getValue(RHS);
3008 BaseOps = {};
3009 }
3010 }
3011
3012 for (unsigned i = 0; i != NumValues; ++i) {
3013 SmallVector<SDValue, 3> Ops(BaseOps.begin(), BaseOps.end());
3014 Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i));
3015 Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i));
3016 Values[i] = DAG.getNode(OpCode, getCurSDLoc(),
3017 LHSVal.getNode()->getValueType(LHSVal.getResNo()+i),
3018 Ops);
3019 }
3020
3021 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3022 DAG.getVTList(ValueVTs), Values));
3023}
3024
3025void SelectionDAGBuilder::visitTrunc(const User &I) {
3026 // TruncInst cannot be a no-op cast because sizeof(src) > sizeof(dest).
3027 SDValue N = getValue(I.getOperand(0));
3028 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3029 I.getType());
3030 setValue(&I, DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), DestVT, N));
3031}
3032
3033void SelectionDAGBuilder::visitZExt(const User &I) {
3034 // ZExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3035 // ZExt also can't be a cast to bool for same reason. So, nothing much to do
3036 SDValue N = getValue(I.getOperand(0));
3037 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3038 I.getType());
3039 setValue(&I, DAG.getNode(ISD::ZERO_EXTEND, getCurSDLoc(), DestVT, N));
3040}
3041
3042void SelectionDAGBuilder::visitSExt(const User &I) {
3043 // SExt cannot be a no-op cast because sizeof(src) < sizeof(dest).
3044 // SExt also can't be a cast to bool for same reason. So, nothing much to do
3045 SDValue N = getValue(I.getOperand(0));
3046 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3047 I.getType());
3048 setValue(&I, DAG.getNode(ISD::SIGN_EXTEND, getCurSDLoc(), DestVT, N));
3049}
3050
3051void SelectionDAGBuilder::visitFPTrunc(const User &I) {
3052 // FPTrunc is never a no-op cast, no need to check
3053 SDValue N = getValue(I.getOperand(0));
3054 SDLoc dl = getCurSDLoc();
3055 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3056 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3057 setValue(&I, DAG.getNode(ISD::FP_ROUND, dl, DestVT, N,
3058 DAG.getTargetConstant(
3059 0, dl, TLI.getPointerTy(DAG.getDataLayout()))));
3060}
3061
3062void SelectionDAGBuilder::visitFPExt(const User &I) {
3063 // FPExt is never a no-op cast, no need to check
3064 SDValue N = getValue(I.getOperand(0));
3065 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3066 I.getType());
3067 setValue(&I, DAG.getNode(ISD::FP_EXTEND, getCurSDLoc(), DestVT, N));
3068}
3069
3070void SelectionDAGBuilder::visitFPToUI(const User &I) {
3071 // FPToUI is never a no-op cast, no need to check
3072 SDValue N = getValue(I.getOperand(0));
3073 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3074 I.getType());
3075 setValue(&I, DAG.getNode(ISD::FP_TO_UINT, getCurSDLoc(), DestVT, N));
3076}
3077
3078void SelectionDAGBuilder::visitFPToSI(const User &I) {
3079 // FPToSI is never a no-op cast, no need to check
3080 SDValue N = getValue(I.getOperand(0));
3081 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3082 I.getType());
3083 setValue(&I, DAG.getNode(ISD::FP_TO_SINT, getCurSDLoc(), DestVT, N));
3084}
3085
3086void SelectionDAGBuilder::visitUIToFP(const User &I) {
3087 // UIToFP is never a no-op cast, no need to check
3088 SDValue N = getValue(I.getOperand(0));
3089 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3090 I.getType());
3091 setValue(&I, DAG.getNode(ISD::UINT_TO_FP, getCurSDLoc(), DestVT, N));
3092}
3093
3094void SelectionDAGBuilder::visitSIToFP(const User &I) {
3095 // SIToFP is never a no-op cast, no need to check
3096 SDValue N = getValue(I.getOperand(0));
3097 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3098 I.getType());
3099 setValue(&I, DAG.getNode(ISD::SINT_TO_FP, getCurSDLoc(), DestVT, N));
3100}
3101
3102void SelectionDAGBuilder::visitPtrToInt(const User &I) {
3103 // What to do depends on the size of the integer and the size of the pointer.
3104 // We can either truncate, zero extend, or no-op, accordingly.
3105 SDValue N = getValue(I.getOperand(0));
3106 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3107 I.getType());
3108 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3109}
3110
3111void SelectionDAGBuilder::visitIntToPtr(const User &I) {
3112 // What to do depends on the size of the integer and the size of the pointer.
3113 // We can either truncate, zero extend, or no-op, accordingly.
3114 SDValue N = getValue(I.getOperand(0));
3115 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3116 I.getType());
3117 setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT));
3118}
3119
3120void SelectionDAGBuilder::visitBitCast(const User &I) {
3121 SDValue N = getValue(I.getOperand(0));
3122 SDLoc dl = getCurSDLoc();
3123 EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
3124 I.getType());
3125
3126 // BitCast assures us that source and destination are the same size so this is
3127 // either a BITCAST or a no-op.
3128 if (DestVT != N.getValueType())
3129 setValue(&I, DAG.getNode(ISD::BITCAST, dl,
3130 DestVT, N)); // convert types.
3131 // Check if the original LLVM IR Operand was a ConstantInt, because getValue()
3132 // might fold any kind of constant expression to an integer constant and that
3133 // is not what we are looking for. Only recognize a bitcast of a genuine
3134 // constant integer as an opaque constant.
3135 else if(ConstantInt *C = dyn_cast<ConstantInt>(I.getOperand(0)))
3136 setValue(&I, DAG.getConstant(C->getValue(), dl, DestVT, /*isTarget=*/false,
3137 /*isOpaque*/true));
3138 else
3139 setValue(&I, N); // noop cast.
3140}
3141
3142void SelectionDAGBuilder::visitAddrSpaceCast(const User &I) {
3143 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3144 const Value *SV = I.getOperand(0);
3145 SDValue N = getValue(SV);
3146 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3147
3148 unsigned SrcAS = SV->getType()->getPointerAddressSpace();
3149 unsigned DestAS = I.getType()->getPointerAddressSpace();
3150
3151 if (!TLI.isNoopAddrSpaceCast(SrcAS, DestAS))
3152 N = DAG.getAddrSpaceCast(getCurSDLoc(), DestVT, N, SrcAS, DestAS);
3153
3154 setValue(&I, N);
3155}
3156
3157void SelectionDAGBuilder::visitInsertElement(const User &I) {
3158 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3159 SDValue InVec = getValue(I.getOperand(0));
3160 SDValue InVal = getValue(I.getOperand(1));
3161 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(2)), getCurSDLoc(),
3162 TLI.getVectorIdxTy(DAG.getDataLayout()));
3163 setValue(&I, DAG.getNode(ISD::INSERT_VECTOR_ELT, getCurSDLoc(),
3164 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3165 InVec, InVal, InIdx));
3166}
3167
3168void SelectionDAGBuilder::visitExtractElement(const User &I) {
3169 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3170 SDValue InVec = getValue(I.getOperand(0));
3171 SDValue InIdx = DAG.getSExtOrTrunc(getValue(I.getOperand(1)), getCurSDLoc(),
3172 TLI.getVectorIdxTy(DAG.getDataLayout()));
3173 setValue(&I, DAG.getNode(ISD::EXTRACT_VECTOR_ELT, getCurSDLoc(),
3174 TLI.getValueType(DAG.getDataLayout(), I.getType()),
3175 InVec, InIdx));
3176}
3177
3178void SelectionDAGBuilder::visitShuffleVector(const User &I) {
3179 SDValue Src1 = getValue(I.getOperand(0));
3180 SDValue Src2 = getValue(I.getOperand(1));
3181 SDLoc DL = getCurSDLoc();
3182
3183 SmallVector<int, 8> Mask;
3184 ShuffleVectorInst::getShuffleMask(cast<Constant>(I.getOperand(2)), Mask);
3185 unsigned MaskNumElts = Mask.size();
3186
3187 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3188 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
3189 EVT SrcVT = Src1.getValueType();
3190 unsigned SrcNumElts = SrcVT.getVectorNumElements();
3191
3192 if (SrcNumElts == MaskNumElts) {
3193 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, Mask));
3194 return;
3195 }
3196
3197 // Normalize the shuffle vector since mask and vector length don't match.
3198 if (SrcNumElts < MaskNumElts) {
3199 // Mask is longer than the source vectors. We can use concatenate vector to
3200 // make the mask and vectors lengths match.
3201
3202 if (MaskNumElts % SrcNumElts == 0) {
3203 // Mask length is a multiple of the source vector length.
3204 // Check if the shuffle is some kind of concatenation of the input
3205 // vectors.
3206 unsigned NumConcat = MaskNumElts / SrcNumElts;
3207 bool IsConcat = true;
3208 SmallVector<int, 8> ConcatSrcs(NumConcat, -1);
3209 for (unsigned i = 0; i != MaskNumElts; ++i) {
3210 int Idx = Mask[i];
3211 if (Idx < 0)
3212 continue;
3213 // Ensure the indices in each SrcVT sized piece are sequential and that
3214 // the same source is used for the whole piece.
3215 if ((Idx % SrcNumElts != (i % SrcNumElts)) ||
3216 (ConcatSrcs[i / SrcNumElts] >= 0 &&
3217 ConcatSrcs[i / SrcNumElts] != (int)(Idx / SrcNumElts))) {
3218 IsConcat = false;
3219 break;
3220 }
3221 // Remember which source this index came from.
3222 ConcatSrcs[i / SrcNumElts] = Idx / SrcNumElts;
3223 }
3224
3225 // The shuffle is concatenating multiple vectors together. Just emit
3226 // a CONCAT_VECTORS operation.
3227 if (IsConcat) {
3228 SmallVector<SDValue, 8> ConcatOps;
3229 for (auto Src : ConcatSrcs) {
3230 if (Src < 0)
3231 ConcatOps.push_back(DAG.getUNDEF(SrcVT));
3232 else if (Src == 0)
3233 ConcatOps.push_back(Src1);
3234 else
3235 ConcatOps.push_back(Src2);
3236 }
3237 setValue(&I, DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ConcatOps));
3238 return;
3239 }
3240 }
3241
3242 unsigned PaddedMaskNumElts = alignTo(MaskNumElts, SrcNumElts);
3243 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
3244 EVT PaddedVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(),
3245 PaddedMaskNumElts);
3246
3247 // Pad both vectors with undefs to make them the same length as the mask.
3248 SDValue UndefVal = DAG.getUNDEF(SrcVT);
3249
3250 SmallVector<SDValue, 8> MOps1(NumConcat, UndefVal);
3251 SmallVector<SDValue, 8> MOps2(NumConcat, UndefVal);
3252 MOps1[0] = Src1;
3253 MOps2[0] = Src2;
3254
3255 Src1 = Src1.isUndef()
3256 ? DAG.getUNDEF(PaddedVT)
3257 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1);
3258 Src2 = Src2.isUndef()
3259 ? DAG.getUNDEF(PaddedVT)
3260 : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2);
3261
3262 // Readjust mask for new input vector length.
3263 SmallVector<int, 8> MappedOps(PaddedMaskNumElts, -1);
3264 for (unsigned i = 0; i != MaskNumElts; ++i) {
3265 int Idx = Mask[i];
3266 if (Idx >= (int)SrcNumElts)
3267 Idx -= SrcNumElts - PaddedMaskNumElts;
3268 MappedOps[i] = Idx;
3269 }
3270
3271 SDValue Result = DAG.getVectorShuffle(PaddedVT, DL, Src1, Src2, MappedOps);
3272
3273 // If the concatenated vector was padded, extract a subvector with the
3274 // correct number of elements.
3275 if (MaskNumElts != PaddedMaskNumElts)
3276 Result = DAG.getNode(
3277 ISD::EXTRACT_SUBVECTOR, DL, VT, Result,
3278 DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())));
3279
3280 setValue(&I, Result);
3281 return;
3282 }
3283
3284 if (SrcNumElts > MaskNumElts) {
3285 // Analyze the access pattern of the vector to see if we can extract
3286 // two subvectors and do the shuffle.
3287 int StartIdx[2] = { -1, -1 }; // StartIdx to extract from
3288 bool CanExtract = true;
3289 for (int Idx : Mask) {
3290 unsigned Input = 0;
3291 if (Idx < 0)
3292 continue;
3293
3294 if (Idx >= (int)SrcNumElts) {
3295 Input = 1;
3296 Idx -= SrcNumElts;
3297 }
3298
3299 // If all the indices come from the same MaskNumElts sized portion of
3300 // the sources we can use extract. Also make sure the extract wouldn't
3301 // extract past the end of the source.
3302 int NewStartIdx = alignDown(Idx, MaskNumElts);
3303 if (NewStartIdx + MaskNumElts > SrcNumElts ||
3304 (StartIdx[Input] >= 0 && StartIdx[Input] != NewStartIdx))
3305 CanExtract = false;
3306 // Make sure we always update StartIdx as we use it to track if all
3307 // elements are undef.
3308 StartIdx[Input] = NewStartIdx;
3309 }
3310
3311 if (StartIdx[0] < 0 && StartIdx[1] < 0) {
3312 setValue(&I, DAG.getUNDEF(VT)); // Vectors are not used.
3313 return;
3314 }
3315 if (CanExtract) {
3316 // Extract appropriate subvector and generate a vector shuffle
3317 for (unsigned Input = 0; Input < 2; ++Input) {
3318 SDValue &Src = Input == 0 ? Src1 : Src2;
3319 if (StartIdx[Input] < 0)
3320 Src = DAG.getUNDEF(VT);
3321 else {
3322 Src = DAG.getNode(
3323 ISD::EXTRACT_SUBVECTOR, DL, VT, Src,
3324 DAG.getConstant(StartIdx[Input], DL,
3325 TLI.getVectorIdxTy(DAG.getDataLayout())));
3326 }
3327 }
3328
3329 // Calculate new mask.
3330 SmallVector<int, 8> MappedOps(Mask.begin(), Mask.end());
3331 for (int &Idx : MappedOps) {
3332 if (Idx >= (int)SrcNumElts)
3333 Idx -= SrcNumElts + StartIdx[1] - MaskNumElts;
3334 else if (Idx >= 0)
3335 Idx -= StartIdx[0];
3336 }
3337
3338 setValue(&I, DAG.getVectorShuffle(VT, DL, Src1, Src2, MappedOps));
3339 return;
3340 }
3341 }
3342
3343 // We can't use either concat vectors or extract subvectors so fall back to
3344 // replacing the shuffle with extract and build vector.
3345 // to insert and build vector.
3346 EVT EltVT = VT.getVectorElementType();
3347 EVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
3348 SmallVector<SDValue,8> Ops;
3349 for (int Idx : Mask) {
3350 SDValue Res;
3351
3352 if (Idx < 0) {
3353 Res = DAG.getUNDEF(EltVT);
3354 } else {
3355 SDValue &Src = Idx < (int)SrcNumElts ? Src1 : Src2;
3356 if (Idx >= (int)SrcNumElts) Idx -= SrcNumElts;
3357
3358 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
3359 EltVT, Src, DAG.getConstant(Idx, DL, IdxVT));
3360 }
3361
3362 Ops.push_back(Res);
3363 }
3364
3365 setValue(&I, DAG.getBuildVector(VT, DL, Ops));
3366}
3367
3368void SelectionDAGBuilder::visitInsertValue(const User &I) {
3369 ArrayRef<unsigned> Indices;
3370 if (const InsertValueInst *IV = dyn_cast<InsertValueInst>(&I))
3371 Indices = IV->getIndices();
3372 else
3373 Indices = cast<ConstantExpr>(&I)->getIndices();
3374
3375 const Value *Op0 = I.getOperand(0);
3376 const Value *Op1 = I.getOperand(1);
3377 Type *AggTy = I.getType();
3378 Type *ValTy = Op1->getType();
3379 bool IntoUndef = isa<UndefValue>(Op0);
3380 bool FromUndef = isa<UndefValue>(Op1);
3381
3382 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3383
3384 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3385 SmallVector<EVT, 4> AggValueVTs;
3386 ComputeValueVTs(TLI, DAG.getDataLayout(), AggTy, AggValueVTs);
3387 SmallVector<EVT, 4> ValValueVTs;
3388 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3389
3390 unsigned NumAggValues = AggValueVTs.size();
3391 unsigned NumValValues = ValValueVTs.size();
3392 SmallVector<SDValue, 4> Values(NumAggValues);
3393
3394 // Ignore an insertvalue that produces an empty object
3395 if (!NumAggValues) {
3396 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3397 return;
3398 }
3399
3400 SDValue Agg = getValue(Op0);
3401 unsigned i = 0;
3402 // Copy the beginning value(s) from the original aggregate.
3403 for (; i != LinearIndex; ++i)
3404 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3405 SDValue(Agg.getNode(), Agg.getResNo() + i);
3406 // Copy values from the inserted value(s).
3407 if (NumValValues) {
3408 SDValue Val = getValue(Op1);
3409 for (; i != LinearIndex + NumValValues; ++i)
3410 Values[i] = FromUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3411 SDValue(Val.getNode(), Val.getResNo() + i - LinearIndex);
3412 }
3413 // Copy remaining value(s) from the original aggregate.
3414 for (; i != NumAggValues; ++i)
3415 Values[i] = IntoUndef ? DAG.getUNDEF(AggValueVTs[i]) :
3416 SDValue(Agg.getNode(), Agg.getResNo() + i);
3417
3418 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3419 DAG.getVTList(AggValueVTs), Values));
3420}
3421
3422void SelectionDAGBuilder::visitExtractValue(const User &I) {
3423 ArrayRef<unsigned> Indices;
3424 if (const ExtractValueInst *EV = dyn_cast<ExtractValueInst>(&I))
3425 Indices = EV->getIndices();
3426 else
3427 Indices = cast<ConstantExpr>(&I)->getIndices();
3428
3429 const Value *Op0 = I.getOperand(0);
3430 Type *AggTy = Op0->getType();
3431 Type *ValTy = I.getType();
3432 bool OutOfUndef = isa<UndefValue>(Op0);
3433
3434 unsigned LinearIndex = ComputeLinearIndex(AggTy, Indices);
3435
3436 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3437 SmallVector<EVT, 4> ValValueVTs;
3438 ComputeValueVTs(TLI, DAG.getDataLayout(), ValTy, ValValueVTs);
3439
3440 unsigned NumValValues = ValValueVTs.size();
3441
3442 // Ignore a extractvalue that produces an empty object
3443 if (!NumValValues) {
3444 setValue(&I, DAG.getUNDEF(MVT(MVT::Other)));
3445 return;
3446 }
3447
3448 SmallVector<SDValue, 4> Values(NumValValues);
3449
3450 SDValue Agg = getValue(Op0);
3451 // Copy out the selected value(s).
3452 for (unsigned i = LinearIndex; i != LinearIndex + NumValValues; ++i)
3453 Values[i - LinearIndex] =
3454 OutOfUndef ?
3455 DAG.getUNDEF(Agg.getNode()->getValueType(Agg.getResNo() + i)) :
3456 SDValue(Agg.getNode(), Agg.getResNo() + i);
3457
3458 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(),
3459 DAG.getVTList(ValValueVTs), Values));
3460}
3461
3462void SelectionDAGBuilder::visitGetElementPtr(const User &I) {
3463 Value *Op0 = I.getOperand(0);
3464 // Note that the pointer operand may be a vector of pointers. Take the scalar
3465 // element which holds a pointer.
3466 unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace();
3467 SDValue N = getValue(Op0);
3468 SDLoc dl = getCurSDLoc();
3469
3470 // Normalize Vector GEP - all scalar operands should be converted to the
3471 // splat vector.
3472 unsigned VectorWidth = I.getType()->isVectorTy() ?
3473 cast<VectorType>(I.getType())->getVectorNumElements() : 0;
3474
3475 if (VectorWidth && !N.getValueType().isVector()) {
3476 LLVMContext &Context = *DAG.getContext();
3477 EVT VT = EVT::getVectorVT(Context, N.getValueType(), VectorWidth);
3478 N = DAG.getSplatBuildVector(VT, dl, N);
3479 }
3480
3481 for (gep_type_iterator GTI = gep_type_begin(&I), E = gep_type_end(&I);
3482 GTI != E; ++GTI) {
3483 const Value *Idx = GTI.getOperand();
3484 if (StructType *StTy = GTI.getStructTypeOrNull()) {
3485 unsigned Field = cast<Constant>(Idx)->getUniqueInteger().getZExtValue();
3486 if (Field) {
3487 // N = N + Offset
3488 uint64_t Offset = DL->getStructLayout(StTy)->getElementOffset(Field);
3489
3490 // In an inbounds GEP with an offset that is nonnegative even when
3491 // interpreted as signed, assume there is no unsigned overflow.
3492 SDNodeFlags Flags;
3493 if (int64_t(Offset) >= 0 && cast<GEPOperator>(I).isInBounds())
3494 Flags.setNoUnsignedWrap(true);
3495
3496 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N,
3497 DAG.getConstant(Offset, dl, N.getValueType()), Flags);
3498 }
3499 } else {
3500 unsigned IdxSize = DAG.getDataLayout().getIndexSizeInBits(AS);
3501 MVT IdxTy = MVT::getIntegerVT(IdxSize);
3502 APInt ElementSize(IdxSize, DL->getTypeAllocSize(GTI.getIndexedType()));
3503
3504 // If this is a scalar constant or a splat vector of constants,
3505 // handle it quickly.
3506 const auto *CI = dyn_cast<ConstantInt>(Idx);
3507 if (!CI && isa<ConstantDataVector>(Idx) &&
3508 cast<ConstantDataVector>(Idx)->getSplatValue())
3509 CI = cast<ConstantInt>(cast<ConstantDataVector>(Idx)->getSplatValue());
3510
3511 if (CI) {
3512 if (CI->isZero())
3513 continue;
3514 APInt Offs = ElementSize * CI->getValue().sextOrTrunc(IdxSize);
3515 LLVMContext &Context = *DAG.getContext();
3516 SDValue OffsVal = VectorWidth ?
3517 DAG.getConstant(Offs, dl, EVT::getVectorVT(Context, IdxTy, VectorWidth)) :
3518 DAG.getConstant(Offs, dl, IdxTy);
3519
3520 // In an inbouds GEP with an offset that is nonnegative even when
3521 // interpreted as signed, assume there is no unsigned overflow.
3522 SDNodeFlags Flags;
3523 if (Offs.isNonNegative() && cast<GEPOperator>(I).isInBounds())
3524 Flags.setNoUnsignedWrap(true);
3525
3526 N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags);
3527 continue;
3528 }
3529
3530 // N = N + Idx * ElementSize;
3531 SDValue IdxN = getValue(Idx);
3532
3533 if (!IdxN.getValueType().isVector() && VectorWidth) {
3534 EVT VT = EVT::getVectorVT(*Context, IdxN.getValueType(), VectorWidth);
3535 IdxN = DAG.getSplatBuildVector(VT, dl, IdxN);
3536 }
3537
3538 // If the index is smaller or larger than intptr_t, truncate or extend
3539 // it.
3540 IdxN = DAG.getSExtOrTrunc(IdxN, dl, N.getValueType());
3541
3542 // If this is a multiply by a power of two, turn it into a shl
3543 // immediately. This is a very common case.
3544 if (ElementSize != 1) {
3545 if (ElementSize.isPowerOf2()) {
3546 unsigned Amt = ElementSize.logBase2();
3547 IdxN = DAG.getNode(ISD::SHL, dl,
3548 N.getValueType(), IdxN,
3549 DAG.getConstant(Amt, dl, IdxN.getValueType()));
3550 } else {
3551 SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType());
3552 IdxN = DAG.getNode(ISD::MUL, dl,
3553 N.getValueType(), IdxN, Scale);
3554 }
3555 }
3556
3557 N = DAG.getNode(ISD::ADD, dl,
3558 N.getValueType(), N, IdxN);
3559 }
3560 }
3561
3562 setValue(&I, N);
3563}
3564
3565void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
3566 // If this is a fixed sized alloca in the entry block of the function,
3567 // allocate it statically on the stack.
3568 if (FuncInfo.StaticAllocaMap.count(&I))
3569 return; // getValue will auto-populate this.
3570
3571 SDLoc dl = getCurSDLoc();
3572 Type *Ty = I.getAllocatedType();
3573 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3574 auto &DL = DAG.getDataLayout();
3575 uint64_t TySize = DL.getTypeAllocSize(Ty);
3576 unsigned Align =
3577 std::max((unsigned)DL.getPrefTypeAlignment(Ty), I.getAlignment());
3578
3579 SDValue AllocSize = getValue(I.getArraySize());
3580
3581 EVT IntPtr = TLI.getPointerTy(DAG.getDataLayout(), DL.getAllocaAddrSpace());
3582 if (AllocSize.getValueType() != IntPtr)
3583 AllocSize = DAG.getZExtOrTrunc(AllocSize, dl, IntPtr);
3584
3585 AllocSize = DAG.getNode(ISD::MUL, dl, IntPtr,
3586 AllocSize,
3587 DAG.getConstant(TySize, dl, IntPtr));
3588
3589 // Handle alignment. If the requested alignment is less than or equal to
3590 // the stack alignment, ignore it. If the size is greater than or equal to
3591 // the stack alignment, we note this in the DYNAMIC_STACKALLOC node.
3592 unsigned StackAlign =
3593 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
3594 if (Align <= StackAlign)
3595 Align = 0;
3596
3597 // Round the size of the allocation up to the stack alignment size
3598 // by add SA-1 to the size. This doesn't overflow because we're computing
3599 // an address inside an alloca.
3600 SDNodeFlags Flags;
3601 Flags.setNoUnsignedWrap(true);
3602 AllocSize = DAG.getNode(ISD::ADD, dl, AllocSize.getValueType(), AllocSize,
3603 DAG.getConstant(StackAlign - 1, dl, IntPtr), Flags);
3604
3605 // Mask out the low bits for alignment purposes.
3606 AllocSize =
3607 DAG.getNode(ISD::AND, dl, AllocSize.getValueType(), AllocSize,
3608 DAG.getConstant(~(uint64_t)(StackAlign - 1), dl, IntPtr));
3609
3610 SDValue Ops[] = {getRoot(), AllocSize, DAG.getConstant(Align, dl, IntPtr)};
3611 SDVTList VTs = DAG.getVTList(AllocSize.getValueType(), MVT::Other);
3612 SDValue DSA = DAG.getNode(ISD::DYNAMIC_STACKALLOC, dl, VTs, Ops);
3613 setValue(&I, DSA);
3614 DAG.setRoot(DSA.getValue(1));
3615
3616 assert(FuncInfo.MF->getFrameInfo().hasVarSizedObjects())(static_cast <bool> (FuncInfo.MF->getFrameInfo().hasVarSizedObjects
()) ? void (0) : __assert_fail ("FuncInfo.MF->getFrameInfo().hasVarSizedObjects()"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3616, __extension__ __PRETTY_FUNCTION__));
3617}
3618
3619void SelectionDAGBuilder::visitLoad(const LoadInst &I) {
3620 if (I.isAtomic())
3621 return visitAtomicLoad(I);
3622
3623 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3624 const Value *SV = I.getOperand(0);
3625 if (TLI.supportSwiftError()) {
3626 // Swifterror values can come from either a function parameter with
3627 // swifterror attribute or an alloca with swifterror attribute.
3628 if (const Argument *Arg = dyn_cast<Argument>(SV)) {
3629 if (Arg->hasSwiftErrorAttr())
3630 return visitLoadFromSwiftError(I);
3631 }
3632
3633 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(SV)) {
3634 if (Alloca->isSwiftError())
3635 return visitLoadFromSwiftError(I);
3636 }
3637 }
3638
3639 SDValue Ptr = getValue(SV);
3640
3641 Type *Ty = I.getType();
3642
3643 bool isVolatile = I.isVolatile();
3644 bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr;
3645 bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr;
3646 bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout());
3647 unsigned Alignment = I.getAlignment();
3648
3649 AAMDNodes AAInfo;
3650 I.getAAMetadata(AAInfo);
3651 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
3652
3653 SmallVector<EVT, 4> ValueVTs;
3654 SmallVector<uint64_t, 4> Offsets;
3655 ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets);
3656 unsigned NumValues = ValueVTs.size();
3657 if (NumValues == 0)
3658 return;
3659
3660 SDValue Root;
3661 bool ConstantMemory = false;
3662 if (isVolatile || NumValues > MaxParallelChains)
3663 // Serialize volatile loads with other side effects.
3664 Root = getRoot();
3665 else if (AA && AA->pointsToConstantMemory(MemoryLocation(
3666 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) {
3667 // Do not serialize (non-volatile) loads of constant memory with anything.
3668 Root = DAG.getEntryNode();
3669 ConstantMemory = true;
3670 } else {
3671 // Do not serialize non-volatile loads against each other.
3672 Root = DAG.getRoot();
3673 }
3674
3675 SDLoc dl = getCurSDLoc();
3676
3677 if (isVolatile)
3678 Root = TLI.prepareVolatileOrAtomicLoad(Root, dl, DAG);
3679
3680 // An aggregate load cannot wrap around the address space, so offsets to its
3681 // parts don't wrap either.
3682 SDNodeFlags Flags;
3683 Flags.setNoUnsignedWrap(true);
3684
3685 SmallVector<SDValue, 4> Values(NumValues);
3686 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3687 EVT PtrVT = Ptr.getValueType();
3688 unsigned ChainI = 0;
3689 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3690 // Serializing loads here may result in excessive register pressure, and
3691 // TokenFactor places arbitrary choke points on the scheduler. SD scheduling
3692 // could recover a bit by hoisting nodes upward in the chain by recognizing
3693 // they are side-effect free or do not alias. The optimizer should really
3694 // avoid this case by converting large object/array copies to llvm.memcpy
3695 // (MaxParallelChains should always remain as failsafe).
3696 if (ChainI == MaxParallelChains) {
3697 assert(PendingLoads.empty() && "PendingLoads must be serialized first")(static_cast <bool> (PendingLoads.empty() && "PendingLoads must be serialized first"
) ? void (0) : __assert_fail ("PendingLoads.empty() && \"PendingLoads must be serialized first\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3697, __extension__ __PRETTY_FUNCTION__));
3698 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3699 makeArrayRef(Chains.data(), ChainI));
3700 Root = Chain;
3701 ChainI = 0;
3702 }
3703 SDValue A = DAG.getNode(ISD::ADD, dl,
3704 PtrVT, Ptr,
3705 DAG.getConstant(Offsets[i], dl, PtrVT),
3706 Flags);
3707 auto MMOFlags = MachineMemOperand::MONone;
3708 if (isVolatile)
3709 MMOFlags |= MachineMemOperand::MOVolatile;
3710 if (isNonTemporal)
3711 MMOFlags |= MachineMemOperand::MONonTemporal;
3712 if (isInvariant)
3713 MMOFlags |= MachineMemOperand::MOInvariant;
3714 if (isDereferenceable)
3715 MMOFlags |= MachineMemOperand::MODereferenceable;
3716 MMOFlags |= TLI.getMMOFlags(I);
3717
3718 SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A,
3719 MachinePointerInfo(SV, Offsets[i]), Alignment,
3720 MMOFlags, AAInfo, Ranges);
3721
3722 Values[i] = L;
3723 Chains[ChainI] = L.getValue(1);
3724 }
3725
3726 if (!ConstantMemory) {
3727 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3728 makeArrayRef(Chains.data(), ChainI));
3729 if (isVolatile)
3730 DAG.setRoot(Chain);
3731 else
3732 PendingLoads.push_back(Chain);
3733 }
3734
3735 setValue(&I, DAG.getNode(ISD::MERGE_VALUES, dl,
3736 DAG.getVTList(ValueVTs), Values));
3737}
3738
3739void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) {
3740 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3741, __extension__ __PRETTY_FUNCTION__))
3741 "call visitStoreToSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitStoreToSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitStoreToSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3741, __extension__ __PRETTY_FUNCTION__));
3742
3743 SmallVector<EVT, 4> ValueVTs;
3744 SmallVector<uint64_t, 4> Offsets;
3745 const Value *SrcV = I.getOperand(0);
3746 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3747 SrcV->getType(), ValueVTs, &Offsets);
3748 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3749, __extension__ __PRETTY_FUNCTION__))
3749 "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3749, __extension__ __PRETTY_FUNCTION__));
3750
3751 SDValue Src = getValue(SrcV);
3752 // Create a virtual register, then update the virtual register.
3753 unsigned VReg; bool CreatedVReg;
3754 std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I);
3755 // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue
3756 // Chain can be getRoot or getControlRoot.
3757 SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg,
3758 SDValue(Src.getNode(), Src.getResNo()));
3759 DAG.setRoot(CopyNode);
3760 if (CreatedVReg)
3761 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg);
3762}
3763
3764void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) {
3765 assert(DAG.getTargetLoweringInfo().supportSwiftError() &&(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3766, __extension__ __PRETTY_FUNCTION__))
3766 "call visitLoadFromSwiftError when backend supports swifterror")(static_cast <bool> (DAG.getTargetLoweringInfo().supportSwiftError
() && "call visitLoadFromSwiftError when backend supports swifterror"
) ? void (0) : __assert_fail ("DAG.getTargetLoweringInfo().supportSwiftError() && \"call visitLoadFromSwiftError when backend supports swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3766, __extension__ __PRETTY_FUNCTION__));
3767
3768 assert(!I.isVolatile() &&(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3771, __extension__ __PRETTY_FUNCTION__))
3769 I.getMetadata(LLVMContext::MD_nontemporal) == nullptr &&(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3771, __extension__ __PRETTY_FUNCTION__))
3770 I.getMetadata(LLVMContext::MD_invariant_load) == nullptr &&(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3771, __extension__ __PRETTY_FUNCTION__))
3771 "Support volatile, non temporal, invariant for load_from_swift_error")(static_cast <bool> (!I.isVolatile() && I.getMetadata
(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata
(LLVMContext::MD_invariant_load) == nullptr && "Support volatile, non temporal, invariant for load_from_swift_error"
) ? void (0) : __assert_fail ("!I.isVolatile() && I.getMetadata(LLVMContext::MD_nontemporal) == nullptr && I.getMetadata(LLVMContext::MD_invariant_load) == nullptr && \"Support volatile, non temporal, invariant for load_from_swift_error\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3771, __extension__ __PRETTY_FUNCTION__));
3772
3773 const Value *SV = I.getOperand(0);
3774 Type *Ty = I.getType();
3775 AAMDNodes AAInfo;
3776 I.getAAMetadata(AAInfo);
3777 assert((!AA || !AA->pointsToConstantMemory(MemoryLocation((static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty)
, AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3779, __extension__ __PRETTY_FUNCTION__))
3778 SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) &&(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty)
, AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3779, __extension__ __PRETTY_FUNCTION__))
3779 "load_from_swift_error should not be constant memory")(static_cast <bool> ((!AA || !AA->pointsToConstantMemory
(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty)
, AAInfo))) && "load_from_swift_error should not be constant memory"
) ? void (0) : __assert_fail ("(!AA || !AA->pointsToConstantMemory(MemoryLocation( SV, DAG.getDataLayout().getTypeStoreSize(Ty), AAInfo))) && \"load_from_swift_error should not be constant memory\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3779, __extension__ __PRETTY_FUNCTION__));
3780
3781 SmallVector<EVT, 4> ValueVTs;
3782 SmallVector<uint64_t, 4> Offsets;
3783 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Ty,
3784 ValueVTs, &Offsets);
3785 assert(ValueVTs.size() == 1 && Offsets[0] == 0 &&(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3786, __extension__ __PRETTY_FUNCTION__))
3786 "expect a single EVT for swifterror")(static_cast <bool> (ValueVTs.size() == 1 && Offsets
[0] == 0 && "expect a single EVT for swifterror") ? void
(0) : __assert_fail ("ValueVTs.size() == 1 && Offsets[0] == 0 && \"expect a single EVT for swifterror\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3786, __extension__ __PRETTY_FUNCTION__));
3787
3788 // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT
3789 SDValue L = DAG.getCopyFromReg(
3790 getRoot(), getCurSDLoc(),
3791 FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first,
3792 ValueVTs[0]);
3793
3794 setValue(&I, L);
3795}
3796
3797void SelectionDAGBuilder::visitStore(const StoreInst &I) {
3798 if (I.isAtomic())
3799 return visitAtomicStore(I);
3800
3801 const Value *SrcV = I.getOperand(0);
3802 const Value *PtrV = I.getOperand(1);
3803
3804 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3805 if (TLI.supportSwiftError()) {
3806 // Swifterror values can come from either a function parameter with
3807 // swifterror attribute or an alloca with swifterror attribute.
3808 if (const Argument *Arg = dyn_cast<Argument>(PtrV)) {
3809 if (Arg->hasSwiftErrorAttr())
3810 return visitStoreToSwiftError(I);
3811 }
3812
3813 if (const AllocaInst *Alloca = dyn_cast<AllocaInst>(PtrV)) {
3814 if (Alloca->isSwiftError())
3815 return visitStoreToSwiftError(I);
3816 }
3817 }
3818
3819 SmallVector<EVT, 4> ValueVTs;
3820 SmallVector<uint64_t, 4> Offsets;
3821 ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(),
3822 SrcV->getType(), ValueVTs, &Offsets);
3823 unsigned NumValues = ValueVTs.size();
3824 if (NumValues == 0)
3825 return;
3826
3827 // Get the lowered operands. Note that we do this after
3828 // checking if NumResults is zero, because with zero results
3829 // the operands won't have values in the map.
3830 SDValue Src = getValue(SrcV);
3831 SDValue Ptr = getValue(PtrV);
3832
3833 SDValue Root = getRoot();
3834 SmallVector<SDValue, 4> Chains(std::min(MaxParallelChains, NumValues));
3835 SDLoc dl = getCurSDLoc();
3836 EVT PtrVT = Ptr.getValueType();
3837 unsigned Alignment = I.getAlignment();
3838 AAMDNodes AAInfo;
3839 I.getAAMetadata(AAInfo);
3840
3841 auto MMOFlags = MachineMemOperand::MONone;
3842 if (I.isVolatile())
3843 MMOFlags |= MachineMemOperand::MOVolatile;
3844 if (I.getMetadata(LLVMContext::MD_nontemporal) != nullptr)
3845 MMOFlags |= MachineMemOperand::MONonTemporal;
3846 MMOFlags |= TLI.getMMOFlags(I);
3847
3848 // An aggregate load cannot wrap around the address space, so offsets to its
3849 // parts don't wrap either.
3850 SDNodeFlags Flags;
3851 Flags.setNoUnsignedWrap(true);
3852
3853 unsigned ChainI = 0;
3854 for (unsigned i = 0; i != NumValues; ++i, ++ChainI) {
3855 // See visitLoad comments.
3856 if (ChainI == MaxParallelChains) {
3857 SDValue Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3858 makeArrayRef(Chains.data(), ChainI));
3859 Root = Chain;
3860 ChainI = 0;
3861 }
3862 SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr,
3863 DAG.getConstant(Offsets[i], dl, PtrVT), Flags);
3864 SDValue St = DAG.getStore(
3865 Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add,
3866 MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo);
3867 Chains[ChainI] = St;
3868 }
3869
3870 SDValue StoreNode = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
3871 makeArrayRef(Chains.data(), ChainI));
3872 DAG.setRoot(StoreNode);
3873}
3874
3875void SelectionDAGBuilder::visitMaskedStore(const CallInst &I,
3876 bool IsCompressing) {
3877 SDLoc sdl = getCurSDLoc();
3878
3879 auto getMaskedStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3880 unsigned& Alignment) {
3881 // llvm.masked.store.*(Src0, Ptr, alignment, Mask)
3882 Src0 = I.getArgOperand(0);
3883 Ptr = I.getArgOperand(1);
3884 Alignment = cast<ConstantInt>(I.getArgOperand(2))->getZExtValue();
3885 Mask = I.getArgOperand(3);
3886 };
3887 auto getCompressingStoreOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
3888 unsigned& Alignment) {
3889 // llvm.masked.compressstore.*(Src0, Ptr, Mask)
3890 Src0 = I.getArgOperand(0);
3891 Ptr = I.getArgOperand(1);
3892 Mask = I.getArgOperand(2);
3893 Alignment = 0;
3894 };
3895
3896 Value *PtrOperand, *MaskOperand, *Src0Operand;
3897 unsigned Alignment;
3898 if (IsCompressing)
3899 getCompressingStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3900 else
3901 getMaskedStoreOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
3902
3903 SDValue Ptr = getValue(PtrOperand);
3904 SDValue Src0 = getValue(Src0Operand);
3905 SDValue Mask = getValue(MaskOperand);
3906
3907 EVT VT = Src0.getValueType();
3908 if (!Alignment)
3909 Alignment = DAG.getEVTAlignment(VT);
3910
3911 AAMDNodes AAInfo;
3912 I.getAAMetadata(AAInfo);
3913
3914 MachineMemOperand *MMO =
3915 DAG.getMachineFunction().
3916 getMachineMemOperand(MachinePointerInfo(PtrOperand),
3917 MachineMemOperand::MOStore, VT.getStoreSize(),
3918 Alignment, AAInfo);
3919 SDValue StoreNode = DAG.getMaskedStore(getRoot(), sdl, Src0, Ptr, Mask, VT,
3920 MMO, false /* Truncating */,
3921 IsCompressing);
3922 DAG.setRoot(StoreNode);
3923 setValue(&I, StoreNode);
3924}
3925
3926// Get a uniform base for the Gather/Scatter intrinsic.
3927// The first argument of the Gather/Scatter intrinsic is a vector of pointers.
3928// We try to represent it as a base pointer + vector of indices.
3929// Usually, the vector of pointers comes from a 'getelementptr' instruction.
3930// The first operand of the GEP may be a single pointer or a vector of pointers
3931// Example:
3932// %gep.ptr = getelementptr i32, <8 x i32*> %vptr, <8 x i32> %ind
3933// or
3934// %gep.ptr = getelementptr i32, i32* %ptr, <8 x i32> %ind
3935// %res = call <8 x i32> @llvm.masked.gather.v8i32(<8 x i32*> %gep.ptr, ..
3936//
3937// When the first GEP operand is a single pointer - it is the uniform base we
3938// are looking for. If first operand of the GEP is a splat vector - we
3939// extract the splat value and use it as a uniform base.
3940// In all other cases the function returns 'false'.
3941static bool getUniformBase(const Value* &Ptr, SDValue& Base, SDValue& Index,
3942 SDValue &Scale, SelectionDAGBuilder* SDB) {
3943 SelectionDAG& DAG = SDB->DAG;
3944 LLVMContext &Context = *DAG.getContext();
3945
3946 assert(Ptr->getType()->isVectorTy() && "Uexpected pointer type")(static_cast <bool> (Ptr->getType()->isVectorTy()
&& "Uexpected pointer type") ? void (0) : __assert_fail
("Ptr->getType()->isVectorTy() && \"Uexpected pointer type\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 3946, __extension__ __PRETTY_FUNCTION__));
3947 const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Ptr);
3948 if (!GEP)
3949 return false;
3950
3951 const Value *GEPPtr = GEP->getPointerOperand();
3952 if (!GEPPtr->getType()->isVectorTy())
3953 Ptr = GEPPtr;
3954 else if (!(Ptr = getSplatValue(GEPPtr)))
3955 return false;
3956
3957 unsigned FinalIndex = GEP->getNumOperands() - 1;
3958 Value *IndexVal = GEP->getOperand(FinalIndex);
3959
3960 // Ensure all the other indices are 0.
3961 for (unsigned i = 1; i < FinalIndex; ++i) {
3962 auto *C = dyn_cast<ConstantInt>(GEP->getOperand(i));
3963 if (!C || !C->isZero())
3964 return false;
3965 }
3966
3967 // The operands of the GEP may be defined in another basic block.
3968 // In this case we'll not find nodes for the operands.
3969 if (!SDB->findValue(Ptr) || !SDB->findValue(IndexVal))
3970 return false;
3971
3972 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3973 const DataLayout &DL = DAG.getDataLayout();
3974 Scale = DAG.getTargetConstant(DL.getTypeAllocSize(GEP->getResultElementType()),
3975 SDB->getCurSDLoc(), TLI.getPointerTy(DL));
3976 Base = SDB->getValue(Ptr);
3977 Index = SDB->getValue(IndexVal);
3978
3979 if (!Index.getValueType().isVector()) {
3980 unsigned GEPWidth = GEP->getType()->getVectorNumElements();
3981 EVT VT = EVT::getVectorVT(Context, Index.getValueType(), GEPWidth);
3982 Index = DAG.getSplatBuildVector(VT, SDLoc(Index), Index);
3983 }
3984 return true;
3985}
3986
3987void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) {
3988 SDLoc sdl = getCurSDLoc();
3989
3990 // llvm.masked.scatter.*(Src0, Ptrs, alignemt, Mask)
3991 const Value *Ptr = I.getArgOperand(1);
3992 SDValue Src0 = getValue(I.getArgOperand(0));
3993 SDValue Mask = getValue(I.getArgOperand(3));
3994 EVT VT = Src0.getValueType();
3995 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(2)))->getZExtValue();
3996 if (!Alignment)
3997 Alignment = DAG.getEVTAlignment(VT);
3998 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
3999
4000 AAMDNodes AAInfo;
4001 I.getAAMetadata(AAInfo);
4002
4003 SDValue Base;
4004 SDValue Index;
4005 SDValue Scale;
4006 const Value *BasePtr = Ptr;
4007 bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4008
4009 const Value *MemOpBasePtr = UniformBase ? BasePtr : nullptr;
4010 MachineMemOperand *MMO = DAG.getMachineFunction().
4011 getMachineMemOperand(MachinePointerInfo(MemOpBasePtr),
4012 MachineMemOperand::MOStore, VT.getStoreSize(),
4013 Alignment, AAInfo);
4014 if (!UniformBase) {
4015 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4016 Index = getValue(Ptr);
4017 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4018 }
4019 SDValue Ops[] = { getRoot(), Src0, Mask, Base, Index, Scale };
4020 SDValue Scatter = DAG.getMaskedScatter(DAG.getVTList(MVT::Other), VT, sdl,
4021 Ops, MMO);
4022 DAG.setRoot(Scatter);
4023 setValue(&I, Scatter);
4024}
4025
4026void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) {
4027 SDLoc sdl = getCurSDLoc();
4028
4029 auto getMaskedLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4030 unsigned& Alignment) {
4031 // @llvm.masked.load.*(Ptr, alignment, Mask, Src0)
4032 Ptr = I.getArgOperand(0);
4033 Alignment = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
4034 Mask = I.getArgOperand(2);
4035 Src0 = I.getArgOperand(3);
4036 };
4037 auto getExpandingLoadOps = [&](Value* &Ptr, Value* &Mask, Value* &Src0,
4038 unsigned& Alignment) {
4039 // @llvm.masked.expandload.*(Ptr, Mask, Src0)
4040 Ptr = I.getArgOperand(0);
4041 Alignment = 0;
4042 Mask = I.getArgOperand(1);
4043 Src0 = I.getArgOperand(2);
4044 };
4045
4046 Value *PtrOperand, *MaskOperand, *Src0Operand;
4047 unsigned Alignment;
4048 if (IsExpanding)
4049 getExpandingLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4050 else
4051 getMaskedLoadOps(PtrOperand, MaskOperand, Src0Operand, Alignment);
4052
4053 SDValue Ptr = getValue(PtrOperand);
4054 SDValue Src0 = getValue(Src0Operand);
4055 SDValue Mask = getValue(MaskOperand);
4056
4057 EVT VT = Src0.getValueType();
4058 if (!Alignment)
4059 Alignment = DAG.getEVTAlignment(VT);
4060
4061 AAMDNodes AAInfo;
4062 I.getAAMetadata(AAInfo);
4063 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4064
4065 // Do not serialize masked loads of constant memory with anything.
4066 bool AddToChain = !AA || !AA->pointsToConstantMemory(MemoryLocation(
4067 PtrOperand, DAG.getDataLayout().getTypeStoreSize(I.getType()), AAInfo));
4068 SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode();
4069
4070 MachineMemOperand *MMO =
4071 DAG.getMachineFunction().
4072 getMachineMemOperand(MachinePointerInfo(PtrOperand),
4073 MachineMemOperand::MOLoad, VT.getStoreSize(),
4074 Alignment, AAInfo, Ranges);
4075
4076 SDValue Load = DAG.getMaskedLoad(VT, sdl, InChain, Ptr, Mask, Src0, VT, MMO,
4077 ISD::NON_EXTLOAD, IsExpanding);
4078 if (AddToChain)
4079 PendingLoads.push_back(Load.getValue(1));
4080 setValue(&I, Load);
4081}
4082
4083void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) {
4084 SDLoc sdl = getCurSDLoc();
4085
4086 // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0)
4087 const Value *Ptr = I.getArgOperand(0);
4088 SDValue Src0 = getValue(I.getArgOperand(3));
4089 SDValue Mask = getValue(I.getArgOperand(2));
4090
4091 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4092 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4093 unsigned Alignment = (cast<ConstantInt>(I.getArgOperand(1)))->getZExtValue();
4094 if (!Alignment)
4095 Alignment = DAG.getEVTAlignment(VT);
4096
4097 AAMDNodes AAInfo;
4098 I.getAAMetadata(AAInfo);
4099 const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range);
4100
4101 SDValue Root = DAG.getRoot();
4102 SDValue Base;
4103 SDValue Index;
4104 SDValue Scale;
4105 const Value *BasePtr = Ptr;
4106 bool UniformBase = getUniformBase(BasePtr, Base, Index, Scale, this);
4107 bool ConstantMemory = false;
4108 if (UniformBase &&
4109 AA && AA->pointsToConstantMemory(MemoryLocation(
4110 BasePtr, DAG.getDataLayout().getTypeStoreSize(I.getType()),
4111 AAInfo))) {
4112 // Do not serialize (non-volatile) loads of constant memory with anything.
4113 Root = DAG.getEntryNode();
4114 ConstantMemory = true;
4115 }
4116
4117 MachineMemOperand *MMO =
4118 DAG.getMachineFunction().
4119 getMachineMemOperand(MachinePointerInfo(UniformBase ? BasePtr : nullptr),
4120 MachineMemOperand::MOLoad, VT.getStoreSize(),
4121 Alignment, AAInfo, Ranges);
4122
4123 if (!UniformBase) {
4124 Base = DAG.getConstant(0, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4125 Index = getValue(Ptr);
4126 Scale = DAG.getTargetConstant(1, sdl, TLI.getPointerTy(DAG.getDataLayout()));
4127 }
4128 SDValue Ops[] = { Root, Src0, Mask, Base, Index, Scale };
4129 SDValue Gather = DAG.getMaskedGather(DAG.getVTList(VT, MVT::Other), VT, sdl,
4130 Ops, MMO);
4131
4132 SDValue OutChain = Gather.getValue(1);
4133 if (!ConstantMemory)
4134 PendingLoads.push_back(OutChain);
4135 setValue(&I, Gather);
4136}
4137
4138void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
4139 SDLoc dl = getCurSDLoc();
4140 AtomicOrdering SuccessOrder = I.getSuccessOrdering();
4141 AtomicOrdering FailureOrder = I.getFailureOrdering();
4142 SyncScope::ID SSID = I.getSyncScopeID();
4143
4144 SDValue InChain = getRoot();
4145
4146 MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType();
4147 SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other);
4148 SDValue L = DAG.getAtomicCmpSwap(
4149 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain,
4150 getValue(I.getPointerOperand()), getValue(I.getCompareOperand()),
4151 getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()),
4152 /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID);
4153
4154 SDValue OutChain = L.getValue(2);
4155
4156 setValue(&I, L);
4157 DAG.setRoot(OutChain);
4158}
4159
4160void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
4161 SDLoc dl = getCurSDLoc();
4162 ISD::NodeType NT;
4163 switch (I.getOperation()) {
4164 default: llvm_unreachable("Unknown atomicrmw operation")::llvm::llvm_unreachable_internal("Unknown atomicrmw operation"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4164);
4165 case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
4166 case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
4167 case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
4168 case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
4169 case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
4170 case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
4171 case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
4172 case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
4173 case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
4174 case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
4175 case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
4176 }
4177 AtomicOrdering Order = I.getOrdering();
4178 SyncScope::ID SSID = I.getSyncScopeID();
4179
4180 SDValue InChain = getRoot();
4181
4182 SDValue L =
4183 DAG.getAtomic(NT, dl,
4184 getValue(I.getValOperand()).getSimpleValueType(),
4185 InChain,
4186 getValue(I.getPointerOperand()),
4187 getValue(I.getValOperand()),
4188 I.getPointerOperand(),
4189 /* Alignment=*/ 0, Order, SSID);
4190
4191 SDValue OutChain = L.getValue(1);
4192
4193 setValue(&I, L);
4194 DAG.setRoot(OutChain);
4195}
4196
4197void SelectionDAGBuilder::visitFence(const FenceInst &I) {
4198 SDLoc dl = getCurSDLoc();
4199 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4200 SDValue Ops[3];
4201 Ops[0] = getRoot();
4202 Ops[1] = DAG.getConstant((unsigned)I.getOrdering(), dl,
4203 TLI.getFenceOperandTy(DAG.getDataLayout()));
4204 Ops[2] = DAG.getConstant(I.getSyncScopeID(), dl,
4205 TLI.getFenceOperandTy(DAG.getDataLayout()));
4206 DAG.setRoot(DAG.getNode(ISD::ATOMIC_FENCE, dl, MVT::Other, Ops));
4207}
4208
4209void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
4210 SDLoc dl = getCurSDLoc();
4211 AtomicOrdering Order = I.getOrdering();
4212 SyncScope::ID SSID = I.getSyncScopeID();
4213
4214 SDValue InChain = getRoot();
4215
4216 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4217 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
4218
4219 if (!TLI.supportsUnalignedAtomics() &&
4220 I.getAlignment() < VT.getStoreSize())
4221 report_fatal_error("Cannot generate unaligned atomic load");
4222
4223 MachineMemOperand *MMO =
4224 DAG.getMachineFunction().
4225 getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()),
4226 MachineMemOperand::MOVolatile |
4227 MachineMemOperand::MOLoad,
4228 VT.getStoreSize(),
4229 I.getAlignment() ? I.getAlignment() :
4230 DAG.getEVTAlignment(VT),
4231 AAMDNodes(), nullptr, SSID, Order);
4232
4233 InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG);
4234 SDValue L =
4235 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain,
4236 getValue(I.getPointerOperand()), MMO);
4237
4238 SDValue OutChain = L.getValue(1);
4239
4240 setValue(&I, L);
4241 DAG.setRoot(OutChain);
4242}
4243
4244void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
4245 SDLoc dl = getCurSDLoc();
4246
4247 AtomicOrdering Order = I.getOrdering();
4248 SyncScope::ID SSID = I.getSyncScopeID();
4249
4250 SDValue InChain = getRoot();
4251
4252 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4253 EVT VT =
4254 TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType());
4255
4256 if (I.getAlignment() < VT.getStoreSize())
4257 report_fatal_error("Cannot generate unaligned atomic store");
4258
4259 SDValue OutChain =
4260 DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT,
4261 InChain,
4262 getValue(I.getPointerOperand()),
4263 getValue(I.getValueOperand()),
4264 I.getPointerOperand(), I.getAlignment(),
4265 Order, SSID);
4266
4267 DAG.setRoot(OutChain);
4268}
4269
4270/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC
4271/// node.
4272void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I,
4273 unsigned Intrinsic) {
4274 // Ignore the callsite's attributes. A specific call site may be marked with
4275 // readnone, but the lowering code will expect the chain based on the
4276 // definition.
4277 const Function *F = I.getCalledFunction();
4278 bool HasChain = !F->doesNotAccessMemory();
4279 bool OnlyLoad = HasChain && F->onlyReadsMemory();
4280
4281 // Build the operand list.
4282 SmallVector<SDValue, 8> Ops;
4283 if (HasChain) { // If this intrinsic has side-effects, chainify it.
4284 if (OnlyLoad) {
4285 // We don't need to serialize loads against other loads.
4286 Ops.push_back(DAG.getRoot());
4287 } else {
4288 Ops.push_back(getRoot());
4289 }
4290 }
4291
4292 // Info is set by getTgtMemInstrinsic
4293 TargetLowering::IntrinsicInfo Info;
4294 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
4295 bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I,
4296 DAG.getMachineFunction(),
4297 Intrinsic);
4298
4299 // Add the intrinsic ID as an integer operand if it's not a target intrinsic.
4300 if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID ||
4301 Info.opc == ISD::INTRINSIC_W_CHAIN)
4302 Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(),
4303 TLI.getPointerTy(DAG.getDataLayout())));
4304
4305 // Add all operands of the call to the operand list.
4306 for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) {
4307 SDValue Op = getValue(I.getArgOperand(i));
4308 Ops.push_back(Op);
4309 }
4310
4311 SmallVector<EVT, 4> ValueVTs;
4312 ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs);
4313
4314 if (HasChain)
4315 ValueVTs.push_back(MVT::Other);
4316
4317 SDVTList VTs = DAG.getVTList(ValueVTs);
4318
4319 // Create the node.
4320 SDValue Result;
4321 if (IsTgtIntrinsic) {
4322 // This is target intrinsic that touches memory
4323 Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs,
4324 Ops, Info.memVT,
4325 MachinePointerInfo(Info.ptrVal, Info.offset), Info.align,
4326 Info.flags, Info.size);
4327 } else if (!HasChain) {
4328 Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops);
4329 } else if (!I.getType()->isVoidTy()) {
4330 Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops);
4331 } else {
4332 Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops);
4333 }
4334
4335 if (HasChain) {
4336 SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1);
4337 if (OnlyLoad)
4338 PendingLoads.push_back(Chain);
4339 else
4340 DAG.setRoot(Chain);
4341 }
4342
4343 if (!I.getType()->isVoidTy()) {
4344 if (VectorType *PTy = dyn_cast<VectorType>(I.getType())) {
4345 EVT VT = TLI.getValueType(DAG.getDataLayout(), PTy);
4346 Result = DAG.getNode(ISD::BITCAST, getCurSDLoc(), VT, Result);
4347 } else
4348 Result = lowerRangeToAssertZExt(DAG, I, Result);
4349
4350 setValue(&I, Result);
4351 }
4352}
4353
4354/// GetSignificand - Get the significand and build it into a floating-point
4355/// number with exponent of 1:
4356///
4357/// Op = (Op & 0x007fffff) | 0x3f800000;
4358///
4359/// where Op is the hexadecimal representation of floating point value.
4360static SDValue GetSignificand(SelectionDAG &DAG, SDValue Op, const SDLoc &dl) {
4361 SDValue t1 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4362 DAG.getConstant(0x007fffff, dl, MVT::i32));
4363 SDValue t2 = DAG.getNode(ISD::OR, dl, MVT::i32, t1,
4364 DAG.getConstant(0x3f800000, dl, MVT::i32));
4365 return DAG.getNode(ISD::BITCAST, dl, MVT::f32, t2);
4366}
4367
4368/// GetExponent - Get the exponent:
4369///
4370/// (float)(int)(((Op & 0x7f800000) >> 23) - 127);
4371///
4372/// where Op is the hexadecimal representation of floating point value.
4373static SDValue GetExponent(SelectionDAG &DAG, SDValue Op,
4374 const TargetLowering &TLI, const SDLoc &dl) {
4375 SDValue t0 = DAG.getNode(ISD::AND, dl, MVT::i32, Op,
4376 DAG.getConstant(0x7f800000, dl, MVT::i32));
4377 SDValue t1 = DAG.getNode(
4378 ISD::SRL, dl, MVT::i32, t0,
4379 DAG.getConstant(23, dl, TLI.getPointerTy(DAG.getDataLayout())));
4380 SDValue t2 = DAG.getNode(ISD::SUB, dl, MVT::i32, t1,
4381 DAG.getConstant(127, dl, MVT::i32));
4382 return DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, t2);
4383}
4384
4385/// getF32Constant - Get 32-bit floating point constant.
4386static SDValue getF32Constant(SelectionDAG &DAG, unsigned Flt,
4387 const SDLoc &dl) {
4388 return DAG.getConstantFP(APFloat(APFloat::IEEEsingle(), APInt(32, Flt)), dl,
4389 MVT::f32);
4390}
4391
4392static SDValue getLimitedPrecisionExp2(SDValue t0, const SDLoc &dl,
4393 SelectionDAG &DAG) {
4394 // TODO: What fast-math-flags should be set on the floating-point nodes?
4395
4396 // IntegerPartOfX = ((int32_t)(t0);
4397 SDValue IntegerPartOfX = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, t0);
4398
4399 // FractionalPartOfX = t0 - (float)IntegerPartOfX;
4400 SDValue t1 = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, IntegerPartOfX);
4401 SDValue X = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0, t1);
4402
4403 // IntegerPartOfX <<= 23;
4404 IntegerPartOfX = DAG.getNode(
4405 ISD::SHL, dl, MVT::i32, IntegerPartOfX,
4406 DAG.getConstant(23, dl, DAG.getTargetLoweringInfo().getPointerTy(
4407 DAG.getDataLayout())));
4408
4409 SDValue TwoToFractionalPartOfX;
4410 if (LimitFloatPrecision <= 6) {
4411 // For floating-point precision of 6:
4412 //
4413 // TwoToFractionalPartOfX =
4414 // 0.997535578f +
4415 // (0.735607626f + 0.252464424f * x) * x;
4416 //
4417 // error 0.0144103317, which is 6 bits
4418 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4419 getF32Constant(DAG, 0x3e814304, dl));
4420 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4421 getF32Constant(DAG, 0x3f3c50c8, dl));
4422 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4423 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4424 getF32Constant(DAG, 0x3f7f5e7e, dl));
4425 } else if (LimitFloatPrecision <= 12) {
4426 // For floating-point precision of 12:
4427 //
4428 // TwoToFractionalPartOfX =
4429 // 0.999892986f +
4430 // (0.696457318f +
4431 // (0.224338339f + 0.792043434e-1f * x) * x) * x;
4432 //
4433 // error 0.000107046256, which is 13 to 14 bits
4434 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4435 getF32Constant(DAG, 0x3da235e3, dl));
4436 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4437 getF32Constant(DAG, 0x3e65b8f3, dl));
4438 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4439 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4440 getF32Constant(DAG, 0x3f324b07, dl));
4441 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4442 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4443 getF32Constant(DAG, 0x3f7ff8fd, dl));
4444 } else { // LimitFloatPrecision <= 18
4445 // For floating-point precision of 18:
4446 //
4447 // TwoToFractionalPartOfX =
4448 // 0.999999982f +
4449 // (0.693148872f +
4450 // (0.240227044f +
4451 // (0.554906021e-1f +
4452 // (0.961591928e-2f +
4453 // (0.136028312e-2f + 0.157059148e-3f *x)*x)*x)*x)*x)*x;
4454 // error 2.47208000*10^(-7), which is better than 18 bits
4455 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4456 getF32Constant(DAG, 0x3924b03e, dl));
4457 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4458 getF32Constant(DAG, 0x3ab24b87, dl));
4459 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4460 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4461 getF32Constant(DAG, 0x3c1d8c17, dl));
4462 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4463 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4464 getF32Constant(DAG, 0x3d634a1d, dl));
4465 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4466 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4467 getF32Constant(DAG, 0x3e75fe14, dl));
4468 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4469 SDValue t11 = DAG.getNode(ISD::FADD, dl, MVT::f32, t10,
4470 getF32Constant(DAG, 0x3f317234, dl));
4471 SDValue t12 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t11, X);
4472 TwoToFractionalPartOfX = DAG.getNode(ISD::FADD, dl, MVT::f32, t12,
4473 getF32Constant(DAG, 0x3f800000, dl));
4474 }
4475
4476 // Add the exponent into the result in integer domain.
4477 SDValue t13 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, TwoToFractionalPartOfX);
4478 return DAG.getNode(ISD::BITCAST, dl, MVT::f32,
4479 DAG.getNode(ISD::ADD, dl, MVT::i32, t13, IntegerPartOfX));
4480}
4481
4482/// expandExp - Lower an exp intrinsic. Handles the special sequences for
4483/// limited-precision mode.
4484static SDValue expandExp(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4485 const TargetLowering &TLI) {
4486 if (Op.getValueType() == MVT::f32 &&
4487 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4488
4489 // Put the exponent in the right bit position for later addition to the
4490 // final result:
4491 //
4492 // #define LOG2OFe 1.4426950f
4493 // t0 = Op * LOG2OFe
4494
4495 // TODO: What fast-math-flags should be set here?
4496 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, Op,
4497 getF32Constant(DAG, 0x3fb8aa3b, dl));
4498 return getLimitedPrecisionExp2(t0, dl, DAG);
4499 }
4500
4501 // No special expansion.
4502 return DAG.getNode(ISD::FEXP, dl, Op.getValueType(), Op);
4503}
4504
4505/// expandLog - Lower a log intrinsic. Handles the special sequences for
4506/// limited-precision mode.
4507static SDValue expandLog(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4508 const TargetLowering &TLI) {
4509 // TODO: What fast-math-flags should be set on the floating-point nodes?
4510
4511 if (Op.getValueType() == MVT::f32 &&
4512 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4513 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4514
4515 // Scale the exponent by log(2) [0.69314718f].
4516 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4517 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4518 getF32Constant(DAG, 0x3f317218, dl));
4519
4520 // Get the significand and build it into a floating-point number with
4521 // exponent of 1.
4522 SDValue X = GetSignificand(DAG, Op1, dl);
4523
4524 SDValue LogOfMantissa;
4525 if (LimitFloatPrecision <= 6) {
4526 // For floating-point precision of 6:
4527 //
4528 // LogofMantissa =
4529 // -1.1609546f +
4530 // (1.4034025f - 0.23903021f * x) * x;
4531 //
4532 // error 0.0034276066, which is better than 8 bits
4533 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4534 getF32Constant(DAG, 0xbe74c456, dl));
4535 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4536 getF32Constant(DAG, 0x3fb3a2b1, dl));
4537 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4538 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4539 getF32Constant(DAG, 0x3f949a29, dl));
4540 } else if (LimitFloatPrecision <= 12) {
4541 // For floating-point precision of 12:
4542 //
4543 // LogOfMantissa =
4544 // -1.7417939f +
4545 // (2.8212026f +
4546 // (-1.4699568f +
4547 // (0.44717955f - 0.56570851e-1f * x) * x) * x) * x;
4548 //
4549 // error 0.000061011436, which is 14 bits
4550 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4551 getF32Constant(DAG, 0xbd67b6d6, dl));
4552 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4553 getF32Constant(DAG, 0x3ee4f4b8, dl));
4554 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4555 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4556 getF32Constant(DAG, 0x3fbc278b, dl));
4557 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4558 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4559 getF32Constant(DAG, 0x40348e95, dl));
4560 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4561 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4562 getF32Constant(DAG, 0x3fdef31a, dl));
4563 } else { // LimitFloatPrecision <= 18
4564 // For floating-point precision of 18:
4565 //
4566 // LogOfMantissa =
4567 // -2.1072184f +
4568 // (4.2372794f +
4569 // (-3.7029485f +
4570 // (2.2781945f +
4571 // (-0.87823314f +
4572 // (0.19073739f - 0.17809712e-1f * x) * x) * x) * x) * x)*x;
4573 //
4574 // error 0.0000023660568, which is better than 18 bits
4575 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4576 getF32Constant(DAG, 0xbc91e5ac, dl));
4577 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4578 getF32Constant(DAG, 0x3e4350aa, dl));
4579 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4580 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4581 getF32Constant(DAG, 0x3f60d3e3, dl));
4582 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4583 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4584 getF32Constant(DAG, 0x4011cdf0, dl));
4585 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4586 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4587 getF32Constant(DAG, 0x406cfd1c, dl));
4588 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4589 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4590 getF32Constant(DAG, 0x408797cb, dl));
4591 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4592 LogOfMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4593 getF32Constant(DAG, 0x4006dcab, dl));
4594 }
4595
4596 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, LogOfMantissa);
4597 }
4598
4599 // No special expansion.
4600 return DAG.getNode(ISD::FLOG, dl, Op.getValueType(), Op);
4601}
4602
4603/// expandLog2 - Lower a log2 intrinsic. Handles the special sequences for
4604/// limited-precision mode.
4605static SDValue expandLog2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4606 const TargetLowering &TLI) {
4607 // TODO: What fast-math-flags should be set on the floating-point nodes?
4608
4609 if (Op.getValueType() == MVT::f32 &&
4610 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4611 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4612
4613 // Get the exponent.
4614 SDValue LogOfExponent = GetExponent(DAG, Op1, TLI, dl);
4615
4616 // Get the significand and build it into a floating-point number with
4617 // exponent of 1.
4618 SDValue X = GetSignificand(DAG, Op1, dl);
4619
4620 // Different possible minimax approximations of significand in
4621 // floating-point for various degrees of accuracy over [1,2].
4622 SDValue Log2ofMantissa;
4623 if (LimitFloatPrecision <= 6) {
4624 // For floating-point precision of 6:
4625 //
4626 // Log2ofMantissa = -1.6749035f + (2.0246817f - .34484768f * x) * x;
4627 //
4628 // error 0.0049451742, which is more than 7 bits
4629 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4630 getF32Constant(DAG, 0xbeb08fe0, dl));
4631 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4632 getF32Constant(DAG, 0x40019463, dl));
4633 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4634 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4635 getF32Constant(DAG, 0x3fd6633d, dl));
4636 } else if (LimitFloatPrecision <= 12) {
4637 // For floating-point precision of 12:
4638 //
4639 // Log2ofMantissa =
4640 // -2.51285454f +
4641 // (4.07009056f +
4642 // (-2.12067489f +
4643 // (.645142248f - 0.816157886e-1f * x) * x) * x) * x;
4644 //
4645 // error 0.0000876136000, which is better than 13 bits
4646 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4647 getF32Constant(DAG, 0xbda7262e, dl));
4648 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4649 getF32Constant(DAG, 0x3f25280b, dl));
4650 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4651 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4652 getF32Constant(DAG, 0x4007b923, dl));
4653 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4654 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4655 getF32Constant(DAG, 0x40823e2f, dl));
4656 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4657 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4658 getF32Constant(DAG, 0x4020d29c, dl));
4659 } else { // LimitFloatPrecision <= 18
4660 // For floating-point precision of 18:
4661 //
4662 // Log2ofMantissa =
4663 // -3.0400495f +
4664 // (6.1129976f +
4665 // (-5.3420409f +
4666 // (3.2865683f +
4667 // (-1.2669343f +
4668 // (0.27515199f -
4669 // 0.25691327e-1f * x) * x) * x) * x) * x) * x;
4670 //
4671 // error 0.0000018516, which is better than 18 bits
4672 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4673 getF32Constant(DAG, 0xbcd2769e, dl));
4674 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4675 getF32Constant(DAG, 0x3e8ce0b9, dl));
4676 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4677 SDValue t3 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4678 getF32Constant(DAG, 0x3fa22ae7, dl));
4679 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4680 SDValue t5 = DAG.getNode(ISD::FADD, dl, MVT::f32, t4,
4681 getF32Constant(DAG, 0x40525723, dl));
4682 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4683 SDValue t7 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t6,
4684 getF32Constant(DAG, 0x40aaf200, dl));
4685 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4686 SDValue t9 = DAG.getNode(ISD::FADD, dl, MVT::f32, t8,
4687 getF32Constant(DAG, 0x40c39dad, dl));
4688 SDValue t10 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t9, X);
4689 Log2ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t10,
4690 getF32Constant(DAG, 0x4042902c, dl));
4691 }
4692
4693 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log2ofMantissa);
4694 }
4695
4696 // No special expansion.
4697 return DAG.getNode(ISD::FLOG2, dl, Op.getValueType(), Op);
4698}
4699
4700/// expandLog10 - Lower a log10 intrinsic. Handles the special sequences for
4701/// limited-precision mode.
4702static SDValue expandLog10(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4703 const TargetLowering &TLI) {
4704 // TODO: What fast-math-flags should be set on the floating-point nodes?
4705
4706 if (Op.getValueType() == MVT::f32 &&
4707 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4708 SDValue Op1 = DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op);
4709
4710 // Scale the exponent by log10(2) [0.30102999f].
4711 SDValue Exp = GetExponent(DAG, Op1, TLI, dl);
4712 SDValue LogOfExponent = DAG.getNode(ISD::FMUL, dl, MVT::f32, Exp,
4713 getF32Constant(DAG, 0x3e9a209a, dl));
4714
4715 // Get the significand and build it into a floating-point number with
4716 // exponent of 1.
4717 SDValue X = GetSignificand(DAG, Op1, dl);
4718
4719 SDValue Log10ofMantissa;
4720 if (LimitFloatPrecision <= 6) {
4721 // For floating-point precision of 6:
4722 //
4723 // Log10ofMantissa =
4724 // -0.50419619f +
4725 // (0.60948995f - 0.10380950f * x) * x;
4726 //
4727 // error 0.0014886165, which is 6 bits
4728 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4729 getF32Constant(DAG, 0xbdd49a13, dl));
4730 SDValue t1 = DAG.getNode(ISD::FADD, dl, MVT::f32, t0,
4731 getF32Constant(DAG, 0x3f1c0789, dl));
4732 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4733 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t2,
4734 getF32Constant(DAG, 0x3f011300, dl));
4735 } else if (LimitFloatPrecision <= 12) {
4736 // For floating-point precision of 12:
4737 //
4738 // Log10ofMantissa =
4739 // -0.64831180f +
4740 // (0.91751397f +
4741 // (-0.31664806f + 0.47637168e-1f * x) * x) * x;
4742 //
4743 // error 0.00019228036, which is better than 12 bits
4744 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4745 getF32Constant(DAG, 0x3d431f31, dl));
4746 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4747 getF32Constant(DAG, 0x3ea21fb2, dl));
4748 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4749 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4750 getF32Constant(DAG, 0x3f6ae232, dl));
4751 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4752 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4753 getF32Constant(DAG, 0x3f25f7c3, dl));
4754 } else { // LimitFloatPrecision <= 18
4755 // For floating-point precision of 18:
4756 //
4757 // Log10ofMantissa =
4758 // -0.84299375f +
4759 // (1.5327582f +
4760 // (-1.0688956f +
4761 // (0.49102474f +
4762 // (-0.12539807f + 0.13508273e-1f * x) * x) * x) * x) * x;
4763 //
4764 // error 0.0000037995730, which is better than 18 bits
4765 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, X,
4766 getF32Constant(DAG, 0x3c5d51ce, dl));
4767 SDValue t1 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t0,
4768 getF32Constant(DAG, 0x3e00685a, dl));
4769 SDValue t2 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t1, X);
4770 SDValue t3 = DAG.getNode(ISD::FADD, dl, MVT::f32, t2,
4771 getF32Constant(DAG, 0x3efb6798, dl));
4772 SDValue t4 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t3, X);
4773 SDValue t5 = DAG.getNode(ISD::FSUB, dl, MVT::f32, t4,
4774 getF32Constant(DAG, 0x3f88d192, dl));
4775 SDValue t6 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t5, X);
4776 SDValue t7 = DAG.getNode(ISD::FADD, dl, MVT::f32, t6,
4777 getF32Constant(DAG, 0x3fc4316c, dl));
4778 SDValue t8 = DAG.getNode(ISD::FMUL, dl, MVT::f32, t7, X);
4779 Log10ofMantissa = DAG.getNode(ISD::FSUB, dl, MVT::f32, t8,
4780 getF32Constant(DAG, 0x3f57ce70, dl));
4781 }
4782
4783 return DAG.getNode(ISD::FADD, dl, MVT::f32, LogOfExponent, Log10ofMantissa);
4784 }
4785
4786 // No special expansion.
4787 return DAG.getNode(ISD::FLOG10, dl, Op.getValueType(), Op);
4788}
4789
4790/// expandExp2 - Lower an exp2 intrinsic. Handles the special sequences for
4791/// limited-precision mode.
4792static SDValue expandExp2(const SDLoc &dl, SDValue Op, SelectionDAG &DAG,
4793 const TargetLowering &TLI) {
4794 if (Op.getValueType() == MVT::f32 &&
4795 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18)
4796 return getLimitedPrecisionExp2(Op, dl, DAG);
4797
4798 // No special expansion.
4799 return DAG.getNode(ISD::FEXP2, dl, Op.getValueType(), Op);
4800}
4801
4802/// visitPow - Lower a pow intrinsic. Handles the special sequences for
4803/// limited-precision mode with x == 10.0f.
4804static SDValue expandPow(const SDLoc &dl, SDValue LHS, SDValue RHS,
4805 SelectionDAG &DAG, const TargetLowering &TLI) {
4806 bool IsExp10 = false;
4807 if (LHS.getValueType() == MVT::f32 && RHS.getValueType() == MVT::f32 &&
4808 LimitFloatPrecision > 0 && LimitFloatPrecision <= 18) {
4809 if (ConstantFPSDNode *LHSC = dyn_cast<ConstantFPSDNode>(LHS)) {
4810 APFloat Ten(10.0f);
4811 IsExp10 = LHSC->isExactlyValue(Ten);
4812 }
4813 }
4814
4815 // TODO: What fast-math-flags should be set on the FMUL node?
4816 if (IsExp10) {
4817 // Put the exponent in the right bit position for later addition to the
4818 // final result:
4819 //
4820 // #define LOG2OF10 3.3219281f
4821 // t0 = Op * LOG2OF10;
4822 SDValue t0 = DAG.getNode(ISD::FMUL, dl, MVT::f32, RHS,
4823 getF32Constant(DAG, 0x40549a78, dl));
4824 return getLimitedPrecisionExp2(t0, dl, DAG);
4825 }
4826
4827 // No special expansion.
4828 return DAG.getNode(ISD::FPOW, dl, LHS.getValueType(), LHS, RHS);
4829}
4830
4831/// ExpandPowI - Expand a llvm.powi intrinsic.
4832static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS,
4833 SelectionDAG &DAG) {
4834 // If RHS is a constant, we can expand this out to a multiplication tree,
4835 // otherwise we end up lowering to a call to __powidf2 (for example). When
4836 // optimizing for size, we only want to do this if the expansion would produce
4837 // a small number of multiplies, otherwise we do the full expansion.
4838 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4839 // Get the exponent as a positive value.
4840 unsigned Val = RHSC->getSExtValue();
4841 if ((int)Val < 0) Val = -Val;
4842
4843 // powi(x, 0) -> 1.0
4844 if (Val == 0)
4845 return DAG.getConstantFP(1.0, DL, LHS.getValueType());
4846
4847 const Function &F = DAG.getMachineFunction().getFunction();
4848 if (!F.optForSize() ||
4849 // If optimizing for size, don't insert too many multiplies.
4850 // This inserts up to 5 multiplies.
4851 countPopulation(Val) + Log2_32(Val) < 7) {
4852 // We use the simple binary decomposition method to generate the multiply
4853 // sequence. There are more optimal ways to do this (for example,
4854 // powi(x,15) generates one more multiply than it should), but this has
4855 // the benefit of being both really simple and much better than a libcall.
4856 SDValue Res; // Logically starts equal to 1.0
4857 SDValue CurSquare = LHS;
4858 // TODO: Intrinsics should have fast-math-flags that propagate to these
4859 // nodes.
4860 while (Val) {
4861 if (Val & 1) {
4862 if (Res.getNode())
4863 Res = DAG.getNode(ISD::FMUL, DL,Res.getValueType(), Res, CurSquare);
4864 else
4865 Res = CurSquare; // 1.0*CurSquare.
4866 }
4867
4868 CurSquare = DAG.getNode(ISD::FMUL, DL, CurSquare.getValueType(),
4869 CurSquare, CurSquare);
4870 Val >>= 1;
4871 }
4872
4873 // If the original was negative, invert the result, producing 1/(x*x*x).
4874 if (RHSC->getSExtValue() < 0)
4875 Res = DAG.getNode(ISD::FDIV, DL, LHS.getValueType(),
4876 DAG.getConstantFP(1.0, DL, LHS.getValueType()), Res);
4877 return Res;
4878 }
4879 }
4880
4881 // Otherwise, expand to a libcall.
4882 return DAG.getNode(ISD::FPOWI, DL, LHS.getValueType(), LHS, RHS);
4883}
4884
4885// getUnderlyingArgReg - Find underlying register used for a truncated or
4886// bitcasted argument.
4887static unsigned getUnderlyingArgReg(const SDValue &N) {
4888 switch (N.getOpcode()) {
4889 case ISD::CopyFromReg:
4890 return cast<RegisterSDNode>(N.getOperand(1))->getReg();
4891 case ISD::BITCAST:
4892 case ISD::AssertZext:
4893 case ISD::AssertSext:
4894 case ISD::TRUNCATE:
4895 return getUnderlyingArgReg(N.getOperand(0));
4896 default:
4897 return 0;
4898 }
4899}
4900
4901/// If the DbgValueInst is a dbg_value of a function argument, create the
4902/// corresponding DBG_VALUE machine instruction for it now. At the end of
4903/// instruction selection, they will be inserted to the entry BB.
4904bool SelectionDAGBuilder::EmitFuncArgumentDbgValue(
4905 const Value *V, DILocalVariable *Variable, DIExpression *Expr,
4906 DILocation *DL, bool IsDbgDeclare, const SDValue &N) {
4907 const Argument *Arg = dyn_cast<Argument>(V);
4908 if (!Arg)
4909 return false;
4910
4911 MachineFunction &MF = DAG.getMachineFunction();
4912 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
4913
4914 bool IsIndirect = false;
4915 Optional<MachineOperand> Op;
4916 // Some arguments' frame index is recorded during argument lowering.
4917 int FI = FuncInfo.getArgumentFrameIndex(Arg);
4918 if (FI != std::numeric_limits<int>::max())
4919 Op = MachineOperand::CreateFI(FI);
4920
4921 if (!Op && N.getNode()) {
4922 unsigned Reg = getUnderlyingArgReg(N);
4923 if (Reg && TargetRegisterInfo::isVirtualRegister(Reg)) {
4924 MachineRegisterInfo &RegInfo = MF.getRegInfo();
4925 unsigned PR = RegInfo.getLiveInPhysReg(Reg);
4926 if (PR)
4927 Reg = PR;
4928 }
4929 if (Reg) {
4930 Op = MachineOperand::CreateReg(Reg, false);
4931 IsIndirect = IsDbgDeclare;
4932 }
4933 }
4934
4935 if (!Op && N.getNode())
4936 // Check if frame index is available.
4937 if (LoadSDNode *LNode = dyn_cast<LoadSDNode>(N.getNode()))
4938 if (FrameIndexSDNode *FINode =
4939 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
4940 Op = MachineOperand::CreateFI(FINode->getIndex());
4941
4942 if (!Op) {
4943 // Check if ValueMap has reg number.
4944 DenseMap<const Value *, unsigned>::iterator VMI = FuncInfo.ValueMap.find(V);
4945 if (VMI != FuncInfo.ValueMap.end()) {
4946 const auto &TLI = DAG.getTargetLoweringInfo();
4947 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), VMI->second,
4948 V->getType(), getABIRegCopyCC(V));
4949 if (RFV.occupiesMultipleRegs()) {
4950 unsigned Offset = 0;
4951 for (auto RegAndSize : RFV.getRegsAndSizes()) {
4952 Op = MachineOperand::CreateReg(RegAndSize.first, false);
4953 auto FragmentExpr = DIExpression::createFragmentExpression(
4954 Expr, Offset, RegAndSize.second);
4955 if (!FragmentExpr)
4956 continue;
4957 FuncInfo.ArgDbgValues.push_back(
4958 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsDbgDeclare,
4959 Op->getReg(), Variable, *FragmentExpr));
4960 Offset += RegAndSize.second;
4961 }
4962 return true;
4963 }
4964 Op = MachineOperand::CreateReg(VMI->second, false);
4965 IsIndirect = IsDbgDeclare;
4966 }
4967 }
4968
4969 if (!Op)
4970 return false;
4971
4972 assert(Variable->isValidLocationForIntrinsic(DL) &&(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4973, __extension__ __PRETTY_FUNCTION__))
4973 "Expected inlined-at fields to agree")(static_cast <bool> (Variable->isValidLocationForIntrinsic
(DL) && "Expected inlined-at fields to agree") ? void
(0) : __assert_fail ("Variable->isValidLocationForIntrinsic(DL) && \"Expected inlined-at fields to agree\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 4973, __extension__ __PRETTY_FUNCTION__));
4974 IsIndirect = (Op->isReg()) ? IsIndirect : true;
4975 FuncInfo.ArgDbgValues.push_back(
4976 BuildMI(MF, DL, TII->get(TargetOpcode::DBG_VALUE), IsIndirect,
4977 *Op, Variable, Expr));
4978
4979 return true;
4980}
4981
4982/// Return the appropriate SDDbgValue based on N.
4983SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N,
4984 DILocalVariable *Variable,
4985 DIExpression *Expr,
4986 const DebugLoc &dl,
4987 unsigned DbgSDNodeOrder) {
4988 if (auto *FISDN = dyn_cast<FrameIndexSDNode>(N.getNode())) {
4989 // Construct a FrameIndexDbgValue for FrameIndexSDNodes so we can describe
4990 // stack slot locations.
4991 //
4992 // Consider "int x = 0; int *px = &x;". There are two kinds of interesting
4993 // debug values here after optimization:
4994 //
4995 // dbg.value(i32* %px, !"int *px", !DIExpression()), and
4996 // dbg.value(i32* %px, !"int x", !DIExpression(DW_OP_deref))
4997 //
4998 // Both describe the direct values of their associated variables.
4999 return DAG.getFrameIndexDbgValue(Variable, Expr, FISDN->getIndex(),
5000 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5001 }
5002 return DAG.getDbgValue(Variable, Expr, N.getNode(), N.getResNo(),
5003 /*IsIndirect*/ false, dl, DbgSDNodeOrder);
5004}
5005
5006// VisualStudio defines setjmp as _setjmp
5007#if defined(_MSC_VER) && defined(setjmp) && \
5008 !defined(setjmp_undefined_for_msvc)
5009# pragma push_macro("setjmp")
5010# undef setjmp
5011# define setjmp_undefined_for_msvc
5012#endif
5013
5014/// Lower the call to the specified intrinsic function. If we want to emit this
5015/// as a call to a named external function, return the name. Otherwise, lower it
5016/// and return null.
5017const char *
5018SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
5019 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
5020 SDLoc sdl = getCurSDLoc();
5021 DebugLoc dl = getCurDebugLoc();
5022 SDValue Res;
5023
5024 switch (Intrinsic) {
5025 default:
5026 // By default, turn this into a target intrinsic node.
5027 visitTargetIntrinsic(I, Intrinsic);
5028 return nullptr;
5029 case Intrinsic::vastart: visitVAStart(I); return nullptr;
5030 case Intrinsic::vaend: visitVAEnd(I); return nullptr;
5031 case Intrinsic::vacopy: visitVACopy(I); return nullptr;
5032 case Intrinsic::returnaddress:
5033 setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl,
5034 TLI.getPointerTy(DAG.getDataLayout()),
5035 getValue(I.getArgOperand(0))));
5036 return nullptr;
5037 case Intrinsic::addressofreturnaddress:
5038 setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl,
5039 TLI.getPointerTy(DAG.getDataLayout())));
5040 return nullptr;
5041 case Intrinsic::frameaddress:
5042 setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl,
5043 TLI.getPointerTy(DAG.getDataLayout()),
5044 getValue(I.getArgOperand(0))));
5045 return nullptr;
5046 case Intrinsic::read_register: {
5047 Value *Reg = I.getArgOperand(0);
5048 SDValue Chain = getRoot();
5049 SDValue RegName =
5050 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5051 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5052 Res = DAG.getNode(ISD::READ_REGISTER, sdl,
5053 DAG.getVTList(VT, MVT::Other), Chain, RegName);
5054 setValue(&I, Res);
5055 DAG.setRoot(Res.getValue(1));
5056 return nullptr;
5057 }
5058 case Intrinsic::write_register: {
5059 Value *Reg = I.getArgOperand(0);
5060 Value *RegValue = I.getArgOperand(1);
5061 SDValue Chain = getRoot();
5062 SDValue RegName =
5063 DAG.getMDNode(cast<MDNode>(cast<MetadataAsValue>(Reg)->getMetadata()));
5064 DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain,
5065 RegName, getValue(RegValue)));
5066 return nullptr;
5067 }
5068 case Intrinsic::setjmp:
5069 return &"_setjmp"[!TLI.usesUnderscoreSetJmp()];
5070 case Intrinsic::longjmp:
5071 return &"_longjmp"[!TLI.usesUnderscoreLongJmp()];
5072 case Intrinsic::memcpy: {
5073 const auto &MCI = cast<MemCpyInst>(I);
5074 SDValue Op1 = getValue(I.getArgOperand(0));
5075 SDValue Op2 = getValue(I.getArgOperand(1));
5076 SDValue Op3 = getValue(I.getArgOperand(2));
5077 // @llvm.memcpy defines 0 and 1 to both mean no alignment.
5078 unsigned DstAlign = std::max<unsigned>(MCI.getDestAlignment(), 1);
5079 unsigned SrcAlign = std::max<unsigned>(MCI.getSourceAlignment(), 1);
5080 unsigned Align = MinAlign(DstAlign, SrcAlign);
5081 bool isVol = MCI.isVolatile();
5082 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5083 // FIXME: Support passing different dest/src alignments to the memcpy DAG
5084 // node.
5085 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5086 false, isTC,
5087 MachinePointerInfo(I.getArgOperand(0)),
5088 MachinePointerInfo(I.getArgOperand(1)));
5089 updateDAGForMaybeTailCall(MC);
5090 return nullptr;
5091 }
5092 case Intrinsic::memset: {
5093 const auto &MSI = cast<MemSetInst>(I);
5094 SDValue Op1 = getValue(I.getArgOperand(0));
5095 SDValue Op2 = getValue(I.getArgOperand(1));
5096 SDValue Op3 = getValue(I.getArgOperand(2));
5097 // @llvm.memset defines 0 and 1 to both mean no alignment.
5098 unsigned Align = std::max<unsigned>(MSI.getDestAlignment(), 1);
5099 bool isVol = MSI.isVolatile();
5100 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5101 SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5102 isTC, MachinePointerInfo(I.getArgOperand(0)));
5103 updateDAGForMaybeTailCall(MS);
5104 return nullptr;
5105 }
5106 case Intrinsic::memmove: {
5107 const auto &MMI = cast<MemMoveInst>(I);
5108 SDValue Op1 = getValue(I.getArgOperand(0));
5109 SDValue Op2 = getValue(I.getArgOperand(1));
5110 SDValue Op3 = getValue(I.getArgOperand(2));
5111 // @llvm.memmove defines 0 and 1 to both mean no alignment.
5112 unsigned DstAlign = std::max<unsigned>(MMI.getDestAlignment(), 1);
5113 unsigned SrcAlign = std::max<unsigned>(MMI.getSourceAlignment(), 1);
5114 unsigned Align = MinAlign(DstAlign, SrcAlign);
5115 bool isVol = MMI.isVolatile();
5116 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5117 // FIXME: Support passing different dest/src alignments to the memmove DAG
5118 // node.
5119 SDValue MM = DAG.getMemmove(getRoot(), sdl, Op1, Op2, Op3, Align, isVol,
5120 isTC, MachinePointerInfo(I.getArgOperand(0)),
5121 MachinePointerInfo(I.getArgOperand(1)));
5122 updateDAGForMaybeTailCall(MM);
5123 return nullptr;
5124 }
5125 case Intrinsic::memcpy_element_unordered_atomic: {
5126 const AtomicMemCpyInst &MI = cast<AtomicMemCpyInst>(I);
5127 SDValue Dst = getValue(MI.getRawDest());
5128 SDValue Src = getValue(MI.getRawSource());
5129 SDValue Length = getValue(MI.getLength());
5130
5131 unsigned DstAlign = MI.getDestAlignment();
5132 unsigned SrcAlign = MI.getSourceAlignment();
5133 Type *LengthTy = MI.getLength()->getType();
5134 unsigned ElemSz = MI.getElementSizeInBytes();
5135 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5136 SDValue MC = DAG.getAtomicMemcpy(getRoot(), sdl, Dst, DstAlign, Src,
5137 SrcAlign, Length, LengthTy, ElemSz, isTC,
5138 MachinePointerInfo(MI.getRawDest()),
5139 MachinePointerInfo(MI.getRawSource()));
5140 updateDAGForMaybeTailCall(MC);
5141 return nullptr;
5142 }
5143 case Intrinsic::memmove_element_unordered_atomic: {
5144 auto &MI = cast<AtomicMemMoveInst>(I);
5145 SDValue Dst = getValue(MI.getRawDest());
5146 SDValue Src = getValue(MI.getRawSource());
5147 SDValue Length = getValue(MI.getLength());
5148
5149 unsigned DstAlign = MI.getDestAlignment();
5150 unsigned SrcAlign = MI.getSourceAlignment();
5151 Type *LengthTy = MI.getLength()->getType();
5152 unsigned ElemSz = MI.getElementSizeInBytes();
5153 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5154 SDValue MC = DAG.getAtomicMemmove(getRoot(), sdl, Dst, DstAlign, Src,
5155 SrcAlign, Length, LengthTy, ElemSz, isTC,
5156 MachinePointerInfo(MI.getRawDest()),
5157 MachinePointerInfo(MI.getRawSource()));
5158 updateDAGForMaybeTailCall(MC);
5159 return nullptr;
5160 }
5161 case Intrinsic::memset_element_unordered_atomic: {
5162 auto &MI = cast<AtomicMemSetInst>(I);
5163 SDValue Dst = getValue(MI.getRawDest());
5164 SDValue Val = getValue(MI.getValue());
5165 SDValue Length = getValue(MI.getLength());
5166
5167 unsigned DstAlign = MI.getDestAlignment();
5168 Type *LengthTy = MI.getLength()->getType();
5169 unsigned ElemSz = MI.getElementSizeInBytes();
5170 bool isTC = I.isTailCall() && isInTailCallPosition(&I, DAG.getTarget());
5171 SDValue MC = DAG.getAtomicMemset(getRoot(), sdl, Dst, DstAlign, Val, Length,
5172 LengthTy, ElemSz, isTC,
5173 MachinePointerInfo(MI.getRawDest()));
5174 updateDAGForMaybeTailCall(MC);
5175 return nullptr;
5176 }
5177 case Intrinsic::dbg_addr:
5178 case Intrinsic::dbg_declare: {
5179 const DbgInfoIntrinsic &DI = cast<DbgInfoIntrinsic>(I);
5180 DILocalVariable *Variable = DI.getVariable();
5181 DIExpression *Expression = DI.getExpression();
5182 dropDanglingDebugInfo(Variable, Expression);
5183 assert(Variable && "Missing variable")(static_cast <bool> (Variable && "Missing variable"
) ? void (0) : __assert_fail ("Variable && \"Missing variable\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5183, __extension__ __PRETTY_FUNCTION__));
5184
5185 // Check if address has undef value.
5186 const Value *Address = DI.getVariableLocation();
5187 if (!Address || isa<UndefValue>(Address) ||
5188 (Address->use_empty() && !isa<Argument>(Address))) {
5189 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false);
5190 return nullptr;
5191 }
5192
5193 bool isParameter = Variable->isParameter() || isa<Argument>(Address);
5194
5195 // Check if this variable can be described by a frame index, typically
5196 // either as a static alloca or a byval parameter.
5197 int FI = std::numeric_limits<int>::max();
5198 if (const auto *AI =
5199 dyn_cast<AllocaInst>(Address->stripInBoundsConstantOffsets())) {
5200 if (AI->isStaticAlloca()) {
5201 auto I = FuncInfo.StaticAllocaMap.find(AI);
5202 if (I != FuncInfo.StaticAllocaMap.end())
5203 FI = I->second;
5204 }
5205 } else if (const auto *Arg = dyn_cast<Argument>(
5206 Address->stripInBoundsConstantOffsets())) {
5207 FI = FuncInfo.getArgumentFrameIndex(Arg);
5208 }
5209
5210 // llvm.dbg.addr is control dependent and always generates indirect
5211 // DBG_VALUE instructions. llvm.dbg.declare is handled as a frame index in
5212 // the MachineFunction variable table.
5213 if (FI != std::numeric_limits<int>::max()) {
5214 if (Intrinsic == Intrinsic::dbg_addr) {
5215 SDDbgValue *SDV = DAG.getFrameIndexDbgValue(
5216 Variable, Expression, FI, /*IsIndirect*/ true, dl, SDNodeOrder);
5217 DAG.AddDbgValue(SDV, getRoot().getNode(), isParameter);
5218 }
5219 return nullptr;
5220 }
5221
5222 SDValue &N = NodeMap[Address];
5223 if (!N.getNode() && isa<Argument>(Address))
5224 // Check unused arguments map.
5225 N = UnusedArgNodeMap[Address];
5226 SDDbgValue *SDV;
5227 if (N.getNode()) {
5228 if (const BitCastInst *BCI = dyn_cast<BitCastInst>(Address))
5229 Address = BCI->getOperand(0);
5230 // Parameters are handled specially.
5231 auto FINode = dyn_cast<FrameIndexSDNode>(N.getNode());
5232 if (isParameter && FINode) {
5233 // Byval parameter. We have a frame index at this point.
5234 SDV =
5235 DAG.getFrameIndexDbgValue(Variable, Expression, FINode->getIndex(),
5236 /*IsIndirect*/ true, dl, SDNodeOrder);
5237 } else if (isa<Argument>(Address)) {
5238 // Address is an argument, so try to emit its dbg value using
5239 // virtual register info from the FuncInfo.ValueMap.
5240 EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N);
5241 return nullptr;
5242 } else {
5243 SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(),
5244 true, dl, SDNodeOrder);
5245 }
5246 DAG.AddDbgValue(SDV, N.getNode(), isParameter);
5247 } else {
5248 // If Address is an argument then try to emit its dbg value using
5249 // virtual register info from the FuncInfo.ValueMap.
5250 if (!EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true,
5251 N)) {
5252 LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug info for " <<
DI << "\n"; } } while (false);
5253 }
5254 }
5255 return nullptr;
5256 }
5257 case Intrinsic::dbg_label: {
5258 const DbgLabelInst &DI = cast<DbgLabelInst>(I);
5259 DILabel *Label = DI.getLabel();
5260 assert(Label && "Missing label")(static_cast <bool> (Label && "Missing label") ?
void (0) : __assert_fail ("Label && \"Missing label\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5260, __extension__ __PRETTY_FUNCTION__));
5261
5262 SDDbgLabel *SDV;
5263 SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder);
5264 DAG.AddDbgLabel(SDV);
5265 return nullptr;
5266 }
5267 case Intrinsic::dbg_value: {
5268 const DbgValueInst &DI = cast<DbgValueInst>(I);
5269 assert(DI.getVariable() && "Missing variable")(static_cast <bool> (DI.getVariable() && "Missing variable"
) ? void (0) : __assert_fail ("DI.getVariable() && \"Missing variable\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5269, __extension__ __PRETTY_FUNCTION__));
5270
5271 DILocalVariable *Variable = DI.getVariable();
5272 DIExpression *Expression = DI.getExpression();
5273 dropDanglingDebugInfo(Variable, Expression);
5274 const Value *V = DI.getValue();
5275 if (!V)
5276 return nullptr;
5277
5278 SDDbgValue *SDV;
5279 if (isa<ConstantInt>(V) || isa<ConstantFP>(V) || isa<UndefValue>(V)) {
5280 SDV = DAG.getConstantDbgValue(Variable, Expression, V, dl, SDNodeOrder);
5281 DAG.AddDbgValue(SDV, nullptr, false);
5282 return nullptr;
5283 }
5284
5285 // Do not use getValue() in here; we don't want to generate code at
5286 // this point if it hasn't been done yet.
5287 SDValue N = NodeMap[V];
5288 if (!N.getNode() && isa<Argument>(V)) // Check unused arguments map.
5289 N = UnusedArgNodeMap[V];
5290 if (N.getNode()) {
5291 if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, false, N))
5292 return nullptr;
5293 SDV = getDbgValue(N, Variable, Expression, dl, SDNodeOrder);
5294 DAG.AddDbgValue(SDV, N.getNode(), false);
5295 return nullptr;
5296 }
5297
5298 // PHI nodes have already been selected, so we should know which VReg that
5299 // is assigns to already.
5300 if (isa<PHINode>(V)) {
5301 auto VMI = FuncInfo.ValueMap.find(V);
5302 if (VMI != FuncInfo.ValueMap.end()) {
5303 unsigned Reg = VMI->second;
5304 // The PHI node may be split up into several MI PHI nodes (in
5305 // FunctionLoweringInfo::set).
5306 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg,
5307 V->getType(), None);
5308 if (RFV.occupiesMultipleRegs()) {
5309 unsigned Offset = 0;
5310 unsigned BitsToDescribe = 0;
5311 if (auto VarSize = Variable->getSizeInBits())
5312 BitsToDescribe = *VarSize;
5313 if (auto Fragment = Expression->getFragmentInfo())
5314 BitsToDescribe = Fragment->SizeInBits;
5315 for (auto RegAndSize : RFV.getRegsAndSizes()) {
5316 unsigned RegisterSize = RegAndSize.second;
5317 // Bail out if all bits are described already.
5318 if (Offset >= BitsToDescribe)
5319 break;
5320 unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe)
5321 ? BitsToDescribe - Offset
5322 : RegisterSize;
5323 auto FragmentExpr = DIExpression::createFragmentExpression(
5324 Expression, Offset, FragmentSize);
5325 if (!FragmentExpr)
5326 continue;
5327 SDV = DAG.getVRegDbgValue(Variable, *FragmentExpr, RegAndSize.first,
5328 false, dl, SDNodeOrder);
5329 DAG.AddDbgValue(SDV, nullptr, false);
5330 Offset += RegisterSize;
5331 }
5332 } else {
5333 SDV = DAG.getVRegDbgValue(Variable, Expression, Reg, false, dl,
5334 SDNodeOrder);
5335 DAG.AddDbgValue(SDV, nullptr, false);
5336 }
5337 return nullptr;
5338 }
5339 }
5340
5341 // TODO: When we get here we will either drop the dbg.value completely, or
5342 // we try to move it forward by letting it dangle for awhile. So we should
5343 // probably add an extra DbgValue to the DAG here, with a reference to
5344 // "noreg", to indicate that we have lost the debug location for the
5345 // variable.
5346
5347 if (!V->use_empty() ) {
5348 // Do not call getValue(V) yet, as we don't want to generate code.
5349 // Remember it for later.
5350 DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder);
5351 return nullptr;
5352 }
5353
5354 LLVM_DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Dropping debug location info for:\n "
<< DI << "\n"; } } while (false);
5355 LLVM_DEBUG(dbgs() << " Last seen at:\n " << *V << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " Last seen at:\n " << *
V << "\n"; } } while (false);
5356 return nullptr;
5357 }
5358
5359 case Intrinsic::eh_typeid_for: {
5360 // Find the type id for the given typeinfo.
5361 GlobalValue *GV = ExtractTypeInfo(I.getArgOperand(0));
5362 unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV);
5363 Res = DAG.getConstant(TypeID, sdl, MVT::i32);
5364 setValue(&I, Res);
5365 return nullptr;
5366 }
5367
5368 case Intrinsic::eh_return_i32:
5369 case Intrinsic::eh_return_i64:
5370 DAG.getMachineFunction().setCallsEHReturn(true);
5371 DAG.setRoot(DAG.getNode(ISD::EH_RETURN, sdl,
5372 MVT::Other,
5373 getControlRoot(),
5374 getValue(I.getArgOperand(0)),
5375 getValue(I.getArgOperand(1))));
5376 return nullptr;
5377 case Intrinsic::eh_unwind_init:
5378 DAG.getMachineFunction().setCallsUnwindInit(true);
5379 return nullptr;
5380 case Intrinsic::eh_dwarf_cfa:
5381 setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl,
5382 TLI.getPointerTy(DAG.getDataLayout()),
5383 getValue(I.getArgOperand(0))));
5384 return nullptr;
5385 case Intrinsic::eh_sjlj_callsite: {
5386 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
5387 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(0));
5388 assert(CI && "Non-constant call site value in eh.sjlj.callsite!")(static_cast <bool> (CI && "Non-constant call site value in eh.sjlj.callsite!"
) ? void (0) : __assert_fail ("CI && \"Non-constant call site value in eh.sjlj.callsite!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5388, __extension__ __PRETTY_FUNCTION__));
5389 assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!")(static_cast <bool> (MMI.getCurrentCallSite() == 0 &&
"Overlapping call sites!") ? void (0) : __assert_fail ("MMI.getCurrentCallSite() == 0 && \"Overlapping call sites!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5389, __extension__ __PRETTY_FUNCTION__));
5390
5391 MMI.setCurrentCallSite(CI->getZExtValue());
5392 return nullptr;
5393 }
5394 case Intrinsic::eh_sjlj_functioncontext: {
5395 // Get and store the index of the function context.
5396 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
5397 AllocaInst *FnCtx =
5398 cast<AllocaInst>(I.getArgOperand(0)->stripPointerCasts());
5399 int FI = FuncInfo.StaticAllocaMap[FnCtx];
5400 MFI.setFunctionContextIndex(FI);
5401 return nullptr;
5402 }
5403 case Intrinsic::eh_sjlj_setjmp: {
5404 SDValue Ops[2];
5405 Ops[0] = getRoot();
5406 Ops[1] = getValue(I.getArgOperand(0));
5407 SDValue Op = DAG.getNode(ISD::EH_SJLJ_SETJMP, sdl,
5408 DAG.getVTList(MVT::i32, MVT::Other), Ops);
5409 setValue(&I, Op.getValue(0));
5410 DAG.setRoot(Op.getValue(1));
5411 return nullptr;
5412 }
5413 case Intrinsic::eh_sjlj_longjmp:
5414 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other,
5415 getRoot(), getValue(I.getArgOperand(0))));
5416 return nullptr;
5417 case Intrinsic::eh_sjlj_setup_dispatch:
5418 DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other,
5419 getRoot()));
5420 return nullptr;
5421 case Intrinsic::masked_gather:
5422 visitMaskedGather(I);
5423 return nullptr;
5424 case Intrinsic::masked_load:
5425 visitMaskedLoad(I);
5426 return nullptr;
5427 case Intrinsic::masked_scatter:
5428 visitMaskedScatter(I);
5429 return nullptr;
5430 case Intrinsic::masked_store:
5431 visitMaskedStore(I);
5432 return nullptr;
5433 case Intrinsic::masked_expandload:
5434 visitMaskedLoad(I, true /* IsExpanding */);
5435 return nullptr;
5436 case Intrinsic::masked_compressstore:
5437 visitMaskedStore(I, true /* IsCompressing */);
5438 return nullptr;
5439 case Intrinsic::x86_mmx_pslli_w:
5440 case Intrinsic::x86_mmx_pslli_d:
5441 case Intrinsic::x86_mmx_pslli_q:
5442 case Intrinsic::x86_mmx_psrli_w:
5443 case Intrinsic::x86_mmx_psrli_d:
5444 case Intrinsic::x86_mmx_psrli_q:
5445 case Intrinsic::x86_mmx_psrai_w:
5446 case Intrinsic::x86_mmx_psrai_d: {
5447 SDValue ShAmt = getValue(I.getArgOperand(1));
5448 if (isa<ConstantSDNode>(ShAmt)) {
5449 visitTargetIntrinsic(I, Intrinsic);
5450 return nullptr;
5451 }
5452 unsigned NewIntrinsic = 0;
5453 EVT ShAmtVT = MVT::v2i32;
5454 switch (Intrinsic) {
5455 case Intrinsic::x86_mmx_pslli_w:
5456 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
5457 break;
5458 case Intrinsic::x86_mmx_pslli_d:
5459 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
5460 break;
5461 case Intrinsic::x86_mmx_pslli_q:
5462 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
5463 break;
5464 case Intrinsic::x86_mmx_psrli_w:
5465 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
5466 break;
5467 case Intrinsic::x86_mmx_psrli_d:
5468 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
5469 break;
5470 case Intrinsic::x86_mmx_psrli_q:
5471 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
5472 break;
5473 case Intrinsic::x86_mmx_psrai_w:
5474 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
5475 break;
5476 case Intrinsic::x86_mmx_psrai_d:
5477 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
5478 break;
5479 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5479); // Can't reach here.
5480 }
5481
5482 // The vector shift intrinsics with scalars uses 32b shift amounts but
5483 // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
5484 // to be zero.
5485 // We must do this early because v2i32 is not a legal type.
5486 SDValue ShOps[2];
5487 ShOps[0] = ShAmt;
5488 ShOps[1] = DAG.getConstant(0, sdl, MVT::i32);
5489 ShAmt = DAG.getBuildVector(ShAmtVT, sdl, ShOps);
5490 EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5491 ShAmt = DAG.getNode(ISD::BITCAST, sdl, DestVT, ShAmt);
5492 Res = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, sdl, DestVT,
5493 DAG.getConstant(NewIntrinsic, sdl, MVT::i32),
5494 getValue(I.getArgOperand(0)), ShAmt);
5495 setValue(&I, Res);
5496 return nullptr;
5497 }
5498 case Intrinsic::powi:
5499 setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)),
5500 getValue(I.getArgOperand(1)), DAG));
5501 return nullptr;
5502 case Intrinsic::log:
5503 setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5504 return nullptr;
5505 case Intrinsic::log2:
5506 setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5507 return nullptr;
5508 case Intrinsic::log10:
5509 setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5510 return nullptr;
5511 case Intrinsic::exp:
5512 setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5513 return nullptr;
5514 case Intrinsic::exp2:
5515 setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI));
5516 return nullptr;
5517 case Intrinsic::pow:
5518 setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)),
5519 getValue(I.getArgOperand(1)), DAG, TLI));
5520 return nullptr;
5521 case Intrinsic::sqrt:
5522 case Intrinsic::fabs:
5523 case Intrinsic::sin:
5524 case Intrinsic::cos:
5525 case Intrinsic::floor:
5526 case Intrinsic::ceil:
5527 case Intrinsic::trunc:
5528 case Intrinsic::rint:
5529 case Intrinsic::nearbyint:
5530 case Intrinsic::round:
5531 case Intrinsic::canonicalize: {
5532 unsigned Opcode;
5533 switch (Intrinsic) {
5534 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5534); // Can't reach here.
5535 case Intrinsic::sqrt: Opcode = ISD::FSQRT; break;
5536 case Intrinsic::fabs: Opcode = ISD::FABS; break;
5537 case Intrinsic::sin: Opcode = ISD::FSIN; break;
5538 case Intrinsic::cos: Opcode = ISD::FCOS; break;
5539 case Intrinsic::floor: Opcode = ISD::FFLOOR; break;
5540 case Intrinsic::ceil: Opcode = ISD::FCEIL; break;
5541 case Intrinsic::trunc: Opcode = ISD::FTRUNC; break;
5542 case Intrinsic::rint: Opcode = ISD::FRINT; break;
5543 case Intrinsic::nearbyint: Opcode = ISD::FNEARBYINT; break;
5544 case Intrinsic::round: Opcode = ISD::FROUND; break;
5545 case Intrinsic::canonicalize: Opcode = ISD::FCANONICALIZE; break;
5546 }
5547
5548 setValue(&I, DAG.getNode(Opcode, sdl,
5549 getValue(I.getArgOperand(0)).getValueType(),
5550 getValue(I.getArgOperand(0))));
5551 return nullptr;
5552 }
5553 case Intrinsic::minnum: {
5554 auto VT = getValue(I.getArgOperand(0)).getValueType();
5555 unsigned Opc =
5556 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINNAN, VT)
5557 ? ISD::FMINNAN
5558 : ISD::FMINNUM;
5559 setValue(&I, DAG.getNode(Opc, sdl, VT,
5560 getValue(I.getArgOperand(0)),
5561 getValue(I.getArgOperand(1))));
5562 return nullptr;
5563 }
5564 case Intrinsic::maxnum: {
5565 auto VT = getValue(I.getArgOperand(0)).getValueType();
5566 unsigned Opc =
5567 I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXNAN, VT)
5568 ? ISD::FMAXNAN
5569 : ISD::FMAXNUM;
5570 setValue(&I, DAG.getNode(Opc, sdl, VT,
5571 getValue(I.getArgOperand(0)),
5572 getValue(I.getArgOperand(1))));
5573 return nullptr;
5574 }
5575 case Intrinsic::copysign:
5576 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl,
5577 getValue(I.getArgOperand(0)).getValueType(),
5578 getValue(I.getArgOperand(0)),
5579 getValue(I.getArgOperand(1))));
5580 return nullptr;
5581 case Intrinsic::fma:
5582 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5583 getValue(I.getArgOperand(0)).getValueType(),
5584 getValue(I.getArgOperand(0)),
5585 getValue(I.getArgOperand(1)),
5586 getValue(I.getArgOperand(2))));
5587 return nullptr;
5588 case Intrinsic::experimental_constrained_fadd:
5589 case Intrinsic::experimental_constrained_fsub:
5590 case Intrinsic::experimental_constrained_fmul:
5591 case Intrinsic::experimental_constrained_fdiv:
5592 case Intrinsic::experimental_constrained_frem:
5593 case Intrinsic::experimental_constrained_fma:
5594 case Intrinsic::experimental_constrained_sqrt:
5595 case Intrinsic::experimental_constrained_pow:
5596 case Intrinsic::experimental_constrained_powi:
5597 case Intrinsic::experimental_constrained_sin:
5598 case Intrinsic::experimental_constrained_cos:
5599 case Intrinsic::experimental_constrained_exp:
5600 case Intrinsic::experimental_constrained_exp2:
5601 case Intrinsic::experimental_constrained_log:
5602 case Intrinsic::experimental_constrained_log10:
5603 case Intrinsic::experimental_constrained_log2:
5604 case Intrinsic::experimental_constrained_rint:
5605 case Intrinsic::experimental_constrained_nearbyint:
5606 visitConstrainedFPIntrinsic(cast<ConstrainedFPIntrinsic>(I));
5607 return nullptr;
5608 case Intrinsic::fmuladd: {
5609 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
5610 if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict &&
5611 TLI.isFMAFasterThanFMulAndFAdd(VT)) {
5612 setValue(&I, DAG.getNode(ISD::FMA, sdl,
5613 getValue(I.getArgOperand(0)).getValueType(),
5614 getValue(I.getArgOperand(0)),
5615 getValue(I.getArgOperand(1)),
5616 getValue(I.getArgOperand(2))));
5617 } else {
5618 // TODO: Intrinsic calls should have fast-math-flags.
5619 SDValue Mul = DAG.getNode(ISD::FMUL, sdl,
5620 getValue(I.getArgOperand(0)).getValueType(),
5621 getValue(I.getArgOperand(0)),
5622 getValue(I.getArgOperand(1)));
5623 SDValue Add = DAG.getNode(ISD::FADD, sdl,
5624 getValue(I.getArgOperand(0)).getValueType(),
5625 Mul,
5626 getValue(I.getArgOperand(2)));
5627 setValue(&I, Add);
5628 }
5629 return nullptr;
5630 }
5631 case Intrinsic::convert_to_fp16:
5632 setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16,
5633 DAG.getNode(ISD::FP_ROUND, sdl, MVT::f16,
5634 getValue(I.getArgOperand(0)),
5635 DAG.getTargetConstant(0, sdl,
5636 MVT::i32))));
5637 return nullptr;
5638 case Intrinsic::convert_from_fp16:
5639 setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl,
5640 TLI.getValueType(DAG.getDataLayout(), I.getType()),
5641 DAG.getNode(ISD::BITCAST, sdl, MVT::f16,
5642 getValue(I.getArgOperand(0)))));
5643 return nullptr;
5644 case Intrinsic::pcmarker: {
5645 SDValue Tmp = getValue(I.getArgOperand(0));
5646 DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp));
5647 return nullptr;
5648 }
5649 case Intrinsic::readcyclecounter: {
5650 SDValue Op = getRoot();
5651 Res = DAG.getNode(ISD::READCYCLECOUNTER, sdl,
5652 DAG.getVTList(MVT::i64, MVT::Other), Op);
5653 setValue(&I, Res);
5654 DAG.setRoot(Res.getValue(1));
5655 return nullptr;
5656 }
5657 case Intrinsic::bitreverse:
5658 setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl,
5659 getValue(I.getArgOperand(0)).getValueType(),
5660 getValue(I.getArgOperand(0))));
5661 return nullptr;
5662 case Intrinsic::bswap:
5663 setValue(&I, DAG.getNode(ISD::BSWAP, sdl,
5664 getValue(I.getArgOperand(0)).getValueType(),
5665 getValue(I.getArgOperand(0))));
5666 return nullptr;
5667 case Intrinsic::cttz: {
5668 SDValue Arg = getValue(I.getArgOperand(0));
5669 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5670 EVT Ty = Arg.getValueType();
5671 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF,
5672 sdl, Ty, Arg));
5673 return nullptr;
5674 }
5675 case Intrinsic::ctlz: {
5676 SDValue Arg = getValue(I.getArgOperand(0));
5677 ConstantInt *CI = cast<ConstantInt>(I.getArgOperand(1));
5678 EVT Ty = Arg.getValueType();
5679 setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF,
5680 sdl, Ty, Arg));
5681 return nullptr;
5682 }
5683 case Intrinsic::ctpop: {
5684 SDValue Arg = getValue(I.getArgOperand(0));
5685 EVT Ty = Arg.getValueType();
5686 setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg));
5687 return nullptr;
5688 }
5689 case Intrinsic::fshl:
5690 case Intrinsic::fshr: {
5691 bool IsFSHL = Intrinsic == Intrinsic::fshl;
5692 SDValue X = getValue(I.getArgOperand(0));
5693 SDValue Y = getValue(I.getArgOperand(1));
5694 SDValue Z = getValue(I.getArgOperand(2));
5695 EVT VT = X.getValueType();
5696
5697 // When X == Y, this is rotate. Create the node directly if legal.
5698 // TODO: This should also be done if the operation is custom, but we have
5699 // to make sure targets are handling the modulo shift amount as expected.
5700 // TODO: If the rotate direction (left or right) corresponding to the shift
5701 // is not available, adjust the shift value and invert the direction.
5702 auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR;
5703 if (X == Y && TLI.isOperationLegal(RotateOpcode, VT)) {
5704 setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z));
5705 return nullptr;
5706 }
5707
5708 // Get the shift amount and inverse shift amount, modulo the bit-width.
5709 SDValue BitWidthC = DAG.getConstant(VT.getScalarSizeInBits(), sdl, VT);
5710 SDValue ShAmt = DAG.getNode(ISD::UREM, sdl, VT, Z, BitWidthC);
5711 SDValue NegZ = DAG.getNode(ISD::SUB, sdl, VT, BitWidthC, Z);
5712 SDValue InvShAmt = DAG.getNode(ISD::UREM, sdl, VT, NegZ, BitWidthC);
5713
5714 // fshl: (X << (Z % BW)) | (Y >> ((BW - Z) % BW))
5715 // fshr: (X << ((BW - Z) % BW)) | (Y >> (Z % BW))
5716 SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : InvShAmt);
5717 SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, Y, IsFSHL ? InvShAmt : ShAmt);
5718 SDValue Res = DAG.getNode(ISD::OR, sdl, VT, ShX, ShY);
5719
5720 // If (Z % BW == 0), then (BW - Z) % BW is also zero, so the result would
5721 // be X | Y. If X == Y (rotate), that's fine. If not, we have to select.
5722 if (X != Y) {
5723 SDValue Zero = DAG.getConstant(0, sdl, VT);
5724 EVT CCVT = MVT::i1;
5725 if (VT.isVector())
5726 CCVT = EVT::getVectorVT(*Context, CCVT, VT.getVectorNumElements());
5727 // For fshl, 0 shift returns the 1st arg (X).
5728 // For fshr, 0 shift returns the 2nd arg (Y).
5729 SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ);
5730 Res = DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Res);
5731 }
5732 setValue(&I, Res);
5733 return nullptr;
5734 }
5735 case Intrinsic::stacksave: {
5736 SDValue Op = getRoot();
5737 Res = DAG.getNode(
5738 ISD::STACKSAVE, sdl,
5739 DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op);
5740 setValue(&I, Res);
5741 DAG.setRoot(Res.getValue(1));
5742 return nullptr;
5743 }
5744 case Intrinsic::stackrestore:
5745 Res = getValue(I.getArgOperand(0));
5746 DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res));
5747 return nullptr;
5748 case Intrinsic::get_dynamic_area_offset: {
5749 SDValue Op = getRoot();
5750 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5751 EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType());
5752 // Result type for @llvm.get.dynamic.area.offset should match PtrTy for
5753 // target.
5754 if (PtrTy != ResTy)
5755 report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset"
5756 " intrinsic!");
5757 Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy),
5758 Op);
5759 DAG.setRoot(Op);
5760 setValue(&I, Res);
5761 return nullptr;
5762 }
5763 case Intrinsic::stackguard: {
5764 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5765 MachineFunction &MF = DAG.getMachineFunction();
5766 const Module &M = *MF.getFunction().getParent();
5767 SDValue Chain = getRoot();
5768 if (TLI.useLoadStackGuardNode()) {
5769 Res = getLoadStackGuard(DAG, sdl, Chain);
5770 } else {
5771 const Value *Global = TLI.getSDagStackGuard(M);
5772 unsigned Align = DL->getPrefTypeAlignment(Global->getType());
5773 Res = DAG.getLoad(PtrTy, sdl, Chain, getValue(Global),
5774 MachinePointerInfo(Global, 0), Align,
5775 MachineMemOperand::MOVolatile);
5776 }
5777 if (TLI.useStackGuardXorFP())
5778 Res = TLI.emitStackGuardXorFP(DAG, Res, sdl);
5779 DAG.setRoot(Chain);
5780 setValue(&I, Res);
5781 return nullptr;
5782 }
5783 case Intrinsic::stackprotector: {
5784 // Emit code into the DAG to store the stack guard onto the stack.
5785 MachineFunction &MF = DAG.getMachineFunction();
5786 MachineFrameInfo &MFI = MF.getFrameInfo();
5787 EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout());
5788 SDValue Src, Chain = getRoot();
5789
5790 if (TLI.useLoadStackGuardNode())
5791 Src = getLoadStackGuard(DAG, sdl, Chain);
5792 else
5793 Src = getValue(I.getArgOperand(0)); // The guard's value.
5794
5795 AllocaInst *Slot = cast<AllocaInst>(I.getArgOperand(1));
5796
5797 int FI = FuncInfo.StaticAllocaMap[Slot];
5798 MFI.setStackProtectorIndex(FI);
5799
5800 SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
5801
5802 // Store the stack protector onto the stack.
5803 Res = DAG.getStore(Chain, sdl, Src, FIN, MachinePointerInfo::getFixedStack(
5804 DAG.getMachineFunction(), FI),
5805 /* Alignment = */ 0, MachineMemOperand::MOVolatile);
5806 setValue(&I, Res);
5807 DAG.setRoot(Res);
5808 return nullptr;
5809 }
5810 case Intrinsic::objectsize: {
5811 // If we don't know by now, we're never going to know.
5812 ConstantInt *CI = dyn_cast<ConstantInt>(I.getArgOperand(1));
5813
5814 assert(CI && "Non-constant type in __builtin_object_size?")(static_cast <bool> (CI && "Non-constant type in __builtin_object_size?"
) ? void (0) : __assert_fail ("CI && \"Non-constant type in __builtin_object_size?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5814, __extension__ __PRETTY_FUNCTION__));
5815
5816 SDValue Arg = getValue(I.getCalledValue());
5817 EVT Ty = Arg.getValueType();
5818
5819 if (CI->isZero())
5820 Res = DAG.getConstant(-1ULL, sdl, Ty);
5821 else
5822 Res = DAG.getConstant(0, sdl, Ty);
5823
5824 setValue(&I, Res);
5825 return nullptr;
5826 }
5827 case Intrinsic::annotation:
5828 case Intrinsic::ptr_annotation:
5829 case Intrinsic::launder_invariant_group:
5830 case Intrinsic::strip_invariant_group:
5831 // Drop the intrinsic, but forward the value
5832 setValue(&I, getValue(I.getOperand(0)));
5833 return nullptr;
5834 case Intrinsic::assume:
5835 case Intrinsic::var_annotation:
5836 case Intrinsic::sideeffect:
5837 // Discard annotate attributes, assumptions, and artificial side-effects.
5838 return nullptr;
5839
5840 case Intrinsic::codeview_annotation: {
5841 // Emit a label associated with this metadata.
5842 MachineFunction &MF = DAG.getMachineFunction();
5843 MCSymbol *Label =
5844 MF.getMMI().getContext().createTempSymbol("annotation", true);
5845 Metadata *MD = cast<MetadataAsValue>(I.getArgOperand(0))->getMetadata();
5846 MF.addCodeViewAnnotation(Label, cast<MDNode>(MD));
5847 Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label);
5848 DAG.setRoot(Res);
5849 return nullptr;
5850 }
5851
5852 case Intrinsic::init_trampoline: {
5853 const Function *F = cast<Function>(I.getArgOperand(1)->stripPointerCasts());
5854
5855 SDValue Ops[6];
5856 Ops[0] = getRoot();
5857 Ops[1] = getValue(I.getArgOperand(0));
5858 Ops[2] = getValue(I.getArgOperand(1));
5859 Ops[3] = getValue(I.getArgOperand(2));
5860 Ops[4] = DAG.getSrcValue(I.getArgOperand(0));
5861 Ops[5] = DAG.getSrcValue(F);
5862
5863 Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops);
5864
5865 DAG.setRoot(Res);
5866 return nullptr;
5867 }
5868 case Intrinsic::adjust_trampoline:
5869 setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl,
5870 TLI.getPointerTy(DAG.getDataLayout()),
5871 getValue(I.getArgOperand(0))));
5872 return nullptr;
5873 case Intrinsic::gcroot: {
5874 assert(DAG.getMachineFunction().getFunction().hasGC() &&(static_cast <bool> (DAG.getMachineFunction().getFunction
().hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5875, __extension__ __PRETTY_FUNCTION__))
5875 "only valid in functions with gc specified, enforced by Verifier")(static_cast <bool> (DAG.getMachineFunction().getFunction
().hasGC() && "only valid in functions with gc specified, enforced by Verifier"
) ? void (0) : __assert_fail ("DAG.getMachineFunction().getFunction().hasGC() && \"only valid in functions with gc specified, enforced by Verifier\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5875, __extension__ __PRETTY_FUNCTION__));
5876 assert(GFI && "implied by previous")(static_cast <bool> (GFI && "implied by previous"
) ? void (0) : __assert_fail ("GFI && \"implied by previous\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5876, __extension__ __PRETTY_FUNCTION__));
5877 const Value *Alloca = I.getArgOperand(0)->stripPointerCasts();
5878 const Constant *TypeMap = cast<Constant>(I.getArgOperand(1));
5879
5880 FrameIndexSDNode *FI = cast<FrameIndexSDNode>(getValue(Alloca).getNode());
5881 GFI->addStackRoot(FI->getIndex(), TypeMap);
5882 return nullptr;
5883 }
5884 case Intrinsic::gcread:
5885 case Intrinsic::gcwrite:
5886 llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!")::llvm::llvm_unreachable_internal("GC failed to lower gcread/gcwrite intrinsics!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5886);
5887 case Intrinsic::flt_rounds:
5888 setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32));
5889 return nullptr;
5890
5891 case Intrinsic::expect:
5892 // Just replace __builtin_expect(exp, c) with EXP.
5893 setValue(&I, getValue(I.getArgOperand(0)));
5894 return nullptr;
5895
5896 case Intrinsic::debugtrap:
5897 case Intrinsic::trap: {
5898 StringRef TrapFuncName =
5899 I.getAttributes()
5900 .getAttribute(AttributeList::FunctionIndex, "trap-func-name")
5901 .getValueAsString();
5902 if (TrapFuncName.empty()) {
5903 ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ?
5904 ISD::TRAP : ISD::DEBUGTRAP;
5905 DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot()));
5906 return nullptr;
5907 }
5908 TargetLowering::ArgListTy Args;
5909
5910 TargetLowering::CallLoweringInfo CLI(DAG);
5911 CLI.setDebugLoc(sdl).setChain(getRoot()).setLibCallee(
5912 CallingConv::C, I.getType(),
5913 DAG.getExternalSymbol(TrapFuncName.data(),
5914 TLI.getPointerTy(DAG.getDataLayout())),
5915 std::move(Args));
5916
5917 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
5918 DAG.setRoot(Result.second);
5919 return nullptr;
5920 }
5921
5922 case Intrinsic::uadd_with_overflow:
5923 case Intrinsic::sadd_with_overflow:
5924 case Intrinsic::usub_with_overflow:
5925 case Intrinsic::ssub_with_overflow:
5926 case Intrinsic::umul_with_overflow:
5927 case Intrinsic::smul_with_overflow: {
5928 ISD::NodeType Op;
5929 switch (Intrinsic) {
5930 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 5930); // Can't reach here.
5931 case Intrinsic::uadd_with_overflow: Op = ISD::UADDO; break;
5932 case Intrinsic::sadd_with_overflow: Op = ISD::SADDO; break;
5933 case Intrinsic::usub_with_overflow: Op = ISD::USUBO; break;
5934 case Intrinsic::ssub_with_overflow: Op = ISD::SSUBO; break;
5935 case Intrinsic::umul_with_overflow: Op = ISD::UMULO; break;
5936 case Intrinsic::smul_with_overflow: Op = ISD::SMULO; break;
5937 }
5938 SDValue Op1 = getValue(I.getArgOperand(0));
5939 SDValue Op2 = getValue(I.getArgOperand(1));
5940
5941 SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1);
5942 setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2));
5943 return nullptr;
5944 }
5945 case Intrinsic::prefetch: {
5946 SDValue Ops[5];
5947 unsigned rw = cast<ConstantInt>(I.getArgOperand(1))->getZExtValue();
5948 auto Flags = rw == 0 ? MachineMemOperand::MOLoad :MachineMemOperand::MOStore;
5949 Ops[0] = DAG.getRoot();
5950 Ops[1] = getValue(I.getArgOperand(0));
5951 Ops[2] = getValue(I.getArgOperand(1));
5952 Ops[3] = getValue(I.getArgOperand(2));
5953 Ops[4] = getValue(I.getArgOperand(3));
5954 SDValue Result = DAG.getMemIntrinsicNode(ISD::PREFETCH, sdl,
5955 DAG.getVTList(MVT::Other), Ops,
5956 EVT::getIntegerVT(*Context, 8),
5957 MachinePointerInfo(I.getArgOperand(0)),
5958 0, /* align */
5959 Flags);
5960
5961 // Chain the prefetch in parallell with any pending loads, to stay out of
5962 // the way of later optimizations.
5963 PendingLoads.push_back(Result);
5964 Result = getRoot();
5965 DAG.setRoot(Result);
5966 return nullptr;
5967 }
5968 case Intrinsic::lifetime_start:
5969 case Intrinsic::lifetime_end: {
5970 bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
5971 // Stack coloring is not enabled in O0, discard region information.
5972 if (TM.getOptLevel() == CodeGenOpt::None)
5973 return nullptr;
5974
5975 SmallVector<Value *, 4> Allocas;
5976 GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL);
5977
5978 for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(),
5979 E = Allocas.end(); Object != E; ++Object) {
5980 AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
5981
5982 // Could not find an Alloca.
5983 if (!LifetimeObject)
5984 continue;
5985
5986 // First check that the Alloca is static, otherwise it won't have a
5987 // valid frame index.
5988 auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject);
5989 if (SI == FuncInfo.StaticAllocaMap.end())
5990 return nullptr;
5991
5992 int FI = SI->second;
5993
5994 SDValue Ops[2];
5995 Ops[0] = getRoot();
5996 Ops[1] =
5997 DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true);
5998 unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
5999
6000 Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops);
6001 DAG.setRoot(Res);
6002 }
6003 return nullptr;
6004 }
6005 case Intrinsic::invariant_start:
6006 // Discard region information.
6007 setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout())));
6008 return nullptr;
6009 case Intrinsic::invariant_end:
6010 // Discard region information.
6011 return nullptr;
6012 case Intrinsic::clear_cache:
6013 return TLI.getClearCacheBuiltinName();
6014 case Intrinsic::donothing:
6015 // ignore
6016 return nullptr;
6017 case Intrinsic::experimental_stackmap:
6018 visitStackmap(I);
6019 return nullptr;
6020 case Intrinsic::experimental_patchpoint_void:
6021 case Intrinsic::experimental_patchpoint_i64:
6022 visitPatchpoint(&I);
6023 return nullptr;
6024 case Intrinsic::experimental_gc_statepoint:
6025 LowerStatepoint(ImmutableStatepoint(&I));
6026 return nullptr;
6027 case Intrinsic::experimental_gc_result:
6028 visitGCResult(cast<GCResultInst>(I));
6029 return nullptr;
6030 case Intrinsic::experimental_gc_relocate:
6031 visitGCRelocate(cast<GCRelocateInst>(I));
6032 return nullptr;
6033 case Intrinsic::instrprof_increment:
6034 llvm_unreachable("instrprof failed to lower an increment")::llvm::llvm_unreachable_internal("instrprof failed to lower an increment"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6034);
6035 case Intrinsic::instrprof_value_profile:
6036 llvm_unreachable("instrprof failed to lower a value profiling call")::llvm::llvm_unreachable_internal("instrprof failed to lower a value profiling call"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6036);
6037 case Intrinsic::localescape: {
6038 MachineFunction &MF = DAG.getMachineFunction();
6039 const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo();
6040
6041 // Directly emit some LOCAL_ESCAPE machine instrs. Label assignment emission
6042 // is the same on all targets.
6043 for (unsigned Idx = 0, E = I.getNumArgOperands(); Idx < E; ++Idx) {
6044 Value *Arg = I.getArgOperand(Idx)->stripPointerCasts();
6045 if (isa<ConstantPointerNull>(Arg))
6046 continue; // Skip null pointers. They represent a hole in index space.
6047 AllocaInst *Slot = cast<AllocaInst>(Arg);
6048 assert(FuncInfo.StaticAllocaMap.count(Slot) &&(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot
) && "can only escape static allocas") ? void (0) : __assert_fail
("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6049, __extension__ __PRETTY_FUNCTION__))
6049 "can only escape static allocas")(static_cast <bool> (FuncInfo.StaticAllocaMap.count(Slot
) && "can only escape static allocas") ? void (0) : __assert_fail
("FuncInfo.StaticAllocaMap.count(Slot) && \"can only escape static allocas\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6049, __extension__ __PRETTY_FUNCTION__));
6050 int FI = FuncInfo.StaticAllocaMap[Slot];
6051 MCSymbol *FrameAllocSym =
6052 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6053 GlobalValue::dropLLVMManglingEscape(MF.getName()), Idx);
6054 BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, dl,
6055 TII->get(TargetOpcode::LOCAL_ESCAPE))
6056 .addSym(FrameAllocSym)
6057 .addFrameIndex(FI);
6058 }
6059
6060 return nullptr;
6061 }
6062
6063 case Intrinsic::localrecover: {
6064 // i8* @llvm.localrecover(i8* %fn, i8* %fp, i32 %idx)
6065 MachineFunction &MF = DAG.getMachineFunction();
6066 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout(), 0);
6067
6068 // Get the symbol that defines the frame offset.
6069 auto *Fn = cast<Function>(I.getArgOperand(0)->stripPointerCasts());
6070 auto *Idx = cast<ConstantInt>(I.getArgOperand(2));
6071 unsigned IdxVal =
6072 unsigned(Idx->getLimitedValue(std::numeric_limits<int>::max()));
6073 MCSymbol *FrameAllocSym =
6074 MF.getMMI().getContext().getOrCreateFrameAllocSymbol(
6075 GlobalValue::dropLLVMManglingEscape(Fn->getName()), IdxVal);
6076
6077 // Create a MCSymbol for the label to avoid any target lowering
6078 // that would make this PC relative.
6079 SDValue OffsetSym = DAG.getMCSymbol(FrameAllocSym, PtrVT);
6080 SDValue OffsetVal =
6081 DAG.getNode(ISD::LOCAL_RECOVER, sdl, PtrVT, OffsetSym);
6082
6083 // Add the offset to the FP.
6084 Value *FP = I.getArgOperand(1);
6085 SDValue FPVal = getValue(FP);
6086 SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal);
6087 setValue(&I, Add);
6088
6089 return nullptr;
6090 }
6091
6092 case Intrinsic::eh_exceptionpointer:
6093 case Intrinsic::eh_exceptioncode: {
6094 // Get the exception pointer vreg, copy from it, and resize it to fit.
6095 const auto *CPI = cast<CatchPadInst>(I.getArgOperand(0));
6096 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
6097 const TargetRegisterClass *PtrRC = TLI.getRegClassFor(PtrVT);
6098 unsigned VReg = FuncInfo.getCatchPadExceptionPointerVReg(CPI, PtrRC);
6099 SDValue N =
6100 DAG.getCopyFromReg(DAG.getEntryNode(), getCurSDLoc(), VReg, PtrVT);
6101 if (Intrinsic == Intrinsic::eh_exceptioncode)
6102 N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32);
6103 setValue(&I, N);
6104 return nullptr;
6105 }
6106 case Intrinsic::xray_customevent: {
6107 // Here we want to make sure that the intrinsic behaves as if it has a
6108 // specific calling convention, and only for x86_64.
6109 // FIXME: Support other platforms later.
6110 const auto &Triple = DAG.getTarget().getTargetTriple();
6111 if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
6112 return nullptr;
6113
6114 SDLoc DL = getCurSDLoc();
6115 SmallVector<SDValue, 8> Ops;
6116
6117 // We want to say that we always want the arguments in registers.
6118 SDValue LogEntryVal = getValue(I.getArgOperand(0));
6119 SDValue StrSizeVal = getValue(I.getArgOperand(1));
6120 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6121 SDValue Chain = getRoot();
6122 Ops.push_back(LogEntryVal);
6123 Ops.push_back(StrSizeVal);
6124 Ops.push_back(Chain);
6125
6126 // We need to enforce the calling convention for the callsite, so that
6127 // argument ordering is enforced correctly, and that register allocation can
6128 // see that some registers may be assumed clobbered and have to preserve
6129 // them across calls to the intrinsic.
6130 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHABLE_EVENT_CALL,
6131 DL, NodeTys, Ops);
6132 SDValue patchableNode = SDValue(MN, 0);
6133 DAG.setRoot(patchableNode);
6134 setValue(&I, patchableNode);
6135 return nullptr;
6136 }
6137 case Intrinsic::xray_typedevent: {
6138 // Here we want to make sure that the intrinsic behaves as if it has a
6139 // specific calling convention, and only for x86_64.
6140 // FIXME: Support other platforms later.
6141 const auto &Triple = DAG.getTarget().getTargetTriple();
6142 if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux())
6143 return nullptr;
6144
6145 SDLoc DL = getCurSDLoc();
6146 SmallVector<SDValue, 8> Ops;
6147
6148 // We want to say that we always want the arguments in registers.
6149 // It's unclear to me how manipulating the selection DAG here forces callers
6150 // to provide arguments in registers instead of on the stack.
6151 SDValue LogTypeId = getValue(I.getArgOperand(0));
6152 SDValue LogEntryVal = getValue(I.getArgOperand(1));
6153 SDValue StrSizeVal = getValue(I.getArgOperand(2));
6154 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
6155 SDValue Chain = getRoot();
6156 Ops.push_back(LogTypeId);
6157 Ops.push_back(LogEntryVal);
6158 Ops.push_back(StrSizeVal);
6159 Ops.push_back(Chain);
6160
6161 // We need to enforce the calling convention for the callsite, so that
6162 // argument ordering is enforced correctly, and that register allocation can
6163 // see that some registers may be assumed clobbered and have to preserve
6164 // them across calls to the intrinsic.
6165 MachineSDNode *MN = DAG.getMachineNode(
6166 TargetOpcode::PATCHABLE_TYPED_EVENT_CALL, DL, NodeTys, Ops);
6167 SDValue patchableNode = SDValue(MN, 0);
6168 DAG.setRoot(patchableNode);
6169 setValue(&I, patchableNode);
6170 return nullptr;
6171 }
6172 case Intrinsic::experimental_deoptimize:
6173 LowerDeoptimizeCall(&I);
6174 return nullptr;
6175
6176 case Intrinsic::experimental_vector_reduce_fadd:
6177 case Intrinsic::experimental_vector_reduce_fmul:
6178 case Intrinsic::experimental_vector_reduce_add:
6179 case Intrinsic::experimental_vector_reduce_mul:
6180 case Intrinsic::experimental_vector_reduce_and:
6181 case Intrinsic::experimental_vector_reduce_or:
6182 case Intrinsic::experimental_vector_reduce_xor:
6183 case Intrinsic::experimental_vector_reduce_smax:
6184 case Intrinsic::experimental_vector_reduce_smin:
6185 case Intrinsic::experimental_vector_reduce_umax:
6186 case Intrinsic::experimental_vector_reduce_umin:
6187 case Intrinsic::experimental_vector_reduce_fmax:
6188 case Intrinsic::experimental_vector_reduce_fmin:
6189 visitVectorReduce(I, Intrinsic);
6190 return nullptr;
6191
6192 case Intrinsic::icall_branch_funnel: {
6193 SmallVector<SDValue, 16> Ops;
6194 Ops.push_back(DAG.getRoot());
6195 Ops.push_back(getValue(I.getArgOperand(0)));
6196
6197 int64_t Offset;
6198 auto *Base = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
6199 I.getArgOperand(1), Offset, DAG.getDataLayout()));
6200 if (!Base)
6201 report_fatal_error(
6202 "llvm.icall.branch.funnel operand must be a GlobalValue");
6203 Ops.push_back(DAG.getTargetGlobalAddress(Base, getCurSDLoc(), MVT::i64, 0));
6204
6205 struct BranchFunnelTarget {
6206 int64_t Offset;
6207 SDValue Target;
6208 };
6209 SmallVector<BranchFunnelTarget, 8> Targets;
6210
6211 for (unsigned Op = 1, N = I.getNumArgOperands(); Op != N; Op += 2) {
6212 auto *ElemBase = dyn_cast<GlobalObject>(GetPointerBaseWithConstantOffset(
6213 I.getArgOperand(Op), Offset, DAG.getDataLayout()));
6214 if (ElemBase != Base)
6215 report_fatal_error("all llvm.icall.branch.funnel operands must refer "
6216 "to the same GlobalValue");
6217
6218 SDValue Val = getValue(I.getArgOperand(Op + 1));
6219 auto *GA = dyn_cast<GlobalAddressSDNode>(Val);
6220 if (!GA)
6221 report_fatal_error(
6222 "llvm.icall.branch.funnel operand must be a GlobalValue");
6223 Targets.push_back({Offset, DAG.getTargetGlobalAddress(
6224 GA->getGlobal(), getCurSDLoc(),
6225 Val.getValueType(), GA->getOffset())});
6226 }
6227 llvm::sort(Targets.begin(), Targets.end(),
6228 [](const BranchFunnelTarget &T1, const BranchFunnelTarget &T2) {
6229 return T1.Offset < T2.Offset;
6230 });
6231
6232 for (auto &T : Targets) {
6233 Ops.push_back(DAG.getTargetConstant(T.Offset, getCurSDLoc(), MVT::i32));
6234 Ops.push_back(T.Target);
6235 }
6236
6237 SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL,
6238 getCurSDLoc(), MVT::Other, Ops),
6239 0);
6240 DAG.setRoot(N);
6241 setValue(&I, N);
6242 HasTailCall = true;
6243 return nullptr;
6244 }
6245
6246 case Intrinsic::wasm_landingpad_index: {
6247 // TODO store landing pad index in a map, which will be used when generating
6248 // LSDA information
6249 return nullptr;
6250 }
6251 }
6252}
6253
6254void SelectionDAGBuilder::visitConstrainedFPIntrinsic(
6255 const ConstrainedFPIntrinsic &FPI) {
6256 SDLoc sdl = getCurSDLoc();
6257 unsigned Opcode;
6258 switch (FPI.getIntrinsicID()) {
6259 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6259); // Can't reach here.
6260 case Intrinsic::experimental_constrained_fadd:
6261 Opcode = ISD::STRICT_FADD;
6262 break;
6263 case Intrinsic::experimental_constrained_fsub:
6264 Opcode = ISD::STRICT_FSUB;
6265 break;
6266 case Intrinsic::experimental_constrained_fmul:
6267 Opcode = ISD::STRICT_FMUL;
6268 break;
6269 case Intrinsic::experimental_constrained_fdiv:
6270 Opcode = ISD::STRICT_FDIV;
6271 break;
6272 case Intrinsic::experimental_constrained_frem:
6273 Opcode = ISD::STRICT_FREM;
6274 break;
6275 case Intrinsic::experimental_constrained_fma:
6276 Opcode = ISD::STRICT_FMA;
6277 break;
6278 case Intrinsic::experimental_constrained_sqrt:
6279 Opcode = ISD::STRICT_FSQRT;
6280 break;
6281 case Intrinsic::experimental_constrained_pow:
6282 Opcode = ISD::STRICT_FPOW;
6283 break;
6284 case Intrinsic::experimental_constrained_powi:
6285 Opcode = ISD::STRICT_FPOWI;
6286 break;
6287 case Intrinsic::experimental_constrained_sin:
6288 Opcode = ISD::STRICT_FSIN;
6289 break;
6290 case Intrinsic::experimental_constrained_cos:
6291 Opcode = ISD::STRICT_FCOS;
6292 break;
6293 case Intrinsic::experimental_constrained_exp:
6294 Opcode = ISD::STRICT_FEXP;
6295 break;
6296 case Intrinsic::experimental_constrained_exp2:
6297 Opcode = ISD::STRICT_FEXP2;
6298 break;
6299 case Intrinsic::experimental_constrained_log:
6300 Opcode = ISD::STRICT_FLOG;
6301 break;
6302 case Intrinsic::experimental_constrained_log10:
6303 Opcode = ISD::STRICT_FLOG10;
6304 break;
6305 case Intrinsic::experimental_constrained_log2:
6306 Opcode = ISD::STRICT_FLOG2;
6307 break;
6308 case Intrinsic::experimental_constrained_rint:
6309 Opcode = ISD::STRICT_FRINT;
6310 break;
6311 case Intrinsic::experimental_constrained_nearbyint:
6312 Opcode = ISD::STRICT_FNEARBYINT;
6313 break;
6314 }
6315 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6316 SDValue Chain = getRoot();
6317 SmallVector<EVT, 4> ValueVTs;
6318 ComputeValueVTs(TLI, DAG.getDataLayout(), FPI.getType(), ValueVTs);
6319 ValueVTs.push_back(MVT::Other); // Out chain
6320
6321 SDVTList VTs = DAG.getVTList(ValueVTs);
6322 SDValue Result;
6323 if (FPI.isUnaryOp())
6324 Result = DAG.getNode(Opcode, sdl, VTs,
6325 { Chain, getValue(FPI.getArgOperand(0)) });
6326 else if (FPI.isTernaryOp())
6327 Result = DAG.getNode(Opcode, sdl, VTs,
6328 { Chain, getValue(FPI.getArgOperand(0)),
6329 getValue(FPI.getArgOperand(1)),
6330 getValue(FPI.getArgOperand(2)) });
6331 else
6332 Result = DAG.getNode(Opcode, sdl, VTs,
6333 { Chain, getValue(FPI.getArgOperand(0)),
6334 getValue(FPI.getArgOperand(1)) });
6335
6336 assert(Result.getNode()->getNumValues() == 2)(static_cast <bool> (Result.getNode()->getNumValues(
) == 2) ? void (0) : __assert_fail ("Result.getNode()->getNumValues() == 2"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6336, __extension__ __PRETTY_FUNCTION__));
6337 SDValue OutChain = Result.getValue(1);
6338 DAG.setRoot(OutChain);
6339 SDValue FPResult = Result.getValue(0);
6340 setValue(&FPI, FPResult);
6341}
6342
6343std::pair<SDValue, SDValue>
6344SelectionDAGBuilder::lowerInvokable(TargetLowering::CallLoweringInfo &CLI,
6345 const BasicBlock *EHPadBB) {
6346 MachineFunction &MF = DAG.getMachineFunction();
6347 MachineModuleInfo &MMI = MF.getMMI();
6348 MCSymbol *BeginLabel = nullptr;
6349
6350 if (EHPadBB) {
6351 // Insert a label before the invoke call to mark the try range. This can be
6352 // used to detect deletion of the invoke via the MachineModuleInfo.
6353 BeginLabel = MMI.getContext().createTempSymbol();
6354
6355 // For SjLj, keep track of which landing pads go with which invokes
6356 // so as to maintain the ordering of pads in the LSDA.
6357 unsigned CallSiteIndex = MMI.getCurrentCallSite();
6358 if (CallSiteIndex) {
6359 MF.setCallSiteBeginLabel(BeginLabel, CallSiteIndex);
6360 LPadToCallSiteMap[FuncInfo.MBBMap[EHPadBB]].push_back(CallSiteIndex);
6361
6362 // Now that the call site is handled, stop tracking it.
6363 MMI.setCurrentCallSite(0);
6364 }
6365
6366 // Both PendingLoads and PendingExports must be flushed here;
6367 // this call might not return.
6368 (void)getRoot();
6369 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getControlRoot(), BeginLabel));
6370
6371 CLI.setChain(getRoot());
6372 }
6373 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6374 std::pair<SDValue, SDValue> Result = TLI.LowerCallTo(CLI);
6375
6376 assert((CLI.IsTailCall || Result.second.getNode()) &&(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode
()) && "Non-null chain expected with non-tail call!")
? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6377, __extension__ __PRETTY_FUNCTION__))
6377 "Non-null chain expected with non-tail call!")(static_cast <bool> ((CLI.IsTailCall || Result.second.getNode
()) && "Non-null chain expected with non-tail call!")
? void (0) : __assert_fail ("(CLI.IsTailCall || Result.second.getNode()) && \"Non-null chain expected with non-tail call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6377, __extension__ __PRETTY_FUNCTION__));
6378 assert((Result.second.getNode() || !Result.first.getNode()) &&(static_cast <bool> ((Result.second.getNode() || !Result
.first.getNode()) && "Null value expected with tail call!"
) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6379, __extension__ __PRETTY_FUNCTION__))
6379 "Null value expected with tail call!")(static_cast <bool> ((Result.second.getNode() || !Result
.first.getNode()) && "Null value expected with tail call!"
) ? void (0) : __assert_fail ("(Result.second.getNode() || !Result.first.getNode()) && \"Null value expected with tail call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6379, __extension__ __PRETTY_FUNCTION__));
6380
6381 if (!Result.second.getNode()) {
6382 // As a special case, a null chain means that a tail call has been emitted
6383 // and the DAG root is already updated.
6384 HasTailCall = true;
6385
6386 // Since there's no actual continuation from this block, nothing can be
6387 // relying on us setting vregs for them.
6388 PendingExports.clear();
6389 } else {
6390 DAG.setRoot(Result.second);
6391 }
6392
6393 if (EHPadBB) {
6394 // Insert a label at the end of the invoke call to mark the try range. This
6395 // can be used to detect deletion of the invoke via the MachineModuleInfo.
6396 MCSymbol *EndLabel = MMI.getContext().createTempSymbol();
6397 DAG.setRoot(DAG.getEHLabel(getCurSDLoc(), getRoot(), EndLabel));
6398
6399 // Inform MachineModuleInfo of range.
6400 auto Pers = classifyEHPersonality(FuncInfo.Fn->getPersonalityFn());
6401 // There is a platform (e.g. wasm) that uses funclet style IR but does not
6402 // actually use outlined funclets and their LSDA info style.
6403 if (MF.hasEHFunclets() && isFuncletEHPersonality(Pers)) {
6404 assert(CLI.CS)(static_cast <bool> (CLI.CS) ? void (0) : __assert_fail
("CLI.CS", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6404, __extension__ __PRETTY_FUNCTION__));
6405 WinEHFuncInfo *EHInfo = DAG.getMachineFunction().getWinEHFuncInfo();
6406 EHInfo->addIPToStateRange(cast<InvokeInst>(CLI.CS.getInstruction()),
6407 BeginLabel, EndLabel);
6408 } else {
6409 MF.addInvoke(FuncInfo.MBBMap[EHPadBB], BeginLabel, EndLabel);
6410 }
6411 }
6412
6413 return Result;
6414}
6415
6416void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
6417 bool isTailCall,
6418 const BasicBlock *EHPadBB) {
6419 auto &DL = DAG.getDataLayout();
6420 FunctionType *FTy = CS.getFunctionType();
6421 Type *RetTy = CS.getType();
6422
6423 TargetLowering::ArgListTy Args;
6424 Args.reserve(CS.arg_size());
6425
6426 const Value *SwiftErrorVal = nullptr;
6427 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6428
6429 // We can't tail call inside a function with a swifterror argument. Lowering
6430 // does not support this yet. It would have to move into the swifterror
6431 // register before the call.
6432 auto *Caller = CS.getInstruction()->getParent()->getParent();
6433 if (TLI.supportSwiftError() &&
6434 Caller->getAttributes().hasAttrSomewhere(Attribute::SwiftError))
6435 isTailCall = false;
6436
6437 for (ImmutableCallSite::arg_iterator i = CS.arg_begin(), e = CS.arg_end();
6438 i != e; ++i) {
6439 TargetLowering::ArgListEntry Entry;
6440 const Value *V = *i;
6441
6442 // Skip empty types
6443 if (V->getType()->isEmptyTy())
6444 continue;
6445
6446 SDValue ArgNode = getValue(V);
6447 Entry.Node = ArgNode; Entry.Ty = V->getType();
6448
6449 Entry.setAttributes(&CS, i - CS.arg_begin());
6450
6451 // Use swifterror virtual register as input to the call.
6452 if (Entry.IsSwiftError && TLI.supportSwiftError()) {
6453 SwiftErrorVal = V;
6454 // We find the virtual register for the actual swifterror argument.
6455 // Instead of using the Value, we use the virtual register instead.
6456 Entry.Node = DAG.getRegister(FuncInfo
6457 .getOrCreateSwiftErrorVRegUseAt(
6458 CS.getInstruction(), FuncInfo.MBB, V)
6459 .first,
6460 EVT(TLI.getPointerTy(DL)));
6461 }
6462
6463 Args.push_back(Entry);
6464
6465 // If we have an explicit sret argument that is an Instruction, (i.e., it
6466 // might point to function-local memory), we can't meaningfully tail-call.
6467 if (Entry.IsSRet && isa<Instruction>(V))
6468 isTailCall = false;
6469 }
6470
6471 // Check if target-independent constraints permit a tail call here.
6472 // Target-dependent constraints are checked within TLI->LowerCallTo.
6473 if (isTailCall && !isInTailCallPosition(CS, DAG.getTarget()))
6474 isTailCall = false;
6475
6476 // Disable tail calls if there is an swifterror argument. Targets have not
6477 // been updated to support tail calls.
6478 if (TLI.supportSwiftError() && SwiftErrorVal)
6479 isTailCall = false;
6480
6481 TargetLowering::CallLoweringInfo CLI(DAG);
6482 CLI.setDebugLoc(getCurSDLoc())
6483 .setChain(getRoot())
6484 .setCallee(RetTy, FTy, Callee, std::move(Args), CS)
6485 .setTailCall(isTailCall)
6486 .setConvergent(CS.isConvergent());
6487 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
6488
6489 if (Result.first.getNode()) {
6490 const Instruction *Inst = CS.getInstruction();
6491 Result.first = lowerRangeToAssertZExt(DAG, *Inst, Result.first);
6492 setValue(Inst, Result.first);
6493 }
6494
6495 // The last element of CLI.InVals has the SDValue for swifterror return.
6496 // Here we copy it to a virtual register and update SwiftErrorMap for
6497 // book-keeping.
6498 if (SwiftErrorVal && TLI.supportSwiftError()) {
6499 // Get the last element of InVals.
6500 SDValue Src = CLI.InVals.back();
6501 unsigned VReg; bool CreatedVReg;
6502 std::tie(VReg, CreatedVReg) =
6503 FuncInfo.getOrCreateSwiftErrorVRegDefAt(CS.getInstruction());
6504 SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src);
6505 // We update the virtual register for the actual swifterror argument.
6506 if (CreatedVReg)
6507 FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg);
6508 DAG.setRoot(CopyNode);
6509 }
6510}
6511
6512static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT,
6513 SelectionDAGBuilder &Builder) {
6514 // Check to see if this load can be trivially constant folded, e.g. if the
6515 // input is from a string literal.
6516 if (const Constant *LoadInput = dyn_cast<Constant>(PtrVal)) {
6517 // Cast pointer to the type we really want to load.
6518 Type *LoadTy =
6519 Type::getIntNTy(PtrVal->getContext(), LoadVT.getScalarSizeInBits());
6520 if (LoadVT.isVector())
6521 LoadTy = VectorType::get(LoadTy, LoadVT.getVectorNumElements());
6522
6523 LoadInput = ConstantExpr::getBitCast(const_cast<Constant *>(LoadInput),
6524 PointerType::getUnqual(LoadTy));
6525
6526 if (const Constant *LoadCst = ConstantFoldLoadFromConstPtr(
6527 const_cast<Constant *>(LoadInput), LoadTy, *Builder.DL))
6528 return Builder.getValue(LoadCst);
6529 }
6530
6531 // Otherwise, we have to emit the load. If the pointer is to unfoldable but
6532 // still constant memory, the input chain can be the entry node.
6533 SDValue Root;
6534 bool ConstantMemory = false;
6535
6536 // Do not serialize (non-volatile) loads of constant memory with anything.
6537 if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) {
6538 Root = Builder.DAG.getEntryNode();
6539 ConstantMemory = true;
6540 } else {
6541 // Do not serialize non-volatile loads against each other.
6542 Root = Builder.DAG.getRoot();
6543 }
6544
6545 SDValue Ptr = Builder.getValue(PtrVal);
6546 SDValue LoadVal = Builder.DAG.getLoad(LoadVT, Builder.getCurSDLoc(), Root,
6547 Ptr, MachinePointerInfo(PtrVal),
6548 /* Alignment = */ 1);
6549
6550 if (!ConstantMemory)
6551 Builder.PendingLoads.push_back(LoadVal.getValue(1));
6552 return LoadVal;
6553}
6554
6555/// Record the value for an instruction that produces an integer result,
6556/// converting the type where necessary.
6557void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I,
6558 SDValue Value,
6559 bool IsSigned) {
6560 EVT VT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6561 I.getType(), true);
6562 if (IsSigned)
6563 Value = DAG.getSExtOrTrunc(Value, getCurSDLoc(), VT);
6564 else
6565 Value = DAG.getZExtOrTrunc(Value, getCurSDLoc(), VT);
6566 setValue(&I, Value);
6567}
6568
6569/// See if we can lower a memcmp call into an optimized form. If so, return
6570/// true and lower it. Otherwise return false, and it will be lowered like a
6571/// normal call.
6572/// The caller already checked that \p I calls the appropriate LibFunc with a
6573/// correct prototype.
6574bool SelectionDAGBuilder::visitMemCmpCall(const CallInst &I) {
6575 const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1);
6576 const Value *Size = I.getArgOperand(2);
6577 const ConstantInt *CSize = dyn_cast<ConstantInt>(Size);
6578 if (CSize && CSize->getZExtValue() == 0) {
6579 EVT CallVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
6580 I.getType(), true);
6581 setValue(&I, DAG.getConstant(0, getCurSDLoc(), CallVT));
6582 return true;
6583 }
6584
6585 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6586 std::pair<SDValue, SDValue> Res = TSI.EmitTargetCodeForMemcmp(
6587 DAG, getCurSDLoc(), DAG.getRoot(), getValue(LHS), getValue(RHS),
6588 getValue(Size), MachinePointerInfo(LHS), MachinePointerInfo(RHS));
6589 if (Res.first.getNode()) {
6590 processIntegerCallValue(I, Res.first, true);
6591 PendingLoads.push_back(Res.second);
6592 return true;
6593 }
6594
6595 // memcmp(S1,S2,2) != 0 -> (*(short*)LHS != *(short*)RHS) != 0
6596 // memcmp(S1,S2,4) != 0 -> (*(int*)LHS != *(int*)RHS) != 0
6597 if (!CSize || !isOnlyUsedInZeroEqualityComparison(&I))
6598 return false;
6599
6600 // If the target has a fast compare for the given size, it will return a
6601 // preferred load type for that size. Require that the load VT is legal and
6602 // that the target supports unaligned loads of that type. Otherwise, return
6603 // INVALID.
6604 auto hasFastLoadsAndCompare = [&](unsigned NumBits) {
6605 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
6606 MVT LVT = TLI.hasFastEqualityCompare(NumBits);
6607 if (LVT != MVT::INVALID_SIMPLE_VALUE_TYPE) {
6608 // TODO: Handle 5 byte compare as 4-byte + 1 byte.
6609 // TODO: Handle 8 byte compare on x86-32 as two 32-bit loads.
6610 // TODO: Check alignment of src and dest ptrs.
6611 unsigned DstAS = LHS->getType()->getPointerAddressSpace();
6612 unsigned SrcAS = RHS->getType()->getPointerAddressSpace();
6613 if (!TLI.isTypeLegal(LVT) ||
6614 !TLI.allowsMisalignedMemoryAccesses(LVT, SrcAS) ||
6615 !TLI.allowsMisalignedMemoryAccesses(LVT, DstAS))
6616 LVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
6617 }
6618
6619 return LVT;
6620 };
6621
6622 // This turns into unaligned loads. We only do this if the target natively
6623 // supports the MVT we'll be loading or if it is small enough (<= 4) that
6624 // we'll only produce a small number of byte loads.
6625 MVT LoadVT;
6626 unsigned NumBitsToCompare = CSize->getZExtValue() * 8;
6627 switch (NumBitsToCompare) {
6628 default:
6629 return false;
6630 case 16:
6631 LoadVT = MVT::i16;
6632 break;
6633 case 32:
6634 LoadVT = MVT::i32;
6635 break;
6636 case 64:
6637 case 128:
6638 case 256:
6639 LoadVT = hasFastLoadsAndCompare(NumBitsToCompare);
6640 break;
6641 }
6642
6643 if (LoadVT == MVT::INVALID_SIMPLE_VALUE_TYPE)
6644 return false;
6645
6646 SDValue LoadL = getMemCmpLoad(LHS, LoadVT, *this);
6647 SDValue LoadR = getMemCmpLoad(RHS, LoadVT, *this);
6648
6649 // Bitcast to a wide integer type if the loads are vectors.
6650 if (LoadVT.isVector()) {
6651 EVT CmpVT = EVT::getIntegerVT(LHS->getContext(), LoadVT.getSizeInBits());
6652 LoadL = DAG.getBitcast(CmpVT, LoadL);
6653 LoadR = DAG.getBitcast(CmpVT, LoadR);
6654 }
6655
6656 SDValue Cmp = DAG.getSetCC(getCurSDLoc(), MVT::i1, LoadL, LoadR, ISD::SETNE);
6657 processIntegerCallValue(I, Cmp, false);
6658 return true;
6659}
6660
6661/// See if we can lower a memchr call into an optimized form. If so, return
6662/// true and lower it. Otherwise return false, and it will be lowered like a
6663/// normal call.
6664/// The caller already checked that \p I calls the appropriate LibFunc with a
6665/// correct prototype.
6666bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) {
6667 const Value *Src = I.getArgOperand(0);
6668 const Value *Char = I.getArgOperand(1);
6669 const Value *Length = I.getArgOperand(2);
6670
6671 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6672 std::pair<SDValue, SDValue> Res =
6673 TSI.EmitTargetCodeForMemchr(DAG, getCurSDLoc(), DAG.getRoot(),
6674 getValue(Src), getValue(Char), getValue(Length),
6675 MachinePointerInfo(Src));
6676 if (Res.first.getNode()) {
6677 setValue(&I, Res.first);
6678 PendingLoads.push_back(Res.second);
6679 return true;
6680 }
6681
6682 return false;
6683}
6684
6685/// See if we can lower a mempcpy call into an optimized form. If so, return
6686/// true and lower it. Otherwise return false, and it will be lowered like a
6687/// normal call.
6688/// The caller already checked that \p I calls the appropriate LibFunc with a
6689/// correct prototype.
6690bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) {
6691 SDValue Dst = getValue(I.getArgOperand(0));
6692 SDValue Src = getValue(I.getArgOperand(1));
6693 SDValue Size = getValue(I.getArgOperand(2));
6694
6695 unsigned DstAlign = DAG.InferPtrAlignment(Dst);
6696 unsigned SrcAlign = DAG.InferPtrAlignment(Src);
6697 unsigned Align = std::min(DstAlign, SrcAlign);
6698 if (Align == 0) // Alignment of one or both could not be inferred.
6699 Align = 1; // 0 and 1 both specify no alignment, but 0 is reserved.
6700
6701 bool isVol = false;
6702 SDLoc sdl = getCurSDLoc();
6703
6704 // In the mempcpy context we need to pass in a false value for isTailCall
6705 // because the return pointer needs to be adjusted by the size of
6706 // the copied memory.
6707 SDValue MC = DAG.getMemcpy(getRoot(), sdl, Dst, Src, Size, Align, isVol,
6708 false, /*isTailCall=*/false,
6709 MachinePointerInfo(I.getArgOperand(0)),
6710 MachinePointerInfo(I.getArgOperand(1)));
6711 assert(MC.getNode() != nullptr &&(static_cast <bool> (MC.getNode() != nullptr &&
"** memcpy should not be lowered as TailCall in mempcpy context **"
) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6712, __extension__ __PRETTY_FUNCTION__))
6712 "** memcpy should not be lowered as TailCall in mempcpy context **")(static_cast <bool> (MC.getNode() != nullptr &&
"** memcpy should not be lowered as TailCall in mempcpy context **"
) ? void (0) : __assert_fail ("MC.getNode() != nullptr && \"** memcpy should not be lowered as TailCall in mempcpy context **\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 6712, __extension__ __PRETTY_FUNCTION__));
6713 DAG.setRoot(MC);
6714
6715 // Check if Size needs to be truncated or extended.
6716 Size = DAG.getSExtOrTrunc(Size, sdl, Dst.getValueType());
6717
6718 // Adjust return pointer to point just past the last dst byte.
6719 SDValue DstPlusSize = DAG.getNode(ISD::ADD, sdl, Dst.getValueType(),
6720 Dst, Size);
6721 setValue(&I, DstPlusSize);
6722 return true;
6723}
6724
6725/// See if we can lower a strcpy call into an optimized form. If so, return
6726/// true and lower it, otherwise return false and it will be lowered like a
6727/// normal call.
6728/// The caller already checked that \p I calls the appropriate LibFunc with a
6729/// correct prototype.
6730bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) {
6731 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6732
6733 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6734 std::pair<SDValue, SDValue> Res =
6735 TSI.EmitTargetCodeForStrcpy(DAG, getCurSDLoc(), getRoot(),
6736 getValue(Arg0), getValue(Arg1),
6737 MachinePointerInfo(Arg0),
6738 MachinePointerInfo(Arg1), isStpcpy);
6739 if (Res.first.getNode()) {
6740 setValue(&I, Res.first);
6741 DAG.setRoot(Res.second);
6742 return true;
6743 }
6744
6745 return false;
6746}
6747
6748/// See if we can lower a strcmp call into an optimized form. If so, return
6749/// true and lower it, otherwise return false and it will be lowered like a
6750/// normal call.
6751/// The caller already checked that \p I calls the appropriate LibFunc with a
6752/// correct prototype.
6753bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) {
6754 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6755
6756 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6757 std::pair<SDValue, SDValue> Res =
6758 TSI.EmitTargetCodeForStrcmp(DAG, getCurSDLoc(), DAG.getRoot(),
6759 getValue(Arg0), getValue(Arg1),
6760 MachinePointerInfo(Arg0),
6761 MachinePointerInfo(Arg1));
6762 if (Res.first.getNode()) {
6763 processIntegerCallValue(I, Res.first, true);
6764 PendingLoads.push_back(Res.second);
6765 return true;
6766 }
6767
6768 return false;
6769}
6770
6771/// See if we can lower a strlen call into an optimized form. If so, return
6772/// true and lower it, otherwise return false and it will be lowered like a
6773/// normal call.
6774/// The caller already checked that \p I calls the appropriate LibFunc with a
6775/// correct prototype.
6776bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) {
6777 const Value *Arg0 = I.getArgOperand(0);
6778
6779 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6780 std::pair<SDValue, SDValue> Res =
6781 TSI.EmitTargetCodeForStrlen(DAG, getCurSDLoc(), DAG.getRoot(),
6782 getValue(Arg0), MachinePointerInfo(Arg0));
6783 if (Res.first.getNode()) {
6784 processIntegerCallValue(I, Res.first, false);
6785 PendingLoads.push_back(Res.second);
6786 return true;
6787 }
6788
6789 return false;
6790}
6791
6792/// See if we can lower a strnlen call into an optimized form. If so, return
6793/// true and lower it, otherwise return false and it will be lowered like a
6794/// normal call.
6795/// The caller already checked that \p I calls the appropriate LibFunc with a
6796/// correct prototype.
6797bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) {
6798 const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1);
6799
6800 const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo();
6801 std::pair<SDValue, SDValue> Res =
6802 TSI.EmitTargetCodeForStrnlen(DAG, getCurSDLoc(), DAG.getRoot(),
6803 getValue(Arg0), getValue(Arg1),
6804 MachinePointerInfo(Arg0));
6805 if (Res.first.getNode()) {
6806 processIntegerCallValue(I, Res.first, false);
6807 PendingLoads.push_back(Res.second);
6808 return true;
6809 }
6810
6811 return false;
6812}
6813
6814/// See if we can lower a unary floating-point operation into an SDNode with
6815/// the specified Opcode. If so, return true and lower it, otherwise return
6816/// false and it will be lowered like a normal call.
6817/// The caller already checked that \p I calls the appropriate LibFunc with a
6818/// correct prototype.
6819bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I,
6820 unsigned Opcode) {
6821 // We already checked this call's prototype; verify it doesn't modify errno.
6822 if (!I.onlyReadsMemory())
6823 return false;
6824
6825 SDValue Tmp = getValue(I.getArgOperand(0));
6826 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), Tmp.getValueType(), Tmp));
6827 return true;
6828}
6829
6830/// See if we can lower a binary floating-point operation into an SDNode with
6831/// the specified Opcode. If so, return true and lower it. Otherwise return
6832/// false, and it will be lowered like a normal call.
6833/// The caller already checked that \p I calls the appropriate LibFunc with a
6834/// correct prototype.
6835bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I,
6836 unsigned Opcode) {
6837 // We already checked this call's prototype; verify it doesn't modify errno.
6838 if (!I.onlyReadsMemory())
6839 return false;
6840
6841 SDValue Tmp0 = getValue(I.getArgOperand(0));
6842 SDValue Tmp1 = getValue(I.getArgOperand(1));
6843 EVT VT = Tmp0.getValueType();
6844 setValue(&I, DAG.getNode(Opcode, getCurSDLoc(), VT, Tmp0, Tmp1));
6845 return true;
6846}
6847
6848void SelectionDAGBuilder::visitCall(const CallInst &I) {
6849 // Handle inline assembly differently.
6850 if (isa<InlineAsm>(I.getCalledValue())) {
6851 visitInlineAsm(&I);
6852 return;
6853 }
6854
6855 MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI();
6856 computeUsesVAFloatArgument(I, MMI);
6857
6858 const char *RenameFn = nullptr;
6859 if (Function *F = I.getCalledFunction()) {
6860 if (F->isDeclaration()) {
6861 // Is this an LLVM intrinsic or a target-specific intrinsic?
6862 unsigned IID = F->getIntrinsicID();
6863 if (!IID)
6864 if (const TargetIntrinsicInfo *II = TM.getIntrinsicInfo())
6865 IID = II->getIntrinsicID(F);
6866
6867 if (IID) {
6868 RenameFn = visitIntrinsicCall(I, IID);
6869 if (!RenameFn)
6870 return;
6871 }
6872 }
6873
6874 // Check for well-known libc/libm calls. If the function is internal, it
6875 // can't be a library call. Don't do the check if marked as nobuiltin for
6876 // some reason or the call site requires strict floating point semantics.
6877 LibFunc Func;
6878 if (!I.isNoBuiltin() && !I.isStrictFP() && !F->hasLocalLinkage() &&
6879 F->hasName() && LibInfo->getLibFunc(*F, Func) &&
6880 LibInfo->hasOptimizedCodeGen(Func)) {
6881 switch (Func) {
6882 default: break;
6883 case LibFunc_copysign:
6884 case LibFunc_copysignf:
6885 case LibFunc_copysignl:
6886 // We already checked this call's prototype; verify it doesn't modify
6887 // errno.
6888 if (I.onlyReadsMemory()) {
6889 SDValue LHS = getValue(I.getArgOperand(0));
6890 SDValue RHS = getValue(I.getArgOperand(1));
6891 setValue(&I, DAG.getNode(ISD::FCOPYSIGN, getCurSDLoc(),
6892 LHS.getValueType(), LHS, RHS));
6893 return;
6894 }
6895 break;
6896 case LibFunc_fabs:
6897 case LibFunc_fabsf:
6898 case LibFunc_fabsl:
6899 if (visitUnaryFloatCall(I, ISD::FABS))
6900 return;
6901 break;
6902 case LibFunc_fmin:
6903 case LibFunc_fminf:
6904 case LibFunc_fminl:
6905 if (visitBinaryFloatCall(I, ISD::FMINNUM))
6906 return;
6907 break;
6908 case LibFunc_fmax:
6909 case LibFunc_fmaxf:
6910 case LibFunc_fmaxl:
6911 if (visitBinaryFloatCall(I, ISD::FMAXNUM))
6912 return;
6913 break;
6914 case LibFunc_sin:
6915 case LibFunc_sinf:
6916 case LibFunc_sinl:
6917 if (visitUnaryFloatCall(I, ISD::FSIN))
6918 return;
6919 break;
6920 case LibFunc_cos:
6921 case LibFunc_cosf:
6922 case LibFunc_cosl:
6923 if (visitUnaryFloatCall(I, ISD::FCOS))
6924 return;
6925 break;
6926 case LibFunc_sqrt:
6927 case LibFunc_sqrtf:
6928 case LibFunc_sqrtl:
6929 case LibFunc_sqrt_finite:
6930 case LibFunc_sqrtf_finite:
6931 case LibFunc_sqrtl_finite:
6932 if (visitUnaryFloatCall(I, ISD::FSQRT))
6933 return;
6934 break;
6935 case LibFunc_floor:
6936 case LibFunc_floorf:
6937 case LibFunc_floorl:
6938 if (visitUnaryFloatCall(I, ISD::FFLOOR))
6939 return;
6940 break;
6941 case LibFunc_nearbyint:
6942 case LibFunc_nearbyintf:
6943 case LibFunc_nearbyintl:
6944 if (visitUnaryFloatCall(I, ISD::FNEARBYINT))
6945 return;
6946 break;
6947 case LibFunc_ceil:
6948 case LibFunc_ceilf:
6949 case LibFunc_ceill:
6950 if (visitUnaryFloatCall(I, ISD::FCEIL))
6951 return;
6952 break;
6953 case LibFunc_rint:
6954 case LibFunc_rintf:
6955 case LibFunc_rintl:
6956 if (visitUnaryFloatCall(I, ISD::FRINT))
6957 return;
6958 break;
6959 case LibFunc_round:
6960 case LibFunc_roundf:
6961 case LibFunc_roundl:
6962 if (visitUnaryFloatCall(I, ISD::FROUND))
6963 return;
6964 break;
6965 case LibFunc_trunc:
6966 case LibFunc_truncf:
6967 case LibFunc_truncl:
6968 if (visitUnaryFloatCall(I, ISD::FTRUNC))
6969 return;
6970 break;
6971 case LibFunc_log2:
6972 case LibFunc_log2f:
6973 case LibFunc_log2l:
6974 if (visitUnaryFloatCall(I, ISD::FLOG2))
6975 return;
6976 break;
6977 case LibFunc_exp2:
6978 case LibFunc_exp2f:
6979 case LibFunc_exp2l:
6980 if (visitUnaryFloatCall(I, ISD::FEXP2))
6981 return;
6982 break;
6983 case LibFunc_memcmp:
6984 if (visitMemCmpCall(I))
6985 return;
6986 break;
6987 case LibFunc_mempcpy:
6988 if (visitMemPCpyCall(I))
6989 return;
6990 break;
6991 case LibFunc_memchr:
6992 if (visitMemChrCall(I))
6993 return;
6994 break;
6995 case LibFunc_strcpy:
6996 if (visitStrCpyCall(I, false))
6997 return;
6998 break;
6999 case LibFunc_stpcpy:
7000 if (visitStrCpyCall(I, true))
7001 return;
7002 break;
7003 case LibFunc_strcmp:
7004 if (visitStrCmpCall(I))
7005 return;
7006 break;
7007 case LibFunc_strlen:
7008 if (visitStrLenCall(I))
7009 return;
7010 break;
7011 case LibFunc_strnlen:
7012 if (visitStrNLenCall(I))
7013 return;
7014 break;
7015 }
7016 }
7017 }
7018
7019 SDValue Callee;
7020 if (!RenameFn)
7021 Callee = getValue(I.getCalledValue());
7022 else
7023 Callee = DAG.getExternalSymbol(
7024 RenameFn,
7025 DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()));
7026
7027 // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't
7028 // have to do anything here to lower funclet bundles.
7029 assert(!I.hasOperandBundlesOtherThan((static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7031, __extension__ __PRETTY_FUNCTION__))
7030 {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) &&(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7031, __extension__ __PRETTY_FUNCTION__))
7031 "Cannot lower calls with arbitrary operand bundles!")(static_cast <bool> (!I.hasOperandBundlesOtherThan( {LLVMContext
::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"
) ? void (0) : __assert_fail ("!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && \"Cannot lower calls with arbitrary operand bundles!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7031, __extension__ __PRETTY_FUNCTION__));
7032
7033 if (I.countOperandBundlesOfType(LLVMContext::OB_deopt))
7034 LowerCallSiteWithDeoptBundle(&I, Callee, nullptr);
7035 else
7036 // Check if we can potentially perform a tail call. More detailed checking
7037 // is be done within LowerCallTo, after more information about the call is
7038 // known.
7039 LowerCallTo(&I, Callee, I.isTailCall());
7040}
7041
7042namespace {
7043
7044/// AsmOperandInfo - This contains information for each constraint that we are
7045/// lowering.
7046class SDISelAsmOperandInfo : public TargetLowering::AsmOperandInfo {
7047public:
7048 /// CallOperand - If this is the result output operand or a clobber
7049 /// this is null, otherwise it is the incoming operand to the CallInst.
7050 /// This gets modified as the asm is processed.
7051 SDValue CallOperand;
7052
7053 /// AssignedRegs - If this is a register or register class operand, this
7054 /// contains the set of register corresponding to the operand.
7055 RegsForValue AssignedRegs;
7056
7057 explicit SDISelAsmOperandInfo(const TargetLowering::AsmOperandInfo &info)
7058 : TargetLowering::AsmOperandInfo(info), CallOperand(nullptr, 0) {
7059 }
7060
7061 /// Whether or not this operand accesses memory
7062 bool hasMemory(const TargetLowering &TLI) const {
7063 // Indirect operand accesses access memory.
7064 if (isIndirect)
7065 return true;
7066
7067 for (const auto &Code : Codes)
7068 if (TLI.getConstraintType(Code) == TargetLowering::C_Memory)
7069 return true;
7070
7071 return false;
7072 }
7073
7074 /// getCallOperandValEVT - Return the EVT of the Value* that this operand
7075 /// corresponds to. If there is no Value* for this operand, it returns
7076 /// MVT::Other.
7077 EVT getCallOperandValEVT(LLVMContext &Context, const TargetLowering &TLI,
7078 const DataLayout &DL) const {
7079 if (!CallOperandVal) return MVT::Other;
7080
7081 if (isa<BasicBlock>(CallOperandVal))
7082 return TLI.getPointerTy(DL);
7083
7084 llvm::Type *OpTy = CallOperandVal->getType();
7085
7086 // FIXME: code duplicated from TargetLowering::ParseConstraints().
7087 // If this is an indirect operand, the operand is a pointer to the
7088 // accessed type.
7089 if (isIndirect) {
7090 PointerType *PtrTy = dyn_cast<PointerType>(OpTy);
7091 if (!PtrTy)
7092 report_fatal_error("Indirect operand for inline asm not a pointer!");
7093 OpTy = PtrTy->getElementType();
7094 }
7095
7096 // Look for vector wrapped in a struct. e.g. { <16 x i8> }.
7097 if (StructType *STy = dyn_cast<StructType>(OpTy))
7098 if (STy->getNumElements() == 1)
7099 OpTy = STy->getElementType(0);
7100
7101 // If OpTy is not a single value, it may be a struct/union that we
7102 // can tile with integers.
7103 if (!OpTy->isSingleValueType() && OpTy->isSized()) {
7104 unsigned BitSize = DL.getTypeSizeInBits(OpTy);
7105 switch (BitSize) {
7106 default: break;
7107 case 1:
7108 case 8:
7109 case 16:
7110 case 32:
7111 case 64:
7112 case 128:
7113 OpTy = IntegerType::get(Context, BitSize);
7114 break;
7115 }
7116 }
7117
7118 return TLI.getValueType(DL, OpTy, true);
7119 }
7120};
7121
7122using SDISelAsmOperandInfoVector = SmallVector<SDISelAsmOperandInfo, 16>;
7123
7124} // end anonymous namespace
7125
7126/// Make sure that the output operand \p OpInfo and its corresponding input
7127/// operand \p MatchingOpInfo have compatible constraint types (otherwise error
7128/// out).
7129static void patchMatchingInput(const SDISelAsmOperandInfo &OpInfo,
7130 SDISelAsmOperandInfo &MatchingOpInfo,
7131 SelectionDAG &DAG) {
7132 if (OpInfo.ConstraintVT == MatchingOpInfo.ConstraintVT)
7133 return;
7134
7135 const TargetRegisterInfo *TRI = DAG.getSubtarget().getRegisterInfo();
7136 const auto &TLI = DAG.getTargetLoweringInfo();
7137
7138 std::pair<unsigned, const TargetRegisterClass *> MatchRC =
7139 TLI.getRegForInlineAsmConstraint(TRI, OpInfo.ConstraintCode,
7140 OpInfo.ConstraintVT);
7141 std::pair<unsigned, const TargetRegisterClass *> InputRC =
7142 TLI.getRegForInlineAsmConstraint(TRI, MatchingOpInfo.ConstraintCode,
7143 MatchingOpInfo.ConstraintVT);
7144 if ((OpInfo.ConstraintVT.isInteger() !=
7145 MatchingOpInfo.ConstraintVT.isInteger()) ||
7146 (MatchRC.second != InputRC.second)) {
7147 // FIXME: error out in a more elegant fashion
7148 report_fatal_error("Unsupported asm: input constraint"
7149 " with a matching output constraint of"
7150 " incompatible type!");
7151 }
7152 MatchingOpInfo.ConstraintVT = OpInfo.ConstraintVT;
7153}
7154
7155/// Get a direct memory input to behave well as an indirect operand.
7156/// This may introduce stores, hence the need for a \p Chain.
7157/// \return The (possibly updated) chain.
7158static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location,
7159 SDISelAsmOperandInfo &OpInfo,
7160 SelectionDAG &DAG) {
7161 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7162
7163 // If we don't have an indirect input, put it in the constpool if we can,
7164 // otherwise spill it to a stack slot.
7165 // TODO: This isn't quite right. We need to handle these according to
7166 // the addressing mode that the constraint wants. Also, this may take
7167 // an additional register for the computation and we don't want that
7168 // either.
7169
7170 // If the operand is a float, integer, or vector constant, spill to a
7171 // constant pool entry to get its address.
7172 const Value *OpVal = OpInfo.CallOperandVal;
7173 if (isa<ConstantFP>(OpVal) || isa<ConstantInt>(OpVal) ||
7174 isa<ConstantVector>(OpVal) || isa<ConstantDataVector>(OpVal)) {
7175 OpInfo.CallOperand = DAG.getConstantPool(
7176 cast<Constant>(OpVal), TLI.getPointerTy(DAG.getDataLayout()));
7177 return Chain;
7178 }
7179
7180 // Otherwise, create a stack slot and emit a store to it before the asm.
7181 Type *Ty = OpVal->getType();
7182 auto &DL = DAG.getDataLayout();
7183 uint64_t TySize = DL.getTypeAllocSize(Ty);
7184 unsigned Align = DL.getPrefTypeAlignment(Ty);
7185 MachineFunction &MF = DAG.getMachineFunction();
7186 int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
7187 SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL));
7188 Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot,
7189 MachinePointerInfo::getFixedStack(MF, SSFI));
7190 OpInfo.CallOperand = StackSlot;
7191
7192 return Chain;
7193}
7194
7195/// GetRegistersForValue - Assign registers (virtual or physical) for the
7196/// specified operand. We prefer to assign virtual registers, to allow the
7197/// register allocator to handle the assignment process. However, if the asm
7198/// uses features that we can't model on machineinstrs, we have SDISel do the
7199/// allocation. This produces generally horrible, but correct, code.
7200///
7201/// OpInfo describes the operand.
7202static void GetRegistersForValue(SelectionDAG &DAG, const TargetLowering &TLI,
7203 const SDLoc &DL,
7204 SDISelAsmOperandInfo &OpInfo) {
7205 LLVMContext &Context = *DAG.getContext();
7206
7207 MachineFunction &MF = DAG.getMachineFunction();
7208 SmallVector<unsigned, 4> Regs;
7209 const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo();
7210
7211 // If this is a constraint for a single physreg, or a constraint for a
7212 // register class, find it.
7213 std::pair<unsigned, const TargetRegisterClass *> PhysReg =
7214 TLI.getRegForInlineAsmConstraint(&TRI, OpInfo.ConstraintCode,
12
←
Value assigned to 'PhysReg.second'
→
7215 OpInfo.ConstraintVT);
7216
7217 unsigned NumRegs = 1;
7218 if (OpInfo.ConstraintVT != MVT::Other) {
13
←
Taking true branch
→
7219 // If this is a FP operand in an integer register (or visa versa), or more
7220 // generally if the operand value disagrees with the register class we plan
7221 // to stick it in, fix the operand type.
7222 //
7223 // If this is an input value, the bitcast to the new type is done now.
7224 // Bitcast for output value is done at the end of visitInlineAsm().
7225 if ((OpInfo.Type == InlineAsm::isOutput ||
14
←
Assuming the condition is false
→
16
←
Assuming pointer value is null
→
7226 OpInfo.Type == InlineAsm::isInput) &&
15
←
Assuming the condition is true
→
7227 PhysReg.second &&
17
←
Assuming the condition is false
→
7228 !TRI.isTypeLegalForClass(*PhysReg.second, OpInfo.ConstraintVT)) {
7229 // Try to convert to the first EVT that the reg class contains. If the
7230 // types are identical size, use a bitcast to convert (e.g. two differing
7231 // vector types). Note: output bitcast is done at the end of
7232 // visitInlineAsm().
7233 MVT RegVT = *TRI.legalclasstypes_begin(*PhysReg.second);
7234 if (RegVT.getSizeInBits() == OpInfo.ConstraintVT.getSizeInBits()) {
7235 // Exclude indirect inputs while they are unsupported because the code
7236 // to perform the load is missing and thus OpInfo.CallOperand still
7237 // refer to the input address rather than the pointed-to value.
7238 if (OpInfo.Type == InlineAsm::isInput && !OpInfo.isIndirect)
7239 OpInfo.CallOperand =
7240 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
7241 OpInfo.ConstraintVT = RegVT;
7242 // If the operand is a FP value and we want it in integer registers,
7243 // use the corresponding integer type. This turns an f64 value into
7244 // i64, which can be passed with two i32 values on a 32-bit machine.
7245 } else if (RegVT.isInteger() && OpInfo.ConstraintVT.isFloatingPoint()) {
7246 RegVT = MVT::getIntegerVT(OpInfo.ConstraintVT.getSizeInBits());
7247 if (OpInfo.Type == InlineAsm::isInput)
7248 OpInfo.CallOperand =
7249 DAG.getNode(ISD::BITCAST, DL, RegVT, OpInfo.CallOperand);
7250 OpInfo.ConstraintVT = RegVT;
7251 }
7252 }
7253
7254 NumRegs = TLI.getNumRegisters(Context, OpInfo.ConstraintVT);
7255 }
7256
7257 MVT RegVT;
7258 EVT ValueVT = OpInfo.ConstraintVT;
7259
7260 // If this is a constraint for a specific physical register, like {r17},
7261 // assign it now.
7262 if (unsigned AssignedReg = PhysReg.first) {
18
←
Assuming 'AssignedReg' is not equal to 0
→
19
←
Taking true branch
→
7263 const TargetRegisterClass *RC = PhysReg.second;
20
←
'RC' initialized to a null pointer value
→
7264 if (OpInfo.ConstraintVT == MVT::Other)
21
←
Taking false branch
→
7265 ValueVT = *TRI.legalclasstypes_begin(*RC);
7266
7267 // Get the actual register value type. This is important, because the user
7268 // may have asked for (e.g.) the AX register in i32 type. We need to
7269 // remember that AX is actually i16 to get the right extension.
7270 RegVT = *TRI.legalclasstypes_begin(*RC);
22
←
Forming reference to null pointer
7271
7272 // This is a explicit reference to a physical register.
7273 Regs.push_back(AssignedReg);
7274
7275 // If this is an expanded reference, add the rest of the regs to Regs.
7276 if (NumRegs != 1) {
7277 TargetRegisterClass::iterator I = RC->begin();
7278 for (; *I != AssignedReg; ++I)
7279 assert(I != RC->end() && "Didn't find reg!")(static_cast <bool> (I != RC->end() && "Didn't find reg!"
) ? void (0) : __assert_fail ("I != RC->end() && \"Didn't find reg!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7279, __extension__ __PRETTY_FUNCTION__));
7280
7281 // Already added the first reg.
7282 --NumRegs; ++I;
7283 for (; NumRegs; --NumRegs, ++I) {
7284 assert(I != RC->end() && "Ran out of registers to allocate!")(static_cast <bool> (I != RC->end() && "Ran out of registers to allocate!"
) ? void (0) : __assert_fail ("I != RC->end() && \"Ran out of registers to allocate!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7284, __extension__ __PRETTY_FUNCTION__));
7285 Regs.push_back(*I);
7286 }
7287 }
7288
7289 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
7290 return;
7291 }
7292
7293 // Otherwise, if this was a reference to an LLVM register class, create vregs
7294 // for this reference.
7295 if (const TargetRegisterClass *RC = PhysReg.second) {
7296 RegVT = *TRI.legalclasstypes_begin(*RC);
7297 if (OpInfo.ConstraintVT == MVT::Other)
7298 ValueVT = RegVT;
7299
7300 // Create the appropriate number of virtual registers.
7301 MachineRegisterInfo &RegInfo = MF.getRegInfo();
7302 for (; NumRegs; --NumRegs)
7303 Regs.push_back(RegInfo.createVirtualRegister(RC));
7304
7305 OpInfo.AssignedRegs = RegsForValue(Regs, RegVT, ValueVT);
7306 return;
7307 }
7308
7309 // Otherwise, we couldn't allocate enough registers for this.
7310}
7311
7312static unsigned
7313findMatchingInlineAsmOperand(unsigned OperandNo,
7314 const std::vector<SDValue> &AsmNodeOperands) {
7315 // Scan until we find the definition we already emitted of this operand.
7316 unsigned CurOp = InlineAsm::Op_FirstOperand;
7317 for (; OperandNo; --OperandNo) {
7318 // Advance to the next operand.
7319 unsigned OpFlag =
7320 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7321 assert((InlineAsm::isRegDefKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
__assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7324, __extension__ __PRETTY_FUNCTION__))
7322 InlineAsm::isRegDefEarlyClobberKind(OpFlag) ||(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
__assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7324, __extension__ __PRETTY_FUNCTION__))
7323 InlineAsm::isMemKind(OpFlag)) &&(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
__assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7324, __extension__ __PRETTY_FUNCTION__))
7324 "Skipped past definitions?")(static_cast <bool> ((InlineAsm::isRegDefKind(OpFlag) ||
InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind
(OpFlag)) && "Skipped past definitions?") ? void (0) :
__assert_fail ("(InlineAsm::isRegDefKind(OpFlag) || InlineAsm::isRegDefEarlyClobberKind(OpFlag) || InlineAsm::isMemKind(OpFlag)) && \"Skipped past definitions?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7324, __extension__ __PRETTY_FUNCTION__));
7325 CurOp += InlineAsm::getNumOperandRegisters(OpFlag) + 1;
7326 }
7327 return CurOp;
7328}
7329
7330/// Fill \p Regs with \p NumRegs new virtual registers of type \p RegVT
7331/// \return true if it has succeeded, false otherwise
7332static bool createVirtualRegs(SmallVector<unsigned, 4> &Regs, unsigned NumRegs,
7333 MVT RegVT, SelectionDAG &DAG) {
7334 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7335 MachineRegisterInfo &RegInfo = DAG.getMachineFunction().getRegInfo();
7336 for (unsigned i = 0, e = NumRegs; i != e; ++i) {
7337 if (const TargetRegisterClass *RC = TLI.getRegClassFor(RegVT))
7338 Regs.push_back(RegInfo.createVirtualRegister(RC));
7339 else
7340 return false;
7341 }
7342 return true;
7343}
7344
7345namespace {
7346
7347class ExtraFlags {
7348 unsigned Flags = 0;
7349
7350public:
7351 explicit ExtraFlags(ImmutableCallSite CS) {
7352 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7353 if (IA->hasSideEffects())
7354 Flags |= InlineAsm::Extra_HasSideEffects;
7355 if (IA->isAlignStack())
7356 Flags |= InlineAsm::Extra_IsAlignStack;
7357 if (CS.isConvergent())
7358 Flags |= InlineAsm::Extra_IsConvergent;
7359 Flags |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
7360 }
7361
7362 void update(const TargetLowering::AsmOperandInfo &OpInfo) {
7363 // Ideally, we would only check against memory constraints. However, the
7364 // meaning of an Other constraint can be target-specific and we can't easily
7365 // reason about it. Therefore, be conservative and set MayLoad/MayStore
7366 // for Other constraints as well.
7367 if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
7368 OpInfo.ConstraintType == TargetLowering::C_Other) {
7369 if (OpInfo.Type == InlineAsm::isInput)
7370 Flags |= InlineAsm::Extra_MayLoad;
7371 else if (OpInfo.Type == InlineAsm::isOutput)
7372 Flags |= InlineAsm::Extra_MayStore;
7373 else if (OpInfo.Type == InlineAsm::isClobber)
7374 Flags |= (InlineAsm::Extra_MayLoad | InlineAsm::Extra_MayStore);
7375 }
7376 }
7377
7378 unsigned get() const { return Flags; }
7379};
7380
7381} // end anonymous namespace
7382
7383/// visitInlineAsm - Handle a call to an InlineAsm object.
7384void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
7385 const InlineAsm *IA = cast<InlineAsm>(CS.getCalledValue());
7386
7387 /// ConstraintOperands - Information about all of the constraints.
7388 SDISelAsmOperandInfoVector ConstraintOperands;
7389
7390 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7391 TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints(
7392 DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS);
7393
7394 bool hasMemory = false;
7395
7396 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7397 ExtraFlags ExtraInfo(CS);
7398
7399 unsigned ArgNo = 0; // ArgNo - The argument of the CallInst.
7400 unsigned ResNo = 0; // ResNo - The result number of the next output.
7401 for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
1
Assuming 'i' is equal to 'e'
→
2
←
Loop condition is false. Execution continues on line 7455
→
7402 ConstraintOperands.push_back(SDISelAsmOperandInfo(TargetConstraints[i]));
7403 SDISelAsmOperandInfo &OpInfo = ConstraintOperands.back();
7404
7405 MVT OpVT = MVT::Other;
7406
7407 // Compute the value type for each operand.
7408 if (OpInfo.Type == InlineAsm::isInput ||
7409 (OpInfo.Type == InlineAsm::isOutput && OpInfo.isIndirect)) {
7410 OpInfo.CallOperandVal = const_cast<Value *>(CS.getArgument(ArgNo++));
7411
7412 // Process the call argument. BasicBlocks are labels, currently appearing
7413 // only in asm's.
7414 if (const BasicBlock *BB = dyn_cast<BasicBlock>(OpInfo.CallOperandVal)) {
7415 OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]);
7416 } else {
7417 OpInfo.CallOperand = getValue(OpInfo.CallOperandVal);
7418 }
7419
7420 OpVT =
7421 OpInfo
7422 .getCallOperandValEVT(*DAG.getContext(), TLI, DAG.getDataLayout())
7423 .getSimpleVT();
7424 }
7425
7426 if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
7427 // The return value of the call is this value. As such, there is no
7428 // corresponding argument.
7429 assert(!CS.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!CS.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7429, __extension__ __PRETTY_FUNCTION__));
7430 if (StructType *STy = dyn_cast<StructType>(CS.getType())) {
7431 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(),
7432 STy->getElementType(ResNo));
7433 } else {
7434 assert(ResNo == 0 && "Asm only has one result!")(static_cast <bool> (ResNo == 0 && "Asm only has one result!"
) ? void (0) : __assert_fail ("ResNo == 0 && \"Asm only has one result!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7434, __extension__ __PRETTY_FUNCTION__));
7435 OpVT = TLI.getSimpleValueType(DAG.getDataLayout(), CS.getType());
7436 }
7437 ++ResNo;
7438 }
7439
7440 OpInfo.ConstraintVT = OpVT;
7441
7442 if (!hasMemory)
7443 hasMemory = OpInfo.hasMemory(TLI);
7444
7445 // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
7446 // FIXME: Could we compute this on OpInfo rather than TargetConstraints[i]?
7447 auto TargetConstraint = TargetConstraints[i];
7448
7449 // Compute the constraint code and ConstraintType to use.
7450 TLI.ComputeConstraintToUse(TargetConstraint, SDValue());
7451
7452 ExtraInfo.update(TargetConstraint);
7453 }
7454
7455 SDValue Chain, Flag;
7456
7457 // We won't need to flush pending loads if this asm doesn't touch
7458 // memory and is nonvolatile.
7459 if (hasMemory || IA->hasSideEffects())
3
←
Assuming the condition is false
→
4
←
Taking false branch
→
7460 Chain = getRoot();
7461 else
7462 Chain = DAG.getRoot();
7463
7464 // Second pass over the constraints: compute which constraint option to use
7465 // and assign registers to constraints that want a specific physreg.
7466 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
5
←
Assuming 'i' is not equal to 'e'
→
6
←
Loop condition is true. Entering loop body
→
7467 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7468
7469 // If this is an output operand with a matching input operand, look up the
7470 // matching input. If their types mismatch, e.g. one is an integer, the
7471 // other is floating point, or their sizes are different, flag it as an
7472 // error.
7473 if (OpInfo.hasMatchingInput()) {
7
←
Taking false branch
→
7474 SDISelAsmOperandInfo &Input = ConstraintOperands[OpInfo.MatchingInput];
7475 patchMatchingInput(OpInfo, Input, DAG);
7476 }
7477
7478 // Compute the constraint code and ConstraintType to use.
7479 TLI.ComputeConstraintToUse(OpInfo, OpInfo.CallOperand, &DAG);
7480
7481 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
8
←
Assuming the condition is false
→
7482 OpInfo.Type == InlineAsm::isClobber)
7483 continue;
7484
7485 // If this is a memory input, and if the operand is not indirect, do what we
7486 // need to provide an address for the memory input.
7487 if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
7488 !OpInfo.isIndirect) {
7489 assert((OpInfo.isMultipleAlternative ||(static_cast <bool> ((OpInfo.isMultipleAlternative || (
OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!"
) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7491, __extension__ __PRETTY_FUNCTION__))
7490 (OpInfo.Type == InlineAsm::isInput)) &&(static_cast <bool> ((OpInfo.isMultipleAlternative || (
OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!"
) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7491, __extension__ __PRETTY_FUNCTION__))
7491 "Can only indirectify direct input operands!")(static_cast <bool> ((OpInfo.isMultipleAlternative || (
OpInfo.Type == InlineAsm::isInput)) && "Can only indirectify direct input operands!"
) ? void (0) : __assert_fail ("(OpInfo.isMultipleAlternative || (OpInfo.Type == InlineAsm::isInput)) && \"Can only indirectify direct input operands!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7491, __extension__ __PRETTY_FUNCTION__));
7492
7493 // Memory operands really want the address of the value.
7494 Chain = getAddressForMemoryInput(Chain, getCurSDLoc(), OpInfo, DAG);
7495
7496 // There is no longer a Value* corresponding to this operand.
7497 OpInfo.CallOperandVal = nullptr;
7498
7499 // It is now an indirect operand.
7500 OpInfo.isIndirect = true;
7501 }
7502
7503 // If this constraint is for a specific register, allocate it before
7504 // anything else.
7505 if (OpInfo.ConstraintType == TargetLowering::C_Register)
9
←
Assuming the condition is true
→
10
←
Taking true branch
→
7506 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
11
←
Calling 'GetRegistersForValue'
→
7507 }
7508
7509 // Third pass - Loop over all of the operands, assigning virtual or physregs
7510 // to register class operands.
7511 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7512 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7513
7514 // C_Register operands have already been allocated, Other/Memory don't need
7515 // to be.
7516 if (OpInfo.ConstraintType == TargetLowering::C_RegisterClass)
7517 GetRegistersForValue(DAG, TLI, getCurSDLoc(), OpInfo);
7518 }
7519
7520 // AsmNodeOperands - The operands for the ISD::INLINEASM node.
7521 std::vector<SDValue> AsmNodeOperands;
7522 AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
7523 AsmNodeOperands.push_back(DAG.getTargetExternalSymbol(
7524 IA->getAsmString().c_str(), TLI.getPointerTy(DAG.getDataLayout())));
7525
7526 // If we have a !srcloc metadata node associated with it, we want to attach
7527 // this to the ultimately generated inline asm machineinstr. To do this, we
7528 // pass in the third operand as this (potentially null) inline asm MDNode.
7529 const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
7530 AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
7531
7532 // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
7533 // bits as operand 3.
7534 AsmNodeOperands.push_back(DAG.getTargetConstant(
7535 ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7536
7537 // Loop over all of the inputs, copying the operand values into the
7538 // appropriate registers and processing the output regs.
7539 RegsForValue RetValRegs;
7540
7541 // IndirectStoresToEmit - The set of stores to emit after the inline asm node.
7542 std::vector<std::pair<RegsForValue, Value *>> IndirectStoresToEmit;
7543
7544 for (unsigned i = 0, e = ConstraintOperands.size(); i != e; ++i) {
7545 SDISelAsmOperandInfo &OpInfo = ConstraintOperands[i];
7546
7547 switch (OpInfo.Type) {
7548 case InlineAsm::isOutput:
7549 if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass &&
7550 OpInfo.ConstraintType != TargetLowering::C_Register) {
7551 // Memory output, or 'other' output (e.g. 'X' constraint).
7552 assert(OpInfo.isIndirect && "Memory output must be indirect operand")(static_cast <bool> (OpInfo.isIndirect && "Memory output must be indirect operand"
) ? void (0) : __assert_fail ("OpInfo.isIndirect && \"Memory output must be indirect operand\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7552, __extension__ __PRETTY_FUNCTION__));
7553
7554 unsigned ConstraintID =
7555 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7556 assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
&& "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7557, __extension__ __PRETTY_FUNCTION__))
7557 "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
&& "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7557, __extension__ __PRETTY_FUNCTION__));
7558
7559 // Add information to the INLINEASM node to know about this output.
7560 unsigned OpFlags = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7561 OpFlags = InlineAsm::getFlagWordForMem(OpFlags, ConstraintID);
7562 AsmNodeOperands.push_back(DAG.getTargetConstant(OpFlags, getCurSDLoc(),
7563 MVT::i32));
7564 AsmNodeOperands.push_back(OpInfo.CallOperand);
7565 break;
7566 }
7567
7568 // Otherwise, this is a register or register class output.
7569
7570 // Copy the output from the appropriate register. Find a register that
7571 // we can use.
7572 if (OpInfo.AssignedRegs.Regs.empty()) {
7573 emitInlineAsmError(
7574 CS, "couldn't allocate output register for constraint '" +
7575 Twine(OpInfo.ConstraintCode) + "'");
7576 return;
7577 }
7578
7579 // If this is an indirect operand, store through the pointer after the
7580 // asm.
7581 if (OpInfo.isIndirect) {
7582 IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs,
7583 OpInfo.CallOperandVal));
7584 } else {
7585 // This is the result value of the call.
7586 assert(!CS.getType()->isVoidTy() && "Bad inline asm!")(static_cast <bool> (!CS.getType()->isVoidTy() &&
"Bad inline asm!") ? void (0) : __assert_fail ("!CS.getType()->isVoidTy() && \"Bad inline asm!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7586, __extension__ __PRETTY_FUNCTION__));
7587 // Concatenate this output onto the outputs list.
7588 RetValRegs.append(OpInfo.AssignedRegs);
7589 }
7590
7591 // Add information to the INLINEASM node to know that this register is
7592 // set.
7593 OpInfo.AssignedRegs
7594 .AddInlineAsmOperands(OpInfo.isEarlyClobber
7595 ? InlineAsm::Kind_RegDefEarlyClobber
7596 : InlineAsm::Kind_RegDef,
7597 false, 0, getCurSDLoc(), DAG, AsmNodeOperands);
7598 break;
7599
7600 case InlineAsm::isInput: {
7601 SDValue InOperandVal = OpInfo.CallOperand;
7602
7603 if (OpInfo.isMatchingInputConstraint()) {
7604 // If this is required to match an output register we have already set,
7605 // just use its register.
7606 auto CurOp = findMatchingInlineAsmOperand(OpInfo.getMatchedOperand(),
7607 AsmNodeOperands);
7608 unsigned OpFlag =
7609 cast<ConstantSDNode>(AsmNodeOperands[CurOp])->getZExtValue();
7610 if (InlineAsm::isRegDefKind(OpFlag) ||
7611 InlineAsm::isRegDefEarlyClobberKind(OpFlag)) {
7612 // Add (OpFlag&0xffff)>>3 registers to MatchedRegs.
7613 if (OpInfo.isIndirect) {
7614 // This happens on gcc/testsuite/gcc.dg/pr8788-1.c
7615 emitInlineAsmError(CS, "inline asm not supported yet:"
7616 " don't know how to handle tied "
7617 "indirect register inputs");
7618 return;
7619 }
7620
7621 MVT RegVT = AsmNodeOperands[CurOp+1].getSimpleValueType();
7622 SmallVector<unsigned, 4> Regs;
7623
7624 if (!createVirtualRegs(Regs,
7625 InlineAsm::getNumOperandRegisters(OpFlag),
7626 RegVT, DAG)) {
7627 emitInlineAsmError(CS, "inline asm error: This value type register "
7628 "class is not natively supported!");
7629 return;
7630 }
7631
7632 RegsForValue MatchedRegs(Regs, RegVT, InOperandVal.getValueType());
7633
7634 SDLoc dl = getCurSDLoc();
7635 // Use the produced MatchedRegs object to
7636 MatchedRegs.getCopyToRegs(InOperandVal, DAG, dl, Chain, &Flag,
7637 CS.getInstruction());
7638 MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
7639 true, OpInfo.getMatchedOperand(), dl,
7640 DAG, AsmNodeOperands);
7641 break;
7642 }
7643
7644 assert(InlineAsm::isMemKind(OpFlag) && "Unknown matching constraint!")(static_cast <bool> (InlineAsm::isMemKind(OpFlag) &&
"Unknown matching constraint!") ? void (0) : __assert_fail (
"InlineAsm::isMemKind(OpFlag) && \"Unknown matching constraint!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7644, __extension__ __PRETTY_FUNCTION__));
7645 assert(InlineAsm::getNumOperandRegisters(OpFlag) == 1 &&(static_cast <bool> (InlineAsm::getNumOperandRegisters(
OpFlag) == 1 && "Unexpected number of operands") ? void
(0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7646, __extension__ __PRETTY_FUNCTION__))
7646 "Unexpected number of operands")(static_cast <bool> (InlineAsm::getNumOperandRegisters(
OpFlag) == 1 && "Unexpected number of operands") ? void
(0) : __assert_fail ("InlineAsm::getNumOperandRegisters(OpFlag) == 1 && \"Unexpected number of operands\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7646, __extension__ __PRETTY_FUNCTION__));
7647 // Add information to the INLINEASM node to know about this input.
7648 // See InlineAsm.h isUseOperandTiedToDef.
7649 OpFlag = InlineAsm::convertMemFlagWordToMatchingFlagWord(OpFlag);
7650 OpFlag = InlineAsm::getFlagWordForMatchingOp(OpFlag,
7651 OpInfo.getMatchedOperand());
7652 AsmNodeOperands.push_back(DAG.getTargetConstant(
7653 OpFlag, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7654 AsmNodeOperands.push_back(AsmNodeOperands[CurOp+1]);
7655 break;
7656 }
7657
7658 // Treat indirect 'X' constraint as memory.
7659 if (OpInfo.ConstraintType == TargetLowering::C_Other &&
7660 OpInfo.isIndirect)
7661 OpInfo.ConstraintType = TargetLowering::C_Memory;
7662
7663 if (OpInfo.ConstraintType == TargetLowering::C_Other) {
7664 std::vector<SDValue> Ops;
7665 TLI.LowerAsmOperandForConstraint(InOperandVal, OpInfo.ConstraintCode,
7666 Ops, DAG);
7667 if (Ops.empty()) {
7668 emitInlineAsmError(CS, "invalid operand for inline asm constraint '" +
7669 Twine(OpInfo.ConstraintCode) + "'");
7670 return;
7671 }
7672
7673 // Add information to the INLINEASM node to know about this input.
7674 unsigned ResOpType =
7675 InlineAsm::getFlagWord(InlineAsm::Kind_Imm, Ops.size());
7676 AsmNodeOperands.push_back(DAG.getTargetConstant(
7677 ResOpType, getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout())));
7678 AsmNodeOperands.insert(AsmNodeOperands.end(), Ops.begin(), Ops.end());
7679 break;
7680 }
7681
7682 if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
7683 assert(OpInfo.isIndirect && "Operand must be indirect to be a mem!")(static_cast <bool> (OpInfo.isIndirect && "Operand must be indirect to be a mem!"
) ? void (0) : __assert_fail ("OpInfo.isIndirect && \"Operand must be indirect to be a mem!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7683, __extension__ __PRETTY_FUNCTION__));
7684 assert(InOperandVal.getValueType() ==(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7686, __extension__ __PRETTY_FUNCTION__))
7685 TLI.getPointerTy(DAG.getDataLayout()) &&(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7686, __extension__ __PRETTY_FUNCTION__))
7686 "Memory operands expect pointer values")(static_cast <bool> (InOperandVal.getValueType() == TLI
.getPointerTy(DAG.getDataLayout()) && "Memory operands expect pointer values"
) ? void (0) : __assert_fail ("InOperandVal.getValueType() == TLI.getPointerTy(DAG.getDataLayout()) && \"Memory operands expect pointer values\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7686, __extension__ __PRETTY_FUNCTION__));
7687
7688 unsigned ConstraintID =
7689 TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode);
7690 assert(ConstraintID != InlineAsm::Constraint_Unknown &&(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
&& "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7691, __extension__ __PRETTY_FUNCTION__))
7691 "Failed to convert memory constraint code to constraint id.")(static_cast <bool> (ConstraintID != InlineAsm::Constraint_Unknown
&& "Failed to convert memory constraint code to constraint id."
) ? void (0) : __assert_fail ("ConstraintID != InlineAsm::Constraint_Unknown && \"Failed to convert memory constraint code to constraint id.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7691, __extension__ __PRETTY_FUNCTION__));
7692
7693 // Add information to the INLINEASM node to know about this input.
7694 unsigned ResOpType = InlineAsm::getFlagWord(InlineAsm::Kind_Mem, 1);
7695 ResOpType = InlineAsm::getFlagWordForMem(ResOpType, ConstraintID);
7696 AsmNodeOperands.push_back(DAG.getTargetConstant(ResOpType,
7697 getCurSDLoc(),
7698 MVT::i32));
7699 AsmNodeOperands.push_back(InOperandVal);
7700 break;
7701 }
7702
7703 assert((OpInfo.ConstraintType == TargetLowering::C_RegisterClass ||(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering
::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::
C_Register) && "Unknown constraint type!") ? void (0)
: __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7705, __extension__ __PRETTY_FUNCTION__))
7704 OpInfo.ConstraintType == TargetLowering::C_Register) &&(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering
::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::
C_Register) && "Unknown constraint type!") ? void (0)
: __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7705, __extension__ __PRETTY_FUNCTION__))
7705 "Unknown constraint type!")(static_cast <bool> ((OpInfo.ConstraintType == TargetLowering
::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::
C_Register) && "Unknown constraint type!") ? void (0)
: __assert_fail ("(OpInfo.ConstraintType == TargetLowering::C_RegisterClass || OpInfo.ConstraintType == TargetLowering::C_Register) && \"Unknown constraint type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7705, __extension__ __PRETTY_FUNCTION__));
7706
7707 // TODO: Support this.
7708 if (OpInfo.isIndirect) {
7709 emitInlineAsmError(
7710 CS, "Don't know how to handle indirect register inputs yet "
7711 "for constraint '" +
7712 Twine(OpInfo.ConstraintCode) + "'");
7713 return;
7714 }
7715
7716 // Copy the input into the appropriate registers.
7717 if (OpInfo.AssignedRegs.Regs.empty()) {
7718 emitInlineAsmError(CS, "couldn't allocate input reg for constraint '" +
7719 Twine(OpInfo.ConstraintCode) + "'");
7720 return;
7721 }
7722
7723 SDLoc dl = getCurSDLoc();
7724
7725 OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, dl,
7726 Chain, &Flag, CS.getInstruction());
7727
7728 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
7729 dl, DAG, AsmNodeOperands);
7730 break;
7731 }
7732 case InlineAsm::isClobber:
7733 // Add the clobbered value to the operand list, so that the register
7734 // allocator is aware that the physreg got clobbered.
7735 if (!OpInfo.AssignedRegs.Regs.empty())
7736 OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_Clobber,
7737 false, 0, getCurSDLoc(), DAG,
7738 AsmNodeOperands);
7739 break;
7740 }
7741 }
7742
7743 // Finish up input operands. Set the input chain and add the flag last.
7744 AsmNodeOperands[InlineAsm::Op_InputChain] = Chain;
7745 if (Flag.getNode()) AsmNodeOperands.push_back(Flag);
7746
7747 Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(),
7748 DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands);
7749 Flag = Chain.getValue(1);
7750
7751 // If this asm returns a register value, copy the result from that register
7752 // and set it as the value of the call.
7753 if (!RetValRegs.Regs.empty()) {
7754 SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7755 Chain, &Flag, CS.getInstruction());
7756
7757 // FIXME: Why don't we do this for inline asms with MRVs?
7758 if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
7759 EVT ResultType = TLI.getValueType(DAG.getDataLayout(), CS.getType());
7760
7761 // If the type of the inline asm call site return value is different but
7762 // has same size as the type of the asm output bitcast it. One example
7763 // of this is for vectors with different width / number of elements.
7764 // This can happen for register classes that can contain multiple
7765 // different value types. The preg or vreg allocated may not have the
7766 // same VT as was expected.
7767 //
7768 // This can also happen for a return value that disagrees with the
7769 // register class it is put in, eg. a double in a general-purpose
7770 // register on a 32-bit machine.
7771 if (ResultType != Val.getValueType() &&
7772 ResultType.getSizeInBits() == Val.getValueSizeInBits()) {
7773 Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(),
7774 ResultType, Val);
7775
7776 } else if (ResultType != Val.getValueType() &&
7777 ResultType.isInteger() && Val.getValueType().isInteger()) {
7778 // If a result value was tied to an input value, the computed result may
7779 // have a wider width than the expected result. Extract the relevant
7780 // portion.
7781 Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultType, Val);
7782 }
7783
7784 assert(ResultType == Val.getValueType() && "Asm result value mismatch!")(static_cast <bool> (ResultType == Val.getValueType() &&
"Asm result value mismatch!") ? void (0) : __assert_fail ("ResultType == Val.getValueType() && \"Asm result value mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7784, __extension__ __PRETTY_FUNCTION__));
7785 }
7786
7787 setValue(CS.getInstruction(), Val);
7788 // Don't need to use this as a chain in this case.
7789 if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty())
7790 return;
7791 }
7792
7793 std::vector<std::pair<SDValue, const Value *>> StoresToEmit;
7794
7795 // Process indirect outputs, first output all of the flagged copies out of
7796 // physregs.
7797 for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) {
7798 RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
7799 const Value *Ptr = IndirectStoresToEmit[i].second;
7800 SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(),
7801 Chain, &Flag, IA);
7802 StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
7803 }
7804
7805 // Emit the non-flagged stores from the physregs.
7806 SmallVector<SDValue, 8> OutChains;
7807 for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) {
7808 SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first,
7809 getValue(StoresToEmit[i].second),
7810 MachinePointerInfo(StoresToEmit[i].second));
7811 OutChains.push_back(Val);
7812 }
7813
7814 if (!OutChains.empty())
7815 Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains);
7816
7817 DAG.setRoot(Chain);
7818}
7819
7820void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS,
7821 const Twine &Message) {
7822 LLVMContext &Ctx = *DAG.getContext();
7823 Ctx.emitError(CS.getInstruction(), Message);
7824
7825 // Make sure we leave the DAG in a valid state
7826 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7827 SmallVector<EVT, 1> ValueVTs;
7828 ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
7829
7830 if (ValueVTs.empty())
7831 return;
7832
7833 SmallVector<SDValue, 1> Ops;
7834 for (unsigned i = 0, e = ValueVTs.size(); i != e; ++i)
7835 Ops.push_back(DAG.getUNDEF(ValueVTs[i]));
7836
7837 setValue(CS.getInstruction(), DAG.getMergeValues(Ops, getCurSDLoc()));
7838}
7839
7840void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
7841 DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(),
7842 MVT::Other, getRoot(),
7843 getValue(I.getArgOperand(0)),
7844 DAG.getSrcValue(I.getArgOperand(0))));
7845}
7846
7847void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
7848 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7849 const DataLayout &DL = DAG.getDataLayout();
7850 SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()),
7851 getCurSDLoc(), getRoot(), getValue(I.getOperand(0)),
7852 DAG.getSrcValue(I.getOperand(0)),
7853 DL.getABITypeAlignment(I.getType()));
7854 setValue(&I, V);
7855 DAG.setRoot(V.getValue(1));
7856}
7857
7858void SelectionDAGBuilder::visitVAEnd(const CallInst &I) {
7859 DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(),
7860 MVT::Other, getRoot(),
7861 getValue(I.getArgOperand(0)),
7862 DAG.getSrcValue(I.getArgOperand(0))));
7863}
7864
7865void SelectionDAGBuilder::visitVACopy(const CallInst &I) {
7866 DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(),
7867 MVT::Other, getRoot(),
7868 getValue(I.getArgOperand(0)),
7869 getValue(I.getArgOperand(1)),
7870 DAG.getSrcValue(I.getArgOperand(0)),
7871 DAG.getSrcValue(I.getArgOperand(1))));
7872}
7873
7874SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG,
7875 const Instruction &I,
7876 SDValue Op) {
7877 const MDNode *Range = I.getMetadata(LLVMContext::MD_range);
7878 if (!Range)
7879 return Op;
7880
7881 ConstantRange CR = getConstantRangeFromMetadata(*Range);
7882 if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet())
7883 return Op;
7884
7885 APInt Lo = CR.getUnsignedMin();
7886 if (!Lo.isMinValue())
7887 return Op;
7888
7889 APInt Hi = CR.getUnsignedMax();
7890 unsigned Bits = Hi.getActiveBits();
7891
7892 EVT SmallVT = EVT::getIntegerVT(*DAG.getContext(), Bits);
7893
7894 SDLoc SL = getCurSDLoc();
7895
7896 SDValue ZExt = DAG.getNode(ISD::AssertZext, SL, Op.getValueType(), Op,
7897 DAG.getValueType(SmallVT));
7898 unsigned NumVals = Op.getNode()->getNumValues();
7899 if (NumVals == 1)
7900 return ZExt;
7901
7902 SmallVector<SDValue, 4> Ops;
7903
7904 Ops.push_back(ZExt);
7905 for (unsigned I = 1; I != NumVals; ++I)
7906 Ops.push_back(Op.getValue(I));
7907
7908 return DAG.getMergeValues(Ops, SL);
7909}
7910
7911/// Populate a CallLowerinInfo (into \p CLI) based on the properties of
7912/// the call being lowered.
7913///
7914/// This is a helper for lowering intrinsics that follow a target calling
7915/// convention or require stack pointer adjustment. Only a subset of the
7916/// intrinsic's operands need to participate in the calling convention.
7917void SelectionDAGBuilder::populateCallLoweringInfo(
7918 TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS,
7919 unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy,
7920 bool IsPatchPoint) {
7921 TargetLowering::ArgListTy Args;
7922 Args.reserve(NumArgs);
7923
7924 // Populate the argument list.
7925 // Attributes for args start at offset 1, after the return attribute.
7926 for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs;
7927 ArgI != ArgE; ++ArgI) {
7928 const Value *V = CS->getOperand(ArgI);
7929
7930 assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic.")(static_cast <bool> (!V->getType()->isEmptyTy() &&
"Empty type passed to intrinsic.") ? void (0) : __assert_fail
("!V->getType()->isEmptyTy() && \"Empty type passed to intrinsic.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7930, __extension__ __PRETTY_FUNCTION__));
7931
7932 TargetLowering::ArgListEntry Entry;
7933 Entry.Node = getValue(V);
7934 Entry.Ty = V->getType();
7935 Entry.setAttributes(&CS, ArgI);
7936 Args.push_back(Entry);
7937 }
7938
7939 CLI.setDebugLoc(getCurSDLoc())
7940 .setChain(getRoot())
7941 .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args))
7942 .setDiscardResult(CS->use_empty())
7943 .setIsPatchPoint(IsPatchPoint);
7944}
7945
7946/// Add a stack map intrinsic call's live variable operands to a stackmap
7947/// or patchpoint target node's operand list.
7948///
7949/// Constants are converted to TargetConstants purely as an optimization to
7950/// avoid constant materialization and register allocation.
7951///
7952/// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not
7953/// generate addess computation nodes, and so ExpandISelPseudo can convert the
7954/// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids
7955/// address materialization and register allocation, but may also be required
7956/// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an
7957/// alloca in the entry block, then the runtime may assume that the alloca's
7958/// StackMap location can be read immediately after compilation and that the
7959/// location is valid at any point during execution (this is similar to the
7960/// assumption made by the llvm.gcroot intrinsic). If the alloca's location were
7961/// only available in a register, then the runtime would need to trap when
7962/// execution reaches the StackMap in order to read the alloca's location.
7963static void addStackMapLiveVars(ImmutableCallSite CS, unsigned StartIdx,
7964 const SDLoc &DL, SmallVectorImpl<SDValue> &Ops,
7965 SelectionDAGBuilder &Builder) {
7966 for (unsigned i = StartIdx, e = CS.arg_size(); i != e; ++i) {
7967 SDValue OpVal = Builder.getValue(CS.getArgument(i));
7968 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(OpVal)) {
7969 Ops.push_back(
7970 Builder.DAG.getTargetConstant(StackMaps::ConstantOp, DL, MVT::i64));
7971 Ops.push_back(
7972 Builder.DAG.getTargetConstant(C->getSExtValue(), DL, MVT::i64));
7973 } else if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(OpVal)) {
7974 const TargetLowering &TLI = Builder.DAG.getTargetLoweringInfo();
7975 Ops.push_back(Builder.DAG.getTargetFrameIndex(
7976 FI->getIndex(), TLI.getFrameIndexTy(Builder.DAG.getDataLayout())));
7977 } else
7978 Ops.push_back(OpVal);
7979 }
7980}
7981
7982/// Lower llvm.experimental.stackmap directly to its target opcode.
7983void SelectionDAGBuilder::visitStackmap(const CallInst &CI) {
7984 // void @llvm.experimental.stackmap(i32 <id>, i32 <numShadowBytes>,
7985 // [live variables...])
7986
7987 assert(CI.getType()->isVoidTy() && "Stackmap cannot return a value.")(static_cast <bool> (CI.getType()->isVoidTy() &&
"Stackmap cannot return a value.") ? void (0) : __assert_fail
("CI.getType()->isVoidTy() && \"Stackmap cannot return a value.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 7987, __extension__ __PRETTY_FUNCTION__));
7988
7989 SDValue Chain, InFlag, Callee, NullPtr;
7990 SmallVector<SDValue, 32> Ops;
7991
7992 SDLoc DL = getCurSDLoc();
7993 Callee = getValue(CI.getCalledValue());
7994 NullPtr = DAG.getIntPtrConstant(0, DL, true);
7995
7996 // The stackmap intrinsic only records the live variables (the arguemnts
7997 // passed to it) and emits NOPS (if requested). Unlike the patchpoint
7998 // intrinsic, this won't be lowered to a function call. This means we don't
7999 // have to worry about calling conventions and target specific lowering code.
8000 // Instead we perform the call lowering right here.
8001 //
8002 // chain, flag = CALLSEQ_START(chain, 0, 0)
8003 // chain, flag = STACKMAP(id, nbytes, ..., chain, flag)
8004 // chain, flag = CALLSEQ_END(chain, 0, 0, flag)
8005 //
8006 Chain = DAG.getCALLSEQ_START(getRoot(), 0, 0, DL);
8007 InFlag = Chain.getValue(1);
8008
8009 // Add the <id> and <numBytes> constants.
8010 SDValue IDVal = getValue(CI.getOperand(PatchPointOpers::IDPos));
8011 Ops.push_back(DAG.getTargetConstant(
8012 cast<ConstantSDNode>(IDVal)->getZExtValue(), DL, MVT::i64));
8013 SDValue NBytesVal = getValue(CI.getOperand(PatchPointOpers::NBytesPos));
8014 Ops.push_back(DAG.getTargetConstant(
8015 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), DL,
8016 MVT::i32));
8017
8018 // Push live variables for the stack map.
8019 addStackMapLiveVars(&CI, 2, DL, Ops, *this);
8020
8021 // We are not pushing any register mask info here on the operands list,
8022 // because the stackmap doesn't clobber anything.
8023
8024 // Push the chain and the glue flag.
8025 Ops.push_back(Chain);
8026 Ops.push_back(InFlag);
8027
8028 // Create the STACKMAP node.
8029 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8030 SDNode *SM = DAG.getMachineNode(TargetOpcode::STACKMAP, DL, NodeTys, Ops);
8031 Chain = SDValue(SM, 0);
8032 InFlag = Chain.getValue(1);
8033
8034 Chain = DAG.getCALLSEQ_END(Chain, NullPtr, NullPtr, InFlag, DL);
8035
8036 // Stackmaps don't generate values, so nothing goes into the NodeMap.
8037
8038 // Set the root to the target-lowered call chain.
8039 DAG.setRoot(Chain);
8040
8041 // Inform the Frame Information that we have a stackmap in this function.
8042 FuncInfo.MF->getFrameInfo().setHasStackMap();
8043}
8044
8045/// Lower llvm.experimental.patchpoint directly to its target opcode.
8046void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS,
8047 const BasicBlock *EHPadBB) {
8048 // void|i64 @llvm.experimental.patchpoint.void|i64(i64 <id>,
8049 // i32 <numBytes>,
8050 // i8* <target>,
8051 // i32 <numArgs>,
8052 // [Args...],
8053 // [live variables...])
8054
8055 CallingConv::ID CC = CS.getCallingConv();
8056 bool IsAnyRegCC = CC == CallingConv::AnyReg;
8057 bool HasDef = !CS->getType()->isVoidTy();
8058 SDLoc dl = getCurSDLoc();
8059 SDValue Callee = getValue(CS->getOperand(PatchPointOpers::TargetPos));
8060
8061 // Handle immediate and symbolic callees.
8062 if (auto* ConstCallee = dyn_cast<ConstantSDNode>(Callee))
8063 Callee = DAG.getIntPtrConstant(ConstCallee->getZExtValue(), dl,
8064 /*isTarget=*/true);
8065 else if (auto* SymbolicCallee = dyn_cast<GlobalAddressSDNode>(Callee))
8066 Callee = DAG.getTargetGlobalAddress(SymbolicCallee->getGlobal(),
8067 SDLoc(SymbolicCallee),
8068 SymbolicCallee->getValueType(0));
8069
8070 // Get the real number of arguments participating in the call <numArgs>
8071 SDValue NArgVal = getValue(CS.getArgument(PatchPointOpers::NArgPos));
8072 unsigned NumArgs = cast<ConstantSDNode>(NArgVal)->getZExtValue();
8073
8074 // Skip the four meta args: <id>, <numNopBytes>, <target>, <numArgs>
8075 // Intrinsics include all meta-operands up to but not including CC.
8076 unsigned NumMetaOpers = PatchPointOpers::CCPos;
8077 assert(CS.arg_size() >= NumMetaOpers + NumArgs &&(static_cast <bool> (CS.arg_size() >= NumMetaOpers +
NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? void (0) : __assert_fail ("CS.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8078, __extension__ __PRETTY_FUNCTION__))
8078 "Not enough arguments provided to the patchpoint intrinsic")(static_cast <bool> (CS.arg_size() >= NumMetaOpers +
NumArgs && "Not enough arguments provided to the patchpoint intrinsic"
) ? void (0) : __assert_fail ("CS.arg_size() >= NumMetaOpers + NumArgs && \"Not enough arguments provided to the patchpoint intrinsic\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8078, __extension__ __PRETTY_FUNCTION__));
8079
8080 // For AnyRegCC the arguments are lowered later on manually.
8081 unsigned NumCallArgs = IsAnyRegCC ? 0 : NumArgs;
8082 Type *ReturnTy =
8083 IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType();
8084
8085 TargetLowering::CallLoweringInfo CLI(DAG);
8086 populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy,
8087 true);
8088 std::pair<SDValue, SDValue> Result = lowerInvokable(CLI, EHPadBB);
8089
8090 SDNode *CallEnd = Result.second.getNode();
8091 if (HasDef && (CallEnd->getOpcode() == ISD::CopyFromReg))
8092 CallEnd = CallEnd->getOperand(0).getNode();
8093
8094 /// Get a call instruction from the call sequence chain.
8095 /// Tail calls are not allowed.
8096 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END &&(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END
&& "Expected a callseq node.") ? void (0) : __assert_fail
("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8097, __extension__ __PRETTY_FUNCTION__))
8097 "Expected a callseq node.")(static_cast <bool> (CallEnd->getOpcode() == ISD::CALLSEQ_END
&& "Expected a callseq node.") ? void (0) : __assert_fail
("CallEnd->getOpcode() == ISD::CALLSEQ_END && \"Expected a callseq node.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8097, __extension__ __PRETTY_FUNCTION__));
8098 SDNode *Call = CallEnd->getOperand(0).getNode();
8099 bool HasGlue = Call->getGluedNode();
8100
8101 // Replace the target specific call node with the patchable intrinsic.
8102 SmallVector<SDValue, 8> Ops;
8103
8104 // Add the <id> and <numBytes> constants.
8105 SDValue IDVal = getValue(CS->getOperand(PatchPointOpers::IDPos));
8106 Ops.push_back(DAG.getTargetConstant(
8107 cast<ConstantSDNode>(IDVal)->getZExtValue(), dl, MVT::i64));
8108 SDValue NBytesVal = getValue(CS->getOperand(PatchPointOpers::NBytesPos));
8109 Ops.push_back(DAG.getTargetConstant(
8110 cast<ConstantSDNode>(NBytesVal)->getZExtValue(), dl,
8111 MVT::i32));
8112
8113 // Add the callee.
8114 Ops.push_back(Callee);
8115
8116 // Adjust <numArgs> to account for any arguments that have been passed on the
8117 // stack instead.
8118 // Call Node: Chain, Target, {Args}, RegMask, [Glue]
8119 unsigned NumCallRegArgs = Call->getNumOperands() - (HasGlue ? 4 : 3);
8120 NumCallRegArgs = IsAnyRegCC ? NumArgs : NumCallRegArgs;
8121 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, dl, MVT::i32));
8122
8123 // Add the calling convention
8124 Ops.push_back(DAG.getTargetConstant((unsigned)CC, dl, MVT::i32));
8125
8126 // Add the arguments we omitted previously. The register allocator should
8127 // place these in any free register.
8128 if (IsAnyRegCC)
8129 for (unsigned i = NumMetaOpers, e = NumMetaOpers + NumArgs; i != e; ++i)
8130 Ops.push_back(getValue(CS.getArgument(i)));
8131
8132 // Push the arguments from the call instruction up to the register mask.
8133 SDNode::op_iterator e = HasGlue ? Call->op_end()-2 : Call->op_end()-1;
8134 Ops.append(Call->op_begin() + 2, e);
8135
8136 // Push live variables for the stack map.
8137 addStackMapLiveVars(CS, NumMetaOpers + NumArgs, dl, Ops, *this);
8138
8139 // Push the register mask info.
8140 if (HasGlue)
8141 Ops.push_back(*(Call->op_end()-2));
8142 else
8143 Ops.push_back(*(Call->op_end()-1));
8144
8145 // Push the chain (this is originally the first operand of the call, but
8146 // becomes now the last or second to last operand).
8147 Ops.push_back(*(Call->op_begin()));
8148
8149 // Push the glue flag (last operand).
8150 if (HasGlue)
8151 Ops.push_back(*(Call->op_end()-1));
8152
8153 SDVTList NodeTys;
8154 if (IsAnyRegCC && HasDef) {
8155 // Create the return types based on the intrinsic definition
8156 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8157 SmallVector<EVT, 3> ValueVTs;
8158 ComputeValueVTs(TLI, DAG.getDataLayout(), CS->getType(), ValueVTs);
8159 assert(ValueVTs.size() == 1 && "Expected only one return value type.")(static_cast <bool> (ValueVTs.size() == 1 && "Expected only one return value type."
) ? void (0) : __assert_fail ("ValueVTs.size() == 1 && \"Expected only one return value type.\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8159, __extension__ __PRETTY_FUNCTION__));
8160
8161 // There is always a chain and a glue type at the end
8162 ValueVTs.push_back(MVT::Other);
8163 ValueVTs.push_back(MVT::Glue);
8164 NodeTys = DAG.getVTList(ValueVTs);
8165 } else
8166 NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
8167
8168 // Replace the target specific call node with a PATCHPOINT node.
8169 MachineSDNode *MN = DAG.getMachineNode(TargetOpcode::PATCHPOINT,
8170 dl, NodeTys, Ops);
8171
8172 // Update the NodeMap.
8173 if (HasDef) {
8174 if (IsAnyRegCC)
8175 setValue(CS.getInstruction(), SDValue(MN, 0));
8176 else
8177 setValue(CS.getInstruction(), Result.first);
8178 }
8179
8180 // Fixup the consumers of the intrinsic. The chain and glue may be used in the
8181 // call sequence. Furthermore the location of the chain and glue can change
8182 // when the AnyReg calling convention is used and the intrinsic returns a
8183 // value.
8184 if (IsAnyRegCC && HasDef) {
8185 SDValue From[] = {SDValue(Call, 0), SDValue(Call, 1)};
8186 SDValue To[] = {SDValue(MN, 1), SDValue(MN, 2)};
8187 DAG.ReplaceAllUsesOfValuesWith(From, To, 2);
8188 } else
8189 DAG.ReplaceAllUsesWith(Call, MN);
8190 DAG.DeleteNode(Call);
8191
8192 // Inform the Frame Information that we have a patchpoint in this function.
8193 FuncInfo.MF->getFrameInfo().setHasPatchPoint();
8194}
8195
8196void SelectionDAGBuilder::visitVectorReduce(const CallInst &I,
8197 unsigned Intrinsic) {
8198 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8199 SDValue Op1 = getValue(I.getArgOperand(0));
8200 SDValue Op2;
8201 if (I.getNumArgOperands() > 1)
8202 Op2 = getValue(I.getArgOperand(1));
8203 SDLoc dl = getCurSDLoc();
8204 EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType());
8205 SDValue Res;
8206 FastMathFlags FMF;
8207 if (isa<FPMathOperator>(I))
8208 FMF = I.getFastMathFlags();
8209
8210 switch (Intrinsic) {
8211 case Intrinsic::experimental_vector_reduce_fadd:
8212 if (FMF.isFast())
8213 Res = DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2);
8214 else
8215 Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2);
8216 break;
8217 case Intrinsic::experimental_vector_reduce_fmul:
8218 if (FMF.isFast())
8219 Res = DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2);
8220 else
8221 Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2);
8222 break;
8223 case Intrinsic::experimental_vector_reduce_add:
8224 Res = DAG.getNode(ISD::VECREDUCE_ADD, dl, VT, Op1);
8225 break;
8226 case Intrinsic::experimental_vector_reduce_mul:
8227 Res = DAG.getNode(ISD::VECREDUCE_MUL, dl, VT, Op1);
8228 break;
8229 case Intrinsic::experimental_vector_reduce_and:
8230 Res = DAG.getNode(ISD::VECREDUCE_AND, dl, VT, Op1);
8231 break;
8232 case Intrinsic::experimental_vector_reduce_or:
8233 Res = DAG.getNode(ISD::VECREDUCE_OR, dl, VT, Op1);
8234 break;
8235 case Intrinsic::experimental_vector_reduce_xor:
8236 Res = DAG.getNode(ISD::VECREDUCE_XOR, dl, VT, Op1);
8237 break;
8238 case Intrinsic::experimental_vector_reduce_smax:
8239 Res = DAG.getNode(ISD::VECREDUCE_SMAX, dl, VT, Op1);
8240 break;
8241 case Intrinsic::experimental_vector_reduce_smin:
8242 Res = DAG.getNode(ISD::VECREDUCE_SMIN, dl, VT, Op1);
8243 break;
8244 case Intrinsic::experimental_vector_reduce_umax:
8245 Res = DAG.getNode(ISD::VECREDUCE_UMAX, dl, VT, Op1);
8246 break;
8247 case Intrinsic::experimental_vector_reduce_umin:
8248 Res = DAG.getNode(ISD::VECREDUCE_UMIN, dl, VT, Op1);
8249 break;
8250 case Intrinsic::experimental_vector_reduce_fmax:
8251 Res = DAG.getNode(ISD::VECREDUCE_FMAX, dl, VT, Op1);
8252 break;
8253 case Intrinsic::experimental_vector_reduce_fmin:
8254 Res = DAG.getNode(ISD::VECREDUCE_FMIN, dl, VT, Op1);
8255 break;
8256 default:
8257 llvm_unreachable("Unhandled vector reduce intrinsic")::llvm::llvm_unreachable_internal("Unhandled vector reduce intrinsic"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8257);
8258 }
8259 setValue(&I, Res);
8260}
8261
8262/// Returns an AttributeList representing the attributes applied to the return
8263/// value of the given call.
8264static AttributeList getReturnAttrs(TargetLowering::CallLoweringInfo &CLI) {
8265 SmallVector<Attribute::AttrKind, 2> Attrs;
8266 if (CLI.RetSExt)
8267 Attrs.push_back(Attribute::SExt);
8268 if (CLI.RetZExt)
8269 Attrs.push_back(Attribute::ZExt);
8270 if (CLI.IsInReg)
8271 Attrs.push_back(Attribute::InReg);
8272
8273 return AttributeList::get(CLI.RetTy->getContext(), AttributeList::ReturnIndex,
8274 Attrs);
8275}
8276
8277/// TargetLowering::LowerCallTo - This is the default LowerCallTo
8278/// implementation, which just calls LowerCall.
8279/// FIXME: When all targets are
8280/// migrated to using LowerCall, this hook should be integrated into SDISel.
8281std::pair<SDValue, SDValue>
8282TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
8283 // Handle the incoming return values from the call.
8284 CLI.Ins.clear();
8285 Type *OrigRetTy = CLI.RetTy;
8286 SmallVector<EVT, 4> RetTys;
8287 SmallVector<uint64_t, 4> Offsets;
8288 auto &DL = CLI.DAG.getDataLayout();
8289 ComputeValueVTs(*this, DL, CLI.RetTy, RetTys, &Offsets);
8290
8291 if (CLI.IsPostTypeLegalization) {
8292 // If we are lowering a libcall after legalization, split the return type.
8293 SmallVector<EVT, 4> OldRetTys = std::move(RetTys);
8294 SmallVector<uint64_t, 4> OldOffsets = std::move(Offsets);
8295 for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) {
8296 EVT RetVT = OldRetTys[i];
8297 uint64_t Offset = OldOffsets[i];
8298 MVT RegisterVT = getRegisterType(CLI.RetTy->getContext(), RetVT);
8299 unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), RetVT);
8300 unsigned RegisterVTByteSZ = RegisterVT.getSizeInBits() / 8;
8301 RetTys.append(NumRegs, RegisterVT);
8302 for (unsigned j = 0; j != NumRegs; ++j)
8303 Offsets.push_back(Offset + j * RegisterVTByteSZ);
8304 }
8305 }
8306
8307 SmallVector<ISD::OutputArg, 4> Outs;
8308 GetReturnInfo(CLI.CallConv, CLI.RetTy, getReturnAttrs(CLI), Outs, *this, DL);
8309
8310 bool CanLowerReturn =
8311 this->CanLowerReturn(CLI.CallConv, CLI.DAG.getMachineFunction(),
8312 CLI.IsVarArg, Outs, CLI.RetTy->getContext());
8313
8314 SDValue DemoteStackSlot;
8315 int DemoteStackIdx = -100;
8316 if (!CanLowerReturn) {
8317 // FIXME: equivalent assert?
8318 // assert(!CS.hasInAllocaArgument() &&
8319 // "sret demotion is incompatible with inalloca");
8320 uint64_t TySize = DL.getTypeAllocSize(CLI.RetTy);
8321 unsigned Align = DL.getPrefTypeAlignment(CLI.RetTy);
8322 MachineFunction &MF = CLI.DAG.getMachineFunction();
8323 DemoteStackIdx = MF.getFrameInfo().CreateStackObject(TySize, Align, false);
8324 Type *StackSlotPtrType = PointerType::getUnqual(CLI.RetTy);
8325
8326 DemoteStackSlot = CLI.DAG.getFrameIndex(DemoteStackIdx, getFrameIndexTy(DL));
8327 ArgListEntry Entry;
8328 Entry.Node = DemoteStackSlot;
8329 Entry.Ty = StackSlotPtrType;
8330 Entry.IsSExt = false;
8331 Entry.IsZExt = false;
8332 Entry.IsInReg = false;
8333 Entry.IsSRet = true;
8334 Entry.IsNest = false;
8335 Entry.IsByVal = false;
8336 Entry.IsReturned = false;
8337 Entry.IsSwiftSelf = false;
8338 Entry.IsSwiftError = false;
8339 Entry.Alignment = Align;
8340 CLI.getArgs().insert(CLI.getArgs().begin(), Entry);
8341 CLI.NumFixedArgs += 1;
8342 CLI.RetTy = Type::getVoidTy(CLI.RetTy->getContext());
8343
8344 // sret demotion isn't compatible with tail-calls, since the sret argument
8345 // points into the callers stack frame.
8346 CLI.IsTailCall = false;
8347 } else {
8348 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8349 EVT VT = RetTys[I];
8350 MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8351 CLI.CallConv, VT);
8352 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8353 CLI.CallConv, VT);
8354 for (unsigned i = 0; i != NumRegs; ++i) {
8355 ISD::InputArg MyFlags;
8356 MyFlags.VT = RegisterVT;
8357 MyFlags.ArgVT = VT;
8358 MyFlags.Used = CLI.IsReturnValueUsed;
8359 if (CLI.RetSExt)
8360 MyFlags.Flags.setSExt();
8361 if (CLI.RetZExt)
8362 MyFlags.Flags.setZExt();
8363 if (CLI.IsInReg)
8364 MyFlags.Flags.setInReg();
8365 CLI.Ins.push_back(MyFlags);
8366 }
8367 }
8368 }
8369
8370 // We push in swifterror return as the last element of CLI.Ins.
8371 ArgListTy &Args = CLI.getArgs();
8372 if (supportSwiftError()) {
8373 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8374 if (Args[i].IsSwiftError) {
8375 ISD::InputArg MyFlags;
8376 MyFlags.VT = getPointerTy(DL);
8377 MyFlags.ArgVT = EVT(getPointerTy(DL));
8378 MyFlags.Flags.setSwiftError();
8379 CLI.Ins.push_back(MyFlags);
8380 }
8381 }
8382 }
8383
8384 // Handle all of the outgoing arguments.
8385 CLI.Outs.clear();
8386 CLI.OutVals.clear();
8387 for (unsigned i = 0, e = Args.size(); i != e; ++i) {
8388 SmallVector<EVT, 4> ValueVTs;
8389 ComputeValueVTs(*this, DL, Args[i].Ty, ValueVTs);
8390 // FIXME: Split arguments if CLI.IsPostTypeLegalization
8391 Type *FinalType = Args[i].Ty;
8392 if (Args[i].IsByVal)
8393 FinalType = cast<PointerType>(Args[i].Ty)->getElementType();
8394 bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters(
8395 FinalType, CLI.CallConv, CLI.IsVarArg);
8396 for (unsigned Value = 0, NumValues = ValueVTs.size(); Value != NumValues;
8397 ++Value) {
8398 EVT VT = ValueVTs[Value];
8399 Type *ArgTy = VT.getTypeForEVT(CLI.RetTy->getContext());
8400 SDValue Op = SDValue(Args[i].Node.getNode(),
8401 Args[i].Node.getResNo() + Value);
8402 ISD::ArgFlagsTy Flags;
8403
8404 // Certain targets (such as MIPS), may have a different ABI alignment
8405 // for a type depending on the context. Give the target a chance to
8406 // specify the alignment it wants.
8407 unsigned OriginalAlignment = getABIAlignmentForCallingConv(ArgTy, DL);
8408
8409 if (Args[i].IsZExt)
8410 Flags.setZExt();
8411 if (Args[i].IsSExt)
8412 Flags.setSExt();
8413 if (Args[i].IsInReg) {
8414 // If we are using vectorcall calling convention, a structure that is
8415 // passed InReg - is surely an HVA
8416 if (CLI.CallConv == CallingConv::X86_VectorCall &&
8417 isa<StructType>(FinalType)) {
8418 // The first value of a structure is marked
8419 if (0 == Value)
8420 Flags.setHvaStart();
8421 Flags.setHva();
8422 }
8423 // Set InReg Flag
8424 Flags.setInReg();
8425 }
8426 if (Args[i].IsSRet)
8427 Flags.setSRet();
8428 if (Args[i].IsSwiftSelf)
8429 Flags.setSwiftSelf();
8430 if (Args[i].IsSwiftError)
8431 Flags.setSwiftError();
8432 if (Args[i].IsByVal)
8433 Flags.setByVal();
8434 if (Args[i].IsInAlloca) {
8435 Flags.setInAlloca();
8436 // Set the byval flag for CCAssignFn callbacks that don't know about
8437 // inalloca. This way we can know how many bytes we should've allocated
8438 // and how many bytes a callee cleanup function will pop. If we port
8439 // inalloca to more targets, we'll have to add custom inalloca handling
8440 // in the various CC lowering callbacks.
8441 Flags.setByVal();
8442 }
8443 if (Args[i].IsByVal || Args[i].IsInAlloca) {
8444 PointerType *Ty = cast<PointerType>(Args[i].Ty);
8445 Type *ElementTy = Ty->getElementType();
8446 Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8447 // For ByVal, alignment should come from FE. BE will guess if this
8448 // info is not there but there are cases it cannot get right.
8449 unsigned FrameAlign;
8450 if (Args[i].Alignment)
8451 FrameAlign = Args[i].Alignment;
8452 else
8453 FrameAlign = getByValTypeAlignment(ElementTy, DL);
8454 Flags.setByValAlign(FrameAlign);
8455 }
8456 if (Args[i].IsNest)
8457 Flags.setNest();
8458 if (NeedsRegBlock)
8459 Flags.setInConsecutiveRegs();
8460 Flags.setOrigAlign(OriginalAlignment);
8461
8462 MVT PartVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8463 CLI.CallConv, VT);
8464 unsigned NumParts = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8465 CLI.CallConv, VT);
8466 SmallVector<SDValue, 4> Parts(NumParts);
8467 ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
8468
8469 if (Args[i].IsSExt)
8470 ExtendKind = ISD::SIGN_EXTEND;
8471 else if (Args[i].IsZExt)
8472 ExtendKind = ISD::ZERO_EXTEND;
8473
8474 // Conservatively only handle 'returned' on non-vectors that can be lowered,
8475 // for now.
8476 if (Args[i].IsReturned && !Op.getValueType().isVector() &&
8477 CanLowerReturn) {
8478 assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues &&(static_cast <bool> (CLI.RetTy == Args[i].Ty &&
RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8479, __extension__ __PRETTY_FUNCTION__))
8479 "unexpected use of 'returned'")(static_cast <bool> (CLI.RetTy == Args[i].Ty &&
RetTys.size() == NumValues && "unexpected use of 'returned'"
) ? void (0) : __assert_fail ("CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues && \"unexpected use of 'returned'\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8479, __extension__ __PRETTY_FUNCTION__));
8480 // Before passing 'returned' to the target lowering code, ensure that
8481 // either the register MVT and the actual EVT are the same size or that
8482 // the return value and argument are extended in the same way; in these
8483 // cases it's safe to pass the argument register value unchanged as the
8484 // return register value (although it's at the target's option whether
8485 // to do so)
8486 // TODO: allow code generation to take advantage of partially preserved
8487 // registers rather than clobbering the entire register when the
8488 // parameter extension method is not compatible with the return
8489 // extension method
8490 if ((NumParts * PartVT.getSizeInBits() == VT.getSizeInBits()) ||
8491 (ExtendKind != ISD::ANY_EXTEND && CLI.RetSExt == Args[i].IsSExt &&
8492 CLI.RetZExt == Args[i].IsZExt))
8493 Flags.setReturned();
8494 }
8495
8496 getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts, PartVT,
8497 CLI.CS.getInstruction(), CLI.CallConv, ExtendKind);
8498
8499 for (unsigned j = 0; j != NumParts; ++j) {
8500 // if it isn't first piece, alignment must be 1
8501 ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(), VT,
8502 i < CLI.NumFixedArgs,
8503 i, j*Parts[j].getValueType().getStoreSize());
8504 if (NumParts > 1 && j == 0)
8505 MyFlags.Flags.setSplit();
8506 else if (j != 0) {
8507 MyFlags.Flags.setOrigAlign(1);
8508 if (j == NumParts - 1)
8509 MyFlags.Flags.setSplitEnd();
8510 }
8511
8512 CLI.Outs.push_back(MyFlags);
8513 CLI.OutVals.push_back(Parts[j]);
8514 }
8515
8516 if (NeedsRegBlock && Value == NumValues - 1)
8517 CLI.Outs[CLI.Outs.size() - 1].Flags.setInConsecutiveRegsLast();
8518 }
8519 }
8520
8521 SmallVector<SDValue, 4> InVals;
8522 CLI.Chain = LowerCall(CLI, InVals);
8523
8524 // Update CLI.InVals to use outside of this function.
8525 CLI.InVals = InVals;
8526
8527 // Verify that the target's LowerCall behaved as expected.
8528 assert(CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other &&(static_cast <bool> (CLI.Chain.getNode() && CLI
.Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!"
) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8529, __extension__ __PRETTY_FUNCTION__))
8529 "LowerCall didn't return a valid chain!")(static_cast <bool> (CLI.Chain.getNode() && CLI
.Chain.getValueType() == MVT::Other && "LowerCall didn't return a valid chain!"
) ? void (0) : __assert_fail ("CLI.Chain.getNode() && CLI.Chain.getValueType() == MVT::Other && \"LowerCall didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8529, __extension__ __PRETTY_FUNCTION__));
8530 assert((!CLI.IsTailCall || InVals.empty()) &&(static_cast <bool> ((!CLI.IsTailCall || InVals.empty()
) && "LowerCall emitted a return value for a tail call!"
) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8531, __extension__ __PRETTY_FUNCTION__))
8531 "LowerCall emitted a return value for a tail call!")(static_cast <bool> ((!CLI.IsTailCall || InVals.empty()
) && "LowerCall emitted a return value for a tail call!"
) ? void (0) : __assert_fail ("(!CLI.IsTailCall || InVals.empty()) && \"LowerCall emitted a return value for a tail call!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8531, __extension__ __PRETTY_FUNCTION__));
8532 assert((CLI.IsTailCall || InVals.size() == CLI.Ins.size()) &&(static_cast <bool> ((CLI.IsTailCall || InVals.size() ==
CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8533, __extension__ __PRETTY_FUNCTION__))
8533 "LowerCall didn't emit the correct number of values!")(static_cast <bool> ((CLI.IsTailCall || InVals.size() ==
CLI.Ins.size()) && "LowerCall didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("(CLI.IsTailCall || InVals.size() == CLI.Ins.size()) && \"LowerCall didn't emit the correct number of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8533, __extension__ __PRETTY_FUNCTION__));
8534
8535 // For a tail call, the return value is merely live-out and there aren't
8536 // any nodes in the DAG representing it. Return a special value to
8537 // indicate that a tail call has been emitted and no more Instructions
8538 // should be processed in the current block.
8539 if (CLI.IsTailCall) {
8540 CLI.DAG.setRoot(CLI.Chain);
8541 return std::make_pair(SDValue(), SDValue());
8542 }
8543
8544#ifndef NDEBUG
8545 for (unsigned i = 0, e = CLI.Ins.size(); i != e; ++i) {
8546 assert(InVals[i].getNode() && "LowerCall emitted a null value!")(static_cast <bool> (InVals[i].getNode() && "LowerCall emitted a null value!"
) ? void (0) : __assert_fail ("InVals[i].getNode() && \"LowerCall emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8546, __extension__ __PRETTY_FUNCTION__));
8547 assert(EVT(CLI.Ins[i].VT) == InVals[i].getValueType() &&(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType
() && "LowerCall emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8548, __extension__ __PRETTY_FUNCTION__))
8548 "LowerCall emitted a value with the wrong type!")(static_cast <bool> (EVT(CLI.Ins[i].VT) == InVals[i].getValueType
() && "LowerCall emitted a value with the wrong type!"
) ? void (0) : __assert_fail ("EVT(CLI.Ins[i].VT) == InVals[i].getValueType() && \"LowerCall emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8548, __extension__ __PRETTY_FUNCTION__));
8549 }
8550#endif
8551
8552 SmallVector<SDValue, 4> ReturnValues;
8553 if (!CanLowerReturn) {
8554 // The instruction result is the result of loading from the
8555 // hidden sret parameter.
8556 SmallVector<EVT, 1> PVTs;
8557 Type *PtrRetTy = OrigRetTy->getPointerTo(DL.getAllocaAddrSpace());
8558
8559 ComputeValueVTs(*this, DL, PtrRetTy, PVTs);
8560 assert(PVTs.size() == 1 && "Pointers should fit in one register")(static_cast <bool> (PVTs.size() == 1 && "Pointers should fit in one register"
) ? void (0) : __assert_fail ("PVTs.size() == 1 && \"Pointers should fit in one register\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8560, __extension__ __PRETTY_FUNCTION__));
8561 EVT PtrVT = PVTs[0];
8562
8563 unsigned NumValues = RetTys.size();
8564 ReturnValues.resize(NumValues);
8565 SmallVector<SDValue, 4> Chains(NumValues);
8566
8567 // An aggregate return value cannot wrap around the address space, so
8568 // offsets to its parts don't wrap either.
8569 SDNodeFlags Flags;
8570 Flags.setNoUnsignedWrap(true);
8571
8572 for (unsigned i = 0; i < NumValues; ++i) {
8573 SDValue Add = CLI.DAG.getNode(ISD::ADD, CLI.DL, PtrVT, DemoteStackSlot,
8574 CLI.DAG.getConstant(Offsets[i], CLI.DL,
8575 PtrVT), Flags);
8576 SDValue L = CLI.DAG.getLoad(
8577 RetTys[i], CLI.DL, CLI.Chain, Add,
8578 MachinePointerInfo::getFixedStack(CLI.DAG.getMachineFunction(),
8579 DemoteStackIdx, Offsets[i]),
8580 /* Alignment = */ 1);
8581 ReturnValues[i] = L;
8582 Chains[i] = L.getValue(1);
8583 }
8584
8585 CLI.Chain = CLI.DAG.getNode(ISD::TokenFactor, CLI.DL, MVT::Other, Chains);
8586 } else {
8587 // Collect the legal value parts into potentially illegal values
8588 // that correspond to the original function's return values.
8589 Optional<ISD::NodeType> AssertOp;
8590 if (CLI.RetSExt)
8591 AssertOp = ISD::AssertSext;
8592 else if (CLI.RetZExt)
8593 AssertOp = ISD::AssertZext;
8594 unsigned CurReg = 0;
8595 for (unsigned I = 0, E = RetTys.size(); I != E; ++I) {
8596 EVT VT = RetTys[I];
8597 MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(),
8598 CLI.CallConv, VT);
8599 unsigned NumRegs = getNumRegistersForCallingConv(CLI.RetTy->getContext(),
8600 CLI.CallConv, VT);
8601
8602 ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
8603 NumRegs, RegisterVT, VT, nullptr,
8604 CLI.CallConv, AssertOp));
8605 CurReg += NumRegs;
8606 }
8607
8608 // For a function returning void, there is no return value. We can't create
8609 // such a node, so we just return a null return value in that case. In
8610 // that case, nothing will actually look at the value.
8611 if (ReturnValues.empty())
8612 return std::make_pair(SDValue(), CLI.Chain);
8613 }
8614
8615 SDValue Res = CLI.DAG.getNode(ISD::MERGE_VALUES, CLI.DL,
8616 CLI.DAG.getVTList(RetTys), ReturnValues);
8617 return std::make_pair(Res, CLI.Chain);
8618}
8619
8620void TargetLowering::LowerOperationWrapper(SDNode *N,
8621 SmallVectorImpl<SDValue> &Results,
8622 SelectionDAG &DAG) const {
8623 if (SDValue Res = LowerOperation(SDValue(N, 0), DAG))
8624 Results.push_back(Res);
8625}
8626
8627SDValue TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
8628 llvm_unreachable("LowerOperation not implemented for this target!")::llvm::llvm_unreachable_internal("LowerOperation not implemented for this target!"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8628);
8629}
8630
8631void
8632SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
8633 SDValue Op = getNonRegisterValue(V);
8634 assert((Op.getOpcode() != ISD::CopyFromReg ||(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg
|| cast<RegisterSDNode>(Op.getOperand(1))->getReg()
!= Reg) && "Copy from a reg to the same reg!") ? void
(0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8636, __extension__ __PRETTY_FUNCTION__))
8635 cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) &&(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg
|| cast<RegisterSDNode>(Op.getOperand(1))->getReg()
!= Reg) && "Copy from a reg to the same reg!") ? void
(0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8636, __extension__ __PRETTY_FUNCTION__))
8636 "Copy from a reg to the same reg!")(static_cast <bool> ((Op.getOpcode() != ISD::CopyFromReg
|| cast<RegisterSDNode>(Op.getOperand(1))->getReg()
!= Reg) && "Copy from a reg to the same reg!") ? void
(0) : __assert_fail ("(Op.getOpcode() != ISD::CopyFromReg || cast<RegisterSDNode>(Op.getOperand(1))->getReg() != Reg) && \"Copy from a reg to the same reg!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8636, __extension__ __PRETTY_FUNCTION__));
8637 assert(!TargetRegisterInfo::isPhysicalRegister(Reg) && "Is a physreg")(static_cast <bool> (!TargetRegisterInfo::isPhysicalRegister
(Reg) && "Is a physreg") ? void (0) : __assert_fail (
"!TargetRegisterInfo::isPhysicalRegister(Reg) && \"Is a physreg\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8637, __extension__ __PRETTY_FUNCTION__));
8638
8639 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8640 // If this is an InlineAsm we have to match the registers required, not the
8641 // notional registers required by the type.
8642
8643 RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, V->getType(),
8644 getABIRegCopyCC(V));
8645 SDValue Chain = DAG.getEntryNode();
8646
8647 ISD::NodeType ExtendType = (FuncInfo.PreferredExtendType.find(V) ==
8648 FuncInfo.PreferredExtendType.end())
8649 ? ISD::ANY_EXTEND
8650 : FuncInfo.PreferredExtendType[V];
8651 RFV.getCopyToRegs(Op, DAG, getCurSDLoc(), Chain, nullptr, V, ExtendType);
8652 PendingExports.push_back(Chain);
8653}
8654
8655#include "llvm/CodeGen/SelectionDAGISel.h"
8656
8657/// isOnlyUsedInEntryBlock - If the specified argument is only used in the
8658/// entry block, return true. This includes arguments used by switches, since
8659/// the switch may expand into multiple basic blocks.
8660static bool isOnlyUsedInEntryBlock(const Argument *A, bool FastISel) {
8661 // With FastISel active, we may be splitting blocks, so force creation
8662 // of virtual registers for all non-dead arguments.
8663 if (FastISel)
8664 return A->use_empty();
8665
8666 const BasicBlock &Entry = A->getParent()->front();
8667 for (const User *U : A->users())
8668 if (cast<Instruction>(U)->getParent() != &Entry || isa<SwitchInst>(U))
8669 return false; // Use not in entry block.
8670
8671 return true;
8672}
8673
8674using ArgCopyElisionMapTy =
8675 DenseMap<const Argument *,
8676 std::pair<const AllocaInst *, const StoreInst *>>;
8677
8678/// Scan the entry block of the function in FuncInfo for arguments that look
8679/// like copies into a local alloca. Record any copied arguments in
8680/// ArgCopyElisionCandidates.
8681static void
8682findArgumentCopyElisionCandidates(const DataLayout &DL,
8683 FunctionLoweringInfo *FuncInfo,
8684 ArgCopyElisionMapTy &ArgCopyElisionCandidates) {
8685 // Record the state of every static alloca used in the entry block. Argument
8686 // allocas are all used in the entry block, so we need approximately as many
8687 // entries as we have arguments.
8688 enum StaticAllocaInfo { Unknown, Clobbered, Elidable };
8689 SmallDenseMap<const AllocaInst *, StaticAllocaInfo, 8> StaticAllocas;
8690 unsigned NumArgs = FuncInfo->Fn->arg_size();
8691 StaticAllocas.reserve(NumArgs * 2);
8692
8693 auto GetInfoIfStaticAlloca = [&](const Value *V) -> StaticAllocaInfo * {
8694 if (!V)
8695 return nullptr;
8696 V = V->stripPointerCasts();
8697 const auto *AI = dyn_cast<AllocaInst>(V);
8698 if (!AI || !AI->isStaticAlloca() || !FuncInfo->StaticAllocaMap.count(AI))
8699 return nullptr;
8700 auto Iter = StaticAllocas.insert({AI, Unknown});
8701 return &Iter.first->second;
8702 };
8703
8704 // Look for stores of arguments to static allocas. Look through bitcasts and
8705 // GEPs to handle type coercions, as long as the alloca is fully initialized
8706 // by the store. Any non-store use of an alloca escapes it and any subsequent
8707 // unanalyzed store might write it.
8708 // FIXME: Handle structs initialized with multiple stores.
8709 for (const Instruction &I : FuncInfo->Fn->getEntryBlock()) {
8710 // Look for stores, and handle non-store uses conservatively.
8711 const auto *SI = dyn_cast<StoreInst>(&I);
8712 if (!SI) {
8713 // We will look through cast uses, so ignore them completely.
8714 if (I.isCast())
8715 continue;
8716 // Ignore debug info intrinsics, they don't escape or store to allocas.
8717 if (isa<DbgInfoIntrinsic>(I))
8718 continue;
8719 // This is an unknown instruction. Assume it escapes or writes to all
8720 // static alloca operands.
8721 for (const Use &U : I.operands()) {
8722 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(U))
8723 *Info = StaticAllocaInfo::Clobbered;
8724 }
8725 continue;
8726 }
8727
8728 // If the stored value is a static alloca, mark it as escaped.
8729 if (StaticAllocaInfo *Info = GetInfoIfStaticAlloca(SI->getValueOperand()))
8730 *Info = StaticAllocaInfo::Clobbered;
8731
8732 // Check if the destination is a static alloca.
8733 const Value *Dst = SI->getPointerOperand()->stripPointerCasts();
8734 StaticAllocaInfo *Info = GetInfoIfStaticAlloca(Dst);
8735 if (!Info)
8736 continue;
8737 const AllocaInst *AI = cast<AllocaInst>(Dst);
8738
8739 // Skip allocas that have been initialized or clobbered.
8740 if (*Info != StaticAllocaInfo::Unknown)
8741 continue;
8742
8743 // Check if the stored value is an argument, and that this store fully
8744 // initializes the alloca. Don't elide copies from the same argument twice.
8745 const Value *Val = SI->getValueOperand()->stripPointerCasts();
8746 const auto *Arg = dyn_cast<Argument>(Val);
8747 if (!Arg || Arg->hasInAllocaAttr() || Arg->hasByValAttr() ||
8748 Arg->getType()->isEmptyTy() ||
8749 DL.getTypeStoreSize(Arg->getType()) !=
8750 DL.getTypeAllocSize(AI->getAllocatedType()) ||
8751 ArgCopyElisionCandidates.count(Arg)) {
8752 *Info = StaticAllocaInfo::Clobbered;
8753 continue;
8754 }
8755
8756 LLVM_DEBUG(dbgs() << "Found argument copy elision candidate: " << *AIdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Found argument copy elision candidate: "
<< *AI << '\n'; } } while (false)
8757 << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Found argument copy elision candidate: "
<< *AI << '\n'; } } while (false);
8758
8759 // Mark this alloca and store for argument copy elision.
8760 *Info = StaticAllocaInfo::Elidable;
8761 ArgCopyElisionCandidates.insert({Arg, {AI, SI}});
8762
8763 // Stop scanning if we've seen all arguments. This will happen early in -O0
8764 // builds, which is useful, because -O0 builds have large entry blocks and
8765 // many allocas.
8766 if (ArgCopyElisionCandidates.size() == NumArgs)
8767 break;
8768 }
8769}
8770
8771/// Try to elide argument copies from memory into a local alloca. Succeeds if
8772/// ArgVal is a load from a suitable fixed stack object.
8773static void tryToElideArgumentCopy(
8774 FunctionLoweringInfo *FuncInfo, SmallVectorImpl<SDValue> &Chains,
8775 DenseMap<int, int> &ArgCopyElisionFrameIndexMap,
8776 SmallPtrSetImpl<const Instruction *> &ElidedArgCopyInstrs,
8777 ArgCopyElisionMapTy &ArgCopyElisionCandidates, const Argument &Arg,
8778 SDValue ArgVal, bool &ArgHasUses) {
8779 // Check if this is a load from a fixed stack object.
8780 auto *LNode = dyn_cast<LoadSDNode>(ArgVal);
8781 if (!LNode)
8782 return;
8783 auto *FINode = dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode());
8784 if (!FINode)
8785 return;
8786
8787 // Check that the fixed stack object is the right size and alignment.
8788 // Look at the alignment that the user wrote on the alloca instead of looking
8789 // at the stack object.
8790 auto ArgCopyIter = ArgCopyElisionCandidates.find(&Arg);
8791 assert(ArgCopyIter != ArgCopyElisionCandidates.end())(static_cast <bool> (ArgCopyIter != ArgCopyElisionCandidates
.end()) ? void (0) : __assert_fail ("ArgCopyIter != ArgCopyElisionCandidates.end()"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8791, __extension__ __PRETTY_FUNCTION__));
8792 const AllocaInst *AI = ArgCopyIter->second.first;
8793 int FixedIndex = FINode->getIndex();
8794 int &AllocaIndex = FuncInfo->StaticAllocaMap[AI];
8795 int OldIndex = AllocaIndex;
8796 MachineFrameInfo &MFI = FuncInfo->MF->getFrameInfo();
8797 if (MFI.getObjectSize(FixedIndex) != MFI.getObjectSize(OldIndex)) {
8798 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack "
"object size\n"; } } while (false)
8799 dbgs() << " argument copy elision failed due to bad fixed stack "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack "
"object size\n"; } } while (false)
8800 "object size\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed due to bad fixed stack "
"object size\n"; } } while (false);
8801 return;
8802 }
8803 unsigned RequiredAlignment = AI->getAlignment();
8804 if (!RequiredAlignment) {
8805 RequiredAlignment = FuncInfo->MF->getDataLayout().getABITypeAlignment(
8806 AI->getAllocatedType());
8807 }
8808 if (MFI.getObjectAlignment(FixedIndex) < RequiredAlignment) {
8809 LLVM_DEBUG(dbgs() << " argument copy elision failed: alignment of alloca "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed: alignment of alloca "
"greater than stack argument alignment (" << RequiredAlignment
<< " vs " << MFI.getObjectAlignment(FixedIndex) <<
")\n"; } } while (false)
8810 "greater than stack argument alignment ("do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed: alignment of alloca "
"greater than stack argument alignment (" << RequiredAlignment
<< " vs " << MFI.getObjectAlignment(FixedIndex) <<
")\n"; } } while (false)
8811 << RequiredAlignment << " vs "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed: alignment of alloca "
"greater than stack argument alignment (" << RequiredAlignment
<< " vs " << MFI.getObjectAlignment(FixedIndex) <<
")\n"; } } while (false)
8812 << MFI.getObjectAlignment(FixedIndex) << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << " argument copy elision failed: alignment of alloca "
"greater than stack argument alignment (" << RequiredAlignment
<< " vs " << MFI.getObjectAlignment(FixedIndex) <<
")\n"; } } while (false);
8813 return;
8814 }
8815
8816 // Perform the elision. Delete the old stack object and replace its only use
8817 // in the variable info map. Mark the stack object as mutable.
8818 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
Arg << " to " << *AI << '\n' << " Replacing frame index "
<< OldIndex << " with " << FixedIndex <<
'\n'; }; } } while (false)
8819 dbgs() << "Eliding argument copy from " << Arg << " to " << *AI << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
Arg << " to " << *AI << '\n' << " Replacing frame index "
<< OldIndex << " with " << FixedIndex <<
'\n'; }; } } while (false)
8820 << " Replacing frame index " << OldIndex << " with " << FixedIndexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
Arg << " to " << *AI << '\n' << " Replacing frame index "
<< OldIndex << " with " << FixedIndex <<
'\n'; }; } } while (false)
8821 << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
Arg << " to " << *AI << '\n' << " Replacing frame index "
<< OldIndex << " with " << FixedIndex <<
'\n'; }; } } while (false)
8822 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Eliding argument copy from " <<
Arg << " to " << *AI << '\n' << " Replacing frame index "
<< OldIndex << " with " << FixedIndex <<
'\n'; }; } } while (false);
8823 MFI.RemoveStackObject(OldIndex);
8824 MFI.setIsImmutableObjectIndex(FixedIndex, false);
8825 AllocaIndex = FixedIndex;
8826 ArgCopyElisionFrameIndexMap.insert({OldIndex, FixedIndex});
8827 Chains.push_back(ArgVal.getValue(1));
8828
8829 // Avoid emitting code for the store implementing the copy.
8830 const StoreInst *SI = ArgCopyIter->second.second;
8831 ElidedArgCopyInstrs.insert(SI);
8832
8833 // Check for uses of the argument again so that we can avoid exporting ArgVal
8834 // if it is't used by anything other than the store.
8835 for (const Value *U : Arg.users()) {
8836 if (U != SI) {
8837 ArgHasUses = true;
8838 break;
8839 }
8840 }
8841}
8842
8843void SelectionDAGISel::LowerArguments(const Function &F) {
8844 SelectionDAG &DAG = SDB->DAG;
8845 SDLoc dl = SDB->getCurSDLoc();
8846 const DataLayout &DL = DAG.getDataLayout();
8847 SmallVector<ISD::InputArg, 16> Ins;
8848
8849 if (!FuncInfo->CanLowerReturn) {
8850 // Put in an sret pointer parameter before all the other parameters.
8851 SmallVector<EVT, 1> ValueVTs;
8852 ComputeValueVTs(*TLI, DAG.getDataLayout(),
8853 F.getReturnType()->getPointerTo(
8854 DAG.getDataLayout().getAllocaAddrSpace()),
8855 ValueVTs);
8856
8857 // NOTE: Assuming that a pointer will never break down to more than one VT
8858 // or one register.
8859 ISD::ArgFlagsTy Flags;
8860 Flags.setSRet();
8861 MVT RegisterVT = TLI->getRegisterType(*DAG.getContext(), ValueVTs[0]);
8862 ISD::InputArg RetArg(Flags, RegisterVT, ValueVTs[0], true,
8863 ISD::InputArg::NoArgIndex, 0);
8864 Ins.push_back(RetArg);
8865 }
8866
8867 // Look for stores of arguments to static allocas. Mark such arguments with a
8868 // flag to ask the target to give us the memory location of that argument if
8869 // available.
8870 ArgCopyElisionMapTy ArgCopyElisionCandidates;
8871 findArgumentCopyElisionCandidates(DL, FuncInfo, ArgCopyElisionCandidates);
8872
8873 // Set up the incoming argument description vector.
8874 for (const Argument &Arg : F.args()) {
8875 unsigned ArgNo = Arg.getArgNo();
8876 SmallVector<EVT, 4> ValueVTs;
8877 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
8878 bool isArgValueUsed = !Arg.use_empty();
8879 unsigned PartBase = 0;
8880 Type *FinalType = Arg.getType();
8881 if (Arg.hasAttribute(Attribute::ByVal))
8882 FinalType = cast<PointerType>(FinalType)->getElementType();
8883 bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters(
8884 FinalType, F.getCallingConv(), F.isVarArg());
8885 for (unsigned Value = 0, NumValues = ValueVTs.size();
8886 Value != NumValues; ++Value) {
8887 EVT VT = ValueVTs[Value];
8888 Type *ArgTy = VT.getTypeForEVT(*DAG.getContext());
8889 ISD::ArgFlagsTy Flags;
8890
8891 // Certain targets (such as MIPS), may have a different ABI alignment
8892 // for a type depending on the context. Give the target a chance to
8893 // specify the alignment it wants.
8894 unsigned OriginalAlignment =
8895 TLI->getABIAlignmentForCallingConv(ArgTy, DL);
8896
8897 if (Arg.hasAttribute(Attribute::ZExt))
8898 Flags.setZExt();
8899 if (Arg.hasAttribute(Attribute::SExt))
8900 Flags.setSExt();
8901 if (Arg.hasAttribute(Attribute::InReg)) {
8902 // If we are using vectorcall calling convention, a structure that is
8903 // passed InReg - is surely an HVA
8904 if (F.getCallingConv() == CallingConv::X86_VectorCall &&
8905 isa<StructType>(Arg.getType())) {
8906 // The first value of a structure is marked
8907 if (0 == Value)
8908 Flags.setHvaStart();
8909 Flags.setHva();
8910 }
8911 // Set InReg Flag
8912 Flags.setInReg();
8913 }
8914 if (Arg.hasAttribute(Attribute::StructRet))
8915 Flags.setSRet();
8916 if (Arg.hasAttribute(Attribute::SwiftSelf))
8917 Flags.setSwiftSelf();
8918 if (Arg.hasAttribute(Attribute::SwiftError))
8919 Flags.setSwiftError();
8920 if (Arg.hasAttribute(Attribute::ByVal))
8921 Flags.setByVal();
8922 if (Arg.hasAttribute(Attribute::InAlloca)) {
8923 Flags.setInAlloca();
8924 // Set the byval flag for CCAssignFn callbacks that don't know about
8925 // inalloca. This way we can know how many bytes we should've allocated
8926 // and how many bytes a callee cleanup function will pop. If we port
8927 // inalloca to more targets, we'll have to add custom inalloca handling
8928 // in the various CC lowering callbacks.
8929 Flags.setByVal();
8930 }
8931 if (F.getCallingConv() == CallingConv::X86_INTR) {
8932 // IA Interrupt passes frame (1st parameter) by value in the stack.
8933 if (ArgNo == 0)
8934 Flags.setByVal();
8935 }
8936 if (Flags.isByVal() || Flags.isInAlloca()) {
8937 PointerType *Ty = cast<PointerType>(Arg.getType());
8938 Type *ElementTy = Ty->getElementType();
8939 Flags.setByValSize(DL.getTypeAllocSize(ElementTy));
8940 // For ByVal, alignment should be passed from FE. BE will guess if
8941 // this info is not there but there are cases it cannot get right.
8942 unsigned FrameAlign;
8943 if (Arg.getParamAlignment())
8944 FrameAlign = Arg.getParamAlignment();
8945 else
8946 FrameAlign = TLI->getByValTypeAlignment(ElementTy, DL);
8947 Flags.setByValAlign(FrameAlign);
8948 }
8949 if (Arg.hasAttribute(Attribute::Nest))
8950 Flags.setNest();
8951 if (NeedsRegBlock)
8952 Flags.setInConsecutiveRegs();
8953 Flags.setOrigAlign(OriginalAlignment);
8954 if (ArgCopyElisionCandidates.count(&Arg))
8955 Flags.setCopyElisionCandidate();
8956
8957 MVT RegisterVT = TLI->getRegisterTypeForCallingConv(
8958 *CurDAG->getContext(), F.getCallingConv(), VT);
8959 unsigned NumRegs = TLI->getNumRegistersForCallingConv(
8960 *CurDAG->getContext(), F.getCallingConv(), VT);
8961 for (unsigned i = 0; i != NumRegs; ++i) {
8962 ISD::InputArg MyFlags(Flags, RegisterVT, VT, isArgValueUsed,
8963 ArgNo, PartBase+i*RegisterVT.getStoreSize());
8964 if (NumRegs > 1 && i == 0)
8965 MyFlags.Flags.setSplit();
8966 // if it isn't first piece, alignment must be 1
8967 else if (i > 0) {
8968 MyFlags.Flags.setOrigAlign(1);
8969 if (i == NumRegs - 1)
8970 MyFlags.Flags.setSplitEnd();
8971 }
8972 Ins.push_back(MyFlags);
8973 }
8974 if (NeedsRegBlock && Value == NumValues - 1)
8975 Ins[Ins.size() - 1].Flags.setInConsecutiveRegsLast();
8976 PartBase += VT.getStoreSize();
8977 }
8978 }
8979
8980 // Call the target to set up the argument values.
8981 SmallVector<SDValue, 8> InVals;
8982 SDValue NewRoot = TLI->LowerFormalArguments(
8983 DAG.getRoot(), F.getCallingConv(), F.isVarArg(), Ins, dl, DAG, InVals);
8984
8985 // Verify that the target's LowerFormalArguments behaved as expected.
8986 assert(NewRoot.getNode() && NewRoot.getValueType() == MVT::Other &&(static_cast <bool> (NewRoot.getNode() && NewRoot
.getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!"
) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8987, __extension__ __PRETTY_FUNCTION__))
8987 "LowerFormalArguments didn't return a valid chain!")(static_cast <bool> (NewRoot.getNode() && NewRoot
.getValueType() == MVT::Other && "LowerFormalArguments didn't return a valid chain!"
) ? void (0) : __assert_fail ("NewRoot.getNode() && NewRoot.getValueType() == MVT::Other && \"LowerFormalArguments didn't return a valid chain!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8987, __extension__ __PRETTY_FUNCTION__));
8988 assert(InVals.size() == Ins.size() &&(static_cast <bool> (InVals.size() == Ins.size() &&
"LowerFormalArguments didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8989, __extension__ __PRETTY_FUNCTION__))
8989 "LowerFormalArguments didn't emit the correct number of values!")(static_cast <bool> (InVals.size() == Ins.size() &&
"LowerFormalArguments didn't emit the correct number of values!"
) ? void (0) : __assert_fail ("InVals.size() == Ins.size() && \"LowerFormalArguments didn't emit the correct number of values!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8989, __extension__ __PRETTY_FUNCTION__));
8990 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8991 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8992 assert(InVals[i].getNode() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8993 "LowerFormalArguments emitted a null value!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8994 assert(EVT(Ins[i].VT) == InVals[i].getValueType() &&do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8995 "LowerFormalArguments emitted a value with the wrong type!");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8996 }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false)
8997 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { for (unsigned i = 0, e = Ins.size(); i != e; ++
i) { (static_cast <bool> (InVals[i].getNode() &&
"LowerFormalArguments emitted a null value!") ? void (0) : __assert_fail
("InVals[i].getNode() && \"LowerFormalArguments emitted a null value!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8993, __extension__ __PRETTY_FUNCTION__)); (static_cast <
bool> (EVT(Ins[i].VT) == InVals[i].getValueType() &&
"LowerFormalArguments emitted a value with the wrong type!")
? void (0) : __assert_fail ("EVT(Ins[i].VT) == InVals[i].getValueType() && \"LowerFormalArguments emitted a value with the wrong type!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 8995, __extension__ __PRETTY_FUNCTION__)); } }; } } while (
false);
8998
8999 // Update the DAG with the new chain value resulting from argument lowering.
9000 DAG.setRoot(NewRoot);
9001
9002 // Set up the argument values.
9003 unsigned i = 0;
9004 if (!FuncInfo->CanLowerReturn) {
9005 // Create a virtual register for the sret pointer, and put in a copy
9006 // from the sret argument into it.
9007 SmallVector<EVT, 1> ValueVTs;
9008 ComputeValueVTs(*TLI, DAG.getDataLayout(),
9009 F.getReturnType()->getPointerTo(
9010 DAG.getDataLayout().getAllocaAddrSpace()),
9011 ValueVTs);
9012 MVT VT = ValueVTs[0].getSimpleVT();
9013 MVT RegVT = TLI->getRegisterType(*CurDAG->getContext(), VT);
9014 Optional<ISD::NodeType> AssertOp = None;
9015 SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1, RegVT, VT,
9016 nullptr, F.getCallingConv(), AssertOp);
9017
9018 MachineFunction& MF = SDB->DAG.getMachineFunction();
9019 MachineRegisterInfo& RegInfo = MF.getRegInfo();
9020 unsigned SRetReg = RegInfo.createVirtualRegister(TLI->getRegClassFor(RegVT));
9021 FuncInfo->DemoteRegister = SRetReg;
9022 NewRoot =
9023 SDB->DAG.getCopyToReg(NewRoot, SDB->getCurSDLoc(), SRetReg, ArgValue);
9024 DAG.setRoot(NewRoot);
9025
9026 // i indexes lowered arguments. Bump it past the hidden sret argument.
9027 ++i;
9028 }
9029
9030 SmallVector<SDValue, 4> Chains;
9031 DenseMap<int, int> ArgCopyElisionFrameIndexMap;
9032 for (const Argument &Arg : F.args()) {
9033 SmallVector<SDValue, 4> ArgValues;
9034 SmallVector<EVT, 4> ValueVTs;
9035 ComputeValueVTs(*TLI, DAG.getDataLayout(), Arg.getType(), ValueVTs);
9036 unsigned NumValues = ValueVTs.size();
9037 if (NumValues == 0)
9038 continue;
9039
9040 bool ArgHasUses = !Arg.use_empty();
9041
9042 // Elide the copying store if the target loaded this argument from a
9043 // suitable fixed stack object.
9044 if (Ins[i].Flags.isCopyElisionCandidate()) {
9045 tryToElideArgumentCopy(FuncInfo, Chains, ArgCopyElisionFrameIndexMap,
9046 ElidedArgCopyInstrs, ArgCopyElisionCandidates, Arg,
9047 InVals[i], ArgHasUses);
9048 }
9049
9050 // If this argument is unused then remember its value. It is used to generate
9051 // debugging information.
9052 bool isSwiftErrorArg =
9053 TLI->supportSwiftError() &&
9054 Arg.hasAttribute(Attribute::SwiftError);
9055 if (!ArgHasUses && !isSwiftErrorArg) {
9056 SDB->setUnusedArgValue(&Arg, InVals[i]);
9057
9058 // Also remember any frame index for use in FastISel.
9059 if (FrameIndexSDNode *FI =
9060 dyn_cast<FrameIndexSDNode>(InVals[i].getNode()))
9061 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9062 }
9063
9064 for (unsigned Val = 0; Val != NumValues; ++Val) {
9065 EVT VT = ValueVTs[Val];
9066 MVT PartVT = TLI->getRegisterTypeForCallingConv(*CurDAG->getContext(),
9067 F.getCallingConv(), VT);
9068 unsigned NumParts = TLI->getNumRegistersForCallingConv(
9069 *CurDAG->getContext(), F.getCallingConv(), VT);
9070
9071 // Even an apparant 'unused' swifterror argument needs to be returned. So
9072 // we do generate a copy for it that can be used on return from the
9073 // function.
9074 if (ArgHasUses || isSwiftErrorArg) {
9075 Optional<ISD::NodeType> AssertOp;
9076 if (Arg.hasAttribute(Attribute::SExt))
9077 AssertOp = ISD::AssertSext;
9078 else if (Arg.hasAttribute(Attribute::ZExt))
9079 AssertOp = ISD::AssertZext;
9080
9081 ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i], NumParts,
9082 PartVT, VT, nullptr,
9083 F.getCallingConv(), AssertOp));
9084 }
9085
9086 i += NumParts;
9087 }
9088
9089 // We don't need to do anything else for unused arguments.
9090 if (ArgValues.empty())
9091 continue;
9092
9093 // Note down frame index.
9094 if (FrameIndexSDNode *FI =
9095 dyn_cast<FrameIndexSDNode>(ArgValues[0].getNode()))
9096 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9097
9098 SDValue Res = DAG.getMergeValues(makeArrayRef(ArgValues.data(), NumValues),
9099 SDB->getCurSDLoc());
9100
9101 SDB->setValue(&Arg, Res);
9102 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::BUILD_PAIR) {
9103 // We want to associate the argument with the frame index, among
9104 // involved operands, that correspond to the lowest address. The
9105 // getCopyFromParts function, called earlier, is swapping the order of
9106 // the operands to BUILD_PAIR depending on endianness. The result of
9107 // that swapping is that the least significant bits of the argument will
9108 // be in the first operand of the BUILD_PAIR node, and the most
9109 // significant bits will be in the second operand.
9110 unsigned LowAddressOp = DAG.getDataLayout().isBigEndian() ? 1 : 0;
9111 if (LoadSDNode *LNode =
9112 dyn_cast<LoadSDNode>(Res.getOperand(LowAddressOp).getNode()))
9113 if (FrameIndexSDNode *FI =
9114 dyn_cast<FrameIndexSDNode>(LNode->getBasePtr().getNode()))
9115 FuncInfo->setArgumentFrameIndex(&Arg, FI->getIndex());
9116 }
9117
9118 // Update the SwiftErrorVRegDefMap.
9119 if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) {
9120 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
9121 if (TargetRegisterInfo::isVirtualRegister(Reg))
9122 FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB,
9123 FuncInfo->SwiftErrorArg, Reg);
9124 }
9125
9126 // If this argument is live outside of the entry block, insert a copy from
9127 // wherever we got it to the vreg that other BB's will reference it as.
9128 if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) {
9129 // If we can, though, try to skip creating an unnecessary vreg.
9130 // FIXME: This isn't very clean... it would be nice to make this more
9131 // general. It's also subtly incompatible with the hacks FastISel
9132 // uses with vregs.
9133 unsigned Reg = cast<RegisterSDNode>(Res.getOperand(1))->getReg();
9134 if (TargetRegisterInfo::isVirtualRegister(Reg)) {
9135 FuncInfo->ValueMap[&Arg] = Reg;
9136 continue;
9137 }
9138 }
9139 if (!isOnlyUsedInEntryBlock(&Arg, TM.Options.EnableFastISel)) {
9140 FuncInfo->InitializeRegForValue(&Arg);
9141 SDB->CopyToExportRegsIfNeeded(&Arg);
9142 }
9143 }
9144
9145 if (!Chains.empty()) {
9146 Chains.push_back(NewRoot);
9147 NewRoot = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains);
9148 }
9149
9150 DAG.setRoot(NewRoot);
9151
9152 assert(i == InVals.size() && "Argument register count mismatch!")(static_cast <bool> (i == InVals.size() && "Argument register count mismatch!"
) ? void (0) : __assert_fail ("i == InVals.size() && \"Argument register count mismatch!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9152, __extension__ __PRETTY_FUNCTION__));
9153
9154 // If any argument copy elisions occurred and we have debug info, update the
9155 // stale frame indices used in the dbg.declare variable info table.
9156 MachineFunction::VariableDbgInfoMapTy &DbgDeclareInfo = MF->getVariableDbgInfo();
9157 if (!DbgDeclareInfo.empty() && !ArgCopyElisionFrameIndexMap.empty()) {
9158 for (MachineFunction::VariableDbgInfo &VI : DbgDeclareInfo) {
9159 auto I = ArgCopyElisionFrameIndexMap.find(VI.Slot);
9160 if (I != ArgCopyElisionFrameIndexMap.end())
9161 VI.Slot = I->second;
9162 }
9163 }
9164
9165 // Finally, if the target has anything special to do, allow it to do so.
9166 EmitFunctionEntryCode();
9167}
9168
9169/// Handle PHI nodes in successor blocks. Emit code into the SelectionDAG to
9170/// ensure constants are generated when needed. Remember the virtual registers
9171/// that need to be added to the Machine PHI nodes as input. We cannot just
9172/// directly add them, because expansion might result in multiple MBB's for one
9173/// BB. As such, the start of the BB might correspond to a different MBB than
9174/// the end.
9175void
9176SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) {
9177 const TerminatorInst *TI = LLVMBB->getTerminator();
9178
9179 SmallPtrSet<MachineBasicBlock *, 4> SuccsHandled;
9180
9181 // Check PHI nodes in successors that expect a value to be available from this
9182 // block.
9183 for (unsigned succ = 0, e = TI->getNumSuccessors(); succ != e; ++succ) {
9184 const BasicBlock *SuccBB = TI->getSuccessor(succ);
9185 if (!isa<PHINode>(SuccBB->begin())) continue;
9186 MachineBasicBlock *SuccMBB = FuncInfo.MBBMap[SuccBB];
9187
9188 // If this terminator has multiple identical successors (common for
9189 // switches), only handle each succ once.
9190 if (!SuccsHandled.insert(SuccMBB).second)
9191 continue;
9192
9193 MachineBasicBlock::iterator MBBI = SuccMBB->begin();
9194
9195 // At this point we know that there is a 1-1 correspondence between LLVM PHI
9196 // nodes and Machine PHI nodes, but the incoming operands have not been
9197 // emitted yet.
9198 for (const PHINode &PN : SuccBB->phis()) {
9199 // Ignore dead phi's.
9200 if (PN.use_empty())
9201 continue;
9202
9203 // Skip empty types
9204 if (PN.getType()->isEmptyTy())
9205 continue;
9206
9207 unsigned Reg;
9208 const Value *PHIOp = PN.getIncomingValueForBlock(LLVMBB);
9209
9210 if (const Constant *C = dyn_cast<Constant>(PHIOp)) {
9211 unsigned &RegOut = ConstantsOut[C];
9212 if (RegOut == 0) {
9213 RegOut = FuncInfo.CreateRegs(C->getType());
9214 CopyValueToVirtualRegister(C, RegOut);
9215 }
9216 Reg = RegOut;
9217 } else {
9218 DenseMap<const Value *, unsigned>::iterator I =
9219 FuncInfo.ValueMap.find(PHIOp);
9220 if (I != FuncInfo.ValueMap.end())
9221 Reg = I->second;
9222 else {
9223 assert(isa<AllocaInst>(PHIOp) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) &&
FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)
) && "Didn't codegen value into a register!??") ? void
(0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9225, __extension__ __PRETTY_FUNCTION__))
9224 FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) &&(static_cast <bool> (isa<AllocaInst>(PHIOp) &&
FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)
) && "Didn't codegen value into a register!??") ? void
(0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9225, __extension__ __PRETTY_FUNCTION__))
9225 "Didn't codegen value into a register!??")(static_cast <bool> (isa<AllocaInst>(PHIOp) &&
FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)
) && "Didn't codegen value into a register!??") ? void
(0) : __assert_fail ("isa<AllocaInst>(PHIOp) && FuncInfo.StaticAllocaMap.count(cast<AllocaInst>(PHIOp)) && \"Didn't codegen value into a register!??\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9225, __extension__ __PRETTY_FUNCTION__));
9226 Reg = FuncInfo.CreateRegs(PHIOp->getType());
9227 CopyValueToVirtualRegister(PHIOp, Reg);
9228 }
9229 }
9230
9231 // Remember that this register needs to added to the machine PHI node as
9232 // the input for this MBB.
9233 SmallVector<EVT, 4> ValueVTs;
9234 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9235 ComputeValueVTs(TLI, DAG.getDataLayout(), PN.getType(), ValueVTs);
9236 for (unsigned vti = 0, vte = ValueVTs.size(); vti != vte; ++vti) {
9237 EVT VT = ValueVTs[vti];
9238 unsigned NumRegisters = TLI.getNumRegisters(*DAG.getContext(), VT);
9239 for (unsigned i = 0, e = NumRegisters; i != e; ++i)
9240 FuncInfo.PHINodesToUpdate.push_back(
9241 std::make_pair(&*MBBI++, Reg + i));
9242 Reg += NumRegisters;
9243 }
9244 }
9245 }
9246
9247 ConstantsOut.clear();
9248}
9249
9250/// Add a successor MBB to ParentMBB< creating a new MachineBB for BB if SuccMBB
9251/// is 0.
9252MachineBasicBlock *
9253SelectionDAGBuilder::StackProtectorDescriptor::
9254AddSuccessorMBB(const BasicBlock *BB,
9255 MachineBasicBlock *ParentMBB,
9256 bool IsLikely,
9257 MachineBasicBlock *SuccMBB) {
9258 // If SuccBB has not been created yet, create it.
9259 if (!SuccMBB) {
9260 MachineFunction *MF = ParentMBB->getParent();
9261 MachineFunction::iterator BBI(ParentMBB);
9262 SuccMBB = MF->CreateMachineBasicBlock(BB);
9263 MF->insert(++BBI, SuccMBB);
9264 }
9265 // Add it as a successor of ParentMBB.
9266 ParentMBB->addSuccessor(
9267 SuccMBB, BranchProbabilityInfo::getBranchProbStackProtector(IsLikely));
9268 return SuccMBB;
9269}
9270
9271MachineBasicBlock *SelectionDAGBuilder::NextBlock(MachineBasicBlock *MBB) {
9272 MachineFunction::iterator I(MBB);
9273 if (++I == FuncInfo.MF->end())
9274 return nullptr;
9275 return &*I;
9276}
9277
9278/// During lowering new call nodes can be created (such as memset, etc.).
9279/// Those will become new roots of the current DAG, but complications arise
9280/// when they are tail calls. In such cases, the call lowering will update
9281/// the root, but the builder still needs to know that a tail call has been
9282/// lowered in order to avoid generating an additional return.
9283void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) {
9284 // If the node is null, we do have a tail call.
9285 if (MaybeTC.getNode() != nullptr)
9286 DAG.setRoot(MaybeTC);
9287 else
9288 HasTailCall = true;
9289}
9290
9291uint64_t
9292SelectionDAGBuilder::getJumpTableRange(const CaseClusterVector &Clusters,
9293 unsigned First, unsigned Last) const {
9294 assert(Last >= First)(static_cast <bool> (Last >= First) ? void (0) : __assert_fail
("Last >= First", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9294, __extension__ __PRETTY_FUNCTION__));
9295 const APInt &LowCase = Clusters[First].Low->getValue();
9296 const APInt &HighCase = Clusters[Last].High->getValue();
9297 assert(LowCase.getBitWidth() == HighCase.getBitWidth())(static_cast <bool> (LowCase.getBitWidth() == HighCase.
getBitWidth()) ? void (0) : __assert_fail ("LowCase.getBitWidth() == HighCase.getBitWidth()"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9297, __extension__ __PRETTY_FUNCTION__));
9298
9299 // FIXME: A range of consecutive cases has 100% density, but only requires one
9300 // comparison to lower. We should discriminate against such consecutive ranges
9301 // in jump tables.
9302
9303 return (HighCase - LowCase).getLimitedValue((UINT64_MAX(18446744073709551615UL) - 1) / 100) + 1;
9304}
9305
9306uint64_t SelectionDAGBuilder::getJumpTableNumCases(
9307 const SmallVectorImpl<unsigned> &TotalCases, unsigned First,
9308 unsigned Last) const {
9309 assert(Last >= First)(static_cast <bool> (Last >= First) ? void (0) : __assert_fail
("Last >= First", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9309, __extension__ __PRETTY_FUNCTION__));
9310 assert(TotalCases[Last] >= TotalCases[First])(static_cast <bool> (TotalCases[Last] >= TotalCases[
First]) ? void (0) : __assert_fail ("TotalCases[Last] >= TotalCases[First]"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9310, __extension__ __PRETTY_FUNCTION__));
9311 uint64_t NumCases =
9312 TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]);
9313 return NumCases;
9314}
9315
9316bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters,
9317 unsigned First, unsigned Last,
9318 const SwitchInst *SI,
9319 MachineBasicBlock *DefaultMBB,
9320 CaseCluster &JTCluster) {
9321 assert(First <= Last)(static_cast <bool> (First <= Last) ? void (0) : __assert_fail
("First <= Last", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9321, __extension__ __PRETTY_FUNCTION__));
9322
9323 auto Prob = BranchProbability::getZero();
9324 unsigned NumCmps = 0;
9325 std::vector<MachineBasicBlock*> Table;
9326 DenseMap<MachineBasicBlock*, BranchProbability> JTProbs;
9327
9328 // Initialize probabilities in JTProbs.
9329 for (unsigned I = First; I <= Last; ++I)
9330 JTProbs[Clusters[I].MBB] = BranchProbability::getZero();
9331
9332 for (unsigned I = First; I <= Last; ++I) {
9333 assert(Clusters[I].Kind == CC_Range)(static_cast <bool> (Clusters[I].Kind == CC_Range) ? void
(0) : __assert_fail ("Clusters[I].Kind == CC_Range", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9333, __extension__ __PRETTY_FUNCTION__));
9334 Prob += Clusters[I].Prob;
9335 const APInt &Low = Clusters[I].Low->getValue();
9336 const APInt &High = Clusters[I].High->getValue();
9337 NumCmps += (Low == High) ? 1 : 2;
9338 if (I != First) {
9339 // Fill the gap between this and the previous cluster.
9340 const APInt &PreviousHigh = Clusters[I - 1].High->getValue();
9341 assert(PreviousHigh.slt(Low))(static_cast <bool> (PreviousHigh.slt(Low)) ? void (0) :
__assert_fail ("PreviousHigh.slt(Low)", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9341, __extension__ __PRETTY_FUNCTION__));
9342 uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1;
9343 for (uint64_t J = 0; J < Gap; J++)
9344 Table.push_back(DefaultMBB);
9345 }
9346 uint64_t ClusterSize = (High - Low).getLimitedValue() + 1;
9347 for (uint64_t J = 0; J < ClusterSize; ++J)
9348 Table.push_back(Clusters[I].MBB);
9349 JTProbs[Clusters[I].MBB] += Clusters[I].Prob;
9350 }
9351
9352 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9353 unsigned NumDests = JTProbs.size();
9354 if (TLI.isSuitableForBitTests(
9355 NumDests, NumCmps, Clusters[First].Low->getValue(),
9356 Clusters[Last].High->getValue(), DAG.getDataLayout())) {
9357 // Clusters[First..Last] should be lowered as bit tests instead.
9358 return false;
9359 }
9360
9361 // Create the MBB that will load from and jump through the table.
9362 // Note: We create it here, but it's not inserted into the function yet.
9363 MachineFunction *CurMF = FuncInfo.MF;
9364 MachineBasicBlock *JumpTableMBB =
9365 CurMF->CreateMachineBasicBlock(SI->getParent());
9366
9367 // Add successors. Note: use table order for determinism.
9368 SmallPtrSet<MachineBasicBlock *, 8> Done;
9369 for (MachineBasicBlock *Succ : Table) {
9370 if (Done.count(Succ))
9371 continue;
9372 addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]);
9373 Done.insert(Succ);
9374 }
9375 JumpTableMBB->normalizeSuccProbs();
9376
9377 unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding())
9378 ->createJumpTableIndex(Table);
9379
9380 // Set up the jump table info.
9381 JumpTable JT(-1U, JTI, JumpTableMBB, nullptr);
9382 JumpTableHeader JTH(Clusters[First].Low->getValue(),
9383 Clusters[Last].High->getValue(), SI->getCondition(),
9384 nullptr, false);
9385 JTCases.emplace_back(std::move(JTH), std::move(JT));
9386
9387 JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High,
9388 JTCases.size() - 1, Prob);
9389 return true;
9390}
9391
9392void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters,
9393 const SwitchInst *SI,
9394 MachineBasicBlock *DefaultMBB) {
9395#ifndef NDEBUG
9396 // Clusters must be non-empty, sorted, and only contain Range clusters.
9397 assert(!Clusters.empty())(static_cast <bool> (!Clusters.empty()) ? void (0) : __assert_fail
("!Clusters.empty()", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9397, __extension__ __PRETTY_FUNCTION__));
9398 for (CaseCluster &C : Clusters)
9399 assert(C.Kind == CC_Range)(static_cast <bool> (C.Kind == CC_Range) ? void (0) : __assert_fail
("C.Kind == CC_Range", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9399, __extension__ __PRETTY_FUNCTION__));
9400 for (unsigned i = 1, e = Clusters.size(); i < e; ++i)
9401 assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue()))(static_cast <bool> (Clusters[i - 1].High->getValue(
).slt(Clusters[i].Low->getValue())) ? void (0) : __assert_fail
("Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue())"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9401, __extension__ __PRETTY_FUNCTION__));
9402#endif
9403
9404 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9405 if (!TLI.areJTsAllowed(SI->getParent()->getParent()))
9406 return;
9407
9408 const int64_t N = Clusters.size();
9409 const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries();
9410 const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2;
9411
9412 if (N < 2 || N < MinJumpTableEntries)
9413 return;
9414
9415 // TotalCases[i]: Total nbr of cases in Clusters[0..i].
9416 SmallVector<unsigned, 8> TotalCases(N);
9417 for (unsigned i = 0; i < N; ++i) {
9418 const APInt &Hi = Clusters[i].High->getValue();
9419 const APInt &Lo = Clusters[i].Low->getValue();
9420 TotalCases[i] = (Hi - Lo).getLimitedValue() + 1;
9421 if (i != 0)
9422 TotalCases[i] += TotalCases[i - 1];
9423 }
9424
9425 // Cheap case: the whole range may be suitable for jump table.
9426 uint64_t Range = getJumpTableRange(Clusters,0, N - 1);
9427 uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1);
9428 assert(NumCases < UINT64_MAX / 100)(static_cast <bool> (NumCases < (18446744073709551615UL
) / 100) ? void (0) : __assert_fail ("NumCases < UINT64_MAX / 100"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9428, __extension__ __PRETTY_FUNCTION__));
9429 assert(Range >= NumCases)(static_cast <bool> (Range >= NumCases) ? void (0) :
__assert_fail ("Range >= NumCases", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9429, __extension__ __PRETTY_FUNCTION__));
9430 if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9431 CaseCluster JTCluster;
9432 if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) {
9433 Clusters[0] = JTCluster;
9434 Clusters.resize(1);
9435 return;
9436 }
9437 }
9438
9439 // The algorithm below is not suitable for -O0.
9440 if (TM.getOptLevel() == CodeGenOpt::None)
9441 return;
9442
9443 // Split Clusters into minimum number of dense partitions. The algorithm uses
9444 // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code
9445 // for the Case Statement'" (1994), but builds the MinPartitions array in
9446 // reverse order to make it easier to reconstruct the partitions in ascending
9447 // order. In the choice between two optimal partitionings, it picks the one
9448 // which yields more jump tables.
9449
9450 // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9451 SmallVector<unsigned, 8> MinPartitions(N);
9452 // LastElement[i] is the last element of the partition starting at i.
9453 SmallVector<unsigned, 8> LastElement(N);
9454 // PartitionsScore[i] is used to break ties when choosing between two
9455 // partitionings resulting in the same number of partitions.
9456 SmallVector<unsigned, 8> PartitionsScore(N);
9457 // For PartitionsScore, a small number of comparisons is considered as good as
9458 // a jump table and a single comparison is considered better than a jump
9459 // table.
9460 enum PartitionScores : unsigned {
9461 NoTable = 0,
9462 Table = 1,
9463 FewCases = 1,
9464 SingleCase = 2
9465 };
9466
9467 // Base case: There is only one way to partition Clusters[N-1].
9468 MinPartitions[N - 1] = 1;
9469 LastElement[N - 1] = N - 1;
9470 PartitionsScore[N - 1] = PartitionScores::SingleCase;
9471
9472 // Note: loop indexes are signed to avoid underflow.
9473 for (int64_t i = N - 2; i >= 0; i--) {
9474 // Find optimal partitioning of Clusters[i..N-1].
9475 // Baseline: Put Clusters[i] into a partition on its own.
9476 MinPartitions[i] = MinPartitions[i + 1] + 1;
9477 LastElement[i] = i;
9478 PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase;
9479
9480 // Search for a solution that results in fewer partitions.
9481 for (int64_t j = N - 1; j > i; j--) {
9482 // Try building a partition from Clusters[i..j].
9483 uint64_t Range = getJumpTableRange(Clusters, i, j);
9484 uint64_t NumCases = getJumpTableNumCases(TotalCases, i, j);
9485 assert(NumCases < UINT64_MAX / 100)(static_cast <bool> (NumCases < (18446744073709551615UL
) / 100) ? void (0) : __assert_fail ("NumCases < UINT64_MAX / 100"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9485, __extension__ __PRETTY_FUNCTION__));
9486 assert(Range >= NumCases)(static_cast <bool> (Range >= NumCases) ? void (0) :
__assert_fail ("Range >= NumCases", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9486, __extension__ __PRETTY_FUNCTION__));
9487 if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) {
9488 unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9489 unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1];
9490 int64_t NumEntries = j - i + 1;
9491
9492 if (NumEntries == 1)
9493 Score += PartitionScores::SingleCase;
9494 else if (NumEntries <= SmallNumberOfEntries)
9495 Score += PartitionScores::FewCases;
9496 else if (NumEntries >= MinJumpTableEntries)
9497 Score += PartitionScores::Table;
9498
9499 // If this leads to fewer partitions, or to the same number of
9500 // partitions with better score, it is a better partitioning.
9501 if (NumPartitions < MinPartitions[i] ||
9502 (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) {
9503 MinPartitions[i] = NumPartitions;
9504 LastElement[i] = j;
9505 PartitionsScore[i] = Score;
9506 }
9507 }
9508 }
9509 }
9510
9511 // Iterate over the partitions, replacing some with jump tables in-place.
9512 unsigned DstIndex = 0;
9513 for (unsigned First = 0, Last; First < N; First = Last + 1) {
9514 Last = LastElement[First];
9515 assert(Last >= First)(static_cast <bool> (Last >= First) ? void (0) : __assert_fail
("Last >= First", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9515, __extension__ __PRETTY_FUNCTION__));
9516 assert(DstIndex <= First)(static_cast <bool> (DstIndex <= First) ? void (0) :
__assert_fail ("DstIndex <= First", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9516, __extension__ __PRETTY_FUNCTION__));
9517 unsigned NumClusters = Last - First + 1;
9518
9519 CaseCluster JTCluster;
9520 if (NumClusters >= MinJumpTableEntries &&
9521 buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) {
9522 Clusters[DstIndex++] = JTCluster;
9523 } else {
9524 for (unsigned I = First; I <= Last; ++I)
9525 std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I]));
9526 }
9527 }
9528 Clusters.resize(DstIndex);
9529}
9530
9531bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters,
9532 unsigned First, unsigned Last,
9533 const SwitchInst *SI,
9534 CaseCluster &BTCluster) {
9535 assert(First <= Last)(static_cast <bool> (First <= Last) ? void (0) : __assert_fail
("First <= Last", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9535, __extension__ __PRETTY_FUNCTION__));
9536 if (First == Last)
9537 return false;
9538
9539 BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9540 unsigned NumCmps = 0;
9541 for (int64_t I = First; I <= Last; ++I) {
9542 assert(Clusters[I].Kind == CC_Range)(static_cast <bool> (Clusters[I].Kind == CC_Range) ? void
(0) : __assert_fail ("Clusters[I].Kind == CC_Range", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9542, __extension__ __PRETTY_FUNCTION__));
9543 Dests.set(Clusters[I].MBB->getNumber());
9544 NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2;
9545 }
9546 unsigned NumDests = Dests.count();
9547
9548 APInt Low = Clusters[First].Low->getValue();
9549 APInt High = Clusters[Last].High->getValue();
9550 assert(Low.slt(High))(static_cast <bool> (Low.slt(High)) ? void (0) : __assert_fail
("Low.slt(High)", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9550, __extension__ __PRETTY_FUNCTION__));
9551
9552 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9553 const DataLayout &DL = DAG.getDataLayout();
9554 if (!TLI.isSuitableForBitTests(NumDests, NumCmps, Low, High, DL))
9555 return false;
9556
9557 APInt LowBound;
9558 APInt CmpRange;
9559
9560 const int BitWidth = TLI.getPointerTy(DL).getSizeInBits();
9561 assert(TLI.rangeFitsInWord(Low, High, DL) &&(static_cast <bool> (TLI.rangeFitsInWord(Low, High, DL)
&& "Case range must fit in bit mask!") ? void (0) : __assert_fail
("TLI.rangeFitsInWord(Low, High, DL) && \"Case range must fit in bit mask!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9562, __extension__ __PRETTY_FUNCTION__))
9562 "Case range must fit in bit mask!")(static_cast <bool> (TLI.rangeFitsInWord(Low, High, DL)
&& "Case range must fit in bit mask!") ? void (0) : __assert_fail
("TLI.rangeFitsInWord(Low, High, DL) && \"Case range must fit in bit mask!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9562, __extension__ __PRETTY_FUNCTION__));
9563
9564 // Check if the clusters cover a contiguous range such that no value in the
9565 // range will jump to the default statement.
9566 bool ContiguousRange = true;
9567 for (int64_t I = First + 1; I <= Last; ++I) {
9568 if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) {
9569 ContiguousRange = false;
9570 break;
9571 }
9572 }
9573
9574 if (Low.isStrictlyPositive() && High.slt(BitWidth)) {
9575 // Optimize the case where all the case values fit in a word without having
9576 // to subtract minValue. In this case, we can optimize away the subtraction.
9577 LowBound = APInt::getNullValue(Low.getBitWidth());
9578 CmpRange = High;
9579 ContiguousRange = false;
9580 } else {
9581 LowBound = Low;
9582 CmpRange = High - Low;
9583 }
9584
9585 CaseBitsVector CBV;
9586 auto TotalProb = BranchProbability::getZero();
9587 for (unsigned i = First; i <= Last; ++i) {
9588 // Find the CaseBits for this destination.
9589 unsigned j;
9590 for (j = 0; j < CBV.size(); ++j)
9591 if (CBV[j].BB == Clusters[i].MBB)
9592 break;
9593 if (j == CBV.size())
9594 CBV.push_back(
9595 CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero()));
9596 CaseBits *CB = &CBV[j];
9597
9598 // Update Mask, Bits and ExtraProb.
9599 uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue();
9600 uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue();
9601 assert(Hi >= Lo && Hi < 64 && "Invalid bit case!")(static_cast <bool> (Hi >= Lo && Hi < 64 &&
"Invalid bit case!") ? void (0) : __assert_fail ("Hi >= Lo && Hi < 64 && \"Invalid bit case!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9601, __extension__ __PRETTY_FUNCTION__));
9602 CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo;
9603 CB->Bits += Hi - Lo + 1;
9604 CB->ExtraProb += Clusters[i].Prob;
9605 TotalProb += Clusters[i].Prob;
9606 }
9607
9608 BitTestInfo BTI;
9609 llvm::sort(CBV.begin(), CBV.end(), [](const CaseBits &a, const CaseBits &b) {
9610 // Sort by probability first, number of bits second, bit mask third.
9611 if (a.ExtraProb != b.ExtraProb)
9612 return a.ExtraProb > b.ExtraProb;
9613 if (a.Bits != b.Bits)
9614 return a.Bits > b.Bits;
9615 return a.Mask < b.Mask;
9616 });
9617
9618 for (auto &CB : CBV) {
9619 MachineBasicBlock *BitTestBB =
9620 FuncInfo.MF->CreateMachineBasicBlock(SI->getParent());
9621 BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb));
9622 }
9623 BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange),
9624 SI->getCondition(), -1U, MVT::Other, false,
9625 ContiguousRange, nullptr, nullptr, std::move(BTI),
9626 TotalProb);
9627
9628 BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High,
9629 BitTestCases.size() - 1, TotalProb);
9630 return true;
9631}
9632
9633void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters,
9634 const SwitchInst *SI) {
9635// Partition Clusters into as few subsets as possible, where each subset has a
9636// range that fits in a machine word and has <= 3 unique destinations.
9637
9638#ifndef NDEBUG
9639 // Clusters must be sorted and contain Range or JumpTable clusters.
9640 assert(!Clusters.empty())(static_cast <bool> (!Clusters.empty()) ? void (0) : __assert_fail
("!Clusters.empty()", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9640, __extension__ __PRETTY_FUNCTION__));
9641 assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable)(static_cast <bool> (Clusters[0].Kind == CC_Range || Clusters
[0].Kind == CC_JumpTable) ? void (0) : __assert_fail ("Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9641, __extension__ __PRETTY_FUNCTION__));
9642 for (const CaseCluster &C : Clusters)
9643 assert(C.Kind == CC_Range || C.Kind == CC_JumpTable)(static_cast <bool> (C.Kind == CC_Range || C.Kind == CC_JumpTable
) ? void (0) : __assert_fail ("C.Kind == CC_Range || C.Kind == CC_JumpTable"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9643, __extension__ __PRETTY_FUNCTION__));
9644 for (unsigned i = 1; i < Clusters.size(); ++i)
9645 assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue()))(static_cast <bool> (Clusters[i-1].High->getValue().
slt(Clusters[i].Low->getValue())) ? void (0) : __assert_fail
("Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue())"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9645, __extension__ __PRETTY_FUNCTION__));
9646#endif
9647
9648 // The algorithm below is not suitable for -O0.
9649 if (TM.getOptLevel() == CodeGenOpt::None)
9650 return;
9651
9652 // If target does not have legal shift left, do not emit bit tests at all.
9653 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9654 const DataLayout &DL = DAG.getDataLayout();
9655
9656 EVT PTy = TLI.getPointerTy(DL);
9657 if (!TLI.isOperationLegal(ISD::SHL, PTy))
9658 return;
9659
9660 int BitWidth = PTy.getSizeInBits();
9661 const int64_t N = Clusters.size();
9662
9663 // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1].
9664 SmallVector<unsigned, 8> MinPartitions(N);
9665 // LastElement[i] is the last element of the partition starting at i.
9666 SmallVector<unsigned, 8> LastElement(N);
9667
9668 // FIXME: This might not be the best algorithm for finding bit test clusters.
9669
9670 // Base case: There is only one way to partition Clusters[N-1].
9671 MinPartitions[N - 1] = 1;
9672 LastElement[N - 1] = N - 1;
9673
9674 // Note: loop indexes are signed to avoid underflow.
9675 for (int64_t i = N - 2; i >= 0; --i) {
9676 // Find optimal partitioning of Clusters[i..N-1].
9677 // Baseline: Put Clusters[i] into a partition on its own.
9678 MinPartitions[i] = MinPartitions[i + 1] + 1;
9679 LastElement[i] = i;
9680
9681 // Search for a solution that results in fewer partitions.
9682 // Note: the search is limited by BitWidth, reducing time complexity.
9683 for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) {
9684 // Try building a partition from Clusters[i..j].
9685
9686 // Check the range.
9687 if (!TLI.rangeFitsInWord(Clusters[i].Low->getValue(),
9688 Clusters[j].High->getValue(), DL))
9689 continue;
9690
9691 // Check nbr of destinations and cluster types.
9692 // FIXME: This works, but doesn't seem very efficient.
9693 bool RangesOnly = true;
9694 BitVector Dests(FuncInfo.MF->getNumBlockIDs());
9695 for (int64_t k = i; k <= j; k++) {
9696 if (Clusters[k].Kind != CC_Range) {
9697 RangesOnly = false;
9698 break;
9699 }
9700 Dests.set(Clusters[k].MBB->getNumber());
9701 }
9702 if (!RangesOnly || Dests.count() > 3)
9703 break;
9704
9705 // Check if it's a better partition.
9706 unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]);
9707 if (NumPartitions < MinPartitions[i]) {
9708 // Found a better partition.
9709 MinPartitions[i] = NumPartitions;
9710 LastElement[i] = j;
9711 }
9712 }
9713 }
9714
9715 // Iterate over the partitions, replacing with bit-test clusters in-place.
9716 unsigned DstIndex = 0;
9717 for (unsigned First = 0, Last; First < N; First = Last + 1) {
9718 Last = LastElement[First];
9719 assert(First <= Last)(static_cast <bool> (First <= Last) ? void (0) : __assert_fail
("First <= Last", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9719, __extension__ __PRETTY_FUNCTION__));
9720 assert(DstIndex <= First)(static_cast <bool> (DstIndex <= First) ? void (0) :
__assert_fail ("DstIndex <= First", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9720, __extension__ __PRETTY_FUNCTION__));
9721
9722 CaseCluster BitTestCluster;
9723 if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) {
9724 Clusters[DstIndex++] = BitTestCluster;
9725 } else {
9726 size_t NumClusters = Last - First + 1;
9727 std::memmove(&Clusters[DstIndex], &Clusters[First],
9728 sizeof(Clusters[0]) * NumClusters);
9729 DstIndex += NumClusters;
9730 }
9731 }
9732 Clusters.resize(DstIndex);
9733}
9734
9735void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond,
9736 MachineBasicBlock *SwitchMBB,
9737 MachineBasicBlock *DefaultMBB) {
9738 MachineFunction *CurMF = FuncInfo.MF;
9739 MachineBasicBlock *NextMBB = nullptr;
9740 MachineFunction::iterator BBI(W.MBB);
9741 if (++BBI != FuncInfo.MF->end())
9742 NextMBB = &*BBI;
9743
9744 unsigned Size = W.LastCluster - W.FirstCluster + 1;
9745
9746 BranchProbabilityInfo *BPI = FuncInfo.BPI;
9747
9748 if (Size == 2 && W.MBB == SwitchMBB) {
9749 // If any two of the cases has the same destination, and if one value
9750 // is the same as the other, but has one bit unset that the other has set,
9751 // use bit manipulation to do two compares at once. For example:
9752 // "if (X == 6 || X == 4)" -> "if ((X|2) == 6)"
9753 // TODO: This could be extended to merge any 2 cases in switches with 3
9754 // cases.
9755 // TODO: Handle cases where W.CaseBB != SwitchBB.
9756 CaseCluster &Small = *W.FirstCluster;
9757 CaseCluster &Big = *W.LastCluster;
9758
9759 if (Small.Low == Small.High && Big.Low == Big.High &&
9760 Small.MBB == Big.MBB) {
9761 const APInt &SmallValue = Small.Low->getValue();
9762 const APInt &BigValue = Big.Low->getValue();
9763
9764 // Check that there is only one bit different.
9765 APInt CommonBit = BigValue ^ SmallValue;
9766 if (CommonBit.isPowerOf2()) {
9767 SDValue CondLHS = getValue(Cond);
9768 EVT VT = CondLHS.getValueType();
9769 SDLoc DL = getCurSDLoc();
9770
9771 SDValue Or = DAG.getNode(ISD::OR, DL, VT, CondLHS,
9772 DAG.getConstant(CommonBit, DL, VT));
9773 SDValue Cond = DAG.getSetCC(
9774 DL, MVT::i1, Or, DAG.getConstant(BigValue | SmallValue, DL, VT),
9775 ISD::SETEQ);
9776
9777 // Update successor info.
9778 // Both Small and Big will jump to Small.BB, so we sum up the
9779 // probabilities.
9780 addSuccessorWithProb(SwitchMBB, Small.MBB, Small.Prob + Big.Prob);
9781 if (BPI)
9782 addSuccessorWithProb(
9783 SwitchMBB, DefaultMBB,
9784 // The default destination is the first successor in IR.
9785 BPI->getEdgeProbability(SwitchMBB->getBasicBlock(), (unsigned)0));
9786 else
9787 addSuccessorWithProb(SwitchMBB, DefaultMBB);
9788
9789 // Insert the true branch.
9790 SDValue BrCond =
9791 DAG.getNode(ISD::BRCOND, DL, MVT::Other, getControlRoot(), Cond,
9792 DAG.getBasicBlock(Small.MBB));
9793 // Insert the false branch.
9794 BrCond = DAG.getNode(ISD::BR, DL, MVT::Other, BrCond,
9795 DAG.getBasicBlock(DefaultMBB));
9796
9797 DAG.setRoot(BrCond);
9798 return;
9799 }
9800 }
9801 }
9802
9803 if (TM.getOptLevel() != CodeGenOpt::None) {
9804 // Here, we order cases by probability so the most likely case will be
9805 // checked first. However, two clusters can have the same probability in
9806 // which case their relative ordering is non-deterministic. So we use Low
9807 // as a tie-breaker as clusters are guaranteed to never overlap.
9808 llvm::sort(W.FirstCluster, W.LastCluster + 1,
9809 [](const CaseCluster &a, const CaseCluster &b) {
9810 return a.Prob != b.Prob ?
9811 a.Prob > b.Prob :
9812 a.Low->getValue().slt(b.Low->getValue());
9813 });
9814
9815 // Rearrange the case blocks so that the last one falls through if possible
9816 // without changing the order of probabilities.
9817 for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster; ) {
9818 --I;
9819 if (I->Prob > W.LastCluster->Prob)
9820 break;
9821 if (I->Kind == CC_Range && I->MBB == NextMBB) {
9822 std::swap(*I, *W.LastCluster);
9823 break;
9824 }
9825 }
9826 }
9827
9828 // Compute total probability.
9829 BranchProbability DefaultProb = W.DefaultProb;
9830 BranchProbability UnhandledProbs = DefaultProb;
9831 for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I)
9832 UnhandledProbs += I->Prob;
9833
9834 MachineBasicBlock *CurMBB = W.MBB;
9835 for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) {
9836 MachineBasicBlock *Fallthrough;
9837 if (I == W.LastCluster) {
9838 // For the last cluster, fall through to the default destination.
9839 Fallthrough = DefaultMBB;
9840 } else {
9841 Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock());
9842 CurMF->insert(BBI, Fallthrough);
9843 // Put Cond in a virtual register to make it available from the new blocks.
9844 ExportFromCurrentBlock(Cond);
9845 }
9846 UnhandledProbs -= I->Prob;
9847
9848 switch (I->Kind) {
9849 case CC_JumpTable: {
9850 // FIXME: Optimize away range check based on pivot comparisons.
9851 JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first;
9852 JumpTable *JT = &JTCases[I->JTCasesIndex].second;
9853
9854 // The jump block hasn't been inserted yet; insert it here.
9855 MachineBasicBlock *JumpMBB = JT->MBB;
9856 CurMF->insert(BBI, JumpMBB);
9857
9858 auto JumpProb = I->Prob;
9859 auto FallthroughProb = UnhandledProbs;
9860
9861 // If the default statement is a target of the jump table, we evenly
9862 // distribute the default probability to successors of CurMBB. Also
9863 // update the probability on the edge from JumpMBB to Fallthrough.
9864 for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(),
9865 SE = JumpMBB->succ_end();
9866 SI != SE; ++SI) {
9867 if (*SI == DefaultMBB) {
9868 JumpProb += DefaultProb / 2;
9869 FallthroughProb -= DefaultProb / 2;
9870 JumpMBB->setSuccProbability(SI, DefaultProb / 2);
9871 JumpMBB->normalizeSuccProbs();
9872 break;
9873 }
9874 }
9875
9876 addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb);
9877 addSuccessorWithProb(CurMBB, JumpMBB, JumpProb);
9878 CurMBB->normalizeSuccProbs();
9879
9880 // The jump table header will be inserted in our current block, do the
9881 // range check, and fall through to our fallthrough block.
9882 JTH->HeaderBB = CurMBB;
9883 JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader.
9884
9885 // If we're in the right place, emit the jump table header right now.
9886 if (CurMBB == SwitchMBB) {
9887 visitJumpTableHeader(*JT, *JTH, SwitchMBB);
9888 JTH->Emitted = true;
9889 }
9890 break;
9891 }
9892 case CC_BitTests: {
9893 // FIXME: Optimize away range check based on pivot comparisons.
9894 BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex];
9895
9896 // The bit test blocks haven't been inserted yet; insert them here.
9897 for (BitTestCase &BTC : BTB->Cases)
9898 CurMF->insert(BBI, BTC.ThisBB);
9899
9900 // Fill in fields of the BitTestBlock.
9901 BTB->Parent = CurMBB;
9902 BTB->Default = Fallthrough;
9903
9904 BTB->DefaultProb = UnhandledProbs;
9905 // If the cases in bit test don't form a contiguous range, we evenly
9906 // distribute the probability on the edge to Fallthrough to two
9907 // successors of CurMBB.
9908 if (!BTB->ContiguousRange) {
9909 BTB->Prob += DefaultProb / 2;
9910 BTB->DefaultProb -= DefaultProb / 2;
9911 }
9912
9913 // If we're in the right place, emit the bit test header right now.
9914 if (CurMBB == SwitchMBB) {
9915 visitBitTestHeader(*BTB, SwitchMBB);
9916 BTB->Emitted = true;
9917 }
9918 break;
9919 }
9920 case CC_Range: {
9921 const Value *RHS, *LHS, *MHS;
9922 ISD::CondCode CC;
9923 if (I->Low == I->High) {
9924 // Check Cond == I->Low.
9925 CC = ISD::SETEQ;
9926 LHS = Cond;
9927 RHS=I->Low;
9928 MHS = nullptr;
9929 } else {
9930 // Check I->Low <= Cond <= I->High.
9931 CC = ISD::SETLE;
9932 LHS = I->Low;
9933 MHS = Cond;
9934 RHS = I->High;
9935 }
9936
9937 // The false probability is the sum of all unhandled cases.
9938 CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB,
9939 getCurSDLoc(), I->Prob, UnhandledProbs);
9940
9941 if (CurMBB == SwitchMBB)
9942 visitSwitchCase(CB, SwitchMBB);
9943 else
9944 SwitchCases.push_back(CB);
9945
9946 break;
9947 }
9948 }
9949 CurMBB = Fallthrough;
9950 }
9951}
9952
9953unsigned SelectionDAGBuilder::caseClusterRank(const CaseCluster &CC,
9954 CaseClusterIt First,
9955 CaseClusterIt Last) {
9956 return std::count_if(First, Last + 1, [&](const CaseCluster &X) {
9957 if (X.Prob != CC.Prob)
9958 return X.Prob > CC.Prob;
9959
9960 // Ties are broken by comparing the case value.
9961 return X.Low->getValue().slt(CC.Low->getValue());
9962 });
9963}
9964
9965void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList,
9966 const SwitchWorkListItem &W,
9967 Value *Cond,
9968 MachineBasicBlock *SwitchMBB) {
9969 assert(W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) &&(static_cast <bool> (W.FirstCluster->Low->getValue
().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?"
) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9970, __extension__ __PRETTY_FUNCTION__))
9970 "Clusters not sorted?")(static_cast <bool> (W.FirstCluster->Low->getValue
().slt(W.LastCluster->Low->getValue()) && "Clusters not sorted?"
) ? void (0) : __assert_fail ("W.FirstCluster->Low->getValue().slt(W.LastCluster->Low->getValue()) && \"Clusters not sorted?\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9970, __extension__ __PRETTY_FUNCTION__));
9971
9972 assert(W.LastCluster - W.FirstCluster + 1 >= 2 && "Too small to split!")(static_cast <bool> (W.LastCluster - W.FirstCluster + 1
>= 2 && "Too small to split!") ? void (0) : __assert_fail
("W.LastCluster - W.FirstCluster + 1 >= 2 && \"Too small to split!\""
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 9972, __extension__ __PRETTY_FUNCTION__));
9973
9974 // Balance the tree based on branch probabilities to create a near-optimal (in
9975 // terms of search time given key frequency) binary search tree. See e.g. Kurt
9976 // Mehlhorn "Nearly Optimal Binary Search Trees" (1975).
9977 CaseClusterIt LastLeft = W.FirstCluster;
9978 CaseClusterIt FirstRight = W.LastCluster;
9979 auto LeftProb = LastLeft->Prob + W.DefaultProb / 2;
9980 auto RightProb = FirstRight->Prob + W.DefaultProb / 2;
9981
9982 // Move LastLeft and FirstRight towards each other from opposite directions to
9983 // find a partitioning of the clusters which balances the probability on both
9984 // sides. If LeftProb and RightProb are equal, alternate which side is
9985 // taken to ensure 0-probability nodes are distributed evenly.
9986 unsigned I = 0;
9987 while (LastLeft + 1 < FirstRight) {
9988 if (LeftProb < RightProb || (LeftProb == RightProb && (I & 1)))
9989 LeftProb += (++LastLeft)->Prob;
9990 else
9991 RightProb += (--FirstRight)->Prob;
9992 I++;
9993 }
9994
9995 while (true) {
9996 // Our binary search tree differs from a typical BST in that ours can have up
9997 // to three values in each leaf. The pivot selection above doesn't take that
9998 // into account, which means the tree might require more nodes and be less
9999 // efficient. We compensate for this here.
10000
10001 unsigned NumLeft = LastLeft - W.FirstCluster + 1;
10002 unsigned NumRight = W.LastCluster - FirstRight + 1;
10003
10004 if (std::min(NumLeft, NumRight) < 3 && std::max(NumLeft, NumRight) > 3) {
10005 // If one side has less than 3 clusters, and the other has more than 3,
10006 // consider taking a cluster from the other side.
10007
10008 if (NumLeft < NumRight) {
10009 // Consider moving the first cluster on the right to the left side.
10010 CaseCluster &CC = *FirstRight;
10011 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
10012 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
10013 if (LeftSideRank <= RightSideRank) {
10014 // Moving the cluster to the left does not demote it.
10015 ++LastLeft;
10016 ++FirstRight;
10017 continue;
10018 }
10019 } else {
10020 assert(NumRight < NumLeft)(static_cast <bool> (NumRight < NumLeft) ? void (0) :
__assert_fail ("NumRight < NumLeft", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10020, __extension__ __PRETTY_FUNCTION__));
10021 // Consider moving the last element on the left to the right side.
10022 CaseCluster &CC = *LastLeft;
10023 unsigned LeftSideRank = caseClusterRank(CC, W.FirstCluster, LastLeft);
10024 unsigned RightSideRank = caseClusterRank(CC, FirstRight, W.LastCluster);
10025 if (RightSideRank <= LeftSideRank) {
10026 // Moving the cluster to the right does not demot it.
10027 --LastLeft;
10028 --FirstRight;
10029 continue;
10030 }
10031 }
10032 }
10033 break;
10034 }
10035
10036 assert(LastLeft + 1 == FirstRight)(static_cast <bool> (LastLeft + 1 == FirstRight) ? void
(0) : __assert_fail ("LastLeft + 1 == FirstRight", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10036, __extension__ __PRETTY_FUNCTION__));
10037 assert(LastLeft >= W.FirstCluster)(static_cast <bool> (LastLeft >= W.FirstCluster) ? void
(0) : __assert_fail ("LastLeft >= W.FirstCluster", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10037, __extension__ __PRETTY_FUNCTION__));
10038 assert(FirstRight <= W.LastCluster)(static_cast <bool> (FirstRight <= W.LastCluster) ? void
(0) : __assert_fail ("FirstRight <= W.LastCluster", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10038, __extension__ __PRETTY_FUNCTION__));
10039
10040 // Use the first element on the right as pivot since we will make less-than
10041 // comparisons against it.
10042 CaseClusterIt PivotCluster = FirstRight;
10043 assert(PivotCluster > W.FirstCluster)(static_cast <bool> (PivotCluster > W.FirstCluster) ?
void (0) : __assert_fail ("PivotCluster > W.FirstCluster"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10043, __extension__ __PRETTY_FUNCTION__));
10044 assert(PivotCluster <= W.LastCluster)(static_cast <bool> (PivotCluster <= W.LastCluster) ?
void (0) : __assert_fail ("PivotCluster <= W.LastCluster"
, "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10044, __extension__ __PRETTY_FUNCTION__));
10045
10046 CaseClusterIt FirstLeft = W.FirstCluster;
10047 CaseClusterIt LastRight = W.LastCluster;
10048
10049 const ConstantInt *Pivot = PivotCluster->Low;
10050
10051 // New blocks will be inserted immediately after the current one.
10052 MachineFunction::iterator BBI(W.MBB);
10053 ++BBI;
10054
10055 // We will branch to the LHS if Value < Pivot. If LHS is a single cluster,
10056 // we can branch to its destination directly if it's squeezed exactly in
10057 // between the known lower bound and Pivot - 1.
10058 MachineBasicBlock *LeftMBB;
10059 if (FirstLeft == LastLeft && FirstLeft->Kind == CC_Range &&
10060 FirstLeft->Low == W.GE &&
10061 (FirstLeft->High->getValue() + 1LL) == Pivot->getValue()) {
10062 LeftMBB = FirstLeft->MBB;
10063 } else {
10064 LeftMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
10065 FuncInfo.MF->insert(BBI, LeftMBB);
10066 WorkList.push_back(
10067 {LeftMBB, FirstLeft, LastLeft, W.GE, Pivot, W.DefaultProb / 2});
10068 // Put Cond in a virtual register to make it available from the new blocks.
10069 ExportFromCurrentBlock(Cond);
10070 }
10071
10072 // Similarly, we will branch to the RHS if Value >= Pivot. If RHS is a
10073 // single cluster, RHS.Low == Pivot, and we can branch to its destination
10074 // directly if RHS.High equals the current upper bound.
10075 MachineBasicBlock *RightMBB;
10076 if (FirstRight == LastRight && FirstRight->Kind == CC_Range &&
10077 W.LT && (FirstRight->High->getValue() + 1ULL) == W.LT->getValue()) {
10078 RightMBB = FirstRight->MBB;
10079 } else {
10080 RightMBB = FuncInfo.MF->CreateMachineBasicBlock(W.MBB->getBasicBlock());
10081 FuncInfo.MF->insert(BBI, RightMBB);
10082 WorkList.push_back(
10083 {RightMBB, FirstRight, LastRight, Pivot, W.LT, W.DefaultProb / 2});
10084 // Put Cond in a virtual register to make it available from the new blocks.
10085 ExportFromCurrentBlock(Cond);
10086 }
10087
10088 // Create the CaseBlock record that will be used to lower the branch.
10089 CaseBlock CB(ISD::SETLT, Cond, Pivot, nullptr, LeftMBB, RightMBB, W.MBB,
10090 getCurSDLoc(), LeftProb, RightProb);
10091
10092 if (W.MBB == SwitchMBB)
10093 visitSwitchCase(CB, SwitchMBB);
10094 else
10095 SwitchCases.push_back(CB);
10096}
10097
10098// Scale CaseProb after peeling a case with the probablity of PeeledCaseProb
10099// from the swith statement.
10100static BranchProbability scaleCaseProbality(BranchProbability CaseProb,
10101 BranchProbability PeeledCaseProb) {
10102 if (PeeledCaseProb == BranchProbability::getOne())
10103 return BranchProbability::getZero();
10104 BranchProbability SwitchProb = PeeledCaseProb.getCompl();
10105
10106 uint32_t Numerator = CaseProb.getNumerator();
10107 uint32_t Denominator = SwitchProb.scale(CaseProb.getDenominator());
10108 return BranchProbability(Numerator, std::max(Numerator, Denominator));
10109}
10110
10111// Try to peel the top probability case if it exceeds the threshold.
10112// Return current MachineBasicBlock for the switch statement if the peeling
10113// does not occur.
10114// If the peeling is performed, return the newly created MachineBasicBlock
10115// for the peeled switch statement. Also update Clusters to remove the peeled
10116// case. PeeledCaseProb is the BranchProbability for the peeled case.
10117MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster(
10118 const SwitchInst &SI, CaseClusterVector &Clusters,
10119 BranchProbability &PeeledCaseProb) {
10120 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
10121 // Don't perform if there is only one cluster or optimizing for size.
10122 if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 ||
10123 TM.getOptLevel() == CodeGenOpt::None ||
10124 SwitchMBB->getParent()->getFunction().optForMinSize())
10125 return SwitchMBB;
10126
10127 BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100);
10128 unsigned PeeledCaseIndex = 0;
10129 bool SwitchPeeled = false;
10130 for (unsigned Index = 0; Index < Clusters.size(); ++Index) {
10131 CaseCluster &CC = Clusters[Index];
10132 if (CC.Prob < TopCaseProb)
10133 continue;
10134 TopCaseProb = CC.Prob;
10135 PeeledCaseIndex = Index;
10136 SwitchPeeled = true;
10137 }
10138 if (!SwitchPeeled)
10139 return SwitchMBB;
10140
10141 LLVM_DEBUG(dbgs() << "Peeled one top case in switch stmt, prob: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: "
<< TopCaseProb << "\n"; } } while (false)
10142 << TopCaseProb << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Peeled one top case in switch stmt, prob: "
<< TopCaseProb << "\n"; } } while (false);
10143
10144 // Record the MBB for the peeled switch statement.
10145 MachineFunction::iterator BBI(SwitchMBB);
10146 ++BBI;
10147 MachineBasicBlock *PeeledSwitchMBB =
10148 FuncInfo.MF->CreateMachineBasicBlock(SwitchMBB->getBasicBlock());
10149 FuncInfo.MF->insert(BBI, PeeledSwitchMBB);
10150
10151 ExportFromCurrentBlock(SI.getCondition());
10152 auto PeeledCaseIt = Clusters.begin() + PeeledCaseIndex;
10153 SwitchWorkListItem W = {SwitchMBB, PeeledCaseIt, PeeledCaseIt,
10154 nullptr, nullptr, TopCaseProb.getCompl()};
10155 lowerWorkItem(W, SI.getCondition(), SwitchMBB, PeeledSwitchMBB);
10156
10157 Clusters.erase(PeeledCaseIt);
10158 for (CaseCluster &CC : Clusters) {
10159 LLVM_DEBUG(do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: "
<< CC.Prob << "\n"; } } while (false)
10160 dbgs() << "Scale the probablity for one cluster, before scaling: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: "
<< CC.Prob << "\n"; } } while (false)
10161 << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "Scale the probablity for one cluster, before scaling: "
<< CC.Prob << "\n"; } } while (false);
10162 CC.Prob = scaleCaseProbality(CC.Prob, TopCaseProb);
10163 LLVM_DEBUG(dbgs() << "After scaling: " << CC.Prob << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { dbgs() << "After scaling: " << CC.Prob
<< "\n"; } } while (false);
10164 }
10165 PeeledCaseProb = TopCaseProb;
10166 return PeeledSwitchMBB;
10167}
10168
10169void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
10170 // Extract cases from the switch.
10171 BranchProbabilityInfo *BPI = FuncInfo.BPI;
10172 CaseClusterVector Clusters;
10173 Clusters.reserve(SI.getNumCases());
10174 for (auto I : SI.cases()) {
10175 MachineBasicBlock *Succ = FuncInfo.MBBMap[I.getCaseSuccessor()];
10176 const ConstantInt *CaseVal = I.getCaseValue();
10177 BranchProbability Prob =
10178 BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex())
10179 : BranchProbability(1, SI.getNumCases() + 1);
10180 Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob));
10181 }
10182
10183 MachineBasicBlock *DefaultMBB = FuncInfo.MBBMap[SI.getDefaultDest()];
10184
10185 // Cluster adjacent cases with the same destination. We do this at all
10186 // optimization levels because it's cheap to do and will make codegen faster
10187 // if there are many clusters.
10188 sortAndRangeify(Clusters);
10189
10190 if (TM.getOptLevel() != CodeGenOpt::None) {
10191 // Replace an unreachable default with the most popular destination.
10192 // FIXME: Exploit unreachable default more aggressively.
10193 bool UnreachableDefault =
10194 isa<UnreachableInst>(SI.getDefaultDest()->getFirstNonPHIOrDbg());
10195 if (UnreachableDefault && !Clusters.empty()) {
10196 DenseMap<const BasicBlock *, unsigned> Popularity;
10197 unsigned MaxPop = 0;
10198 const BasicBlock *MaxBB = nullptr;
10199 for (auto I : SI.cases()) {
10200 const BasicBlock *BB = I.getCaseSuccessor();
10201 if (++Popularity[BB] > MaxPop) {
10202 MaxPop = Popularity[BB];
10203 MaxBB = BB;
10204 }
10205 }
10206 // Set new default.
10207 assert(MaxPop > 0 && MaxBB)(static_cast <bool> (MaxPop > 0 && MaxBB) ? void
(0) : __assert_fail ("MaxPop > 0 && MaxBB", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10207, __extension__ __PRETTY_FUNCTION__));
10208 DefaultMBB = FuncInfo.MBBMap[MaxBB];
10209
10210 // Remove cases that were pointing to the destination that is now the
10211 // default.
10212 CaseClusterVector New;
10213 New.reserve(Clusters.size());
10214 for (CaseCluster &CC : Clusters) {
10215 if (CC.MBB != DefaultMBB)
10216 New.push_back(CC);
10217 }
10218 Clusters = std::move(New);
10219 }
10220 }
10221
10222 // The branch probablity of the peeled case.
10223 BranchProbability PeeledCaseProb = BranchProbability::getZero();
10224 MachineBasicBlock *PeeledSwitchMBB =
10225 peelDominantCaseCluster(SI, Clusters, PeeledCaseProb);
10226
10227 // If there is only the default destination, jump there directly.
10228 MachineBasicBlock *SwitchMBB = FuncInfo.MBB;
10229 if (Clusters.empty()) {
10230 assert(PeeledSwitchMBB == SwitchMBB)(static_cast <bool> (PeeledSwitchMBB == SwitchMBB) ? void
(0) : __assert_fail ("PeeledSwitchMBB == SwitchMBB", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10230, __extension__ __PRETTY_FUNCTION__));
10231 SwitchMBB->addSuccessor(DefaultMBB);
10232 if (DefaultMBB != NextBlock(SwitchMBB)) {
10233 DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other,
10234 getControlRoot(), DAG.getBasicBlock(DefaultMBB)));
10235 }
10236 return;
10237 }
10238
10239 findJumpTables(Clusters, &SI, DefaultMBB);
10240 findBitTestClusters(Clusters, &SI);
10241
10242 LLVM_DEBUG({do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10243 dbgs() << "Case clusters: ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10244 for (const CaseCluster &C : Clusters) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10245 if (C.Kind == CC_JumpTable)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10246 dbgs() << "JT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10247 if (C.Kind == CC_BitTests)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10248 dbgs() << "BT:";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10249
10250 C.Low->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10251 if (C.Low != C.High) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10252 dbgs() << '-';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10253 C.High->getValue().print(dbgs(), true);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10254 }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10255 dbgs() << ' ';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10256 }do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10257 dbgs() << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
)
10258 })do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("isel")) { { dbgs() << "Case clusters: "; for (const CaseCluster
&C : Clusters) { if (C.Kind == CC_JumpTable) dbgs() <<
"JT:"; if (C.Kind == CC_BitTests) dbgs() << "BT:"; C.Low
->getValue().print(dbgs(), true); if (C.Low != C.High) { dbgs
() << '-'; C.High->getValue().print(dbgs(), true); }
dbgs() << ' '; } dbgs() << '\n'; }; } } while (false
);
10259
10260 assert(!Clusters.empty())(static_cast <bool> (!Clusters.empty()) ? void (0) : __assert_fail
("!Clusters.empty()", "/build/llvm-toolchain-snapshot-7~svn338205/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp"
, 10260, __extension__ __PRETTY_FUNCTION__));
10261 SwitchWorkList WorkList;
10262 CaseClusterIt First = Clusters.begin();
10263 CaseClusterIt Last = Clusters.end() - 1;
10264 auto DefaultProb = getEdgeProbability(PeeledSwitchMBB, DefaultMBB);
10265 // Scale the branchprobability for DefaultMBB if the peel occurs and
10266 // DefaultMBB is not replaced.
10267 if (PeeledCaseProb != BranchProbability::getZero() &&
10268 DefaultMBB == FuncInfo.MBBMap[SI.getDefaultDest()])
10269 DefaultProb = scaleCaseProbality(DefaultProb, PeeledCaseProb);
10270 WorkList.push_back(
10271 {PeeledSwitchMBB, First, Last, nullptr, nullptr, DefaultProb});
10272
10273 while (!WorkList.empty()) {
10274 SwitchWorkListItem W = WorkList.back();
10275 WorkList.pop_back();
10276 unsigned NumClusters = W.LastCluster - W.FirstCluster + 1;
10277
10278 if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None &&
10279 !DefaultMBB->getParent()->getFunction().optForMinSize()) {
10280 // For optimized builds, lower large range as a balanced binary tree.
10281 splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB);
10282 continue;
10283 }
10284
10285 lowerWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB);
10286 }
10287}