Bug Summary

File:lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
Warning:line 756, column 29
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name LegalizeDAG.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-9/lib/clang/9.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG -I /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/include -I /build/llvm-toolchain-snapshot-9~svn359999/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/9.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-9/lib/clang/9.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-9~svn359999/build-llvm/lib/CodeGen/SelectionDAG -fdebug-prefix-map=/build/llvm-toolchain-snapshot-9~svn359999=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2019-05-06-051613-19774-1 -x c++ /build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp -faddrsig

/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp

1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
16#include "llvm/ADT/SetVector.h"
17#include "llvm/ADT/SmallPtrSet.h"
18#include "llvm/ADT/SmallSet.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/CodeGen/ISDOpcodes.h"
21#include "llvm/CodeGen/MachineFunction.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineMemOperand.h"
24#include "llvm/CodeGen/RuntimeLibcalls.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/SelectionDAGNodes.h"
27#include "llvm/CodeGen/TargetFrameLowering.h"
28#include "llvm/CodeGen/TargetLowering.h"
29#include "llvm/CodeGen/TargetSubtargetInfo.h"
30#include "llvm/CodeGen/ValueTypes.h"
31#include "llvm/IR/CallingConv.h"
32#include "llvm/IR/Constants.h"
33#include "llvm/IR/DataLayout.h"
34#include "llvm/IR/DerivedTypes.h"
35#include "llvm/IR/Function.h"
36#include "llvm/IR/Metadata.h"
37#include "llvm/IR/Type.h"
38#include "llvm/Support/Casting.h"
39#include "llvm/Support/Compiler.h"
40#include "llvm/Support/Debug.h"
41#include "llvm/Support/ErrorHandling.h"
42#include "llvm/Support/MachineValueType.h"
43#include "llvm/Support/MathExtras.h"
44#include "llvm/Support/raw_ostream.h"
45#include "llvm/Target/TargetMachine.h"
46#include "llvm/Target/TargetOptions.h"
47#include <algorithm>
48#include <cassert>
49#include <cstdint>
50#include <tuple>
51#include <utility>
52
53using namespace llvm;
54
55#define DEBUG_TYPE"legalizedag" "legalizedag"
56
57namespace {
58
59/// Keeps track of state when getting the sign of a floating-point value as an
60/// integer.
61struct FloatSignAsInt {
62 EVT FloatVT;
63 SDValue Chain;
64 SDValue FloatPtr;
65 SDValue IntPtr;
66 MachinePointerInfo IntPointerInfo;
67 MachinePointerInfo FloatPointerInfo;
68 SDValue IntValue;
69 APInt SignMask;
70 uint8_t SignBit;
71};
72
73//===----------------------------------------------------------------------===//
74/// This takes an arbitrary SelectionDAG as input and
75/// hacks on it until the target machine can handle it. This involves
76/// eliminating value sizes the machine cannot handle (promoting small sizes to
77/// large sizes or splitting up large values into small values) as well as
78/// eliminating operations the machine cannot handle.
79///
80/// This code also does a small amount of optimization and recognition of idioms
81/// as part of its processing. For example, if a target does not support a
82/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
83/// will attempt merge setcc and brc instructions into brcc's.
84class SelectionDAGLegalize {
85 const TargetMachine &TM;
86 const TargetLowering &TLI;
87 SelectionDAG &DAG;
88
89 /// The set of nodes which have already been legalized. We hold a
90 /// reference to it in order to update as necessary on node deletion.
91 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
92
93 /// A set of all the nodes updated during legalization.
94 SmallSetVector<SDNode *, 16> *UpdatedNodes;
95
96 EVT getSetCCResultType(EVT VT) const {
97 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
98 }
99
100 // Libcall insertion helpers.
101
102public:
103 SelectionDAGLegalize(SelectionDAG &DAG,
104 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
105 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
106 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
107 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
108
109 /// Legalizes the given operation.
110 void LegalizeOp(SDNode *Node);
111
112private:
113 SDValue OptimizeFloatStore(StoreSDNode *ST);
114
115 void LegalizeLoadOps(SDNode *Node);
116 void LegalizeStoreOps(SDNode *Node);
117
118 /// Some targets cannot handle a variable
119 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
120 /// is necessary to spill the vector being inserted into to memory, perform
121 /// the insert there, and then read the result back.
122 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
123 const SDLoc &dl);
124 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
125 const SDLoc &dl);
126
127 /// Return a vector shuffle operation which
128 /// performs the same shuffe in terms of order or result bytes, but on a type
129 /// whose vector element type is narrower than the original shuffle type.
130 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
131 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
132 SDValue N1, SDValue N2,
133 ArrayRef<int> Mask) const;
134
135 bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC,
136 bool &NeedInvert, const SDLoc &dl);
137
138 SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
139
140 std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
141 SDNode *Node, bool isSigned);
142 SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
143 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
144 RTLIB::Libcall Call_F128,
145 RTLIB::Libcall Call_PPCF128);
146 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
147 RTLIB::Libcall Call_I8,
148 RTLIB::Libcall Call_I16,
149 RTLIB::Libcall Call_I32,
150 RTLIB::Libcall Call_I64,
151 RTLIB::Libcall Call_I128);
152 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
153 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
154
155 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
156 const SDLoc &dl);
157 SDValue ExpandBUILD_VECTOR(SDNode *Node);
158 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
159 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
160 SmallVectorImpl<SDValue> &Results);
161 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
162 SDValue Value) const;
163 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
164 SDValue NewIntValue) const;
165 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
166 SDValue ExpandFABS(SDNode *Node) const;
167 SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT,
168 const SDLoc &dl);
169 SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned,
170 const SDLoc &dl);
171 SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned,
172 const SDLoc &dl);
173
174 SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl);
175 SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl);
176
177 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
178 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
179 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
180
181 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
182 SDValue ExpandConstant(ConstantSDNode *CP);
183
184 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
185 bool ExpandNode(SDNode *Node);
186 void ConvertNodeToLibcall(SDNode *Node);
187 void PromoteNode(SDNode *Node);
188
189public:
190 // Node replacement helpers
191
192 void ReplacedNode(SDNode *N) {
193 LegalizedNodes.erase(N);
194 if (UpdatedNodes)
195 UpdatedNodes->insert(N);
196 }
197
198 void ReplaceNode(SDNode *Old, SDNode *New) {
199 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
200 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
201
202 assert(Old->getNumValues() == New->getNumValues() &&((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 204, __PRETTY_FUNCTION__))
203 "Replacing one node with another that produces a different number "((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 204, __PRETTY_FUNCTION__))
204 "of values!")((Old->getNumValues() == New->getNumValues() &&
"Replacing one node with another that produces a different number "
"of values!") ? static_cast<void> (0) : __assert_fail (
"Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 204, __PRETTY_FUNCTION__))
;
205 DAG.ReplaceAllUsesWith(Old, New);
206 if (UpdatedNodes)
207 UpdatedNodes->insert(New);
208 ReplacedNode(Old);
209 }
210
211 void ReplaceNode(SDValue Old, SDValue New) {
212 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
213 dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); dbgs() << " with: "; New->dump
(&DAG); } } while (false)
;
214
215 DAG.ReplaceAllUsesWith(Old, New);
216 if (UpdatedNodes)
217 UpdatedNodes->insert(New.getNode());
218 ReplacedNode(Old.getNode());
219 }
220
221 void ReplaceNode(SDNode *Old, const SDValue *New) {
222 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << " ... replacing: "; Old->
dump(&DAG); } } while (false)
;
223
224 DAG.ReplaceAllUsesWith(Old, New);
225 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
226 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
227 New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << (i == 0 ? " with: "
: " and: "); New[i]->dump(&DAG); } } while (
false)
;
228 if (UpdatedNodes)
229 UpdatedNodes->insert(New[i].getNode());
230 }
231 ReplacedNode(Old);
232 }
233};
234
235} // end anonymous namespace
236
237/// Return a vector shuffle operation which
238/// performs the same shuffle in terms of order or result bytes, but on a type
239/// whose vector element type is narrower than the original shuffle type.
240/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
241SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
242 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
243 ArrayRef<int> Mask) const {
244 unsigned NumMaskElts = VT.getVectorNumElements();
245 unsigned NumDestElts = NVT.getVectorNumElements();
246 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
247
248 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")((NumEltsGrowth && "Cannot promote to vector type with fewer elts!"
) ? static_cast<void> (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 248, __PRETTY_FUNCTION__))
;
249
250 if (NumEltsGrowth == 1)
251 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
252
253 SmallVector<int, 8> NewMask;
254 for (unsigned i = 0; i != NumMaskElts; ++i) {
255 int Idx = Mask[i];
256 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
257 if (Idx < 0)
258 NewMask.push_back(-1);
259 else
260 NewMask.push_back(Idx * NumEltsGrowth + j);
261 }
262 }
263 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")((NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?"
) ? static_cast<void> (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 263, __PRETTY_FUNCTION__))
;
264 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")((TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?"
) ? static_cast<void> (0) : __assert_fail ("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 264, __PRETTY_FUNCTION__))
;
265 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
266}
267
268/// Expands the ConstantFP node to an integer constant or
269/// a load from the constant pool.
270SDValue
271SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
272 bool Extend = false;
273 SDLoc dl(CFP);
274
275 // If a FP immediate is precise when represented as a float and if the
276 // target can do an extending load from float to double, we put it into
277 // the constant pool as a float, even if it's is statically typed as a
278 // double. This shrinks FP constants and canonicalizes them for targets where
279 // an FP extending load is the same cost as a normal load (such as on the x87
280 // fp stack or PPC FP unit).
281 EVT VT = CFP->getValueType(0);
282 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
283 if (!UseCP) {
284 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 284, __PRETTY_FUNCTION__))
;
285 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
286 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
287 }
288
289 APFloat APF = CFP->getValueAPF();
290 EVT OrigVT = VT;
291 EVT SVT = VT;
292
293 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
294 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
295 if (!APF.isSignaling()) {
296 while (SVT != MVT::f32 && SVT != MVT::f16) {
297 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
298 if (ConstantFPSDNode::isValueValidForType(SVT, APF) &&
299 // Only do this if the target has a native EXTLOAD instruction from
300 // smaller type.
301 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
302 TLI.ShouldShrinkFPConstant(OrigVT)) {
303 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
304 LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
305 VT = SVT;
306 Extend = true;
307 }
308 }
309 }
310
311 SDValue CPIdx =
312 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
313 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
314 if (Extend) {
315 SDValue Result = DAG.getExtLoad(
316 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
317 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
318 Alignment);
319 return Result;
320 }
321 SDValue Result = DAG.getLoad(
322 OrigVT, dl, DAG.getEntryNode(), CPIdx,
323 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
324 return Result;
325}
326
327/// Expands the Constant node to a load from the constant pool.
328SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
329 SDLoc dl(CP);
330 EVT VT = CP->getValueType(0);
331 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
332 TLI.getPointerTy(DAG.getDataLayout()));
333 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
334 SDValue Result = DAG.getLoad(
335 VT, dl, DAG.getEntryNode(), CPIdx,
336 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
337 return Result;
338}
339
340/// Some target cannot handle a variable insertion index for the
341/// INSERT_VECTOR_ELT instruction. In this case, it
342/// is necessary to spill the vector being inserted into to memory, perform
343/// the insert there, and then read the result back.
344SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
345 SDValue Val,
346 SDValue Idx,
347 const SDLoc &dl) {
348 SDValue Tmp1 = Vec;
349 SDValue Tmp2 = Val;
350 SDValue Tmp3 = Idx;
351
352 // If the target doesn't support this, we have to spill the input vector
353 // to a temporary stack slot, update the element, then reload it. This is
354 // badness. We could also load the value into a vector register (either
355 // with a "move to register" or "extload into register" instruction, then
356 // permute it into place, if the idx is a constant and if the idx is
357 // supported by the target.
358 EVT VT = Tmp1.getValueType();
359 EVT EltVT = VT.getVectorElementType();
360 SDValue StackPtr = DAG.CreateStackTemporary(VT);
361
362 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
363
364 // Store the vector.
365 SDValue Ch = DAG.getStore(
366 DAG.getEntryNode(), dl, Tmp1, StackPtr,
367 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
368
369 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
370
371 // Store the scalar value.
372 Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT);
373 // Load the updated vector.
374 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
375 DAG.getMachineFunction(), SPFI));
376}
377
378SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
379 SDValue Idx,
380 const SDLoc &dl) {
381 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
382 // SCALAR_TO_VECTOR requires that the type of the value being inserted
383 // match the element type of the vector being created, except for
384 // integers in which case the inserted value can be over width.
385 EVT EltVT = Vec.getValueType().getVectorElementType();
386 if (Val.getValueType() == EltVT ||
387 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
388 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
389 Vec.getValueType(), Val);
390
391 unsigned NumElts = Vec.getValueType().getVectorNumElements();
392 // We generate a shuffle of InVec and ScVec, so the shuffle mask
393 // should be 0,1,2,3,4,5... with the appropriate element replaced with
394 // elt 0 of the RHS.
395 SmallVector<int, 8> ShufOps;
396 for (unsigned i = 0; i != NumElts; ++i)
397 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
398
399 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
400 }
401 }
402 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
403}
404
405SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
406 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Optimizing float store operations\n"
; } } while (false)
;
407 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
408 // FIXME: We shouldn't do this for TargetConstantFP's.
409 // FIXME: move this to the DAG Combiner! Note that we can't regress due
410 // to phase ordering between legalized code and the dag combiner. This
411 // probably means that we need to integrate dag combiner and legalizer
412 // together.
413 // We generally can't do this one for long doubles.
414 SDValue Chain = ST->getChain();
415 SDValue Ptr = ST->getBasePtr();
416 unsigned Alignment = ST->getAlignment();
417 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
418 AAMDNodes AAInfo = ST->getAAInfo();
419 SDLoc dl(ST);
420 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) {
421 if (CFP->getValueType(0) == MVT::f32 &&
422 TLI.isTypeLegal(MVT::i32)) {
423 SDValue Con = DAG.getConstant(CFP->getValueAPF().
424 bitcastToAPInt().zextOrTrunc(32),
425 SDLoc(CFP), MVT::i32);
426 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment,
427 MMOFlags, AAInfo);
428 }
429
430 if (CFP->getValueType(0) == MVT::f64) {
431 // If this target supports 64-bit registers, do a single 64-bit store.
432 if (TLI.isTypeLegal(MVT::i64)) {
433 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
434 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
435 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
436 Alignment, MMOFlags, AAInfo);
437 }
438
439 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
440 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
441 // stores. If the target supports neither 32- nor 64-bits, this
442 // xform is certainly not worth it.
443 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
444 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
445 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
446 if (DAG.getDataLayout().isBigEndian())
447 std::swap(Lo, Hi);
448
449 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment,
450 MMOFlags, AAInfo);
451 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
452 DAG.getConstant(4, dl, Ptr.getValueType()));
453 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
454 ST->getPointerInfo().getWithOffset(4),
455 MinAlign(Alignment, 4U), MMOFlags, AAInfo);
456
457 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
458 }
459 }
460 }
461 return SDValue(nullptr, 0);
462}
463
464void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
465 StoreSDNode *ST = cast<StoreSDNode>(Node);
466 SDValue Chain = ST->getChain();
467 SDValue Ptr = ST->getBasePtr();
468 SDLoc dl(Node);
469
470 unsigned Alignment = ST->getAlignment();
471 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
472 AAMDNodes AAInfo = ST->getAAInfo();
473
474 if (!ST->isTruncatingStore()) {
475 LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing store operation\n"
; } } while (false)
;
476 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
477 ReplaceNode(ST, OptStore);
478 return;
479 }
480
481 SDValue Value = ST->getValue();
482 MVT VT = Value.getSimpleValueType();
483 switch (TLI.getOperationAction(ISD::STORE, VT)) {
484 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 484)
;
485 case TargetLowering::Legal: {
486 // If this is an unaligned store and the target doesn't support it,
487 // expand it.
488 EVT MemVT = ST->getMemoryVT();
489 unsigned AS = ST->getAddressSpace();
490 unsigned Align = ST->getAlignment();
491 const DataLayout &DL = DAG.getDataLayout();
492 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
493 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n"
; } } while (false)
;
494 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
495 ReplaceNode(SDValue(ST, 0), Result);
496 } else
497 LLVM_DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal store\n"; } } while
(false)
;
498 break;
499 }
500 case TargetLowering::Custom: {
501 LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom lowering\n"
; } } while (false)
;
502 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
503 if (Res && Res != SDValue(Node, 0))
504 ReplaceNode(SDValue(Node, 0), Res);
505 return;
506 }
507 case TargetLowering::Promote: {
508 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
509 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 510, __PRETTY_FUNCTION__))
510 "Can only promote stores to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 510, __PRETTY_FUNCTION__))
;
511 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
512 SDValue Result =
513 DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
514 Alignment, MMOFlags, AAInfo);
515 ReplaceNode(SDValue(Node, 0), Result);
516 break;
517 }
518 }
519 return;
520 }
521
522 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing truncating store operations\n"
; } } while (false)
;
523 SDValue Value = ST->getValue();
524 EVT StVT = ST->getMemoryVT();
525 unsigned StWidth = StVT.getSizeInBits();
526 auto &DL = DAG.getDataLayout();
527
528 if (StWidth != StVT.getStoreSizeInBits()) {
529 // Promote to a byte-sized store with upper bits zero if not
530 // storing an integral number of bytes. For example, promote
531 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
532 EVT NVT = EVT::getIntegerVT(*DAG.getContext(),
533 StVT.getStoreSizeInBits());
534 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
535 SDValue Result =
536 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
537 Alignment, MMOFlags, AAInfo);
538 ReplaceNode(SDValue(Node, 0), Result);
539 } else if (StWidth & (StWidth - 1)) {
540 // If not storing a power-of-2 number of bits, expand as two stores.
541 assert(!StVT.isVector() && "Unsupported truncstore!")((!StVT.isVector() && "Unsupported truncstore!") ? static_cast
<void> (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 541, __PRETTY_FUNCTION__))
;
542 unsigned RoundWidth = 1 << Log2_32(StWidth);
543 assert(RoundWidth < StWidth)((RoundWidth < StWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 543, __PRETTY_FUNCTION__))
;
544 unsigned ExtraWidth = StWidth - RoundWidth;
545 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 545, __PRETTY_FUNCTION__))
;
546 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 547, __PRETTY_FUNCTION__))
547 "Store size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 547, __PRETTY_FUNCTION__))
;
548 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
549 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
550 SDValue Lo, Hi;
551 unsigned IncrementSize;
552
553 if (DL.isLittleEndian()) {
554 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
555 // Store the bottom RoundWidth bits.
556 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
557 RoundVT, Alignment, MMOFlags, AAInfo);
558
559 // Store the remaining ExtraWidth bits.
560 IncrementSize = RoundWidth / 8;
561 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
562 DAG.getConstant(IncrementSize, dl,
563 Ptr.getValueType()));
564 Hi = DAG.getNode(
565 ISD::SRL, dl, Value.getValueType(), Value,
566 DAG.getConstant(RoundWidth, dl,
567 TLI.getShiftAmountTy(Value.getValueType(), DL)));
568 Hi = DAG.getTruncStore(
569 Chain, dl, Hi, Ptr,
570 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
571 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
572 } else {
573 // Big endian - avoid unaligned stores.
574 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
575 // Store the top RoundWidth bits.
576 Hi = DAG.getNode(
577 ISD::SRL, dl, Value.getValueType(), Value,
578 DAG.getConstant(ExtraWidth, dl,
579 TLI.getShiftAmountTy(Value.getValueType(), DL)));
580 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(),
581 RoundVT, Alignment, MMOFlags, AAInfo);
582
583 // Store the remaining ExtraWidth bits.
584 IncrementSize = RoundWidth / 8;
585 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
586 DAG.getConstant(IncrementSize, dl,
587 Ptr.getValueType()));
588 Lo = DAG.getTruncStore(
589 Chain, dl, Value, Ptr,
590 ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT,
591 MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
592 }
593
594 // The order of the stores doesn't matter.
595 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
596 ReplaceNode(SDValue(Node, 0), Result);
597 } else {
598 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
599 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 599)
;
600 case TargetLowering::Legal: {
601 EVT MemVT = ST->getMemoryVT();
602 unsigned AS = ST->getAddressSpace();
603 unsigned Align = ST->getAlignment();
604 // If this is an unaligned store and the target doesn't support it,
605 // expand it.
606 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
607 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
608 ReplaceNode(SDValue(ST, 0), Result);
609 }
610 break;
611 }
612 case TargetLowering::Custom: {
613 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
614 if (Res && Res != SDValue(Node, 0))
615 ReplaceNode(SDValue(Node, 0), Res);
616 return;
617 }
618 case TargetLowering::Expand:
619 assert(!StVT.isVector() &&((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 620, __PRETTY_FUNCTION__))
620 "Vector Stores are handled in LegalizeVectorOps")((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 620, __PRETTY_FUNCTION__))
;
621
622 SDValue Result;
623
624 // TRUNCSTORE:i16 i32 -> STORE i16
625 if (TLI.isTypeLegal(StVT)) {
626 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
627 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
628 Alignment, MMOFlags, AAInfo);
629 } else {
630 // The in-memory type isn't legal. Truncate to the type it would promote
631 // to, and then do a truncstore.
632 Value = DAG.getNode(ISD::TRUNCATE, dl,
633 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
634 Value);
635 Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
636 StVT, Alignment, MMOFlags, AAInfo);
637 }
638
639 ReplaceNode(SDValue(Node, 0), Result);
640 break;
641 }
642 }
643}
644
645void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
646 LoadSDNode *LD = cast<LoadSDNode>(Node);
647 SDValue Chain = LD->getChain(); // The chain.
648 SDValue Ptr = LD->getBasePtr(); // The base pointer.
649 SDValue Value; // The value returned by the load op.
650 SDLoc dl(Node);
651
652 ISD::LoadExtType ExtType = LD->getExtensionType();
653 if (ExtType == ISD::NON_EXTLOAD) {
1
Assuming 'ExtType' is not equal to NON_EXTLOAD
2
Taking false branch
654 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n"
; } } while (false)
;
655 MVT VT = Node->getSimpleValueType(0);
656 SDValue RVal = SDValue(Node, 0);
657 SDValue RChain = SDValue(Node, 1);
658
659 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
660 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 660)
;
661 case TargetLowering::Legal: {
662 EVT MemVT = LD->getMemoryVT();
663 unsigned AS = LD->getAddressSpace();
664 unsigned Align = LD->getAlignment();
665 const DataLayout &DL = DAG.getDataLayout();
666 // If this is an unaligned load and the target doesn't support it,
667 // expand it.
668 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
669 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
670 }
671 break;
672 }
673 case TargetLowering::Custom:
674 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
675 RVal = Res;
676 RChain = Res.getValue(1);
677 }
678 break;
679
680 case TargetLowering::Promote: {
681 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
682 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 683, __PRETTY_FUNCTION__))
683 "Can only promote loads to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 683, __PRETTY_FUNCTION__))
;
684
685 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
686 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
687 RChain = Res.getValue(1);
688 break;
689 }
690 }
691 if (RChain.getNode() != Node) {
692 assert(RVal.getNode() != Node && "Load must be completely replaced")((RVal.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 692, __PRETTY_FUNCTION__))
;
693 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
694 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
695 if (UpdatedNodes) {
696 UpdatedNodes->insert(RVal.getNode());
697 UpdatedNodes->insert(RChain.getNode());
698 }
699 ReplacedNode(Node);
700 }
701 return;
702 }
703
704 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing extending load operation\n"
; } } while (false)
;
3
Assuming 'DebugFlag' is 0
4
Loop condition is false. Exiting loop
705 EVT SrcVT = LD->getMemoryVT();
706 unsigned SrcWidth = SrcVT.getSizeInBits();
707 unsigned Alignment = LD->getAlignment();
708 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
709 AAMDNodes AAInfo = LD->getAAInfo();
710
711 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
5
Assuming the condition is false
6
Taking false branch
712 // Some targets pretend to have an i1 loading operation, and actually
713 // load an i8. This trick is correct for ZEXTLOAD because the top 7
714 // bits are guaranteed to be zero; it helps the optimizers understand
715 // that these bits are zero. It is also useful for EXTLOAD, since it
716 // tells the optimizers that those bits are undefined. It would be
717 // nice to have an effective generic way of getting these benefits...
718 // Until such a way is found, don't insist on promoting i1 here.
719 (SrcVT != MVT::i1 ||
720 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
721 TargetLowering::Promote)) {
722 // Promote to a byte-sized load if not loading an integral number of
723 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
724 unsigned NewWidth = SrcVT.getStoreSizeInBits();
725 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
726 SDValue Ch;
727
728 // The extra bits are guaranteed to be zero, since we stored them that
729 // way. A zext load from NVT thus automatically gives zext from SrcVT.
730
731 ISD::LoadExtType NewExtType =
732 ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD;
733
734 SDValue Result =
735 DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr,
736 LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo);
737
738 Ch = Result.getValue(1); // The chain.
739
740 if (ExtType == ISD::SEXTLOAD)
741 // Having the top bits zero doesn't help when sign extending.
742 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
743 Result.getValueType(),
744 Result, DAG.getValueType(SrcVT));
745 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
746 // All the top bits are guaranteed to be zero - inform the optimizers.
747 Result = DAG.getNode(ISD::AssertZext, dl,
748 Result.getValueType(), Result,
749 DAG.getValueType(SrcVT));
750
751 Value = Result;
752 Chain = Ch;
753 } else if (SrcWidth & (SrcWidth - 1)) {
7
Assuming the condition is true
8
Taking true branch
754 // If not loading a power-of-2 number of bits, expand as two loads.
755 assert(!SrcVT.isVector() && "Unsupported extload!")((!SrcVT.isVector() && "Unsupported extload!") ? static_cast
<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 755, __PRETTY_FUNCTION__))
;
9
Assuming the condition is true
10
'?' condition is true
756 unsigned RoundWidth = 1 << Log2_32(SrcWidth);
11
Calling 'Log2_32'
13
Returning from 'Log2_32'
14
The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int'
757 assert(RoundWidth < SrcWidth)((RoundWidth < SrcWidth) ? static_cast<void> (0) : __assert_fail
("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 757, __PRETTY_FUNCTION__))
;
758 unsigned ExtraWidth = SrcWidth - RoundWidth;
759 assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) :
__assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 759, __PRETTY_FUNCTION__))
;
760 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 761, __PRETTY_FUNCTION__))
761 "Load size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!"
) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 761, __PRETTY_FUNCTION__))
;
762 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
763 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
764 SDValue Lo, Hi, Ch;
765 unsigned IncrementSize;
766 auto &DL = DAG.getDataLayout();
767
768 if (DL.isLittleEndian()) {
769 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
770 // Load the bottom RoundWidth bits.
771 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
772 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
773 AAInfo);
774
775 // Load the remaining ExtraWidth bits.
776 IncrementSize = RoundWidth / 8;
777 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
778 DAG.getConstant(IncrementSize, dl,
779 Ptr.getValueType()));
780 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
781 LD->getPointerInfo().getWithOffset(IncrementSize),
782 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
783 AAInfo);
784
785 // Build a factor node to remember that this load is independent of
786 // the other one.
787 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
788 Hi.getValue(1));
789
790 // Move the top bits to the right place.
791 Hi = DAG.getNode(
792 ISD::SHL, dl, Hi.getValueType(), Hi,
793 DAG.getConstant(RoundWidth, dl,
794 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
795
796 // Join the hi and lo parts.
797 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
798 } else {
799 // Big endian - avoid unaligned loads.
800 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
801 // Load the top RoundWidth bits.
802 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
803 LD->getPointerInfo(), RoundVT, Alignment, MMOFlags,
804 AAInfo);
805
806 // Load the remaining ExtraWidth bits.
807 IncrementSize = RoundWidth / 8;
808 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
809 DAG.getConstant(IncrementSize, dl,
810 Ptr.getValueType()));
811 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
812 LD->getPointerInfo().getWithOffset(IncrementSize),
813 ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags,
814 AAInfo);
815
816 // Build a factor node to remember that this load is independent of
817 // the other one.
818 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
819 Hi.getValue(1));
820
821 // Move the top bits to the right place.
822 Hi = DAG.getNode(
823 ISD::SHL, dl, Hi.getValueType(), Hi,
824 DAG.getConstant(ExtraWidth, dl,
825 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
826
827 // Join the hi and lo parts.
828 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
829 }
830
831 Chain = Ch;
832 } else {
833 bool isCustom = false;
834 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
835 SrcVT.getSimpleVT())) {
836 default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 836)
;
837 case TargetLowering::Custom:
838 isCustom = true;
839 LLVM_FALLTHROUGH[[clang::fallthrough]];
840 case TargetLowering::Legal:
841 Value = SDValue(Node, 0);
842 Chain = SDValue(Node, 1);
843
844 if (isCustom) {
845 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
846 Value = Res;
847 Chain = Res.getValue(1);
848 }
849 } else {
850 // If this is an unaligned load and the target doesn't support it,
851 // expand it.
852 EVT MemVT = LD->getMemoryVT();
853 unsigned AS = LD->getAddressSpace();
854 unsigned Align = LD->getAlignment();
855 const DataLayout &DL = DAG.getDataLayout();
856 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) {
857 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
858 }
859 }
860 break;
861
862 case TargetLowering::Expand: {
863 EVT DestVT = Node->getValueType(0);
864 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
865 // If the source type is not legal, see if there is a legal extload to
866 // an intermediate type that we can then extend further.
867 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
868 if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
869 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
870 // If we are loading a legal type, this is a non-extload followed by a
871 // full extend.
872 ISD::LoadExtType MidExtType =
873 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
874
875 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
876 SrcVT, LD->getMemOperand());
877 unsigned ExtendOp =
878 ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
879 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
880 Chain = Load.getValue(1);
881 break;
882 }
883
884 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
885 // normal undefined upper bits behavior to allow using an in-reg extend
886 // with the illegal FP type, so load as an integer and do the
887 // from-integer conversion.
888 if (SrcVT.getScalarType() == MVT::f16) {
889 EVT ISrcVT = SrcVT.changeTypeToInteger();
890 EVT IDestVT = DestVT.changeTypeToInteger();
891 EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
892
893 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT,
894 Chain, Ptr, ISrcVT,
895 LD->getMemOperand());
896 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
897 Chain = Result.getValue(1);
898 break;
899 }
900 }
901
902 assert(!SrcVT.isVector() &&((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 903, __PRETTY_FUNCTION__))
903 "Vector Loads are handled in LegalizeVectorOps")((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps"
) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 903, __PRETTY_FUNCTION__))
;
904
905 // FIXME: This does not work for vectors on most targets. Sign-
906 // and zero-extend operations are currently folded into extending
907 // loads, whether they are legal or not, and then we end up here
908 // without any support for legalizing them.
909 assert(ExtType != ISD::EXTLOAD &&((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 910, __PRETTY_FUNCTION__))
910 "EXTLOAD should always be supported!")((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!"
) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 910, __PRETTY_FUNCTION__))
;
911 // Turn the unsupported load into an EXTLOAD followed by an
912 // explicit zero/sign extend inreg.
913 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
914 Node->getValueType(0),
915 Chain, Ptr, SrcVT,
916 LD->getMemOperand());
917 SDValue ValRes;
918 if (ExtType == ISD::SEXTLOAD)
919 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
920 Result.getValueType(),
921 Result, DAG.getValueType(SrcVT));
922 else
923 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType());
924 Value = ValRes;
925 Chain = Result.getValue(1);
926 break;
927 }
928 }
929 }
930
931 // Since loads produce two values, make sure to remember that we legalized
932 // both of them.
933 if (Chain.getNode() != Node) {
934 assert(Value.getNode() != Node && "Load must be completely replaced")((Value.getNode() != Node && "Load must be completely replaced"
) ? static_cast<void> (0) : __assert_fail ("Value.getNode() != Node && \"Load must be completely replaced\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 934, __PRETTY_FUNCTION__))
;
935 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
936 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
937 if (UpdatedNodes) {
938 UpdatedNodes->insert(Value.getNode());
939 UpdatedNodes->insert(Chain.getNode());
940 }
941 ReplacedNode(Node);
942 }
943}
944
945/// Return a legal replacement for the given operation, with all legal operands.
946void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
947 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "\nLegalizing: "; Node->
dump(&DAG); } } while (false)
;
948
949 // Allow illegal target nodes and illegal registers.
950 if (Node->getOpcode() == ISD::TargetConstant ||
951 Node->getOpcode() == ISD::Register)
952 return;
953
954#ifndef NDEBUG
955 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
956 assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 959, __PRETTY_FUNCTION__))
957 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 959, __PRETTY_FUNCTION__))
958 TLI.isTypeLegal(Node->getValueType(i))) &&(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 959, __PRETTY_FUNCTION__))
959 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType
(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->
getValueType(i))) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 959, __PRETTY_FUNCTION__))
;
960
961 for (const SDValue &Op : Node->op_values())
962 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 967, __PRETTY_FUNCTION__))
963 TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 967, __PRETTY_FUNCTION__))
964 TLI.isTypeLegal(Op.getValueType()) ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 967, __PRETTY_FUNCTION__))
965 Op.getOpcode() == ISD::TargetConstant ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 967, __PRETTY_FUNCTION__))
966 Op.getOpcode() == ISD::Register) &&(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 967, __PRETTY_FUNCTION__))
967 "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType
()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode(
) == ISD::Register) && "Unexpected illegal type!") ? static_cast
<void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 967, __PRETTY_FUNCTION__))
;
968#endif
969
970 // Figure out the correct action; the way to query this varies by opcode
971 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
972 bool SimpleFinishLegalizing = true;
973 switch (Node->getOpcode()) {
974 case ISD::INTRINSIC_W_CHAIN:
975 case ISD::INTRINSIC_WO_CHAIN:
976 case ISD::INTRINSIC_VOID:
977 case ISD::STACKSAVE:
978 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
979 break;
980 case ISD::GET_DYNAMIC_AREA_OFFSET:
981 Action = TLI.getOperationAction(Node->getOpcode(),
982 Node->getValueType(0));
983 break;
984 case ISD::VAARG:
985 Action = TLI.getOperationAction(Node->getOpcode(),
986 Node->getValueType(0));
987 if (Action != TargetLowering::Promote)
988 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
989 break;
990 case ISD::FP_TO_FP16:
991 case ISD::SINT_TO_FP:
992 case ISD::UINT_TO_FP:
993 case ISD::EXTRACT_VECTOR_ELT:
994 Action = TLI.getOperationAction(Node->getOpcode(),
995 Node->getOperand(0).getValueType());
996 break;
997 case ISD::FP_ROUND_INREG:
998 case ISD::SIGN_EXTEND_INREG: {
999 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1000 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1001 break;
1002 }
1003 case ISD::ATOMIC_STORE:
1004 Action = TLI.getOperationAction(Node->getOpcode(),
1005 Node->getOperand(2).getValueType());
1006 break;
1007 case ISD::SELECT_CC:
1008 case ISD::SETCC:
1009 case ISD::BR_CC: {
1010 unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1011 Node->getOpcode() == ISD::SETCC ? 2 : 1;
1012 unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0;
1013 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1014 ISD::CondCode CCCode =
1015 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1016 Action = TLI.getCondCodeAction(CCCode, OpVT);
1017 if (Action == TargetLowering::Legal) {
1018 if (Node->getOpcode() == ISD::SELECT_CC)
1019 Action = TLI.getOperationAction(Node->getOpcode(),
1020 Node->getValueType(0));
1021 else
1022 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1023 }
1024 break;
1025 }
1026 case ISD::LOAD:
1027 case ISD::STORE:
1028 // FIXME: Model these properly. LOAD and STORE are complicated, and
1029 // STORE expects the unlegalized operand in some cases.
1030 SimpleFinishLegalizing = false;
1031 break;
1032 case ISD::CALLSEQ_START:
1033 case ISD::CALLSEQ_END:
1034 // FIXME: This shouldn't be necessary. These nodes have special properties
1035 // dealing with the recursive nature of legalization. Removing this
1036 // special case should be done as part of making LegalizeDAG non-recursive.
1037 SimpleFinishLegalizing = false;
1038 break;
1039 case ISD::EXTRACT_ELEMENT:
1040 case ISD::FLT_ROUNDS_:
1041 case ISD::MERGE_VALUES:
1042 case ISD::EH_RETURN:
1043 case ISD::FRAME_TO_ARGS_OFFSET:
1044 case ISD::EH_DWARF_CFA:
1045 case ISD::EH_SJLJ_SETJMP:
1046 case ISD::EH_SJLJ_LONGJMP:
1047 case ISD::EH_SJLJ_SETUP_DISPATCH:
1048 // These operations lie about being legal: when they claim to be legal,
1049 // they should actually be expanded.
1050 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1051 if (Action == TargetLowering::Legal)
1052 Action = TargetLowering::Expand;
1053 break;
1054 case ISD::INIT_TRAMPOLINE:
1055 case ISD::ADJUST_TRAMPOLINE:
1056 case ISD::FRAMEADDR:
1057 case ISD::RETURNADDR:
1058 case ISD::ADDROFRETURNADDR:
1059 case ISD::SPONENTRY:
1060 // These operations lie about being legal: when they claim to be legal,
1061 // they should actually be custom-lowered.
1062 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1063 if (Action == TargetLowering::Legal)
1064 Action = TargetLowering::Custom;
1065 break;
1066 case ISD::READCYCLECOUNTER:
1067 // READCYCLECOUNTER returns an i64, even if type legalization might have
1068 // expanded that to several smaller types.
1069 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1070 break;
1071 case ISD::READ_REGISTER:
1072 case ISD::WRITE_REGISTER:
1073 // Named register is legal in the DAG, but blocked by register name
1074 // selection if not implemented by target (to chose the correct register)
1075 // They'll be converted to Copy(To/From)Reg.
1076 Action = TargetLowering::Legal;
1077 break;
1078 case ISD::DEBUGTRAP:
1079 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1080 if (Action == TargetLowering::Expand) {
1081 // replace ISD::DEBUGTRAP with ISD::TRAP
1082 SDValue NewVal;
1083 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1084 Node->getOperand(0));
1085 ReplaceNode(Node, NewVal.getNode());
1086 LegalizeOp(NewVal.getNode());
1087 return;
1088 }
1089 break;
1090 case ISD::STRICT_FADD:
1091 case ISD::STRICT_FSUB:
1092 case ISD::STRICT_FMUL:
1093 case ISD::STRICT_FDIV:
1094 case ISD::STRICT_FREM:
1095 case ISD::STRICT_FSQRT:
1096 case ISD::STRICT_FMA:
1097 case ISD::STRICT_FPOW:
1098 case ISD::STRICT_FPOWI:
1099 case ISD::STRICT_FSIN:
1100 case ISD::STRICT_FCOS:
1101 case ISD::STRICT_FEXP:
1102 case ISD::STRICT_FEXP2:
1103 case ISD::STRICT_FLOG:
1104 case ISD::STRICT_FLOG10:
1105 case ISD::STRICT_FLOG2:
1106 case ISD::STRICT_FRINT:
1107 case ISD::STRICT_FNEARBYINT:
1108 case ISD::STRICT_FMAXNUM:
1109 case ISD::STRICT_FMINNUM:
1110 case ISD::STRICT_FCEIL:
1111 case ISD::STRICT_FFLOOR:
1112 case ISD::STRICT_FROUND:
1113 case ISD::STRICT_FTRUNC:
1114 // These pseudo-ops get legalized as if they were their non-strict
1115 // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT
1116 // is also legal, but if ISD::FSQRT requires expansion then so does
1117 // ISD::STRICT_FSQRT.
1118 Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
1119 Node->getValueType(0));
1120 break;
1121 case ISD::SADDSAT:
1122 case ISD::UADDSAT:
1123 case ISD::SSUBSAT:
1124 case ISD::USUBSAT: {
1125 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1126 break;
1127 }
1128 case ISD::SMULFIX:
1129 case ISD::UMULFIX: {
1130 unsigned Scale = Node->getConstantOperandVal(2);
1131 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1132 Node->getValueType(0), Scale);
1133 break;
1134 }
1135 case ISD::MSCATTER:
1136 Action = TLI.getOperationAction(Node->getOpcode(),
1137 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1138 break;
1139 case ISD::MSTORE:
1140 Action = TLI.getOperationAction(Node->getOpcode(),
1141 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1142 break;
1143 case ISD::VECREDUCE_FADD:
1144 case ISD::VECREDUCE_FMUL:
1145 case ISD::VECREDUCE_ADD:
1146 case ISD::VECREDUCE_MUL:
1147 case ISD::VECREDUCE_AND:
1148 case ISD::VECREDUCE_OR:
1149 case ISD::VECREDUCE_XOR:
1150 case ISD::VECREDUCE_SMAX:
1151 case ISD::VECREDUCE_SMIN:
1152 case ISD::VECREDUCE_UMAX:
1153 case ISD::VECREDUCE_UMIN:
1154 case ISD::VECREDUCE_FMAX:
1155 case ISD::VECREDUCE_FMIN:
1156 Action = TLI.getOperationAction(
1157 Node->getOpcode(), Node->getOperand(0).getValueType());
1158 break;
1159 default:
1160 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1161 Action = TargetLowering::Legal;
1162 } else {
1163 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1164 }
1165 break;
1166 }
1167
1168 if (SimpleFinishLegalizing) {
1169 SDNode *NewNode = Node;
1170 switch (Node->getOpcode()) {
1171 default: break;
1172 case ISD::SHL:
1173 case ISD::SRL:
1174 case ISD::SRA:
1175 case ISD::ROTL:
1176 case ISD::ROTR: {
1177 // Legalizing shifts/rotates requires adjusting the shift amount
1178 // to the appropriate width.
1179 SDValue Op0 = Node->getOperand(0);
1180 SDValue Op1 = Node->getOperand(1);
1181 if (!Op1.getValueType().isVector()) {
1182 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1183 // The getShiftAmountOperand() may create a new operand node or
1184 // return the existing one. If new operand is created we need
1185 // to update the parent node.
1186 // Do not try to legalize SAO here! It will be automatically legalized
1187 // in the next round.
1188 if (SAO != Op1)
1189 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1190 }
1191 }
1192 break;
1193 case ISD::FSHL:
1194 case ISD::FSHR:
1195 case ISD::SRL_PARTS:
1196 case ISD::SRA_PARTS:
1197 case ISD::SHL_PARTS: {
1198 // Legalizing shifts/rotates requires adjusting the shift amount
1199 // to the appropriate width.
1200 SDValue Op0 = Node->getOperand(0);
1201 SDValue Op1 = Node->getOperand(1);
1202 SDValue Op2 = Node->getOperand(2);
1203 if (!Op2.getValueType().isVector()) {
1204 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1205 // The getShiftAmountOperand() may create a new operand node or
1206 // return the existing one. If new operand is created we need
1207 // to update the parent node.
1208 if (SAO != Op2)
1209 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1210 }
1211 break;
1212 }
1213 }
1214
1215 if (NewNode != Node) {
1216 ReplaceNode(Node, NewNode);
1217 Node = NewNode;
1218 }
1219 switch (Action) {
1220 case TargetLowering::Legal:
1221 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legal node: nothing to do\n"
; } } while (false)
;
1222 return;
1223 case TargetLowering::Custom:
1224 LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying custom legalization\n"
; } } while (false)
;
1225 // FIXME: The handling for custom lowering with multiple results is
1226 // a complete mess.
1227 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1228 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1229 return;
1230
1231 if (Node->getNumValues() == 1) {
1232 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1233 // We can just directly replace this node with the lowered value.
1234 ReplaceNode(SDValue(Node, 0), Res);
1235 return;
1236 }
1237
1238 SmallVector<SDValue, 8> ResultVals;
1239 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
1240 ResultVals.push_back(Res.getValue(i));
1241 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully custom legalized node\n"
; } } while (false)
;
1242 ReplaceNode(Node, ResultVals.data());
1243 return;
1244 }
1245 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not custom legalize node\n"
; } } while (false)
;
1246 LLVM_FALLTHROUGH[[clang::fallthrough]];
1247 case TargetLowering::Expand:
1248 if (ExpandNode(Node))
1249 return;
1250 LLVM_FALLTHROUGH[[clang::fallthrough]];
1251 case TargetLowering::LibCall:
1252 ConvertNodeToLibcall(Node);
1253 return;
1254 case TargetLowering::Promote:
1255 PromoteNode(Node);
1256 return;
1257 }
1258 }
1259
1260 switch (Node->getOpcode()) {
1261 default:
1262#ifndef NDEBUG
1263 dbgs() << "NODE: ";
1264 Node->dump( &DAG);
1265 dbgs() << "\n";
1266#endif
1267 llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1267)
;
1268
1269 case ISD::CALLSEQ_START:
1270 case ISD::CALLSEQ_END:
1271 break;
1272 case ISD::LOAD:
1273 return LegalizeLoadOps(Node);
1274 case ISD::STORE:
1275 return LegalizeStoreOps(Node);
1276 }
1277}
1278
1279SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1280 SDValue Vec = Op.getOperand(0);
1281 SDValue Idx = Op.getOperand(1);
1282 SDLoc dl(Op);
1283
1284 // Before we generate a new store to a temporary stack slot, see if there is
1285 // already one that we can use. There often is because when we scalarize
1286 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1287 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1288 // the vector. If all are expanded here, we don't want one store per vector
1289 // element.
1290
1291 // Caches for hasPredecessorHelper
1292 SmallPtrSet<const SDNode *, 32> Visited;
1293 SmallVector<const SDNode *, 16> Worklist;
1294 Visited.insert(Op.getNode());
1295 Worklist.push_back(Idx.getNode());
1296 SDValue StackPtr, Ch;
1297 for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1298 UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1299 SDNode *User = *UI;
1300 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1301 if (ST->isIndexed() || ST->isTruncatingStore() ||
1302 ST->getValue() != Vec)
1303 continue;
1304
1305 // Make sure that nothing else could have stored into the destination of
1306 // this store.
1307 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1308 continue;
1309
1310 // If the index is dependent on the store we will introduce a cycle when
1311 // creating the load (the load uses the index, and by replacing the chain
1312 // we will make the index dependent on the load). Also, the store might be
1313 // dependent on the extractelement and introduce a cycle when creating
1314 // the load.
1315 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1316 ST->hasPredecessor(Op.getNode()))
1317 continue;
1318
1319 StackPtr = ST->getBasePtr();
1320 Ch = SDValue(ST, 0);
1321 break;
1322 }
1323 }
1324
1325 EVT VecVT = Vec.getValueType();
1326
1327 if (!Ch.getNode()) {
1328 // Store the value to a temporary stack slot, then LOAD the returned part.
1329 StackPtr = DAG.CreateStackTemporary(VecVT);
1330 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1331 MachinePointerInfo());
1332 }
1333
1334 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1335
1336 SDValue NewLoad;
1337
1338 if (Op.getValueType().isVector())
1339 NewLoad =
1340 DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1341 else
1342 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1343 MachinePointerInfo(),
1344 VecVT.getVectorElementType());
1345
1346 // Replace the chain going out of the store, by the one out of the load.
1347 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1348
1349 // We introduced a cycle though, so update the loads operands, making sure
1350 // to use the original store's chain as an incoming chain.
1351 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1352 NewLoad->op_end());
1353 NewLoadOperands[0] = Ch;
1354 NewLoad =
1355 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1356 return NewLoad;
1357}
1358
1359SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1360 assert(Op.getValueType().isVector() && "Non-vector insert subvector!")((Op.getValueType().isVector() && "Non-vector insert subvector!"
) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Non-vector insert subvector!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1360, __PRETTY_FUNCTION__))
;
1361
1362 SDValue Vec = Op.getOperand(0);
1363 SDValue Part = Op.getOperand(1);
1364 SDValue Idx = Op.getOperand(2);
1365 SDLoc dl(Op);
1366
1367 // Store the value to a temporary stack slot, then LOAD the returned part.
1368 EVT VecVT = Vec.getValueType();
1369 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1370 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1371 MachinePointerInfo PtrInfo =
1372 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1373
1374 // First store the whole vector.
1375 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1376
1377 // Then store the inserted part.
1378 SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1379
1380 // Store the subvector.
1381 Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo());
1382
1383 // Finally, load the updated vector.
1384 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1385}
1386
1387SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1388 // We can't handle this case efficiently. Allocate a sufficiently
1389 // aligned object on the stack, store each element into it, then load
1390 // the result as a vector.
1391 // Create the stack frame object.
1392 EVT VT = Node->getValueType(0);
1393 EVT EltVT = VT.getVectorElementType();
1394 SDLoc dl(Node);
1395 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1396 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1397 MachinePointerInfo PtrInfo =
1398 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1399
1400 // Emit a store of each element to the stack slot.
1401 SmallVector<SDValue, 8> Stores;
1402 unsigned TypeByteSize = EltVT.getSizeInBits() / 8;
1403 // Store (in the right endianness) the elements to memory.
1404 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1405 // Ignore undef elements.
1406 if (Node->getOperand(i).isUndef()) continue;
1407
1408 unsigned Offset = TypeByteSize*i;
1409
1410 SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType());
1411 Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx);
1412
1413 // If the destination vector element type is narrower than the source
1414 // element type, only store the bits necessary.
1415 if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) {
1416 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1417 Node->getOperand(i), Idx,
1418 PtrInfo.getWithOffset(Offset), EltVT));
1419 } else
1420 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1421 Idx, PtrInfo.getWithOffset(Offset)));
1422 }
1423
1424 SDValue StoreChain;
1425 if (!Stores.empty()) // Not all undef elements?
1426 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1427 else
1428 StoreChain = DAG.getEntryNode();
1429
1430 // Result is a load from the stack slot.
1431 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1432}
1433
1434/// Bitcast a floating-point value to an integer value. Only bitcast the part
1435/// containing the sign bit if the target has no integer value capable of
1436/// holding all bits of the floating-point value.
1437void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1438 const SDLoc &DL,
1439 SDValue Value) const {
1440 EVT FloatVT = Value.getValueType();
1441 unsigned NumBits = FloatVT.getSizeInBits();
1442 State.FloatVT = FloatVT;
1443 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1444 // Convert to an integer of the same size.
1445 if (TLI.isTypeLegal(IVT)) {
1446 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1447 State.SignMask = APInt::getSignMask(NumBits);
1448 State.SignBit = NumBits - 1;
1449 return;
1450 }
1451
1452 auto &DataLayout = DAG.getDataLayout();
1453 // Store the float to memory, then load the sign part out as an integer.
1454 MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1455 // First create a temporary that is aligned for both the load and store.
1456 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1457 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1458 // Then store the float to it.
1459 State.FloatPtr = StackPtr;
1460 MachineFunction &MF = DAG.getMachineFunction();
1461 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1462 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1463 State.FloatPointerInfo);
1464
1465 SDValue IntPtr;
1466 if (DataLayout.isBigEndian()) {
1467 assert(FloatVT.isByteSized() && "Unsupported floating point type!")((FloatVT.isByteSized() && "Unsupported floating point type!"
) ? static_cast<void> (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1467, __PRETTY_FUNCTION__))
;
1468 // Load out a legal integer with the same sign bit as the float.
1469 IntPtr = StackPtr;
1470 State.IntPointerInfo = State.FloatPointerInfo;
1471 } else {
1472 // Advance the pointer so that the loaded byte will contain the sign bit.
1473 unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1;
1474 IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr,
1475 DAG.getConstant(ByteOffset, DL, StackPtr.getValueType()));
1476 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1477 ByteOffset);
1478 }
1479
1480 State.IntPtr = IntPtr;
1481 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1482 State.IntPointerInfo, MVT::i8);
1483 State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7);
1484 State.SignBit = 7;
1485}
1486
1487/// Replace the integer value produced by getSignAsIntValue() with a new value
1488/// and cast the result back to a floating-point type.
1489SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1490 const SDLoc &DL,
1491 SDValue NewIntValue) const {
1492 if (!State.Chain)
1493 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1494
1495 // Override the part containing the sign bit in the value stored on the stack.
1496 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1497 State.IntPointerInfo, MVT::i8);
1498 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1499 State.FloatPointerInfo);
1500}
1501
1502SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1503 SDLoc DL(Node);
1504 SDValue Mag = Node->getOperand(0);
1505 SDValue Sign = Node->getOperand(1);
1506
1507 // Get sign bit into an integer value.
1508 FloatSignAsInt SignAsInt;
1509 getSignAsIntValue(SignAsInt, DL, Sign);
1510
1511 EVT IntVT = SignAsInt.IntValue.getValueType();
1512 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1513 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1514 SignMask);
1515
1516 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1517 EVT FloatVT = Mag.getValueType();
1518 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1519 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1520 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1521 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1522 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1523 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1524 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1525 }
1526
1527 // Transform Mag value to integer, and clear the sign bit.
1528 FloatSignAsInt MagAsInt;
1529 getSignAsIntValue(MagAsInt, DL, Mag);
1530 EVT MagVT = MagAsInt.IntValue.getValueType();
1531 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1532 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1533 ClearSignMask);
1534
1535 // Get the signbit at the right position for MagAsInt.
1536 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1537 EVT ShiftVT = IntVT;
1538 if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) {
1539 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1540 ShiftVT = MagVT;
1541 }
1542 if (ShiftAmount > 0) {
1543 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1544 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1545 } else if (ShiftAmount < 0) {
1546 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1547 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1548 }
1549 if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) {
1550 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1551 }
1552
1553 // Store the part with the modified sign and convert back to float.
1554 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1555 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1556}
1557
1558SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1559 SDLoc DL(Node);
1560 SDValue Value = Node->getOperand(0);
1561
1562 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1563 EVT FloatVT = Value.getValueType();
1564 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1565 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1566 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1567 }
1568
1569 // Transform value to integer, clear the sign bit and transform back.
1570 FloatSignAsInt ValueAsInt;
1571 getSignAsIntValue(ValueAsInt, DL, Value);
1572 EVT IntVT = ValueAsInt.IntValue.getValueType();
1573 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1574 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1575 ClearSignMask);
1576 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1577}
1578
1579void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1580 SmallVectorImpl<SDValue> &Results) {
1581 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
1582 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1583, __PRETTY_FUNCTION__))
1583 " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1583, __PRETTY_FUNCTION__))
;
1584 SDLoc dl(Node);
1585 EVT VT = Node->getValueType(0);
1586 SDValue Tmp1 = SDValue(Node, 0);
1587 SDValue Tmp2 = SDValue(Node, 1);
1588 SDValue Tmp3 = Node->getOperand(2);
1589 SDValue Chain = Tmp1.getOperand(0);
1590
1591 // Chain the dynamic stack allocation so that it doesn't modify the stack
1592 // pointer when other instructions are using the stack.
1593 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1594
1595 SDValue Size = Tmp2.getOperand(1);
1596 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1597 Chain = SP.getValue(1);
1598 unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue();
1599 unsigned StackAlign =
1600 DAG.getSubtarget().getFrameLowering()->getStackAlignment();
1601 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
1602 if (Align > StackAlign)
1603 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1604 DAG.getConstant(-(uint64_t)Align, dl, VT));
1605 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1606
1607 Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1608 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1609
1610 Results.push_back(Tmp1);
1611 Results.push_back(Tmp2);
1612}
1613
1614/// Legalize a SETCC with given LHS and RHS and condition code CC on the current
1615/// target.
1616///
1617/// If the SETCC has been legalized using AND / OR, then the legalized node
1618/// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert
1619/// will be set to false.
1620///
1621/// If the SETCC has been legalized by using getSetCCSwappedOperands(),
1622/// then the values of LHS and RHS will be swapped, CC will be set to the
1623/// new condition, and NeedInvert will be set to false.
1624///
1625/// If the SETCC has been legalized using the inverse condcode, then LHS and
1626/// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert
1627/// will be set to true. The caller must invert the result of the SETCC with
1628/// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect
1629/// of a true/false result.
1630///
1631/// \returns true if the SetCC has been legalized, false if it hasn't.
1632bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS,
1633 SDValue &RHS, SDValue &CC,
1634 bool &NeedInvert,
1635 const SDLoc &dl) {
1636 MVT OpVT = LHS.getSimpleValueType();
1637 ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get();
1638 NeedInvert = false;
1639 bool NeedSwap = false;
1640 switch (TLI.getCondCodeAction(CCCode, OpVT)) {
1641 default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1641)
;
1642 case TargetLowering::Legal:
1643 // Nothing to do.
1644 break;
1645 case TargetLowering::Expand: {
1646 ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode);
1647 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1648 std::swap(LHS, RHS);
1649 CC = DAG.getCondCode(InvCC);
1650 return true;
1651 }
1652 // Swapping operands didn't work. Try inverting the condition.
1653 InvCC = getSetCCInverse(CCCode, OpVT.isInteger());
1654 if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1655 // If inverting the condition is not enough, try swapping operands
1656 // on top of it.
1657 InvCC = ISD::getSetCCSwappedOperands(InvCC);
1658 NeedSwap = true;
1659 }
1660 if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) {
1661 CC = DAG.getCondCode(InvCC);
1662 NeedInvert = true;
1663 if (NeedSwap)
1664 std::swap(LHS, RHS);
1665 return true;
1666 }
1667
1668 ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
1669 unsigned Opc = 0;
1670 switch (CCCode) {
1671 default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1671)
;
1672 case ISD::SETO:
1673 assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1674, __PRETTY_FUNCTION__))
1674 && "If SETO is expanded, SETOEQ must be legal!")((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1674, __PRETTY_FUNCTION__))
;
1675 CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
1676 case ISD::SETUO:
1677 assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1678, __PRETTY_FUNCTION__))
1678 && "If SETUO is expanded, SETUNE must be legal!")((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1678, __PRETTY_FUNCTION__))
;
1679 CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break;
1680 case ISD::SETOEQ:
1681 case ISD::SETOGT:
1682 case ISD::SETOGE:
1683 case ISD::SETOLT:
1684 case ISD::SETOLE:
1685 case ISD::SETONE:
1686 case ISD::SETUEQ:
1687 case ISD::SETUNE:
1688 case ISD::SETUGT:
1689 case ISD::SETUGE:
1690 case ISD::SETULT:
1691 case ISD::SETULE:
1692 // If we are floating point, assign and break, otherwise fall through.
1693 if (!OpVT.isInteger()) {
1694 // We can use the 4th bit to tell if we are the unordered
1695 // or ordered version of the opcode.
1696 CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
1697 Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
1698 CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
1699 break;
1700 }
1701 // Fallthrough if we are unsigned integer.
1702 LLVM_FALLTHROUGH[[clang::fallthrough]];
1703 case ISD::SETLE:
1704 case ISD::SETGT:
1705 case ISD::SETGE:
1706 case ISD::SETLT:
1707 case ISD::SETNE:
1708 case ISD::SETEQ:
1709 // If all combinations of inverting the condition and swapping operands
1710 // didn't work then we have no means to expand the condition.
1711 llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1711)
;
1712 }
1713
1714 SDValue SetCC1, SetCC2;
1715 if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
1716 // If we aren't the ordered or unorder operation,
1717 // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
1718 SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
1719 SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
1720 } else {
1721 // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
1722 SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
1723 SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
1724 }
1725 LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
1726 RHS = SDValue();
1727 CC = SDValue();
1728 return true;
1729 }
1730 }
1731 return false;
1732}
1733
1734/// Emit a store/load combination to the stack. This stores
1735/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1736/// a load from the stack slot to DestVT, extending it if needed.
1737/// The resultant code need not be legal.
1738SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1739 EVT DestVT, const SDLoc &dl) {
1740 // Create the stack frame object.
1741 unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment(
1742 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1743 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
1744
1745 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1746 int SPFI = StackPtrFI->getIndex();
1747 MachinePointerInfo PtrInfo =
1748 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1749
1750 unsigned SrcSize = SrcOp.getValueSizeInBits();
1751 unsigned SlotSize = SlotVT.getSizeInBits();
1752 unsigned DestSize = DestVT.getSizeInBits();
1753 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1754 unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType);
1755
1756 // Emit a store to the stack slot. Use a truncstore if the input value is
1757 // later than DestVT.
1758 SDValue Store;
1759
1760 if (SrcSize > SlotSize)
1761 Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo,
1762 SlotVT, SrcAlign);
1763 else {
1764 assert(SrcSize == SlotSize && "Invalid store")((SrcSize == SlotSize && "Invalid store") ? static_cast
<void> (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1764, __PRETTY_FUNCTION__))
;
1765 Store =
1766 DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1767 }
1768
1769 // Result is a load from the stack slot.
1770 if (SlotSize == DestSize)
1771 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1772
1773 assert(SlotSize < DestSize && "Unknown extension!")((SlotSize < DestSize && "Unknown extension!") ? static_cast
<void> (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1773, __PRETTY_FUNCTION__))
;
1774 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1775 DestAlign);
1776}
1777
1778SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1779 SDLoc dl(Node);
1780 // Create a vector sized/aligned stack slot, store the value to element #0,
1781 // then load the whole vector back out.
1782 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1783
1784 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1785 int SPFI = StackPtrFI->getIndex();
1786
1787 SDValue Ch = DAG.getTruncStore(
1788 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1789 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1790 Node->getValueType(0).getVectorElementType());
1791 return DAG.getLoad(
1792 Node->getValueType(0), dl, Ch, StackPtr,
1793 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1794}
1795
1796static bool
1797ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG,
1798 const TargetLowering &TLI, SDValue &Res) {
1799 unsigned NumElems = Node->getNumOperands();
1800 SDLoc dl(Node);
1801 EVT VT = Node->getValueType(0);
1802
1803 // Try to group the scalars into pairs, shuffle the pairs together, then
1804 // shuffle the pairs of pairs together, etc. until the vector has
1805 // been built. This will work only if all of the necessary shuffle masks
1806 // are legal.
1807
1808 // We do this in two phases; first to check the legality of the shuffles,
1809 // and next, assuming that all shuffles are legal, to create the new nodes.
1810 for (int Phase = 0; Phase < 2; ++Phase) {
1811 SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals,
1812 NewIntermedVals;
1813 for (unsigned i = 0; i < NumElems; ++i) {
1814 SDValue V = Node->getOperand(i);
1815 if (V.isUndef())
1816 continue;
1817
1818 SDValue Vec;
1819 if (Phase)
1820 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1821 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1822 }
1823
1824 while (IntermedVals.size() > 2) {
1825 NewIntermedVals.clear();
1826 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1827 // This vector and the next vector are shuffled together (simply to
1828 // append the one to the other).
1829 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1830
1831 SmallVector<int, 16> FinalIndices;
1832 FinalIndices.reserve(IntermedVals[i].second.size() +
1833 IntermedVals[i+1].second.size());
1834
1835 int k = 0;
1836 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1837 ++j, ++k) {
1838 ShuffleVec[k] = j;
1839 FinalIndices.push_back(IntermedVals[i].second[j]);
1840 }
1841 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1842 ++j, ++k) {
1843 ShuffleVec[k] = NumElems + j;
1844 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1845 }
1846
1847 SDValue Shuffle;
1848 if (Phase)
1849 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1850 IntermedVals[i+1].first,
1851 ShuffleVec);
1852 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1853 return false;
1854 NewIntermedVals.push_back(
1855 std::make_pair(Shuffle, std::move(FinalIndices)));
1856 }
1857
1858 // If we had an odd number of defined values, then append the last
1859 // element to the array of new vectors.
1860 if ((IntermedVals.size() & 1) != 0)
1861 NewIntermedVals.push_back(IntermedVals.back());
1862
1863 IntermedVals.swap(NewIntermedVals);
1864 }
1865
1866 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1867, __PRETTY_FUNCTION__))
1867 "Invalid number of intermediate vectors")((IntermedVals.size() <= 2 && IntermedVals.size() >
0 && "Invalid number of intermediate vectors") ? static_cast
<void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1867, __PRETTY_FUNCTION__))
;
1868 SDValue Vec1 = IntermedVals[0].first;
1869 SDValue Vec2;
1870 if (IntermedVals.size() > 1)
1871 Vec2 = IntermedVals[1].first;
1872 else if (Phase)
1873 Vec2 = DAG.getUNDEF(VT);
1874
1875 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1876 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1877 ShuffleVec[IntermedVals[0].second[i]] = i;
1878 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1879 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1880
1881 if (Phase)
1882 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1883 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1884 return false;
1885 }
1886
1887 return true;
1888}
1889
1890/// Expand a BUILD_VECTOR node on targets that don't
1891/// support the operation, but do support the resultant vector type.
1892SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1893 unsigned NumElems = Node->getNumOperands();
1894 SDValue Value1, Value2;
1895 SDLoc dl(Node);
1896 EVT VT = Node->getValueType(0);
1897 EVT OpVT = Node->getOperand(0).getValueType();
1898 EVT EltVT = VT.getVectorElementType();
1899
1900 // If the only non-undef value is the low element, turn this into a
1901 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1902 bool isOnlyLowElement = true;
1903 bool MoreThanTwoValues = false;
1904 bool isConstant = true;
1905 for (unsigned i = 0; i < NumElems; ++i) {
1906 SDValue V = Node->getOperand(i);
1907 if (V.isUndef())
1908 continue;
1909 if (i > 0)
1910 isOnlyLowElement = false;
1911 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1912 isConstant = false;
1913
1914 if (!Value1.getNode()) {
1915 Value1 = V;
1916 } else if (!Value2.getNode()) {
1917 if (V != Value1)
1918 Value2 = V;
1919 } else if (V != Value1 && V != Value2) {
1920 MoreThanTwoValues = true;
1921 }
1922 }
1923
1924 if (!Value1.getNode())
1925 return DAG.getUNDEF(VT);
1926
1927 if (isOnlyLowElement)
1928 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1929
1930 // If all elements are constants, create a load from the constant pool.
1931 if (isConstant) {
1932 SmallVector<Constant*, 16> CV;
1933 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1934 if (ConstantFPSDNode *V =
1935 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1936 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1937 } else if (ConstantSDNode *V =
1938 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1939 if (OpVT==EltVT)
1940 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1941 else {
1942 // If OpVT and EltVT don't match, EltVT is not legal and the
1943 // element values have been promoted/truncated earlier. Undo this;
1944 // we don't want a v16i8 to become a v16i32 for example.
1945 const ConstantInt *CI = V->getConstantIntValue();
1946 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1947 CI->getZExtValue()));
1948 }
1949 } else {
1950 assert(Node->getOperand(i).isUndef())((Node->getOperand(i).isUndef()) ? static_cast<void>
(0) : __assert_fail ("Node->getOperand(i).isUndef()", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 1950, __PRETTY_FUNCTION__))
;
1951 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1952 CV.push_back(UndefValue::get(OpNTy));
1953 }
1954 }
1955 Constant *CP = ConstantVector::get(CV);
1956 SDValue CPIdx =
1957 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1958 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
1959 return DAG.getLoad(
1960 VT, dl, DAG.getEntryNode(), CPIdx,
1961 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1962 Alignment);
1963 }
1964
1965 SmallSet<SDValue, 16> DefinedValues;
1966 for (unsigned i = 0; i < NumElems; ++i) {
1967 if (Node->getOperand(i).isUndef())
1968 continue;
1969 DefinedValues.insert(Node->getOperand(i));
1970 }
1971
1972 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1973 if (!MoreThanTwoValues) {
1974 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1975 for (unsigned i = 0; i < NumElems; ++i) {
1976 SDValue V = Node->getOperand(i);
1977 if (V.isUndef())
1978 continue;
1979 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1980 }
1981 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1982 // Get the splatted value into the low element of a vector register.
1983 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1984 SDValue Vec2;
1985 if (Value2.getNode())
1986 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1987 else
1988 Vec2 = DAG.getUNDEF(VT);
1989
1990 // Return shuffle(LowValVec, undef, <0,0,0,0>)
1991 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1992 }
1993 } else {
1994 SDValue Res;
1995 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1996 return Res;
1997 }
1998 }
1999
2000 // Otherwise, we can't handle this case efficiently.
2001 return ExpandVectorBuildThroughStack(Node);
2002}
2003
2004// Expand a node into a call to a libcall. If the result value
2005// does not fit into a register, return the lo part and set the hi part to the
2006// by-reg argument. If it does fit into a single register, return the result
2007// and leave the Hi part unset.
2008SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2009 bool isSigned) {
2010 TargetLowering::ArgListTy Args;
2011 TargetLowering::ArgListEntry Entry;
2012 for (const SDValue &Op : Node->op_values()) {
2013 EVT ArgVT = Op.getValueType();
2014 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2015 Entry.Node = Op;
2016 Entry.Ty = ArgTy;
2017 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2018 Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2019 Args.push_back(Entry);
2020 }
2021 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2022 TLI.getPointerTy(DAG.getDataLayout()));
2023
2024 EVT RetVT = Node->getValueType(0);
2025 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2026
2027 // By default, the input chain to this libcall is the entry node of the
2028 // function. If the libcall is going to be emitted as a tail call then
2029 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2030 // node which is being folded has a non-entry input chain.
2031 SDValue InChain = DAG.getEntryNode();
2032
2033 // isTailCall may be true since the callee does not reference caller stack
2034 // frame. Check if it's in the right position and that the return types match.
2035 SDValue TCChain = InChain;
2036 const Function &F = DAG.getMachineFunction().getFunction();
2037 bool isTailCall =
2038 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2039 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2040 if (isTailCall)
2041 InChain = TCChain;
2042
2043 TargetLowering::CallLoweringInfo CLI(DAG);
2044 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2045 CLI.setDebugLoc(SDLoc(Node))
2046 .setChain(InChain)
2047 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2048 std::move(Args))
2049 .setTailCall(isTailCall)
2050 .setSExtResult(signExtend)
2051 .setZExtResult(!signExtend)
2052 .setIsPostTypeLegalization(true);
2053
2054 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2055
2056 if (!CallInfo.second.getNode()) {
2057 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created tailcall: "; DAG.
getRoot().dump(); } } while (false)
;
2058 // It's a tailcall, return the chain (which is the DAG root).
2059 return DAG.getRoot();
2060 }
2061
2062 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Created libcall: "; CallInfo
.first.dump(); } } while (false)
;
2063 return CallInfo.first;
2064}
2065
2066// Expand a node into a call to a libcall. Similar to
2067// ExpandLibCall except that the first operand is the in-chain.
2068std::pair<SDValue, SDValue>
2069SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC,
2070 SDNode *Node,
2071 bool isSigned) {
2072 SDValue InChain = Node->getOperand(0);
2073
2074 TargetLowering::ArgListTy Args;
2075 TargetLowering::ArgListEntry Entry;
2076 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) {
2077 EVT ArgVT = Node->getOperand(i).getValueType();
2078 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2079 Entry.Node = Node->getOperand(i);
2080 Entry.Ty = ArgTy;
2081 Entry.IsSExt = isSigned;
2082 Entry.IsZExt = !isSigned;
2083 Args.push_back(Entry);
2084 }
2085 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2086 TLI.getPointerTy(DAG.getDataLayout()));
2087
2088 Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext());
2089
2090 TargetLowering::CallLoweringInfo CLI(DAG);
2091 CLI.setDebugLoc(SDLoc(Node))
2092 .setChain(InChain)
2093 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2094 std::move(Args))
2095 .setSExtResult(isSigned)
2096 .setZExtResult(!isSigned);
2097
2098 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2099
2100 return CallInfo;
2101}
2102
2103SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2104 RTLIB::Libcall Call_F32,
2105 RTLIB::Libcall Call_F64,
2106 RTLIB::Libcall Call_F80,
2107 RTLIB::Libcall Call_F128,
2108 RTLIB::Libcall Call_PPCF128) {
2109 if (Node->isStrictFPOpcode())
2110 Node = DAG.mutateStrictFPToFP(Node);
2111
2112 RTLIB::Libcall LC;
2113 switch (Node->getSimpleValueType(0).SimpleTy) {
2114 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2114)
;
2115 case MVT::f32: LC = Call_F32; break;
2116 case MVT::f64: LC = Call_F64; break;
2117 case MVT::f80: LC = Call_F80; break;
2118 case MVT::f128: LC = Call_F128; break;
2119 case MVT::ppcf128: LC = Call_PPCF128; break;
2120 }
2121 return ExpandLibCall(LC, Node, false);
2122}
2123
2124SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2125 RTLIB::Libcall Call_I8,
2126 RTLIB::Libcall Call_I16,
2127 RTLIB::Libcall Call_I32,
2128 RTLIB::Libcall Call_I64,
2129 RTLIB::Libcall Call_I128) {
2130 RTLIB::Libcall LC;
2131 switch (Node->getSimpleValueType(0).SimpleTy) {
2132 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2132)
;
2133 case MVT::i8: LC = Call_I8; break;
2134 case MVT::i16: LC = Call_I16; break;
2135 case MVT::i32: LC = Call_I32; break;
2136 case MVT::i64: LC = Call_I64; break;
2137 case MVT::i128: LC = Call_I128; break;
2138 }
2139 return ExpandLibCall(LC, Node, isSigned);
2140}
2141
2142/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2143void
2144SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2145 SmallVectorImpl<SDValue> &Results) {
2146 unsigned Opcode = Node->getOpcode();
2147 bool isSigned = Opcode == ISD::SDIVREM;
2148
2149 RTLIB::Libcall LC;
2150 switch (Node->getSimpleValueType(0).SimpleTy) {
2151 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2151)
;
2152 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2153 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2154 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2155 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2156 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2157 }
2158
2159 // The input chain to this libcall is the entry node of the function.
2160 // Legalizing the call will automatically add the previous call to the
2161 // dependence.
2162 SDValue InChain = DAG.getEntryNode();
2163
2164 EVT RetVT = Node->getValueType(0);
2165 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2166
2167 TargetLowering::ArgListTy Args;
2168 TargetLowering::ArgListEntry Entry;
2169 for (const SDValue &Op : Node->op_values()) {
2170 EVT ArgVT = Op.getValueType();
2171 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2172 Entry.Node = Op;
2173 Entry.Ty = ArgTy;
2174 Entry.IsSExt = isSigned;
2175 Entry.IsZExt = !isSigned;
2176 Args.push_back(Entry);
2177 }
2178
2179 // Also pass the return address of the remainder.
2180 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2181 Entry.Node = FIPtr;
2182 Entry.Ty = RetTy->getPointerTo();
2183 Entry.IsSExt = isSigned;
2184 Entry.IsZExt = !isSigned;
2185 Args.push_back(Entry);
2186
2187 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2188 TLI.getPointerTy(DAG.getDataLayout()));
2189
2190 SDLoc dl(Node);
2191 TargetLowering::CallLoweringInfo CLI(DAG);
2192 CLI.setDebugLoc(dl)
2193 .setChain(InChain)
2194 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2195 std::move(Args))
2196 .setSExtResult(isSigned)
2197 .setZExtResult(!isSigned);
2198
2199 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2200
2201 // Remainder is loaded back from the stack frame.
2202 SDValue Rem =
2203 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2204 Results.push_back(CallInfo.first);
2205 Results.push_back(Rem);
2206}
2207
2208/// Return true if sincos libcall is available.
2209static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2210 RTLIB::Libcall LC;
2211 switch (Node->getSimpleValueType(0).SimpleTy) {
2212 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2212)
;
2213 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2214 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2215 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2216 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2217 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2218 }
2219 return TLI.getLibcallName(LC) != nullptr;
2220}
2221
2222/// Only issue sincos libcall if both sin and cos are needed.
2223static bool useSinCos(SDNode *Node) {
2224 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2225 ? ISD::FCOS : ISD::FSIN;
2226
2227 SDValue Op0 = Node->getOperand(0);
2228 for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2229 UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2230 SDNode *User = *UI;
2231 if (User == Node)
2232 continue;
2233 // The other user might have been turned into sincos already.
2234 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2235 return true;
2236 }
2237 return false;
2238}
2239
2240/// Issue libcalls to sincos to compute sin / cos pairs.
2241void
2242SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2243 SmallVectorImpl<SDValue> &Results) {
2244 RTLIB::Libcall LC;
2245 switch (Node->getSimpleValueType(0).SimpleTy) {
2246 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2246)
;
2247 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2248 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2249 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2250 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2251 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2252 }
2253
2254 // The input chain to this libcall is the entry node of the function.
2255 // Legalizing the call will automatically add the previous call to the
2256 // dependence.
2257 SDValue InChain = DAG.getEntryNode();
2258
2259 EVT RetVT = Node->getValueType(0);
2260 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2261
2262 TargetLowering::ArgListTy Args;
2263 TargetLowering::ArgListEntry Entry;
2264
2265 // Pass the argument.
2266 Entry.Node = Node->getOperand(0);
2267 Entry.Ty = RetTy;
2268 Entry.IsSExt = false;
2269 Entry.IsZExt = false;
2270 Args.push_back(Entry);
2271
2272 // Pass the return address of sin.
2273 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2274 Entry.Node = SinPtr;
2275 Entry.Ty = RetTy->getPointerTo();
2276 Entry.IsSExt = false;
2277 Entry.IsZExt = false;
2278 Args.push_back(Entry);
2279
2280 // Also pass the return address of the cos.
2281 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2282 Entry.Node = CosPtr;
2283 Entry.Ty = RetTy->getPointerTo();
2284 Entry.IsSExt = false;
2285 Entry.IsZExt = false;
2286 Args.push_back(Entry);
2287
2288 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2289 TLI.getPointerTy(DAG.getDataLayout()));
2290
2291 SDLoc dl(Node);
2292 TargetLowering::CallLoweringInfo CLI(DAG);
2293 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2294 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2295 std::move(Args));
2296
2297 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2298
2299 Results.push_back(
2300 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2301 Results.push_back(
2302 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2303}
2304
2305/// This function is responsible for legalizing a
2306/// INT_TO_FP operation of the specified operand when the target requests that
2307/// we expand it. At this point, we know that the result and operand types are
2308/// legal for the target.
2309SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0,
2310 EVT DestVT,
2311 const SDLoc &dl) {
2312 EVT SrcVT = Op0.getValueType();
2313
2314 // TODO: Should any fast-math-flags be set for the created nodes?
2315 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; }
} while (false)
;
2316 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
2317 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
2318 "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double "
"expansion\n"; } } while (false)
;
2319
2320 // Get the stack frame index of a 8 byte buffer.
2321 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2322
2323 // word offset constant for Hi/Lo address computation
2324 SDValue WordOff = DAG.getConstant(sizeof(int), dl,
2325 StackSlot.getValueType());
2326 // set up Hi and Lo (into buffer) address based on endian
2327 SDValue Hi = StackSlot;
2328 SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(),
2329 StackSlot, WordOff);
2330 if (DAG.getDataLayout().isLittleEndian())
2331 std::swap(Hi, Lo);
2332
2333 // if signed map to unsigned space
2334 SDValue Op0Mapped;
2335 if (isSigned) {
2336 // constant used to invert sign bit (signed to unsigned mapping)
2337 SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32);
2338 Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit);
2339 } else {
2340 Op0Mapped = Op0;
2341 }
2342 // store the lo of the constructed double - based on integer input
2343 SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo,
2344 MachinePointerInfo());
2345 // initial hi portion of constructed double
2346 SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2347 // store the hi of the constructed double - biased exponent
2348 SDValue Store2 =
2349 DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo());
2350 // load the constructed double
2351 SDValue Load =
2352 DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo());
2353 // FP constant to bias correct the final result
2354 SDValue Bias = DAG.getConstantFP(isSigned ?
2355 BitsToDouble(0x4330000080000000ULL) :
2356 BitsToDouble(0x4330000000000000ULL),
2357 dl, MVT::f64);
2358 // subtract the bias
2359 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2360 // final result
2361 SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2362 return Result;
2363 }
2364 assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")((!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet"
) ? static_cast<void> (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2364, __PRETTY_FUNCTION__))
;
2365 // Code below here assumes !isSigned without checking again.
2366
2367 SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2368
2369 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2370 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2371 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2372 Four = DAG.getIntPtrConstant(4, dl);
2373 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2374 SignSet, Four, Zero);
2375
2376 // If the sign bit of the integer is set, the large number will be treated
2377 // as a negative number. To counteract this, the dynamic code adds an
2378 // offset depending on the data type.
2379 uint64_t FF;
2380 switch (SrcVT.getSimpleVT().SimpleTy) {
2381 default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2381)
;
2382 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2383 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2384 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2385 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2386 }
2387 if (DAG.getDataLayout().isLittleEndian())
2388 FF <<= 32;
2389 Constant *FudgeFactor = ConstantInt::get(
2390 Type::getInt64Ty(*DAG.getContext()), FF);
2391
2392 SDValue CPIdx =
2393 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2394 unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
2395 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2396 Alignment = std::min(Alignment, 4u);
2397 SDValue FudgeInReg;
2398 if (DestVT == MVT::f32)
2399 FudgeInReg = DAG.getLoad(
2400 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2401 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2402 Alignment);
2403 else {
2404 SDValue Load = DAG.getExtLoad(
2405 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2406 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2407 Alignment);
2408 HandleSDNode Handle(Load);
2409 LegalizeOp(Load.getNode());
2410 FudgeInReg = Handle.getValue();
2411 }
2412
2413 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2414}
2415
2416/// This function is responsible for legalizing a
2417/// *INT_TO_FP operation of the specified operand when the target requests that
2418/// we promote it. At this point, we know that the result and operand types are
2419/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2420/// operation that takes a larger input.
2421SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT,
2422 bool isSigned,
2423 const SDLoc &dl) {
2424 // First step, figure out the appropriate *INT_TO_FP operation to use.
2425 EVT NewInTy = LegalOp.getValueType();
2426
2427 unsigned OpToUse = 0;
2428
2429 // Scan for the appropriate larger type to use.
2430 while (true) {
2431 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2432 assert(NewInTy.isInteger() && "Ran out of possibilities!")((NewInTy.isInteger() && "Ran out of possibilities!")
? static_cast<void> (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2432, __PRETTY_FUNCTION__))
;
2433
2434 // If the target supports SINT_TO_FP of this type, use it.
2435 if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) {
2436 OpToUse = ISD::SINT_TO_FP;
2437 break;
2438 }
2439 if (isSigned) continue;
2440
2441 // If the target supports UINT_TO_FP of this type, use it.
2442 if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) {
2443 OpToUse = ISD::UINT_TO_FP;
2444 break;
2445 }
2446
2447 // Otherwise, try a larger type.
2448 }
2449
2450 // Okay, we found the operation and type to use. Zero extend our input to the
2451 // desired type then run the operation on it.
2452 return DAG.getNode(OpToUse, dl, DestVT,
2453 DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2454 dl, NewInTy, LegalOp));
2455}
2456
2457/// This function is responsible for legalizing a
2458/// FP_TO_*INT operation of the specified operand when the target requests that
2459/// we promote it. At this point, we know that the result and operand types are
2460/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2461/// operation that returns a larger result.
2462SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT,
2463 bool isSigned,
2464 const SDLoc &dl) {
2465 // First step, figure out the appropriate FP_TO*INT operation to use.
2466 EVT NewOutTy = DestVT;
2467
2468 unsigned OpToUse = 0;
2469
2470 // Scan for the appropriate larger type to use.
2471 while (true) {
2472 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2473 assert(NewOutTy.isInteger() && "Ran out of possibilities!")((NewOutTy.isInteger() && "Ran out of possibilities!"
) ? static_cast<void> (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2473, __PRETTY_FUNCTION__))
;
2474
2475 // A larger signed type can hold all unsigned values of the requested type,
2476 // so using FP_TO_SINT is valid
2477 if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) {
2478 OpToUse = ISD::FP_TO_SINT;
2479 break;
2480 }
2481
2482 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2483 if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) {
2484 OpToUse = ISD::FP_TO_UINT;
2485 break;
2486 }
2487
2488 // Otherwise, try a larger type.
2489 }
2490
2491 // Okay, we found the operation and type to use.
2492 SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2493
2494 // Truncate the result of the extended FP_TO_*INT operation to the desired
2495 // size.
2496 return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2497}
2498
2499/// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts.
2500SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) {
2501 EVT VT = Op.getValueType();
2502 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2503 unsigned Sz = VT.getScalarSizeInBits();
2504
2505 SDValue Tmp, Tmp2, Tmp3;
2506
2507 // If we can, perform BSWAP first and then the mask+swap the i4, then i2
2508 // and finally the i1 pairs.
2509 // TODO: We can easily support i4/i2 legal types if any target ever does.
2510 if (Sz >= 8 && isPowerOf2_32(Sz)) {
2511 // Create the masks - repeating the pattern every byte.
2512 APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0));
2513 APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC));
2514 APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA));
2515 APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F));
2516 APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33));
2517 APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55));
2518
2519 // BSWAP if the type is wider than a single byte.
2520 Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op);
2521
2522 // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4)
2523 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT));
2524 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT));
2525 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT));
2526 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT));
2527 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2528
2529 // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2)
2530 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT));
2531 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT));
2532 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT));
2533 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT));
2534 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2535
2536 // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1)
2537 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT));
2538 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT));
2539 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT));
2540 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT));
2541 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3);
2542 return Tmp;
2543 }
2544
2545 Tmp = DAG.getConstant(0, dl, VT);
2546 for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) {
2547 if (I < J)
2548 Tmp2 =
2549 DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT));
2550 else
2551 Tmp2 =
2552 DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT));
2553
2554 APInt Shift(Sz, 1);
2555 Shift <<= J;
2556 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT));
2557 Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2);
2558 }
2559
2560 return Tmp;
2561}
2562
2563/// Open code the operations for BSWAP of the specified operation.
2564SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) {
2565 EVT VT = Op.getValueType();
2566 EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2567 SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8;
2568 switch (VT.getSimpleVT().getScalarType().SimpleTy) {
2569 default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2569)
;
2570 case MVT::i16:
2571 // Use a rotate by 8. This can be further expanded if necessary.
2572 return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2573 case MVT::i32:
2574 Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2575 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2576 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2577 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2578 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2579 DAG.getConstant(0xFF0000, dl, VT));
2580 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT));
2581 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2582 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2583 return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2584 case MVT::i64:
2585 Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2586 Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2587 Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2588 Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2589 Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT));
2590 Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT));
2591 Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT));
2592 Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT));
2593 Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7,
2594 DAG.getConstant(255ULL<<48, dl, VT));
2595 Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6,
2596 DAG.getConstant(255ULL<<40, dl, VT));
2597 Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5,
2598 DAG.getConstant(255ULL<<32, dl, VT));
2599 Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4,
2600 DAG.getConstant(255ULL<<24, dl, VT));
2601 Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3,
2602 DAG.getConstant(255ULL<<16, dl, VT));
2603 Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2,
2604 DAG.getConstant(255ULL<<8 , dl, VT));
2605 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7);
2606 Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5);
2607 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3);
2608 Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1);
2609 Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6);
2610 Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2);
2611 return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4);
2612 }
2613}
2614
2615bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2616 LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to expand node\n";
} } while (false)
;
2617 SmallVector<SDValue, 8> Results;
2618 SDLoc dl(Node);
2619 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2620 bool NeedInvert;
2621 switch (Node->getOpcode()) {
2622 case ISD::ABS:
2623 if (TLI.expandABS(Node, Tmp1, DAG))
2624 Results.push_back(Tmp1);
2625 break;
2626 case ISD::CTPOP:
2627 if (TLI.expandCTPOP(Node, Tmp1, DAG))
2628 Results.push_back(Tmp1);
2629 break;
2630 case ISD::CTLZ:
2631 case ISD::CTLZ_ZERO_UNDEF:
2632 if (TLI.expandCTLZ(Node, Tmp1, DAG))
2633 Results.push_back(Tmp1);
2634 break;
2635 case ISD::CTTZ:
2636 case ISD::CTTZ_ZERO_UNDEF:
2637 if (TLI.expandCTTZ(Node, Tmp1, DAG))
2638 Results.push_back(Tmp1);
2639 break;
2640 case ISD::BITREVERSE:
2641 Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl));
2642 break;
2643 case ISD::BSWAP:
2644 Results.push_back(ExpandBSWAP(Node->getOperand(0), dl));
2645 break;
2646 case ISD::FRAMEADDR:
2647 case ISD::RETURNADDR:
2648 case ISD::FRAME_TO_ARGS_OFFSET:
2649 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2650 break;
2651 case ISD::EH_DWARF_CFA: {
2652 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2653 TLI.getPointerTy(DAG.getDataLayout()));
2654 SDValue Offset = DAG.getNode(ISD::ADD, dl,
2655 CfaArg.getValueType(),
2656 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2657 CfaArg.getValueType()),
2658 CfaArg);
2659 SDValue FA = DAG.getNode(
2660 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2661 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2662 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2663 FA, Offset));
2664 break;
2665 }
2666 case ISD::FLT_ROUNDS_:
2667 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2668 break;
2669 case ISD::EH_RETURN:
2670 case ISD::EH_LABEL:
2671 case ISD::PREFETCH:
2672 case ISD::VAEND:
2673 case ISD::EH_SJLJ_LONGJMP:
2674 // If the target didn't expand these, there's nothing to do, so just
2675 // preserve the chain and be done.
2676 Results.push_back(Node->getOperand(0));
2677 break;
2678 case ISD::READCYCLECOUNTER:
2679 // If the target didn't expand this, just return 'zero' and preserve the
2680 // chain.
2681 Results.append(Node->getNumValues() - 1,
2682 DAG.getConstant(0, dl, Node->getValueType(0)));
2683 Results.push_back(Node->getOperand(0));
2684 break;
2685 case ISD::EH_SJLJ_SETJMP:
2686 // If the target didn't expand this, just return 'zero' and preserve the
2687 // chain.
2688 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2689 Results.push_back(Node->getOperand(0));
2690 break;
2691 case ISD::ATOMIC_LOAD: {
2692 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2693 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2694 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2695 SDValue Swap = DAG.getAtomicCmpSwap(
2696 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2697 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2698 cast<AtomicSDNode>(Node)->getMemOperand());
2699 Results.push_back(Swap.getValue(0));
2700 Results.push_back(Swap.getValue(1));
2701 break;
2702 }
2703 case ISD::ATOMIC_STORE: {
2704 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2705 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2706 cast<AtomicSDNode>(Node)->getMemoryVT(),
2707 Node->getOperand(0),
2708 Node->getOperand(1), Node->getOperand(2),
2709 cast<AtomicSDNode>(Node)->getMemOperand());
2710 Results.push_back(Swap.getValue(1));
2711 break;
2712 }
2713 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
2714 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2715 // splits out the success value as a comparison. Expanding the resulting
2716 // ATOMIC_CMP_SWAP will produce a libcall.
2717 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2718 SDValue Res = DAG.getAtomicCmpSwap(
2719 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2720 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2721 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2722
2723 SDValue ExtRes = Res;
2724 SDValue LHS = Res;
2725 SDValue RHS = Node->getOperand(1);
2726
2727 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2728 EVT OuterType = Node->getValueType(0);
2729 switch (TLI.getExtendForAtomicOps()) {
2730 case ISD::SIGN_EXTEND:
2731 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2732 DAG.getValueType(AtomicType));
2733 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2734 Node->getOperand(2), DAG.getValueType(AtomicType));
2735 ExtRes = LHS;
2736 break;
2737 case ISD::ZERO_EXTEND:
2738 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2739 DAG.getValueType(AtomicType));
2740 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2741 ExtRes = LHS;
2742 break;
2743 case ISD::ANY_EXTEND:
2744 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2745 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2746 break;
2747 default:
2748 llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2748)
;
2749 }
2750
2751 SDValue Success =
2752 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2753
2754 Results.push_back(ExtRes.getValue(0));
2755 Results.push_back(Success);
2756 Results.push_back(Res.getValue(1));
2757 break;
2758 }
2759 case ISD::DYNAMIC_STACKALLOC:
2760 ExpandDYNAMIC_STACKALLOC(Node, Results);
2761 break;
2762 case ISD::MERGE_VALUES:
2763 for (unsigned i = 0; i < Node->getNumValues(); i++)
2764 Results.push_back(Node->getOperand(i));
2765 break;
2766 case ISD::UNDEF: {
2767 EVT VT = Node->getValueType(0);
2768 if (VT.isInteger())
2769 Results.push_back(DAG.getConstant(0, dl, VT));
2770 else {
2771 assert(VT.isFloatingPoint() && "Unknown value type!")((VT.isFloatingPoint() && "Unknown value type!") ? static_cast
<void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2771, __PRETTY_FUNCTION__))
;
2772 Results.push_back(DAG.getConstantFP(0, dl, VT));
2773 }
2774 break;
2775 }
2776 case ISD::FP_ROUND:
2777 case ISD::BITCAST:
2778 Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2779 Node->getValueType(0), dl);
2780 Results.push_back(Tmp1);
2781 break;
2782 case ISD::FP_EXTEND:
2783 Tmp1 = EmitStackConvert(Node->getOperand(0),
2784 Node->getOperand(0).getValueType(),
2785 Node->getValueType(0), dl);
2786 Results.push_back(Tmp1);
2787 break;
2788 case ISD::SIGN_EXTEND_INREG: {
2789 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2790 EVT VT = Node->getValueType(0);
2791
2792 // An in-register sign-extend of a boolean is a negation:
2793 // 'true' (1) sign-extended is -1.
2794 // 'false' (0) sign-extended is 0.
2795 // However, we must mask the high bits of the source operand because the
2796 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2797
2798 // TODO: Do this for vectors too?
2799 if (ExtraVT.getSizeInBits() == 1) {
2800 SDValue One = DAG.getConstant(1, dl, VT);
2801 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2802 SDValue Zero = DAG.getConstant(0, dl, VT);
2803 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2804 Results.push_back(Neg);
2805 break;
2806 }
2807
2808 // NOTE: we could fall back on load/store here too for targets without
2809 // SRA. However, it is doubtful that any exist.
2810 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2811 unsigned BitsDiff = VT.getScalarSizeInBits() -
2812 ExtraVT.getScalarSizeInBits();
2813 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2814 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2815 Node->getOperand(0), ShiftCst);
2816 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2817 Results.push_back(Tmp1);
2818 break;
2819 }
2820 case ISD::FP_ROUND_INREG: {
2821 // The only way we can lower this is to turn it into a TRUNCSTORE,
2822 // EXTLOAD pair, targeting a temporary location (a stack slot).
2823
2824 // NOTE: there is a choice here between constantly creating new stack
2825 // slots and always reusing the same one. We currently always create
2826 // new ones, as reuse may inhibit scheduling.
2827 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2828 Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT,
2829 Node->getValueType(0), dl);
2830 Results.push_back(Tmp1);
2831 break;
2832 }
2833 case ISD::UINT_TO_FP:
2834 if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) {
2835 Results.push_back(Tmp1);
2836 break;
2837 }
2838 LLVM_FALLTHROUGH[[clang::fallthrough]];
2839 case ISD::SINT_TO_FP:
2840 Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP,
2841 Node->getOperand(0), Node->getValueType(0), dl);
2842 Results.push_back(Tmp1);
2843 break;
2844 case ISD::FP_TO_SINT:
2845 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2846 Results.push_back(Tmp1);
2847 break;
2848 case ISD::FP_TO_UINT:
2849 if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG))
2850 Results.push_back(Tmp1);
2851 break;
2852 case ISD::VAARG:
2853 Results.push_back(DAG.expandVAArg(Node));
2854 Results.push_back(Results[0].getValue(1));
2855 break;
2856 case ISD::VACOPY:
2857 Results.push_back(DAG.expandVACopy(Node));
2858 break;
2859 case ISD::EXTRACT_VECTOR_ELT:
2860 if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2861 // This must be an access of the only element. Return it.
2862 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2863 Node->getOperand(0));
2864 else
2865 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2866 Results.push_back(Tmp1);
2867 break;
2868 case ISD::EXTRACT_SUBVECTOR:
2869 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2870 break;
2871 case ISD::INSERT_SUBVECTOR:
2872 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2873 break;
2874 case ISD::CONCAT_VECTORS:
2875 Results.push_back(ExpandVectorBuildThroughStack(Node));
2876 break;
2877 case ISD::SCALAR_TO_VECTOR:
2878 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2879 break;
2880 case ISD::INSERT_VECTOR_ELT:
2881 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2882 Node->getOperand(1),
2883 Node->getOperand(2), dl));
2884 break;
2885 case ISD::VECTOR_SHUFFLE: {
2886 SmallVector<int, 32> NewMask;
2887 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
2888
2889 EVT VT = Node->getValueType(0);
2890 EVT EltVT = VT.getVectorElementType();
2891 SDValue Op0 = Node->getOperand(0);
2892 SDValue Op1 = Node->getOperand(1);
2893 if (!TLI.isTypeLegal(EltVT)) {
2894 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
2895
2896 // BUILD_VECTOR operands are allowed to be wider than the element type.
2897 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
2898 // it.
2899 if (NewEltVT.bitsLT(EltVT)) {
2900 // Convert shuffle node.
2901 // If original node was v4i64 and the new EltVT is i32,
2902 // cast operands to v8i32 and re-build the mask.
2903
2904 // Calculate new VT, the size of the new VT should be equal to original.
2905 EVT NewVT =
2906 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
2907 VT.getSizeInBits() / NewEltVT.getSizeInBits());
2908 assert(NewVT.bitsEq(VT))((NewVT.bitsEq(VT)) ? static_cast<void> (0) : __assert_fail
("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2908, __PRETTY_FUNCTION__))
;
2909
2910 // cast operands to new VT
2911 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
2912 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
2913
2914 // Convert the shuffle mask
2915 unsigned int factor =
2916 NewVT.getVectorNumElements()/VT.getVectorNumElements();
2917
2918 // EltVT gets smaller
2919 assert(factor > 0)((factor > 0) ? static_cast<void> (0) : __assert_fail
("factor > 0", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 2919, __PRETTY_FUNCTION__))
;
2920
2921 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
2922 if (Mask[i] < 0) {
2923 for (unsigned fi = 0; fi < factor; ++fi)
2924 NewMask.push_back(Mask[i]);
2925 }
2926 else {
2927 for (unsigned fi = 0; fi < factor; ++fi)
2928 NewMask.push_back(Mask[i]*factor+fi);
2929 }
2930 }
2931 Mask = NewMask;
2932 VT = NewVT;
2933 }
2934 EltVT = NewEltVT;
2935 }
2936 unsigned NumElems = VT.getVectorNumElements();
2937 SmallVector<SDValue, 16> Ops;
2938 for (unsigned i = 0; i != NumElems; ++i) {
2939 if (Mask[i] < 0) {
2940 Ops.push_back(DAG.getUNDEF(EltVT));
2941 continue;
2942 }
2943 unsigned Idx = Mask[i];
2944 if (Idx < NumElems)
2945 Ops.push_back(DAG.getNode(
2946 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
2947 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
2948 else
2949 Ops.push_back(DAG.getNode(
2950 ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
2951 DAG.getConstant(Idx - NumElems, dl,
2952 TLI.getVectorIdxTy(DAG.getDataLayout()))));
2953 }
2954
2955 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
2956 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
2957 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
2958 Results.push_back(Tmp1);
2959 break;
2960 }
2961 case ISD::EXTRACT_ELEMENT: {
2962 EVT OpTy = Node->getOperand(0).getValueType();
2963 if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
2964 // 1 -> Hi
2965 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
2966 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
2967 TLI.getShiftAmountTy(
2968 Node->getOperand(0).getValueType(),
2969 DAG.getDataLayout())));
2970 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
2971 } else {
2972 // 0 -> Lo
2973 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
2974 Node->getOperand(0));
2975 }
2976 Results.push_back(Tmp1);
2977 break;
2978 }
2979 case ISD::STACKSAVE:
2980 // Expand to CopyFromReg if the target set
2981 // StackPointerRegisterToSaveRestore.
2982 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2983 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
2984 Node->getValueType(0)));
2985 Results.push_back(Results[0].getValue(1));
2986 } else {
2987 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
2988 Results.push_back(Node->getOperand(0));
2989 }
2990 break;
2991 case ISD::STACKRESTORE:
2992 // Expand to CopyToReg if the target set
2993 // StackPointerRegisterToSaveRestore.
2994 if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) {
2995 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
2996 Node->getOperand(1)));
2997 } else {
2998 Results.push_back(Node->getOperand(0));
2999 }
3000 break;
3001 case ISD::GET_DYNAMIC_AREA_OFFSET:
3002 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3003 Results.push_back(Results[0].getValue(0));
3004 break;
3005 case ISD::FCOPYSIGN:
3006 Results.push_back(ExpandFCOPYSIGN(Node));
3007 break;
3008 case ISD::FNEG:
3009 // Expand Y = FNEG(X) -> Y = SUB -0.0, X
3010 Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0));
3011 // TODO: If FNEG has fast-math-flags, propagate them to the FSUB.
3012 Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1,
3013 Node->getOperand(0));
3014 Results.push_back(Tmp1);
3015 break;
3016 case ISD::FABS:
3017 Results.push_back(ExpandFABS(Node));
3018 break;
3019 case ISD::SMIN:
3020 case ISD::SMAX:
3021 case ISD::UMIN:
3022 case ISD::UMAX: {
3023 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3024 ISD::CondCode Pred;
3025 switch (Node->getOpcode()) {
3026 default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3026)
;
3027 case ISD::SMAX: Pred = ISD::SETGT; break;
3028 case ISD::SMIN: Pred = ISD::SETLT; break;
3029 case ISD::UMAX: Pred = ISD::SETUGT; break;
3030 case ISD::UMIN: Pred = ISD::SETULT; break;
3031 }
3032 Tmp1 = Node->getOperand(0);
3033 Tmp2 = Node->getOperand(1);
3034 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3035 Results.push_back(Tmp1);
3036 break;
3037 }
3038 case ISD::FMINNUM:
3039 case ISD::FMAXNUM: {
3040 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3041 Results.push_back(Expanded);
3042 break;
3043 }
3044 case ISD::FSIN:
3045 case ISD::FCOS: {
3046 EVT VT = Node->getValueType(0);
3047 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3048 // fcos which share the same operand and both are used.
3049 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3050 isSinCosLibcallAvailable(Node, TLI))
3051 && useSinCos(Node)) {
3052 SDVTList VTs = DAG.getVTList(VT, VT);
3053 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3054 if (Node->getOpcode() == ISD::FCOS)
3055 Tmp1 = Tmp1.getValue(1);
3056 Results.push_back(Tmp1);
3057 }
3058 break;
3059 }
3060 case ISD::FMAD:
3061 llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3061)
;
3062
3063 case ISD::FP16_TO_FP:
3064 if (Node->getValueType(0) != MVT::f32) {
3065 // We can extend to types bigger than f32 in two steps without changing
3066 // the result. Since "f16 -> f32" is much more commonly available, give
3067 // CodeGen the option of emitting that before resorting to a libcall.
3068 SDValue Res =
3069 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3070 Results.push_back(
3071 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3072 }
3073 break;
3074 case ISD::FP_TO_FP16:
3075 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n";
} } while (false)
;
3076 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3077 SDValue Op = Node->getOperand(0);
3078 MVT SVT = Op.getSimpleValueType();
3079 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3080 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3081 // Under fastmath, we can expand this node into a fround followed by
3082 // a float-half conversion.
3083 SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3084 DAG.getIntPtrConstant(0, dl));
3085 Results.push_back(
3086 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3087 }
3088 }
3089 break;
3090 case ISD::ConstantFP: {
3091 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3092 // Check to see if this FP immediate is already legal.
3093 // If this is a legal constant, turn it into a TargetConstantFP node.
3094 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3095 DAG.getMachineFunction().getFunction().hasOptSize()))
3096 Results.push_back(ExpandConstantFP(CFP, true));
3097 break;
3098 }
3099 case ISD::Constant: {
3100 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3101 Results.push_back(ExpandConstant(CP));
3102 break;
3103 }
3104 case ISD::FSUB: {
3105 EVT VT = Node->getValueType(0);
3106 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3107 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3108 const SDNodeFlags Flags = Node->getFlags();
3109 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3110 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3111 Results.push_back(Tmp1);
3112 }
3113 break;
3114 }
3115 case ISD::SUB: {
3116 EVT VT = Node->getValueType(0);
3117 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3119, __PRETTY_FUNCTION__))
3118 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3119, __PRETTY_FUNCTION__))
3119 "Don't know how to expand this subtraction!")((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom
(ISD::XOR, VT) && "Don't know how to expand this subtraction!"
) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3119, __PRETTY_FUNCTION__))
;
3120 Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3121 DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3122 VT));
3123 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3124 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3125 break;
3126 }
3127 case ISD::UREM:
3128 case ISD::SREM: {
3129 EVT VT = Node->getValueType(0);
3130 bool isSigned = Node->getOpcode() == ISD::SREM;
3131 unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV;
3132 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3133 Tmp2 = Node->getOperand(0);
3134 Tmp3 = Node->getOperand(1);
3135 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3136 SDVTList VTs = DAG.getVTList(VT, VT);
3137 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
3138 Results.push_back(Tmp1);
3139 } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
3140 // X % Y -> X-X/Y*Y
3141 Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3);
3142 Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3);
3143 Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1);
3144 Results.push_back(Tmp1);
3145 }
3146 break;
3147 }
3148 case ISD::UDIV:
3149 case ISD::SDIV: {
3150 bool isSigned = Node->getOpcode() == ISD::SDIV;
3151 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3152 EVT VT = Node->getValueType(0);
3153 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3154 SDVTList VTs = DAG.getVTList(VT, VT);
3155 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3156 Node->getOperand(1));
3157 Results.push_back(Tmp1);
3158 }
3159 break;
3160 }
3161 case ISD::MULHU:
3162 case ISD::MULHS: {
3163 unsigned ExpandOpcode =
3164 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3165 EVT VT = Node->getValueType(0);
3166 SDVTList VTs = DAG.getVTList(VT, VT);
3167
3168 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3169 Node->getOperand(1));
3170 Results.push_back(Tmp1.getValue(1));
3171 break;
3172 }
3173 case ISD::UMUL_LOHI:
3174 case ISD::SMUL_LOHI: {
3175 SDValue LHS = Node->getOperand(0);
3176 SDValue RHS = Node->getOperand(1);
3177 MVT VT = LHS.getSimpleValueType();
3178 unsigned MULHOpcode =
3179 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3180
3181 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3182 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3183 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3184 break;
3185 }
3186
3187 SmallVector<SDValue, 4> Halves;
3188 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3189 assert(TLI.isTypeLegal(HalfType))((TLI.isTypeLegal(HalfType)) ? static_cast<void> (0) : __assert_fail
("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3189, __PRETTY_FUNCTION__))
;
3190 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves,
3191 HalfType, DAG,
3192 TargetLowering::MulExpansionKind::Always)) {
3193 for (unsigned i = 0; i < 2; ++i) {
3194 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3195 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3196 SDValue Shift = DAG.getConstant(
3197 HalfType.getScalarSizeInBits(), dl,
3198 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3199 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3200 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3201 }
3202 break;
3203 }
3204 break;
3205 }
3206 case ISD::MUL: {
3207 EVT VT = Node->getValueType(0);
3208 SDVTList VTs = DAG.getVTList(VT, VT);
3209 // See if multiply or divide can be lowered using two-result operations.
3210 // We just need the low half of the multiply; try both the signed
3211 // and unsigned forms. If the target supports both SMUL_LOHI and
3212 // UMUL_LOHI, form a preference by checking which forms of plain
3213 // MULH it supports.
3214 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3215 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3216 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3217 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3218 unsigned OpToUse = 0;
3219 if (HasSMUL_LOHI && !HasMULHS) {
3220 OpToUse = ISD::SMUL_LOHI;
3221 } else if (HasUMUL_LOHI && !HasMULHU) {
3222 OpToUse = ISD::UMUL_LOHI;
3223 } else if (HasSMUL_LOHI) {
3224 OpToUse = ISD::SMUL_LOHI;
3225 } else if (HasUMUL_LOHI) {
3226 OpToUse = ISD::UMUL_LOHI;
3227 }
3228 if (OpToUse) {
3229 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3230 Node->getOperand(1)));
3231 break;
3232 }
3233
3234 SDValue Lo, Hi;
3235 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3236 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3237 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3238 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3239 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3240 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3241 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3242 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3243 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3244 SDValue Shift =
3245 DAG.getConstant(HalfType.getSizeInBits(), dl,
3246 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3247 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3248 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3249 }
3250 break;
3251 }
3252 case ISD::FSHL:
3253 case ISD::FSHR:
3254 if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3255 Results.push_back(Tmp1);
3256 break;
3257 case ISD::ROTL:
3258 case ISD::ROTR:
3259 if (TLI.expandROT(Node, Tmp1, DAG))
3260 Results.push_back(Tmp1);
3261 break;
3262 case ISD::SADDSAT:
3263 case ISD::UADDSAT:
3264 case ISD::SSUBSAT:
3265 case ISD::USUBSAT:
3266 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3267 break;
3268 case ISD::SMULFIX:
3269 case ISD::UMULFIX:
3270 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3271 break;
3272 case ISD::SADDO:
3273 case ISD::SSUBO: {
3274 SDValue LHS = Node->getOperand(0);
3275 SDValue RHS = Node->getOperand(1);
3276 bool IsAdd = Node->getOpcode() == ISD::SADDO;
3277
3278 SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3279 LHS.getValueType(), LHS, RHS);
3280 Results.push_back(Sum);
3281
3282 EVT ResultType = Node->getValueType(1);
3283 EVT OType = getSetCCResultType(Node->getValueType(0));
3284
3285 // If SADDSAT/SSUBSAT is legal, compare results to detect overflow.
3286 unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT;
3287 if (TLI.isOperationLegalOrCustom(OpcSat, LHS.getValueType())) {
3288 SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS);
3289 SDValue SetCC = DAG.getSetCC(dl, OType, Sum, Sat, ISD::SETNE);
3290 Results.push_back(
3291 DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3292 break;
3293 }
3294
3295 SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3296
3297 // LHSSign -> LHS >= 0
3298 // RHSSign -> RHS >= 0
3299 // SumSign -> Sum >= 0
3300 //
3301 // Add:
3302 // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3303 // Sub:
3304 // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3305 SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3306 SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3307 SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3308 IsAdd ? ISD::SETEQ : ISD::SETNE);
3309
3310 SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3311 SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3312
3313 SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3314 Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType));
3315 break;
3316 }
3317 case ISD::UADDO:
3318 case ISD::USUBO: {
3319 SDValue LHS = Node->getOperand(0);
3320 SDValue RHS = Node->getOperand(1);
3321 bool IsAdd = Node->getOpcode() == ISD::UADDO;
3322
3323 // If ADD/SUBCARRY is legal, use that instead.
3324 unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY;
3325 if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) {
3326 SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1));
3327 SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(),
3328 { LHS, RHS, CarryIn });
3329 Results.push_back(SDValue(NodeCarry.getNode(), 0));
3330 Results.push_back(SDValue(NodeCarry.getNode(), 1));
3331 break;
3332 }
3333
3334 SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl,
3335 LHS.getValueType(), LHS, RHS);
3336 Results.push_back(Sum);
3337
3338 EVT ResultType = Node->getValueType(1);
3339 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3340 ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3341 SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3342
3343 Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType));
3344 break;
3345 }
3346 case ISD::UMULO:
3347 case ISD::SMULO: {
3348 SDValue Result, Overflow;
3349 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3350 Results.push_back(Result);
3351 Results.push_back(Overflow);
3352 }
3353 break;
3354 }
3355 case ISD::BUILD_PAIR: {
3356 EVT PairTy = Node->getValueType(0);
3357 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3358 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3359 Tmp2 = DAG.getNode(
3360 ISD::SHL, dl, PairTy, Tmp2,
3361 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3362 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3363 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3364 break;
3365 }
3366 case ISD::SELECT:
3367 Tmp1 = Node->getOperand(0);
3368 Tmp2 = Node->getOperand(1);
3369 Tmp3 = Node->getOperand(2);
3370 if (Tmp1.getOpcode() == ISD::SETCC) {
3371 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3372 Tmp2, Tmp3,
3373 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3374 } else {
3375 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3376 DAG.getConstant(0, dl, Tmp1.getValueType()),
3377 Tmp2, Tmp3, ISD::SETNE);
3378 }
3379 Results.push_back(Tmp1);
3380 break;
3381 case ISD::BR_JT: {
3382 SDValue Chain = Node->getOperand(0);
3383 SDValue Table = Node->getOperand(1);
3384 SDValue Index = Node->getOperand(2);
3385
3386 const DataLayout &TD = DAG.getDataLayout();
3387 EVT PTy = TLI.getPointerTy(TD);
3388
3389 unsigned EntrySize =
3390 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3391
3392 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3393 // This transformation needs to be done here since otherwise the MIPS
3394 // backend will end up emitting a three instruction multiply sequence
3395 // instead of a single shift and MSP430 will call a runtime function.
3396 if (llvm::isPowerOf2_32(EntrySize))
3397 Index = DAG.getNode(
3398 ISD::SHL, dl, Index.getValueType(), Index,
3399 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3400 else
3401 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3402 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3403 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3404 Index, Table);
3405
3406 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3407 SDValue LD = DAG.getExtLoad(
3408 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3409 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3410 Addr = LD;
3411 if (TLI.isJumpTableRelative()) {
3412 // For PIC, the sequence is:
3413 // BRIND(load(Jumptable + index) + RelocBase)
3414 // RelocBase can be JumpTable, GOT or some sort of global base.
3415 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3416 TLI.getPICJumpTableRelocBase(Table, DAG));
3417 }
3418
3419 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3420 Results.push_back(Tmp1);
3421 break;
3422 }
3423 case ISD::BRCOND:
3424 // Expand brcond's setcc into its constituent parts and create a BR_CC
3425 // Node.
3426 Tmp1 = Node->getOperand(0);
3427 Tmp2 = Node->getOperand(1);
3428 if (Tmp2.getOpcode() == ISD::SETCC) {
3429 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3430 Tmp1, Tmp2.getOperand(2),
3431 Tmp2.getOperand(0), Tmp2.getOperand(1),
3432 Node->getOperand(2));
3433 } else {
3434 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3435 if (Tmp2.isUndef() ||
3436 (Tmp2.getOpcode() == ISD::AND &&
3437 isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3438 cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3439 Tmp3 = Tmp2;
3440 else
3441 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3442 DAG.getConstant(1, dl, Tmp2.getValueType()));
3443 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3444 DAG.getCondCode(ISD::SETNE), Tmp3,
3445 DAG.getConstant(0, dl, Tmp3.getValueType()),
3446 Node->getOperand(2));
3447 }
3448 Results.push_back(Tmp1);
3449 break;
3450 case ISD::SETCC: {
3451 Tmp1 = Node->getOperand(0);
3452 Tmp2 = Node->getOperand(1);
3453 Tmp3 = Node->getOperand(2);
3454 bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2,
3455 Tmp3, NeedInvert, dl);
3456
3457 if (Legalized) {
3458 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3459 // condition code, create a new SETCC node.
3460 if (Tmp3.getNode())
3461 Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3462 Tmp1, Tmp2, Tmp3);
3463
3464 // If we expanded the SETCC by inverting the condition code, then wrap
3465 // the existing SETCC in a NOT to restore the intended condition.
3466 if (NeedInvert)
3467 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3468
3469 Results.push_back(Tmp1);
3470 break;
3471 }
3472
3473 // Otherwise, SETCC for the given comparison type must be completely
3474 // illegal; expand it into a SELECT_CC.
3475 EVT VT = Node->getValueType(0);
3476 int TrueValue;
3477 switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3478 case TargetLowering::ZeroOrOneBooleanContent:
3479 case TargetLowering::UndefinedBooleanContent:
3480 TrueValue = 1;
3481 break;
3482 case TargetLowering::ZeroOrNegativeOneBooleanContent:
3483 TrueValue = -1;
3484 break;
3485 }
3486 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3487 DAG.getConstant(TrueValue, dl, VT),
3488 DAG.getConstant(0, dl, VT),
3489 Tmp3);
3490 Results.push_back(Tmp1);
3491 break;
3492 }
3493 case ISD::SELECT_CC: {
3494 Tmp1 = Node->getOperand(0); // LHS
3495 Tmp2 = Node->getOperand(1); // RHS
3496 Tmp3 = Node->getOperand(2); // True
3497 Tmp4 = Node->getOperand(3); // False
3498 EVT VT = Node->getValueType(0);
3499 SDValue CC = Node->getOperand(4);
3500 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3501
3502 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3503 // If the condition code is legal, then we need to expand this
3504 // node using SETCC and SELECT.
3505 EVT CmpVT = Tmp1.getValueType();
3506 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3508, __PRETTY_FUNCTION__))
3507 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3508, __PRETTY_FUNCTION__))
3508 "expanded.")((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
"expanded.") ? static_cast<void> (0) : __assert_fail (
"!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3508, __PRETTY_FUNCTION__))
;
3509 EVT CCVT =
3510 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
3511 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC);
3512 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3513 break;
3514 }
3515
3516 // SELECT_CC is legal, so the condition code must not be.
3517 bool Legalized = false;
3518 // Try to legalize by inverting the condition. This is for targets that
3519 // might support an ordered version of a condition, but not the unordered
3520 // version (or vice versa).
3521 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp,
3522 Tmp1.getValueType().isInteger());
3523 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3524 // Use the new condition code and swap true and false
3525 Legalized = true;
3526 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3527 } else {
3528 // If The inverse is not legal, then try to swap the arguments using
3529 // the inverse condition code.
3530 ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3531 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3532 // The swapped inverse condition is legal, so swap true and false,
3533 // lhs and rhs.
3534 Legalized = true;
3535 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3536 }
3537 }
3538
3539 if (!Legalized) {
3540 Legalized = LegalizeSetCCCondCode(
3541 getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert,
3542 dl);
3543
3544 assert(Legalized && "Can't legalize SELECT_CC with legal condition!")((Legalized && "Can't legalize SELECT_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3544, __PRETTY_FUNCTION__))
;
3545
3546 // If we expanded the SETCC by inverting the condition code, then swap
3547 // the True/False operands to match.
3548 if (NeedInvert)
3549 std::swap(Tmp3, Tmp4);
3550
3551 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3552 // condition code, create a new SELECT_CC node.
3553 if (CC.getNode()) {
3554 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3555 Tmp1, Tmp2, Tmp3, Tmp4, CC);
3556 } else {
3557 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3558 CC = DAG.getCondCode(ISD::SETNE);
3559 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3560 Tmp2, Tmp3, Tmp4, CC);
3561 }
3562 }
3563 Results.push_back(Tmp1);
3564 break;
3565 }
3566 case ISD::BR_CC: {
3567 Tmp1 = Node->getOperand(0); // Chain
3568 Tmp2 = Node->getOperand(2); // LHS
3569 Tmp3 = Node->getOperand(3); // RHS
3570 Tmp4 = Node->getOperand(1); // CC
3571
3572 bool Legalized = LegalizeSetCCCondCode(getSetCCResultType(
3573 Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl);
3574 (void)Legalized;
3575 assert(Legalized && "Can't legalize BR_CC with legal condition!")((Legalized && "Can't legalize BR_CC with legal condition!"
) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3575, __PRETTY_FUNCTION__))
;
3576
3577 assert(!NeedInvert && "Don't know how to invert BR_CC!")((!NeedInvert && "Don't know how to invert BR_CC!") ?
static_cast<void> (0) : __assert_fail ("!NeedInvert && \"Don't know how to invert BR_CC!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3577, __PRETTY_FUNCTION__))
;
3578
3579 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3580 // node.
3581 if (Tmp4.getNode()) {
3582 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3583 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3584 } else {
3585 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3586 Tmp4 = DAG.getCondCode(ISD::SETNE);
3587 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3588 Tmp2, Tmp3, Node->getOperand(4));
3589 }
3590 Results.push_back(Tmp1);
3591 break;
3592 }
3593 case ISD::BUILD_VECTOR:
3594 Results.push_back(ExpandBUILD_VECTOR(Node));
3595 break;
3596 case ISD::SRA:
3597 case ISD::SRL:
3598 case ISD::SHL: {
3599 // Scalarize vector SRA/SRL/SHL.
3600 EVT VT = Node->getValueType(0);
3601 assert(VT.isVector() && "Unable to legalize non-vector shift")((VT.isVector() && "Unable to legalize non-vector shift"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3601, __PRETTY_FUNCTION__))
;
3602 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")((TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal"
) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3602, __PRETTY_FUNCTION__))
;
3603 unsigned NumElem = VT.getVectorNumElements();
3604
3605 SmallVector<SDValue, 8> Scalars;
3606 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3607 SDValue Ex = DAG.getNode(
3608 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0),
3609 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3610 SDValue Sh = DAG.getNode(
3611 ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1),
3612 DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
3613 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3614 VT.getScalarType(), Ex, Sh));
3615 }
3616
3617 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3618 ReplaceNode(SDValue(Node, 0), Result);
3619 break;
3620 }
3621 case ISD::VECREDUCE_FADD:
3622 case ISD::VECREDUCE_FMUL:
3623 case ISD::VECREDUCE_ADD:
3624 case ISD::VECREDUCE_MUL:
3625 case ISD::VECREDUCE_AND:
3626 case ISD::VECREDUCE_OR:
3627 case ISD::VECREDUCE_XOR:
3628 case ISD::VECREDUCE_SMAX:
3629 case ISD::VECREDUCE_SMIN:
3630 case ISD::VECREDUCE_UMAX:
3631 case ISD::VECREDUCE_UMIN:
3632 case ISD::VECREDUCE_FMAX:
3633 case ISD::VECREDUCE_FMIN:
3634 Results.push_back(TLI.expandVecReduce(Node, DAG));
3635 break;
3636 case ISD::GLOBAL_OFFSET_TABLE:
3637 case ISD::GlobalAddress:
3638 case ISD::GlobalTLSAddress:
3639 case ISD::ExternalSymbol:
3640 case ISD::ConstantPool:
3641 case ISD::JumpTable:
3642 case ISD::INTRINSIC_W_CHAIN:
3643 case ISD::INTRINSIC_WO_CHAIN:
3644 case ISD::INTRINSIC_VOID:
3645 // FIXME: Custom lowering for these operations shouldn't return null!
3646 break;
3647 }
3648
3649 // Replace the original node with the legalized result.
3650 if (Results.empty()) {
3651 LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Cannot expand node\n"; } }
while (false)
;
3652 return false;
3653 }
3654
3655 LLVM_DEBUG(dbgs() << "Successfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully expanded node\n"
; } } while (false)
;
3656 ReplaceNode(Node, Results.data());
3657 return true;
3658}
3659
3660void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3661 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to convert node to libcall\n"
; } } while (false)
;
3662 SmallVector<SDValue, 8> Results;
3663 SDLoc dl(Node);
3664 // FIXME: Check flags on the node to see if we can use a finite call.
3665 bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath;
3666 unsigned Opc = Node->getOpcode();
3667 switch (Opc) {
3668 case ISD::ATOMIC_FENCE: {
3669 // If the target didn't lower this, lower it to '__sync_synchronize()' call
3670 // FIXME: handle "fence singlethread" more efficiently.
3671 TargetLowering::ArgListTy Args;
3672
3673 TargetLowering::CallLoweringInfo CLI(DAG);
3674 CLI.setDebugLoc(dl)
3675 .setChain(Node->getOperand(0))
3676 .setLibCallee(
3677 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3678 DAG.getExternalSymbol("__sync_synchronize",
3679 TLI.getPointerTy(DAG.getDataLayout())),
3680 std::move(Args));
3681
3682 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3683
3684 Results.push_back(CallResult.second);
3685 break;
3686 }
3687 // By default, atomic intrinsics are marked Legal and lowered. Targets
3688 // which don't support them directly, however, may want libcalls, in which
3689 // case they mark them Expand, and we get here.
3690 case ISD::ATOMIC_SWAP:
3691 case ISD::ATOMIC_LOAD_ADD:
3692 case ISD::ATOMIC_LOAD_SUB:
3693 case ISD::ATOMIC_LOAD_AND:
3694 case ISD::ATOMIC_LOAD_CLR:
3695 case ISD::ATOMIC_LOAD_OR:
3696 case ISD::ATOMIC_LOAD_XOR:
3697 case ISD::ATOMIC_LOAD_NAND:
3698 case ISD::ATOMIC_LOAD_MIN:
3699 case ISD::ATOMIC_LOAD_MAX:
3700 case ISD::ATOMIC_LOAD_UMIN:
3701 case ISD::ATOMIC_LOAD_UMAX:
3702 case ISD::ATOMIC_CMP_SWAP: {
3703 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3704 RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
3705 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")((LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3705, __PRETTY_FUNCTION__))
;
3706
3707 std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false);
3708 Results.push_back(Tmp.first);
3709 Results.push_back(Tmp.second);
3710 break;
3711 }
3712 case ISD::TRAP: {
3713 // If this operation is not supported, lower it to 'abort()' call
3714 TargetLowering::ArgListTy Args;
3715 TargetLowering::CallLoweringInfo CLI(DAG);
3716 CLI.setDebugLoc(dl)
3717 .setChain(Node->getOperand(0))
3718 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3719 DAG.getExternalSymbol(
3720 "abort", TLI.getPointerTy(DAG.getDataLayout())),
3721 std::move(Args));
3722 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3723
3724 Results.push_back(CallResult.second);
3725 break;
3726 }
3727 case ISD::FMINNUM:
3728 case ISD::STRICT_FMINNUM:
3729 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3730 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3731 RTLIB::FMIN_PPCF128));
3732 break;
3733 case ISD::FMAXNUM:
3734 case ISD::STRICT_FMAXNUM:
3735 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3736 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3737 RTLIB::FMAX_PPCF128));
3738 break;
3739 case ISD::FSQRT:
3740 case ISD::STRICT_FSQRT:
3741 Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3742 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3743 RTLIB::SQRT_PPCF128));
3744 break;
3745 case ISD::FCBRT:
3746 Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3747 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3748 RTLIB::CBRT_PPCF128));
3749 break;
3750 case ISD::FSIN:
3751 case ISD::STRICT_FSIN:
3752 Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3753 RTLIB::SIN_F80, RTLIB::SIN_F128,
3754 RTLIB::SIN_PPCF128));
3755 break;
3756 case ISD::FCOS:
3757 case ISD::STRICT_FCOS:
3758 Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3759 RTLIB::COS_F80, RTLIB::COS_F128,
3760 RTLIB::COS_PPCF128));
3761 break;
3762 case ISD::FSINCOS:
3763 // Expand into sincos libcall.
3764 ExpandSinCosLibCall(Node, Results);
3765 break;
3766 case ISD::FLOG:
3767 case ISD::STRICT_FLOG:
3768 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite))
3769 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32,
3770 RTLIB::LOG_FINITE_F64,
3771 RTLIB::LOG_FINITE_F80,
3772 RTLIB::LOG_FINITE_F128,
3773 RTLIB::LOG_FINITE_PPCF128));
3774 else
3775 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64,
3776 RTLIB::LOG_F80, RTLIB::LOG_F128,
3777 RTLIB::LOG_PPCF128));
3778 break;
3779 case ISD::FLOG2:
3780 case ISD::STRICT_FLOG2:
3781 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite))
3782 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32,
3783 RTLIB::LOG2_FINITE_F64,
3784 RTLIB::LOG2_FINITE_F80,
3785 RTLIB::LOG2_FINITE_F128,
3786 RTLIB::LOG2_FINITE_PPCF128));
3787 else
3788 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64,
3789 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
3790 RTLIB::LOG2_PPCF128));
3791 break;
3792 case ISD::FLOG10:
3793 case ISD::STRICT_FLOG10:
3794 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite))
3795 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32,
3796 RTLIB::LOG10_FINITE_F64,
3797 RTLIB::LOG10_FINITE_F80,
3798 RTLIB::LOG10_FINITE_F128,
3799 RTLIB::LOG10_FINITE_PPCF128));
3800 else
3801 Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64,
3802 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
3803 RTLIB::LOG10_PPCF128));
3804 break;
3805 case ISD::FEXP:
3806 case ISD::STRICT_FEXP:
3807 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite))
3808 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32,
3809 RTLIB::EXP_FINITE_F64,
3810 RTLIB::EXP_FINITE_F80,
3811 RTLIB::EXP_FINITE_F128,
3812 RTLIB::EXP_FINITE_PPCF128));
3813 else
3814 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64,
3815 RTLIB::EXP_F80, RTLIB::EXP_F128,
3816 RTLIB::EXP_PPCF128));
3817 break;
3818 case ISD::FEXP2:
3819 case ISD::STRICT_FEXP2:
3820 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite))
3821 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32,
3822 RTLIB::EXP2_FINITE_F64,
3823 RTLIB::EXP2_FINITE_F80,
3824 RTLIB::EXP2_FINITE_F128,
3825 RTLIB::EXP2_FINITE_PPCF128));
3826 else
3827 Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64,
3828 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
3829 RTLIB::EXP2_PPCF128));
3830 break;
3831 case ISD::FTRUNC:
3832 case ISD::STRICT_FTRUNC:
3833 Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
3834 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
3835 RTLIB::TRUNC_PPCF128));
3836 break;
3837 case ISD::FFLOOR:
3838 case ISD::STRICT_FFLOOR:
3839 Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
3840 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
3841 RTLIB::FLOOR_PPCF128));
3842 break;
3843 case ISD::FCEIL:
3844 case ISD::STRICT_FCEIL:
3845 Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
3846 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
3847 RTLIB::CEIL_PPCF128));
3848 break;
3849 case ISD::FRINT:
3850 case ISD::STRICT_FRINT:
3851 Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
3852 RTLIB::RINT_F80, RTLIB::RINT_F128,
3853 RTLIB::RINT_PPCF128));
3854 break;
3855 case ISD::FNEARBYINT:
3856 case ISD::STRICT_FNEARBYINT:
3857 Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
3858 RTLIB::NEARBYINT_F64,
3859 RTLIB::NEARBYINT_F80,
3860 RTLIB::NEARBYINT_F128,
3861 RTLIB::NEARBYINT_PPCF128));
3862 break;
3863 case ISD::FROUND:
3864 case ISD::STRICT_FROUND:
3865 Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32,
3866 RTLIB::ROUND_F64,
3867 RTLIB::ROUND_F80,
3868 RTLIB::ROUND_F128,
3869 RTLIB::ROUND_PPCF128));
3870 break;
3871 case ISD::FPOWI:
3872 case ISD::STRICT_FPOWI:
3873 Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
3874 RTLIB::POWI_F80, RTLIB::POWI_F128,
3875 RTLIB::POWI_PPCF128));
3876 break;
3877 case ISD::FPOW:
3878 case ISD::STRICT_FPOW:
3879 if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite))
3880 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32,
3881 RTLIB::POW_FINITE_F64,
3882 RTLIB::POW_FINITE_F80,
3883 RTLIB::POW_FINITE_F128,
3884 RTLIB::POW_FINITE_PPCF128));
3885 else
3886 Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64,
3887 RTLIB::POW_F80, RTLIB::POW_F128,
3888 RTLIB::POW_PPCF128));
3889 break;
3890 case ISD::FDIV:
3891 Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
3892 RTLIB::DIV_F80, RTLIB::DIV_F128,
3893 RTLIB::DIV_PPCF128));
3894 break;
3895 case ISD::FREM:
3896 case ISD::STRICT_FREM:
3897 Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
3898 RTLIB::REM_F80, RTLIB::REM_F128,
3899 RTLIB::REM_PPCF128));
3900 break;
3901 case ISD::FMA:
3902 case ISD::STRICT_FMA:
3903 Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
3904 RTLIB::FMA_F80, RTLIB::FMA_F128,
3905 RTLIB::FMA_PPCF128));
3906 break;
3907 case ISD::FADD:
3908 Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
3909 RTLIB::ADD_F80, RTLIB::ADD_F128,
3910 RTLIB::ADD_PPCF128));
3911 break;
3912 case ISD::FMUL:
3913 Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
3914 RTLIB::MUL_F80, RTLIB::MUL_F128,
3915 RTLIB::MUL_PPCF128));
3916 break;
3917 case ISD::FP16_TO_FP:
3918 if (Node->getValueType(0) == MVT::f32) {
3919 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
3920 }
3921 break;
3922 case ISD::FP_TO_FP16: {
3923 RTLIB::Libcall LC =
3924 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
3925 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")((LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16"
) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3925, __PRETTY_FUNCTION__))
;
3926 Results.push_back(ExpandLibCall(LC, Node, false));
3927 break;
3928 }
3929 case ISD::FSUB:
3930 Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
3931 RTLIB::SUB_F80, RTLIB::SUB_F128,
3932 RTLIB::SUB_PPCF128));
3933 break;
3934 case ISD::SREM:
3935 Results.push_back(ExpandIntLibCall(Node, true,
3936 RTLIB::SREM_I8,
3937 RTLIB::SREM_I16, RTLIB::SREM_I32,
3938 RTLIB::SREM_I64, RTLIB::SREM_I128));
3939 break;
3940 case ISD::UREM:
3941 Results.push_back(ExpandIntLibCall(Node, false,
3942 RTLIB::UREM_I8,
3943 RTLIB::UREM_I16, RTLIB::UREM_I32,
3944 RTLIB::UREM_I64, RTLIB::UREM_I128));
3945 break;
3946 case ISD::SDIV:
3947 Results.push_back(ExpandIntLibCall(Node, true,
3948 RTLIB::SDIV_I8,
3949 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
3950 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
3951 break;
3952 case ISD::UDIV:
3953 Results.push_back(ExpandIntLibCall(Node, false,
3954 RTLIB::UDIV_I8,
3955 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
3956 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
3957 break;
3958 case ISD::SDIVREM:
3959 case ISD::UDIVREM:
3960 // Expand into divrem libcall
3961 ExpandDivRemLibCall(Node, Results);
3962 break;
3963 case ISD::MUL:
3964 Results.push_back(ExpandIntLibCall(Node, false,
3965 RTLIB::MUL_I8,
3966 RTLIB::MUL_I16, RTLIB::MUL_I32,
3967 RTLIB::MUL_I64, RTLIB::MUL_I128));
3968 break;
3969 case ISD::CTLZ_ZERO_UNDEF:
3970 switch (Node->getSimpleValueType(0).SimpleTy) {
3971 default:
3972 llvm_unreachable("LibCall explicitly requested, but not available")::llvm::llvm_unreachable_internal("LibCall explicitly requested, but not available"
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 3972)
;
3973 case MVT::i32:
3974 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
3975 break;
3976 case MVT::i64:
3977 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
3978 break;
3979 case MVT::i128:
3980 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
3981 break;
3982 }
3983 break;
3984 }
3985
3986 // Replace the original node with the legalized result.
3987 if (!Results.empty()) {
3988 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully converted node to libcall\n"
; } } while (false)
;
3989 ReplaceNode(Node, Results.data());
3990 } else
3991 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not convert node to libcall\n"
; } } while (false)
;
3992}
3993
3994// Determine the vector type to use in place of an original scalar element when
3995// promoting equally sized vectors.
3996static MVT getPromotedVectorElementType(const TargetLowering &TLI,
3997 MVT EltVT, MVT NewEltVT) {
3998 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
3999 MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4000 assert(TLI.isTypeLegal(MidVT) && "unexpected")((TLI.isTypeLegal(MidVT) && "unexpected") ? static_cast
<void> (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4000, __PRETTY_FUNCTION__))
;
4001 return MidVT;
4002}
4003
4004void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4005 LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Trying to promote node\n"
; } } while (false)
;
4006 SmallVector<SDValue, 8> Results;
4007 MVT OVT = Node->getSimpleValueType(0);
4008 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4009 Node->getOpcode() == ISD::SINT_TO_FP ||
4010 Node->getOpcode() == ISD::SETCC ||
4011 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4012 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4013 OVT = Node->getOperand(0).getSimpleValueType();
4014 }
4015 if (Node->getOpcode() == ISD::BR_CC)
4016 OVT = Node->getOperand(2).getSimpleValueType();
4017 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4018 SDLoc dl(Node);
4019 SDValue Tmp1, Tmp2, Tmp3;
4020 switch (Node->getOpcode()) {
4021 case ISD::CTTZ:
4022 case ISD::CTTZ_ZERO_UNDEF:
4023 case ISD::CTLZ:
4024 case ISD::CTLZ_ZERO_UNDEF:
4025 case ISD::CTPOP:
4026 // Zero extend the argument.
4027 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4028 if (Node->getOpcode() == ISD::CTTZ) {
4029 // The count is the same in the promoted type except if the original
4030 // value was zero. This can be handled by setting the bit just off
4031 // the top of the original type.
4032 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4033 OVT.getSizeInBits());
4034 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4035 DAG.getConstant(TopBit, dl, NVT));
4036 }
4037 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4038 // already the correct result.
4039 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4040 if (Node->getOpcode() == ISD::CTLZ ||
4041 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4042 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4043 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4044 DAG.getConstant(NVT.getSizeInBits() -
4045 OVT.getSizeInBits(), dl, NVT));
4046 }
4047 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4048 break;
4049 case ISD::BITREVERSE:
4050 case ISD::BSWAP: {
4051 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4052 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4053 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4054 Tmp1 = DAG.getNode(
4055 ISD::SRL, dl, NVT, Tmp1,
4056 DAG.getConstant(DiffBits, dl,
4057 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4058
4059 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4060 break;
4061 }
4062 case ISD::FP_TO_UINT:
4063 case ISD::FP_TO_SINT:
4064 Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0),
4065 Node->getOpcode() == ISD::FP_TO_SINT, dl);
4066 Results.push_back(Tmp1);
4067 break;
4068 case ISD::UINT_TO_FP:
4069 case ISD::SINT_TO_FP:
4070 Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0),
4071 Node->getOpcode() == ISD::SINT_TO_FP, dl);
4072 Results.push_back(Tmp1);
4073 break;
4074 case ISD::VAARG: {
4075 SDValue Chain = Node->getOperand(0); // Get the chain.
4076 SDValue Ptr = Node->getOperand(1); // Get the pointer.
4077
4078 unsigned TruncOp;
4079 if (OVT.isVector()) {
4080 TruncOp = ISD::BITCAST;
4081 } else {
4082 assert(OVT.isInteger()((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4083, __PRETTY_FUNCTION__))
4083 && "VAARG promotion is supported only for vectors or integer types")((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4083, __PRETTY_FUNCTION__))
;
4084 TruncOp = ISD::TRUNCATE;
4085 }
4086
4087 // Perform the larger operation, then convert back
4088 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4089 Node->getConstantOperandVal(3));
4090 Chain = Tmp1.getValue(1);
4091
4092 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4093
4094 // Modified the chain result - switch anything that used the old chain to
4095 // use the new one.
4096 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4097 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4098 if (UpdatedNodes) {
4099 UpdatedNodes->insert(Tmp2.getNode());
4100 UpdatedNodes->insert(Chain.getNode());
4101 }
4102 ReplacedNode(Node);
4103 break;
4104 }
4105 case ISD::MUL:
4106 case ISD::SDIV:
4107 case ISD::SREM:
4108 case ISD::UDIV:
4109 case ISD::UREM:
4110 case ISD::AND:
4111 case ISD::OR:
4112 case ISD::XOR: {
4113 unsigned ExtOp, TruncOp;
4114 if (OVT.isVector()) {
4115 ExtOp = ISD::BITCAST;
4116 TruncOp = ISD::BITCAST;
4117 } else {
4118 assert(OVT.isInteger() && "Cannot promote logic operation")((OVT.isInteger() && "Cannot promote logic operation"
) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4118, __PRETTY_FUNCTION__))
;
4119
4120 switch (Node->getOpcode()) {
4121 default:
4122 ExtOp = ISD::ANY_EXTEND;
4123 break;
4124 case ISD::SDIV:
4125 case ISD::SREM:
4126 ExtOp = ISD::SIGN_EXTEND;
4127 break;
4128 case ISD::UDIV:
4129 case ISD::UREM:
4130 ExtOp = ISD::ZERO_EXTEND;
4131 break;
4132 }
4133 TruncOp = ISD::TRUNCATE;
4134 }
4135 // Promote each of the values to the new type.
4136 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4137 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4138 // Perform the larger operation, then convert back
4139 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4140 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4141 break;
4142 }
4143 case ISD::UMUL_LOHI:
4144 case ISD::SMUL_LOHI: {
4145 // Promote to a multiply in a wider integer type.
4146 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4147 : ISD::SIGN_EXTEND;
4148 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4149 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4150 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4151
4152 auto &DL = DAG.getDataLayout();
4153 unsigned OriginalSize = OVT.getScalarSizeInBits();
4154 Tmp2 = DAG.getNode(
4155 ISD::SRL, dl, NVT, Tmp1,
4156 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4157 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4158 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4159 break;
4160 }
4161 case ISD::SELECT: {
4162 unsigned ExtOp, TruncOp;
4163 if (Node->getValueType(0).isVector() ||
4164 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4165 ExtOp = ISD::BITCAST;
4166 TruncOp = ISD::BITCAST;
4167 } else if (Node->getValueType(0).isInteger()) {
4168 ExtOp = ISD::ANY_EXTEND;
4169 TruncOp = ISD::TRUNCATE;
4170 } else {
4171 ExtOp = ISD::FP_EXTEND;
4172 TruncOp = ISD::FP_ROUND;
4173 }
4174 Tmp1 = Node->getOperand(0);
4175 // Promote each of the values to the new type.
4176 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4177 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4178 // Perform the larger operation, then round down.
4179 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4180 if (TruncOp != ISD::FP_ROUND)
4181 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4182 else
4183 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4184 DAG.getIntPtrConstant(0, dl));
4185 Results.push_back(Tmp1);
4186 break;
4187 }
4188 case ISD::VECTOR_SHUFFLE: {
4189 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4190
4191 // Cast the two input vectors.
4192 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4193 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4194
4195 // Convert the shuffle mask to the right # elements.
4196 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4197 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4198 Results.push_back(Tmp1);
4199 break;
4200 }
4201 case ISD::SETCC: {
4202 unsigned ExtOp = ISD::FP_EXTEND;
4203 if (NVT.isInteger()) {
4204 ISD::CondCode CCCode =
4205 cast<CondCodeSDNode>(Node->getOperand(2))->get();
4206 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4207 }
4208 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4209 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4210 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
4211 Tmp1, Tmp2, Node->getOperand(2)));
4212 break;
4213 }
4214 case ISD::BR_CC: {
4215 unsigned ExtOp = ISD::FP_EXTEND;
4216 if (NVT.isInteger()) {
4217 ISD::CondCode CCCode =
4218 cast<CondCodeSDNode>(Node->getOperand(1))->get();
4219 ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
4220 }
4221 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4222 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4223 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4224 Node->getOperand(0), Node->getOperand(1),
4225 Tmp1, Tmp2, Node->getOperand(4)));
4226 break;
4227 }
4228 case ISD::FADD:
4229 case ISD::FSUB:
4230 case ISD::FMUL:
4231 case ISD::FDIV:
4232 case ISD::FREM:
4233 case ISD::FMINNUM:
4234 case ISD::FMAXNUM:
4235 case ISD::FPOW:
4236 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4237 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4238 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4239 Node->getFlags());
4240 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4241 Tmp3, DAG.getIntPtrConstant(0, dl)));
4242 break;
4243 case ISD::FMA:
4244 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4245 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4246 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4247 Results.push_back(
4248 DAG.getNode(ISD::FP_ROUND, dl, OVT,
4249 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4250 DAG.getIntPtrConstant(0, dl)));
4251 break;
4252 case ISD::FCOPYSIGN:
4253 case ISD::FPOWI: {
4254 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4255 Tmp2 = Node->getOperand(1);
4256 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4257
4258 // fcopysign doesn't change anything but the sign bit, so
4259 // (fp_round (fcopysign (fpext a), b))
4260 // is as precise as
4261 // (fp_round (fpext a))
4262 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4263 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4264 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4265 Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4266 break;
4267 }
4268 case ISD::FFLOOR:
4269 case ISD::FCEIL:
4270 case ISD::FRINT:
4271 case ISD::FNEARBYINT:
4272 case ISD::FROUND:
4273 case ISD::FTRUNC:
4274 case ISD::FNEG:
4275 case ISD::FSQRT:
4276 case ISD::FSIN:
4277 case ISD::FCOS:
4278 case ISD::FLOG:
4279 case ISD::FLOG2:
4280 case ISD::FLOG10:
4281 case ISD::FABS:
4282 case ISD::FEXP:
4283 case ISD::FEXP2:
4284 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4285 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4286 Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4287 Tmp2, DAG.getIntPtrConstant(0, dl)));
4288 break;
4289 case ISD::BUILD_VECTOR: {
4290 MVT EltVT = OVT.getVectorElementType();
4291 MVT NewEltVT = NVT.getVectorElementType();
4292
4293 // Handle bitcasts to a different vector type with the same total bit size
4294 //
4295 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4296 // =>
4297 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4298
4299 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4300, __PRETTY_FUNCTION__))
4300 "Invalid promote type for build_vector")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for build_vector") ? static_cast
<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4300, __PRETTY_FUNCTION__))
;
4301 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4301, __PRETTY_FUNCTION__))
;
4302
4303 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4304
4305 SmallVector<SDValue, 8> NewOps;
4306 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4307 SDValue Op = Node->getOperand(I);
4308 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4309 }
4310
4311 SDLoc SL(Node);
4312 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4313 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4314 Results.push_back(CvtVec);
4315 break;
4316 }
4317 case ISD::EXTRACT_VECTOR_ELT: {
4318 MVT EltVT = OVT.getVectorElementType();
4319 MVT NewEltVT = NVT.getVectorElementType();
4320
4321 // Handle bitcasts to a different vector type with the same total bit size.
4322 //
4323 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4324 // =>
4325 // v4i32:castx = bitcast x:v2i64
4326 //
4327 // i64 = bitcast
4328 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4329 // (i32 (extract_vector_elt castx, (2 * y + 1)))
4330 //
4331
4332 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4333, __PRETTY_FUNCTION__))
4333 "Invalid promote type for extract_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for extract_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4333, __PRETTY_FUNCTION__))
;
4334 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4334, __PRETTY_FUNCTION__))
;
4335
4336 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4337 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4338
4339 SDValue Idx = Node->getOperand(1);
4340 EVT IdxVT = Idx.getValueType();
4341 SDLoc SL(Node);
4342 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4343 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4344
4345 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4346
4347 SmallVector<SDValue, 8> NewOps;
4348 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4349 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4350 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4351
4352 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4353 CastVec, TmpIdx);
4354 NewOps.push_back(Elt);
4355 }
4356
4357 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4358 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4359 break;
4360 }
4361 case ISD::INSERT_VECTOR_ELT: {
4362 MVT EltVT = OVT.getVectorElementType();
4363 MVT NewEltVT = NVT.getVectorElementType();
4364
4365 // Handle bitcasts to a different vector type with the same total bit size
4366 //
4367 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4368 // =>
4369 // v4i32:castx = bitcast x:v2i64
4370 // v2i32:casty = bitcast y:i64
4371 //
4372 // v2i64 = bitcast
4373 // (v4i32 insert_vector_elt
4374 // (v4i32 insert_vector_elt v4i32:castx,
4375 // (extract_vector_elt casty, 0), 2 * z),
4376 // (extract_vector_elt casty, 1), (2 * z + 1))
4377
4378 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4379, __PRETTY_FUNCTION__))
4379 "Invalid promote type for insert_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits
() && "Invalid promote type for insert_vector_elt") ?
static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4379, __PRETTY_FUNCTION__))
;
4380 assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast
<void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4380, __PRETTY_FUNCTION__))
;
4381
4382 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4383 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4384
4385 SDValue Val = Node->getOperand(1);
4386 SDValue Idx = Node->getOperand(2);
4387 EVT IdxVT = Idx.getValueType();
4388 SDLoc SL(Node);
4389
4390 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4391 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4392
4393 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4394 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4395
4396 SDValue NewVec = CastVec;
4397 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4398 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4399 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4400
4401 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4402 CastVal, IdxOffset);
4403
4404 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4405 NewVec, Elt, InEltIdx);
4406 }
4407
4408 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4409 break;
4410 }
4411 case ISD::SCALAR_TO_VECTOR: {
4412 MVT EltVT = OVT.getVectorElementType();
4413 MVT NewEltVT = NVT.getVectorElementType();
4414
4415 // Handle bitcasts to different vector type with the same total bit size.
4416 //
4417 // e.g. v2i64 = scalar_to_vector x:i64
4418 // =>
4419 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4420 //
4421
4422 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4423 SDValue Val = Node->getOperand(0);
4424 SDLoc SL(Node);
4425
4426 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4427 SDValue Undef = DAG.getUNDEF(MidVT);
4428
4429 SmallVector<SDValue, 8> NewElts;
4430 NewElts.push_back(CastVal);
4431 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4432 NewElts.push_back(Undef);
4433
4434 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4435 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4436 Results.push_back(CvtVec);
4437 break;
4438 }
4439 case ISD::ATOMIC_SWAP: {
4440 AtomicSDNode *AM = cast<AtomicSDNode>(Node);
4441 SDLoc SL(Node);
4442 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4443 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&((NVT.getSizeInBits() == OVT.getSizeInBits() && "unexpected promotion type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == OVT.getSizeInBits() && \"unexpected promotion type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4444, __PRETTY_FUNCTION__))
4444 "unexpected promotion type")((NVT.getSizeInBits() == OVT.getSizeInBits() && "unexpected promotion type"
) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == OVT.getSizeInBits() && \"unexpected promotion type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4444, __PRETTY_FUNCTION__))
;
4445 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&((AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits()
&& "unexpected atomic_swap with illegal type") ? static_cast
<void> (0) : __assert_fail ("AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && \"unexpected atomic_swap with illegal type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4446, __PRETTY_FUNCTION__))
4446 "unexpected atomic_swap with illegal type")((AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits()
&& "unexpected atomic_swap with illegal type") ? static_cast
<void> (0) : __assert_fail ("AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && \"unexpected atomic_swap with illegal type\""
, "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp"
, 4446, __PRETTY_FUNCTION__))
;
4447
4448 SDValue NewAtomic
4449 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
4450 DAG.getVTList(NVT, MVT::Other),
4451 { AM->getChain(), AM->getBasePtr(), CastVal },
4452 AM->getMemOperand());
4453 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
4454 Results.push_back(NewAtomic.getValue(1));
4455 break;
4456 }
4457 }
4458
4459 // Replace the original node with the legalized result.
4460 if (!Results.empty()) {
4461 LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Successfully promoted node\n"
; } } while (false)
;
4462 ReplaceNode(Node, Results.data());
4463 } else
4464 LLVM_DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("legalizedag")) { dbgs() << "Could not promote node\n"
; } } while (false)
;
4465}
4466
4467/// This is the entry point for the file.
4468void SelectionDAG::Legalize() {
4469 AssignTopologicalOrder();
4470
4471 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4472 // Use a delete listener to remove nodes which were deleted during
4473 // legalization from LegalizeNodes. This is needed to handle the situation
4474 // where a new node is allocated by the object pool to the same address of a
4475 // previously deleted node.
4476 DAGNodeDeletedListener DeleteListener(
4477 *this,
4478 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4479
4480 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4481
4482 // Visit all the nodes. We start in topological order, so that we see
4483 // nodes with their original operands intact. Legalization can produce
4484 // new nodes which may themselves need to be legalized. Iterate until all
4485 // nodes have been legalized.
4486 while (true) {
4487 bool AnyLegalized = false;
4488 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4489 --NI;
4490
4491 SDNode *N = &*NI;
4492 if (N->use_empty() && N != getRoot().getNode()) {
4493 ++NI;
4494 DeleteNode(N);
4495 continue;
4496 }
4497
4498 if (LegalizedNodes.insert(N).second) {
4499 AnyLegalized = true;
4500 Legalizer.LegalizeOp(N);
4501
4502 if (N->use_empty() && N != getRoot().getNode()) {
4503 ++NI;
4504 DeleteNode(N);
4505 }
4506 }
4507 }
4508 if (!AnyLegalized)
4509 break;
4510
4511 }
4512
4513 // Remove dead nodes now.
4514 RemoveDeadNodes();
4515}
4516
4517bool SelectionDAG::LegalizeOp(SDNode *N,
4518 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4519 SmallPtrSet<SDNode *, 16> LegalizedNodes;
4520 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4521
4522 // Directly insert the node in question, and legalize it. This will recurse
4523 // as needed through operands.
4524 LegalizedNodes.insert(N);
4525 Legalizer.LegalizeOp(N);
4526
4527 return LegalizedNodes.count(N);
4528}

/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h

1//===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains some functions that are useful for math stuff.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_SUPPORT_MATHEXTRAS_H
14#define LLVM_SUPPORT_MATHEXTRAS_H
15
16#include "llvm/Support/Compiler.h"
17#include "llvm/Support/SwapByteOrder.h"
18#include <algorithm>
19#include <cassert>
20#include <climits>
21#include <cstring>
22#include <limits>
23#include <type_traits>
24
25#ifdef __ANDROID_NDK__
26#include <android/api-level.h>
27#endif
28
29#ifdef _MSC_VER
30// Declare these intrinsics manually rather including intrin.h. It's very
31// expensive, and MathExtras.h is popular.
32// #include <intrin.h>
33extern "C" {
34unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask);
35unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask);
36unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask);
37unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask);
38}
39#endif
40
41namespace llvm {
42/// The behavior an operation has on an input of 0.
43enum ZeroBehavior {
44 /// The returned value is undefined.
45 ZB_Undefined,
46 /// The returned value is numeric_limits<T>::max()
47 ZB_Max,
48 /// The returned value is numeric_limits<T>::digits
49 ZB_Width
50};
51
52namespace detail {
53template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter {
54 static std::size_t count(T Val, ZeroBehavior) {
55 if (!Val)
56 return std::numeric_limits<T>::digits;
57 if (Val & 0x1)
58 return 0;
59
60 // Bisection method.
61 std::size_t ZeroBits = 0;
62 T Shift = std::numeric_limits<T>::digits >> 1;
63 T Mask = std::numeric_limits<T>::max() >> Shift;
64 while (Shift) {
65 if ((Val & Mask) == 0) {
66 Val >>= Shift;
67 ZeroBits |= Shift;
68 }
69 Shift >>= 1;
70 Mask >>= Shift;
71 }
72 return ZeroBits;
73 }
74};
75
76#if __GNUC__4 >= 4 || defined(_MSC_VER)
77template <typename T> struct TrailingZerosCounter<T, 4> {
78 static std::size_t count(T Val, ZeroBehavior ZB) {
79 if (ZB != ZB_Undefined && Val == 0)
80 return 32;
81
82#if __has_builtin(__builtin_ctz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
83 return __builtin_ctz(Val);
84#elif defined(_MSC_VER)
85 unsigned long Index;
86 _BitScanForward(&Index, Val);
87 return Index;
88#endif
89 }
90};
91
92#if !defined(_MSC_VER) || defined(_M_X64)
93template <typename T> struct TrailingZerosCounter<T, 8> {
94 static std::size_t count(T Val, ZeroBehavior ZB) {
95 if (ZB != ZB_Undefined && Val == 0)
96 return 64;
97
98#if __has_builtin(__builtin_ctzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
99 return __builtin_ctzll(Val);
100#elif defined(_MSC_VER)
101 unsigned long Index;
102 _BitScanForward64(&Index, Val);
103 return Index;
104#endif
105 }
106};
107#endif
108#endif
109} // namespace detail
110
111/// Count number of 0's from the least significant bit to the most
112/// stopping at the first 1.
113///
114/// Only unsigned integral types are allowed.
115///
116/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
117/// valid arguments.
118template <typename T>
119std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
120 static_assert(std::numeric_limits<T>::is_integer &&
121 !std::numeric_limits<T>::is_signed,
122 "Only unsigned integral types are allowed.");
123 return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB);
124}
125
126namespace detail {
127template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter {
128 static std::size_t count(T Val, ZeroBehavior) {
129 if (!Val)
130 return std::numeric_limits<T>::digits;
131
132 // Bisection method.
133 std::size_t ZeroBits = 0;
134 for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) {
135 T Tmp = Val >> Shift;
136 if (Tmp)
137 Val = Tmp;
138 else
139 ZeroBits |= Shift;
140 }
141 return ZeroBits;
142 }
143};
144
145#if __GNUC__4 >= 4 || defined(_MSC_VER)
146template <typename T> struct LeadingZerosCounter<T, 4> {
147 static std::size_t count(T Val, ZeroBehavior ZB) {
148 if (ZB != ZB_Undefined && Val == 0)
149 return 32;
150
151#if __has_builtin(__builtin_clz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
152 return __builtin_clz(Val);
153#elif defined(_MSC_VER)
154 unsigned long Index;
155 _BitScanReverse(&Index, Val);
156 return Index ^ 31;
157#endif
158 }
159};
160
161#if !defined(_MSC_VER) || defined(_M_X64)
162template <typename T> struct LeadingZerosCounter<T, 8> {
163 static std::size_t count(T Val, ZeroBehavior ZB) {
164 if (ZB != ZB_Undefined && Val == 0)
165 return 64;
166
167#if __has_builtin(__builtin_clzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20
) + ((0) << 10) + (0))
168 return __builtin_clzll(Val);
169#elif defined(_MSC_VER)
170 unsigned long Index;
171 _BitScanReverse64(&Index, Val);
172 return Index ^ 63;
173#endif
174 }
175};
176#endif
177#endif
178} // namespace detail
179
180/// Count number of 0's from the most significant bit to the least
181/// stopping at the first 1.
182///
183/// Only unsigned integral types are allowed.
184///
185/// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are
186/// valid arguments.
187template <typename T>
188std::size_t countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) {
189 static_assert(std::numeric_limits<T>::is_integer &&
190 !std::numeric_limits<T>::is_signed,
191 "Only unsigned integral types are allowed.");
192 return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB);
193}
194
195/// Get the index of the first set bit starting from the least
196/// significant bit.
197///
198/// Only unsigned integral types are allowed.
199///
200/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
201/// valid arguments.
202template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) {
203 if (ZB == ZB_Max && Val == 0)
204 return std::numeric_limits<T>::max();
205
206 return countTrailingZeros(Val, ZB_Undefined);
207}
208
209/// Create a bitmask with the N right-most bits set to 1, and all other
210/// bits set to 0. Only unsigned types are allowed.
211template <typename T> T maskTrailingOnes(unsigned N) {
212 static_assert(std::is_unsigned<T>::value, "Invalid type!");
213 const unsigned Bits = CHAR_BIT8 * sizeof(T);
214 assert(N <= Bits && "Invalid bit index")((N <= Bits && "Invalid bit index") ? static_cast<
void> (0) : __assert_fail ("N <= Bits && \"Invalid bit index\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 214, __PRETTY_FUNCTION__))
;
215 return N == 0 ? 0 : (T(-1) >> (Bits - N));
216}
217
218/// Create a bitmask with the N left-most bits set to 1, and all other
219/// bits set to 0. Only unsigned types are allowed.
220template <typename T> T maskLeadingOnes(unsigned N) {
221 return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
222}
223
224/// Create a bitmask with the N right-most bits set to 0, and all other
225/// bits set to 1. Only unsigned types are allowed.
226template <typename T> T maskTrailingZeros(unsigned N) {
227 return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
228}
229
230/// Create a bitmask with the N left-most bits set to 0, and all other
231/// bits set to 1. Only unsigned types are allowed.
232template <typename T> T maskLeadingZeros(unsigned N) {
233 return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N);
234}
235
236/// Get the index of the last set bit starting from the least
237/// significant bit.
238///
239/// Only unsigned integral types are allowed.
240///
241/// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are
242/// valid arguments.
243template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) {
244 if (ZB == ZB_Max && Val == 0)
245 return std::numeric_limits<T>::max();
246
247 // Use ^ instead of - because both gcc and llvm can remove the associated ^
248 // in the __builtin_clz intrinsic on x86.
249 return countLeadingZeros(Val, ZB_Undefined) ^
250 (std::numeric_limits<T>::digits - 1);
251}
252
253/// Macro compressed bit reversal table for 256 bits.
254///
255/// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable
256static const unsigned char BitReverseTable256[256] = {
257#define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64
258#define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16)
259#define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4)
260 R6(0), R6(2), R6(1), R6(3)
261#undef R2
262#undef R4
263#undef R6
264};
265
266/// Reverse the bits in \p Val.
267template <typename T>
268T reverseBits(T Val) {
269 unsigned char in[sizeof(Val)];
270 unsigned char out[sizeof(Val)];
271 std::memcpy(in, &Val, sizeof(Val));
272 for (unsigned i = 0; i < sizeof(Val); ++i)
273 out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]];
274 std::memcpy(&Val, out, sizeof(Val));
275 return Val;
276}
277
278// NOTE: The following support functions use the _32/_64 extensions instead of
279// type overloading so that signed and unsigned integers can be used without
280// ambiguity.
281
282/// Return the high 32 bits of a 64 bit value.
283constexpr inline uint32_t Hi_32(uint64_t Value) {
284 return static_cast<uint32_t>(Value >> 32);
285}
286
287/// Return the low 32 bits of a 64 bit value.
288constexpr inline uint32_t Lo_32(uint64_t Value) {
289 return static_cast<uint32_t>(Value);
290}
291
292/// Make a 64-bit integer from a high / low pair of 32-bit integers.
293constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) {
294 return ((uint64_t)High << 32) | (uint64_t)Low;
295}
296
297/// Checks if an integer fits into the given bit width.
298template <unsigned N> constexpr inline bool isInt(int64_t x) {
299 return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1)));
300}
301// Template specializations to get better code for common cases.
302template <> constexpr inline bool isInt<8>(int64_t x) {
303 return static_cast<int8_t>(x) == x;
304}
305template <> constexpr inline bool isInt<16>(int64_t x) {
306 return static_cast<int16_t>(x) == x;
307}
308template <> constexpr inline bool isInt<32>(int64_t x) {
309 return static_cast<int32_t>(x) == x;
310}
311
312/// Checks if a signed integer is an N bit number shifted left by S.
313template <unsigned N, unsigned S>
314constexpr inline bool isShiftedInt(int64_t x) {
315 static_assert(
316 N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number.");
317 static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide.");
318 return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
319}
320
321/// Checks if an unsigned integer fits into the given bit width.
322///
323/// This is written as two functions rather than as simply
324///
325/// return N >= 64 || X < (UINT64_C(1) << N);
326///
327/// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting
328/// left too many places.
329template <unsigned N>
330constexpr inline typename std::enable_if<(N < 64), bool>::type
331isUInt(uint64_t X) {
332 static_assert(N > 0, "isUInt<0> doesn't make sense");
333 return X < (UINT64_C(1)1UL << (N));
334}
335template <unsigned N>
336constexpr inline typename std::enable_if<N >= 64, bool>::type
337isUInt(uint64_t X) {
338 return true;
339}
340
341// Template specializations to get better code for common cases.
342template <> constexpr inline bool isUInt<8>(uint64_t x) {
343 return static_cast<uint8_t>(x) == x;
344}
345template <> constexpr inline bool isUInt<16>(uint64_t x) {
346 return static_cast<uint16_t>(x) == x;
347}
348template <> constexpr inline bool isUInt<32>(uint64_t x) {
349 return static_cast<uint32_t>(x) == x;
350}
351
352/// Checks if a unsigned integer is an N bit number shifted left by S.
353template <unsigned N, unsigned S>
354constexpr inline bool isShiftedUInt(uint64_t x) {
355 static_assert(
356 N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)");
357 static_assert(N + S <= 64,
358 "isShiftedUInt<N, S> with N + S > 64 is too wide.");
359 // Per the two static_asserts above, S must be strictly less than 64. So
360 // 1 << S is not undefined behavior.
361 return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0);
362}
363
364/// Gets the maximum value for a N-bit unsigned integer.
365inline uint64_t maxUIntN(uint64_t N) {
366 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 366, __PRETTY_FUNCTION__))
;
367
368 // uint64_t(1) << 64 is undefined behavior, so we can't do
369 // (uint64_t(1) << N) - 1
370 // without checking first that N != 64. But this works and doesn't have a
371 // branch.
372 return UINT64_MAX(18446744073709551615UL) >> (64 - N);
373}
374
375/// Gets the minimum value for a N-bit signed integer.
376inline int64_t minIntN(int64_t N) {
377 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 377, __PRETTY_FUNCTION__))
;
378
379 return -(UINT64_C(1)1UL<<(N-1));
380}
381
382/// Gets the maximum value for a N-bit signed integer.
383inline int64_t maxIntN(int64_t N) {
384 assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range"
) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 384, __PRETTY_FUNCTION__))
;
385
386 // This relies on two's complement wraparound when N == 64, so we convert to
387 // int64_t only at the very end to avoid UB.
388 return (UINT64_C(1)1UL << (N - 1)) - 1;
389}
390
391/// Checks if an unsigned integer fits into the given (dynamic) bit width.
392inline bool isUIntN(unsigned N, uint64_t x) {
393 return N >= 64 || x <= maxUIntN(N);
394}
395
396/// Checks if an signed integer fits into the given (dynamic) bit width.
397inline bool isIntN(unsigned N, int64_t x) {
398 return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N));
399}
400
401/// Return true if the argument is a non-empty sequence of ones starting at the
402/// least significant bit with the remainder zero (32 bit version).
403/// Ex. isMask_32(0x0000FFFFU) == true.
404constexpr inline bool isMask_32(uint32_t Value) {
405 return Value && ((Value + 1) & Value) == 0;
406}
407
408/// Return true if the argument is a non-empty sequence of ones starting at the
409/// least significant bit with the remainder zero (64 bit version).
410constexpr inline bool isMask_64(uint64_t Value) {
411 return Value && ((Value + 1) & Value) == 0;
412}
413
414/// Return true if the argument contains a non-empty sequence of ones with the
415/// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true.
416constexpr inline bool isShiftedMask_32(uint32_t Value) {
417 return Value && isMask_32((Value - 1) | Value);
418}
419
420/// Return true if the argument contains a non-empty sequence of ones with the
421/// remainder zero (64 bit version.)
422constexpr inline bool isShiftedMask_64(uint64_t Value) {
423 return Value && isMask_64((Value - 1) | Value);
424}
425
426/// Return true if the argument is a power of two > 0.
427/// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.)
428constexpr inline bool isPowerOf2_32(uint32_t Value) {
429 return Value && !(Value & (Value - 1));
430}
431
432/// Return true if the argument is a power of two > 0 (64 bit edition.)
433constexpr inline bool isPowerOf2_64(uint64_t Value) {
434 return Value && !(Value & (Value - 1));
435}
436
437/// Return a byte-swapped representation of the 16-bit argument.
438inline uint16_t ByteSwap_16(uint16_t Value) {
439 return sys::SwapByteOrder_16(Value);
440}
441
442/// Return a byte-swapped representation of the 32-bit argument.
443inline uint32_t ByteSwap_32(uint32_t Value) {
444 return sys::SwapByteOrder_32(Value);
445}
446
447/// Return a byte-swapped representation of the 64-bit argument.
448inline uint64_t ByteSwap_64(uint64_t Value) {
449 return sys::SwapByteOrder_64(Value);
450}
451
452/// Count the number of ones from the most significant bit to the first
453/// zero bit.
454///
455/// Ex. countLeadingOnes(0xFF0FFF00) == 8.
456/// Only unsigned integral types are allowed.
457///
458/// \param ZB the behavior on an input of all ones. Only ZB_Width and
459/// ZB_Undefined are valid arguments.
460template <typename T>
461std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
462 static_assert(std::numeric_limits<T>::is_integer &&
463 !std::numeric_limits<T>::is_signed,
464 "Only unsigned integral types are allowed.");
465 return countLeadingZeros<T>(~Value, ZB);
466}
467
468/// Count the number of ones from the least significant bit to the first
469/// zero bit.
470///
471/// Ex. countTrailingOnes(0x00FF00FF) == 8.
472/// Only unsigned integral types are allowed.
473///
474/// \param ZB the behavior on an input of all ones. Only ZB_Width and
475/// ZB_Undefined are valid arguments.
476template <typename T>
477std::size_t countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) {
478 static_assert(std::numeric_limits<T>::is_integer &&
479 !std::numeric_limits<T>::is_signed,
480 "Only unsigned integral types are allowed.");
481 return countTrailingZeros<T>(~Value, ZB);
482}
483
484namespace detail {
485template <typename T, std::size_t SizeOfT> struct PopulationCounter {
486 static unsigned count(T Value) {
487 // Generic version, forward to 32 bits.
488 static_assert(SizeOfT <= 4, "Not implemented!");
489#if __GNUC__4 >= 4
490 return __builtin_popcount(Value);
491#else
492 uint32_t v = Value;
493 v = v - ((v >> 1) & 0x55555555);
494 v = (v & 0x33333333) + ((v >> 2) & 0x33333333);
495 return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24;
496#endif
497 }
498};
499
500template <typename T> struct PopulationCounter<T, 8> {
501 static unsigned count(T Value) {
502#if __GNUC__4 >= 4
503 return __builtin_popcountll(Value);
504#else
505 uint64_t v = Value;
506 v = v - ((v >> 1) & 0x5555555555555555ULL);
507 v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL);
508 v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL;
509 return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56);
510#endif
511 }
512};
513} // namespace detail
514
515/// Count the number of set bits in a value.
516/// Ex. countPopulation(0xF000F000) = 8
517/// Returns 0 if the word is zero.
518template <typename T>
519inline unsigned countPopulation(T Value) {
520 static_assert(std::numeric_limits<T>::is_integer &&
521 !std::numeric_limits<T>::is_signed,
522 "Only unsigned integral types are allowed.");
523 return detail::PopulationCounter<T, sizeof(T)>::count(Value);
524}
525
526/// Return the log base 2 of the specified value.
527inline double Log2(double Value) {
528#if defined(__ANDROID_API__) && __ANDROID_API__ < 18
529 return __builtin_log(Value) / __builtin_log(2.0);
530#else
531 return log2(Value);
532#endif
533}
534
535/// Return the floor log base 2 of the specified value, -1 if the value is zero.
536/// (32 bit edition.)
537/// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2
538inline unsigned Log2_32(uint32_t Value) {
539 return 31 - countLeadingZeros(Value);
12
Returning the value 4294967295
540}
541
542/// Return the floor log base 2 of the specified value, -1 if the value is zero.
543/// (64 bit edition.)
544inline unsigned Log2_64(uint64_t Value) {
545 return 63 - countLeadingZeros(Value);
546}
547
548/// Return the ceil log base 2 of the specified value, 32 if the value is zero.
549/// (32 bit edition).
550/// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3
551inline unsigned Log2_32_Ceil(uint32_t Value) {
552 return 32 - countLeadingZeros(Value - 1);
553}
554
555/// Return the ceil log base 2 of the specified value, 64 if the value is zero.
556/// (64 bit edition.)
557inline unsigned Log2_64_Ceil(uint64_t Value) {
558 return 64 - countLeadingZeros(Value - 1);
559}
560
561/// Return the greatest common divisor of the values using Euclid's algorithm.
562inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) {
563 while (B) {
564 uint64_t T = B;
565 B = A % B;
566 A = T;
567 }
568 return A;
569}
570
571/// This function takes a 64-bit integer and returns the bit equivalent double.
572inline double BitsToDouble(uint64_t Bits) {
573 double D;
574 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
575 memcpy(&D, &Bits, sizeof(Bits));
576 return D;
577}
578
579/// This function takes a 32-bit integer and returns the bit equivalent float.
580inline float BitsToFloat(uint32_t Bits) {
581 float F;
582 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
583 memcpy(&F, &Bits, sizeof(Bits));
584 return F;
585}
586
587/// This function takes a double and returns the bit equivalent 64-bit integer.
588/// Note that copying doubles around changes the bits of NaNs on some hosts,
589/// notably x86, so this routine cannot be used if these bits are needed.
590inline uint64_t DoubleToBits(double Double) {
591 uint64_t Bits;
592 static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes");
593 memcpy(&Bits, &Double, sizeof(Double));
594 return Bits;
595}
596
597/// This function takes a float and returns the bit equivalent 32-bit integer.
598/// Note that copying floats around changes the bits of NaNs on some hosts,
599/// notably x86, so this routine cannot be used if these bits are needed.
600inline uint32_t FloatToBits(float Float) {
601 uint32_t Bits;
602 static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes");
603 memcpy(&Bits, &Float, sizeof(Float));
604 return Bits;
605}
606
607/// A and B are either alignments or offsets. Return the minimum alignment that
608/// may be assumed after adding the two together.
609constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) {
610 // The largest power of 2 that divides both A and B.
611 //
612 // Replace "-Value" by "1+~Value" in the following commented code to avoid
613 // MSVC warning C4146
614 // return (A | B) & -(A | B);
615 return (A | B) & (1 + ~(A | B));
616}
617
618/// Aligns \c Addr to \c Alignment bytes, rounding up.
619///
620/// Alignment should be a power of two. This method rounds up, so
621/// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8.
622inline uintptr_t alignAddr(const void *Addr, size_t Alignment) {
623 assert(Alignment && isPowerOf2_64((uint64_t)Alignment) &&((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 624, __PRETTY_FUNCTION__))
624 "Alignment is not a power of two!")((Alignment && isPowerOf2_64((uint64_t)Alignment) &&
"Alignment is not a power of two!") ? static_cast<void>
(0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 624, __PRETTY_FUNCTION__))
;
625
626 assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr)(((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr) ? static_cast
<void> (0) : __assert_fail ("(uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr"
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 626, __PRETTY_FUNCTION__))
;
627
628 return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1));
629}
630
631/// Returns the necessary adjustment for aligning \c Ptr to \c Alignment
632/// bytes, rounding up.
633inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) {
634 return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr;
635}
636
637/// Returns the next power of two (in 64-bits) that is strictly greater than A.
638/// Returns zero on overflow.
639inline uint64_t NextPowerOf2(uint64_t A) {
640 A |= (A >> 1);
641 A |= (A >> 2);
642 A |= (A >> 4);
643 A |= (A >> 8);
644 A |= (A >> 16);
645 A |= (A >> 32);
646 return A + 1;
647}
648
649/// Returns the power of two which is less than or equal to the given value.
650/// Essentially, it is a floor operation across the domain of powers of two.
651inline uint64_t PowerOf2Floor(uint64_t A) {
652 if (!A) return 0;
653 return 1ull << (63 - countLeadingZeros(A, ZB_Undefined));
654}
655
656/// Returns the power of two which is greater than or equal to the given value.
657/// Essentially, it is a ceil operation across the domain of powers of two.
658inline uint64_t PowerOf2Ceil(uint64_t A) {
659 if (!A)
660 return 0;
661 return NextPowerOf2(A - 1);
662}
663
664/// Returns the next integer (mod 2**64) that is greater than or equal to
665/// \p Value and is a multiple of \p Align. \p Align must be non-zero.
666///
667/// If non-zero \p Skew is specified, the return value will be a minimal
668/// integer that is greater than or equal to \p Value and equal to
669/// \p Align * N + \p Skew for some integer N. If \p Skew is larger than
670/// \p Align, its value is adjusted to '\p Skew mod \p Align'.
671///
672/// Examples:
673/// \code
674/// alignTo(5, 8) = 8
675/// alignTo(17, 8) = 24
676/// alignTo(~0LL, 8) = 0
677/// alignTo(321, 255) = 510
678///
679/// alignTo(5, 8, 7) = 7
680/// alignTo(17, 8, 1) = 17
681/// alignTo(~0LL, 8, 3) = 3
682/// alignTo(321, 255, 42) = 552
683/// \endcode
684inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
685 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 685, __PRETTY_FUNCTION__))
;
686 Skew %= Align;
687 return (Value + Align - 1 - Skew) / Align * Align + Skew;
688}
689
690/// Returns the next integer (mod 2**64) that is greater than or equal to
691/// \p Value and is a multiple of \c Align. \c Align must be non-zero.
692template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) {
693 static_assert(Align != 0u, "Align must be non-zero");
694 return (Value + Align - 1) / Align * Align;
695}
696
697/// Returns the integer ceil(Numerator / Denominator).
698inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) {
699 return alignTo(Numerator, Denominator) / Denominator;
700}
701
702/// \c alignTo for contexts where a constant expression is required.
703/// \sa alignTo
704///
705/// \todo FIXME: remove when \c constexpr becomes really \c constexpr
706template <uint64_t Align>
707struct AlignTo {
708 static_assert(Align != 0u, "Align must be non-zero");
709 template <uint64_t Value>
710 struct from_value {
711 static const uint64_t value = (Value + Align - 1) / Align * Align;
712 };
713};
714
715/// Returns the largest uint64_t less than or equal to \p Value and is
716/// \p Skew mod \p Align. \p Align must be non-zero
717inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) {
718 assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast<
void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 718, __PRETTY_FUNCTION__))
;
719 Skew %= Align;
720 return (Value - Skew) / Align * Align + Skew;
721}
722
723/// Returns the offset to the next integer (mod 2**64) that is greater than
724/// or equal to \p Value and is a multiple of \p Align. \p Align must be
725/// non-zero.
726inline uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align) {
727 return alignTo(Value, Align) - Value;
728}
729
730/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
731/// Requires 0 < B <= 32.
732template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) {
733 static_assert(B > 0, "Bit width can't be 0.");
734 static_assert(B <= 32, "Bit width out of range.");
735 return int32_t(X << (32 - B)) >> (32 - B);
736}
737
738/// Sign-extend the number in the bottom B bits of X to a 32-bit integer.
739/// Requires 0 < B < 32.
740inline int32_t SignExtend32(uint32_t X, unsigned B) {
741 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 741, __PRETTY_FUNCTION__))
;
742 assert(B <= 32 && "Bit width out of range.")((B <= 32 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 742, __PRETTY_FUNCTION__))
;
743 return int32_t(X << (32 - B)) >> (32 - B);
744}
745
746/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
747/// Requires 0 < B < 64.
748template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) {
749 static_assert(B > 0, "Bit width can't be 0.");
750 static_assert(B <= 64, "Bit width out of range.");
751 return int64_t(x << (64 - B)) >> (64 - B);
752}
753
754/// Sign-extend the number in the bottom B bits of X to a 64-bit integer.
755/// Requires 0 < B < 64.
756inline int64_t SignExtend64(uint64_t X, unsigned B) {
757 assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast<
void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 757, __PRETTY_FUNCTION__))
;
758 assert(B <= 64 && "Bit width out of range.")((B <= 64 && "Bit width out of range.") ? static_cast
<void> (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\""
, "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h"
, 758, __PRETTY_FUNCTION__))
;
759 return int64_t(X << (64 - B)) >> (64 - B);
760}
761
762/// Subtract two unsigned integers, X and Y, of type T and return the absolute
763/// value of the result.
764template <typename T>
765typename std::enable_if<std::is_unsigned<T>::value, T>::type
766AbsoluteDifference(T X, T Y) {
767 return std::max(X, Y) - std::min(X, Y);
768}
769
770/// Add two unsigned integers, X and Y, of type T. Clamp the result to the
771/// maximum representable value of T on overflow. ResultOverflowed indicates if
772/// the result is larger than the maximum representable value of type T.
773template <typename T>
774typename std::enable_if<std::is_unsigned<T>::value, T>::type
775SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) {
776 bool Dummy;
777 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
778 // Hacker's Delight, p. 29
779 T Z = X + Y;
780 Overflowed = (Z < X || Z < Y);
781 if (Overflowed)
782 return std::numeric_limits<T>::max();
783 else
784 return Z;
785}
786
787/// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the
788/// maximum representable value of T on overflow. ResultOverflowed indicates if
789/// the result is larger than the maximum representable value of type T.
790template <typename T>
791typename std::enable_if<std::is_unsigned<T>::value, T>::type
792SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) {
793 bool Dummy;
794 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
795
796 // Hacker's Delight, p. 30 has a different algorithm, but we don't use that
797 // because it fails for uint16_t (where multiplication can have undefined
798 // behavior due to promotion to int), and requires a division in addition
799 // to the multiplication.
800
801 Overflowed = false;
802
803 // Log2(Z) would be either Log2Z or Log2Z + 1.
804 // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z
805 // will necessarily be less than Log2Max as desired.
806 int Log2Z = Log2_64(X) + Log2_64(Y);
807 const T Max = std::numeric_limits<T>::max();
808 int Log2Max = Log2_64(Max);
809 if (Log2Z < Log2Max) {
810 return X * Y;
811 }
812 if (Log2Z > Log2Max) {
813 Overflowed = true;
814 return Max;
815 }
816
817 // We're going to use the top bit, and maybe overflow one
818 // bit past it. Multiply all but the bottom bit then add
819 // that on at the end.
820 T Z = (X >> 1) * Y;
821 if (Z & ~(Max >> 1)) {
822 Overflowed = true;
823 return Max;
824 }
825 Z <<= 1;
826 if (X & 1)
827 return SaturatingAdd(Z, Y, ResultOverflowed);
828
829 return Z;
830}
831
832/// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to
833/// the product. Clamp the result to the maximum representable value of T on
834/// overflow. ResultOverflowed indicates if the result is larger than the
835/// maximum representable value of type T.
836template <typename T>
837typename std::enable_if<std::is_unsigned<T>::value, T>::type
838SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) {
839 bool Dummy;
840 bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy;
841
842 T Product = SaturatingMultiply(X, Y, &Overflowed);
843 if (Overflowed)
844 return Product;
845
846 return SaturatingAdd(A, Product, &Overflowed);
847}
848
849/// Use this rather than HUGE_VALF; the latter causes warnings on MSVC.
850extern const float huge_valf;
851} // End llvm namespace
852
853#endif