File: | lib/CodeGen/SelectionDAG/LegalizeDAG.cpp |
Warning: | line 756, column 29 The result of the left shift is undefined due to shifting by '4294967295', which is greater or equal to the width of type 'int' |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // | |||
9 | // This file implements the SelectionDAG::Legalize method. | |||
10 | // | |||
11 | //===----------------------------------------------------------------------===// | |||
12 | ||||
13 | #include "llvm/ADT/APFloat.h" | |||
14 | #include "llvm/ADT/APInt.h" | |||
15 | #include "llvm/ADT/ArrayRef.h" | |||
16 | #include "llvm/ADT/SetVector.h" | |||
17 | #include "llvm/ADT/SmallPtrSet.h" | |||
18 | #include "llvm/ADT/SmallSet.h" | |||
19 | #include "llvm/ADT/SmallVector.h" | |||
20 | #include "llvm/CodeGen/ISDOpcodes.h" | |||
21 | #include "llvm/CodeGen/MachineFunction.h" | |||
22 | #include "llvm/CodeGen/MachineJumpTableInfo.h" | |||
23 | #include "llvm/CodeGen/MachineMemOperand.h" | |||
24 | #include "llvm/CodeGen/RuntimeLibcalls.h" | |||
25 | #include "llvm/CodeGen/SelectionDAG.h" | |||
26 | #include "llvm/CodeGen/SelectionDAGNodes.h" | |||
27 | #include "llvm/CodeGen/TargetFrameLowering.h" | |||
28 | #include "llvm/CodeGen/TargetLowering.h" | |||
29 | #include "llvm/CodeGen/TargetSubtargetInfo.h" | |||
30 | #include "llvm/CodeGen/ValueTypes.h" | |||
31 | #include "llvm/IR/CallingConv.h" | |||
32 | #include "llvm/IR/Constants.h" | |||
33 | #include "llvm/IR/DataLayout.h" | |||
34 | #include "llvm/IR/DerivedTypes.h" | |||
35 | #include "llvm/IR/Function.h" | |||
36 | #include "llvm/IR/Metadata.h" | |||
37 | #include "llvm/IR/Type.h" | |||
38 | #include "llvm/Support/Casting.h" | |||
39 | #include "llvm/Support/Compiler.h" | |||
40 | #include "llvm/Support/Debug.h" | |||
41 | #include "llvm/Support/ErrorHandling.h" | |||
42 | #include "llvm/Support/MachineValueType.h" | |||
43 | #include "llvm/Support/MathExtras.h" | |||
44 | #include "llvm/Support/raw_ostream.h" | |||
45 | #include "llvm/Target/TargetMachine.h" | |||
46 | #include "llvm/Target/TargetOptions.h" | |||
47 | #include <algorithm> | |||
48 | #include <cassert> | |||
49 | #include <cstdint> | |||
50 | #include <tuple> | |||
51 | #include <utility> | |||
52 | ||||
53 | using namespace llvm; | |||
54 | ||||
55 | #define DEBUG_TYPE"legalizedag" "legalizedag" | |||
56 | ||||
57 | namespace { | |||
58 | ||||
59 | /// Keeps track of state when getting the sign of a floating-point value as an | |||
60 | /// integer. | |||
61 | struct FloatSignAsInt { | |||
62 | EVT FloatVT; | |||
63 | SDValue Chain; | |||
64 | SDValue FloatPtr; | |||
65 | SDValue IntPtr; | |||
66 | MachinePointerInfo IntPointerInfo; | |||
67 | MachinePointerInfo FloatPointerInfo; | |||
68 | SDValue IntValue; | |||
69 | APInt SignMask; | |||
70 | uint8_t SignBit; | |||
71 | }; | |||
72 | ||||
73 | //===----------------------------------------------------------------------===// | |||
74 | /// This takes an arbitrary SelectionDAG as input and | |||
75 | /// hacks on it until the target machine can handle it. This involves | |||
76 | /// eliminating value sizes the machine cannot handle (promoting small sizes to | |||
77 | /// large sizes or splitting up large values into small values) as well as | |||
78 | /// eliminating operations the machine cannot handle. | |||
79 | /// | |||
80 | /// This code also does a small amount of optimization and recognition of idioms | |||
81 | /// as part of its processing. For example, if a target does not support a | |||
82 | /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this | |||
83 | /// will attempt merge setcc and brc instructions into brcc's. | |||
84 | class SelectionDAGLegalize { | |||
85 | const TargetMachine &TM; | |||
86 | const TargetLowering &TLI; | |||
87 | SelectionDAG &DAG; | |||
88 | ||||
89 | /// The set of nodes which have already been legalized. We hold a | |||
90 | /// reference to it in order to update as necessary on node deletion. | |||
91 | SmallPtrSetImpl<SDNode *> &LegalizedNodes; | |||
92 | ||||
93 | /// A set of all the nodes updated during legalization. | |||
94 | SmallSetVector<SDNode *, 16> *UpdatedNodes; | |||
95 | ||||
96 | EVT getSetCCResultType(EVT VT) const { | |||
97 | return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); | |||
98 | } | |||
99 | ||||
100 | // Libcall insertion helpers. | |||
101 | ||||
102 | public: | |||
103 | SelectionDAGLegalize(SelectionDAG &DAG, | |||
104 | SmallPtrSetImpl<SDNode *> &LegalizedNodes, | |||
105 | SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr) | |||
106 | : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG), | |||
107 | LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {} | |||
108 | ||||
109 | /// Legalizes the given operation. | |||
110 | void LegalizeOp(SDNode *Node); | |||
111 | ||||
112 | private: | |||
113 | SDValue OptimizeFloatStore(StoreSDNode *ST); | |||
114 | ||||
115 | void LegalizeLoadOps(SDNode *Node); | |||
116 | void LegalizeStoreOps(SDNode *Node); | |||
117 | ||||
118 | /// Some targets cannot handle a variable | |||
119 | /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it | |||
120 | /// is necessary to spill the vector being inserted into to memory, perform | |||
121 | /// the insert there, and then read the result back. | |||
122 | SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx, | |||
123 | const SDLoc &dl); | |||
124 | SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx, | |||
125 | const SDLoc &dl); | |||
126 | ||||
127 | /// Return a vector shuffle operation which | |||
128 | /// performs the same shuffe in terms of order or result bytes, but on a type | |||
129 | /// whose vector element type is narrower than the original shuffle type. | |||
130 | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> | |||
131 | SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl, | |||
132 | SDValue N1, SDValue N2, | |||
133 | ArrayRef<int> Mask) const; | |||
134 | ||||
135 | bool LegalizeSetCCCondCode(EVT VT, SDValue &LHS, SDValue &RHS, SDValue &CC, | |||
136 | bool &NeedInvert, const SDLoc &dl); | |||
137 | ||||
138 | SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); | |||
139 | ||||
140 | std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC, | |||
141 | SDNode *Node, bool isSigned); | |||
142 | SDValue ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32, | |||
143 | RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, | |||
144 | RTLIB::Libcall Call_F128, | |||
145 | RTLIB::Libcall Call_PPCF128); | |||
146 | SDValue ExpandIntLibCall(SDNode *Node, bool isSigned, | |||
147 | RTLIB::Libcall Call_I8, | |||
148 | RTLIB::Libcall Call_I16, | |||
149 | RTLIB::Libcall Call_I32, | |||
150 | RTLIB::Libcall Call_I64, | |||
151 | RTLIB::Libcall Call_I128); | |||
152 | void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); | |||
153 | void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results); | |||
154 | ||||
155 | SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, | |||
156 | const SDLoc &dl); | |||
157 | SDValue ExpandBUILD_VECTOR(SDNode *Node); | |||
158 | SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); | |||
159 | void ExpandDYNAMIC_STACKALLOC(SDNode *Node, | |||
160 | SmallVectorImpl<SDValue> &Results); | |||
161 | void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL, | |||
162 | SDValue Value) const; | |||
163 | SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL, | |||
164 | SDValue NewIntValue) const; | |||
165 | SDValue ExpandFCOPYSIGN(SDNode *Node) const; | |||
166 | SDValue ExpandFABS(SDNode *Node) const; | |||
167 | SDValue ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, EVT DestVT, | |||
168 | const SDLoc &dl); | |||
169 | SDValue PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, bool isSigned, | |||
170 | const SDLoc &dl); | |||
171 | SDValue PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, bool isSigned, | |||
172 | const SDLoc &dl); | |||
173 | ||||
174 | SDValue ExpandBITREVERSE(SDValue Op, const SDLoc &dl); | |||
175 | SDValue ExpandBSWAP(SDValue Op, const SDLoc &dl); | |||
176 | ||||
177 | SDValue ExpandExtractFromVectorThroughStack(SDValue Op); | |||
178 | SDValue ExpandInsertToVectorThroughStack(SDValue Op); | |||
179 | SDValue ExpandVectorBuildThroughStack(SDNode* Node); | |||
180 | ||||
181 | SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP); | |||
182 | SDValue ExpandConstant(ConstantSDNode *CP); | |||
183 | ||||
184 | // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall | |||
185 | bool ExpandNode(SDNode *Node); | |||
186 | void ConvertNodeToLibcall(SDNode *Node); | |||
187 | void PromoteNode(SDNode *Node); | |||
188 | ||||
189 | public: | |||
190 | // Node replacement helpers | |||
191 | ||||
192 | void ReplacedNode(SDNode *N) { | |||
193 | LegalizedNodes.erase(N); | |||
194 | if (UpdatedNodes) | |||
195 | UpdatedNodes->insert(N); | |||
196 | } | |||
197 | ||||
198 | void ReplaceNode(SDNode *Old, SDNode *New) { | |||
199 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << " ... replacing: "; Old-> dump(&DAG); dbgs() << " with: "; New->dump (&DAG); } } while (false) | |||
200 | dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << " ... replacing: "; Old-> dump(&DAG); dbgs() << " with: "; New->dump (&DAG); } } while (false); | |||
201 | ||||
202 | assert(Old->getNumValues() == New->getNumValues() &&((Old->getNumValues() == New->getNumValues() && "Replacing one node with another that produces a different number " "of values!") ? static_cast<void> (0) : __assert_fail ( "Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 204, __PRETTY_FUNCTION__)) | |||
203 | "Replacing one node with another that produces a different number "((Old->getNumValues() == New->getNumValues() && "Replacing one node with another that produces a different number " "of values!") ? static_cast<void> (0) : __assert_fail ( "Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 204, __PRETTY_FUNCTION__)) | |||
204 | "of values!")((Old->getNumValues() == New->getNumValues() && "Replacing one node with another that produces a different number " "of values!") ? static_cast<void> (0) : __assert_fail ( "Old->getNumValues() == New->getNumValues() && \"Replacing one node with another that produces a different number \" \"of values!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 204, __PRETTY_FUNCTION__)); | |||
205 | DAG.ReplaceAllUsesWith(Old, New); | |||
206 | if (UpdatedNodes) | |||
207 | UpdatedNodes->insert(New); | |||
208 | ReplacedNode(Old); | |||
209 | } | |||
210 | ||||
211 | void ReplaceNode(SDValue Old, SDValue New) { | |||
212 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << " ... replacing: "; Old-> dump(&DAG); dbgs() << " with: "; New->dump (&DAG); } } while (false) | |||
213 | dbgs() << " with: "; New->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << " ... replacing: "; Old-> dump(&DAG); dbgs() << " with: "; New->dump (&DAG); } } while (false); | |||
214 | ||||
215 | DAG.ReplaceAllUsesWith(Old, New); | |||
216 | if (UpdatedNodes) | |||
217 | UpdatedNodes->insert(New.getNode()); | |||
218 | ReplacedNode(Old.getNode()); | |||
219 | } | |||
220 | ||||
221 | void ReplaceNode(SDNode *Old, const SDValue *New) { | |||
222 | LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << " ... replacing: "; Old-> dump(&DAG); } } while (false); | |||
223 | ||||
224 | DAG.ReplaceAllUsesWith(Old, New); | |||
225 | for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) { | |||
226 | LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << (i == 0 ? " with: " : " and: "); New[i]->dump(&DAG); } } while ( false) | |||
227 | New[i]->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << (i == 0 ? " with: " : " and: "); New[i]->dump(&DAG); } } while ( false); | |||
228 | if (UpdatedNodes) | |||
229 | UpdatedNodes->insert(New[i].getNode()); | |||
230 | } | |||
231 | ReplacedNode(Old); | |||
232 | } | |||
233 | }; | |||
234 | ||||
235 | } // end anonymous namespace | |||
236 | ||||
237 | /// Return a vector shuffle operation which | |||
238 | /// performs the same shuffle in terms of order or result bytes, but on a type | |||
239 | /// whose vector element type is narrower than the original shuffle type. | |||
240 | /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3> | |||
241 | SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType( | |||
242 | EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, | |||
243 | ArrayRef<int> Mask) const { | |||
244 | unsigned NumMaskElts = VT.getVectorNumElements(); | |||
245 | unsigned NumDestElts = NVT.getVectorNumElements(); | |||
246 | unsigned NumEltsGrowth = NumDestElts / NumMaskElts; | |||
247 | ||||
248 | assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!")((NumEltsGrowth && "Cannot promote to vector type with fewer elts!" ) ? static_cast<void> (0) : __assert_fail ("NumEltsGrowth && \"Cannot promote to vector type with fewer elts!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 248, __PRETTY_FUNCTION__)); | |||
249 | ||||
250 | if (NumEltsGrowth == 1) | |||
251 | return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask); | |||
252 | ||||
253 | SmallVector<int, 8> NewMask; | |||
254 | for (unsigned i = 0; i != NumMaskElts; ++i) { | |||
255 | int Idx = Mask[i]; | |||
256 | for (unsigned j = 0; j != NumEltsGrowth; ++j) { | |||
257 | if (Idx < 0) | |||
258 | NewMask.push_back(-1); | |||
259 | else | |||
260 | NewMask.push_back(Idx * NumEltsGrowth + j); | |||
261 | } | |||
262 | } | |||
263 | assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?")((NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?" ) ? static_cast<void> (0) : __assert_fail ("NewMask.size() == NumDestElts && \"Non-integer NumEltsGrowth?\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 263, __PRETTY_FUNCTION__)); | |||
264 | assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?")((TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?" ) ? static_cast<void> (0) : __assert_fail ("TLI.isShuffleMaskLegal(NewMask, NVT) && \"Shuffle not legal?\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 264, __PRETTY_FUNCTION__)); | |||
265 | return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask); | |||
266 | } | |||
267 | ||||
268 | /// Expands the ConstantFP node to an integer constant or | |||
269 | /// a load from the constant pool. | |||
270 | SDValue | |||
271 | SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) { | |||
272 | bool Extend = false; | |||
273 | SDLoc dl(CFP); | |||
274 | ||||
275 | // If a FP immediate is precise when represented as a float and if the | |||
276 | // target can do an extending load from float to double, we put it into | |||
277 | // the constant pool as a float, even if it's is statically typed as a | |||
278 | // double. This shrinks FP constants and canonicalizes them for targets where | |||
279 | // an FP extending load is the same cost as a normal load (such as on the x87 | |||
280 | // fp stack or PPC FP unit). | |||
281 | EVT VT = CFP->getValueType(0); | |||
282 | ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue()); | |||
283 | if (!UseCP) { | |||
284 | assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion")(((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion" ) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32) && \"Invalid type expansion\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 284, __PRETTY_FUNCTION__)); | |||
285 | return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl, | |||
286 | (VT == MVT::f64) ? MVT::i64 : MVT::i32); | |||
287 | } | |||
288 | ||||
289 | APFloat APF = CFP->getValueAPF(); | |||
290 | EVT OrigVT = VT; | |||
291 | EVT SVT = VT; | |||
292 | ||||
293 | // We don't want to shrink SNaNs. Converting the SNaN back to its real type | |||
294 | // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ). | |||
295 | if (!APF.isSignaling()) { | |||
296 | while (SVT != MVT::f32 && SVT != MVT::f16) { | |||
297 | SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1); | |||
298 | if (ConstantFPSDNode::isValueValidForType(SVT, APF) && | |||
299 | // Only do this if the target has a native EXTLOAD instruction from | |||
300 | // smaller type. | |||
301 | TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) && | |||
302 | TLI.ShouldShrinkFPConstant(OrigVT)) { | |||
303 | Type *SType = SVT.getTypeForEVT(*DAG.getContext()); | |||
304 | LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType)); | |||
305 | VT = SVT; | |||
306 | Extend = true; | |||
307 | } | |||
308 | } | |||
309 | } | |||
310 | ||||
311 | SDValue CPIdx = | |||
312 | DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout())); | |||
313 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |||
314 | if (Extend) { | |||
315 | SDValue Result = DAG.getExtLoad( | |||
316 | ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx, | |||
317 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT, | |||
318 | Alignment); | |||
319 | return Result; | |||
320 | } | |||
321 | SDValue Result = DAG.getLoad( | |||
322 | OrigVT, dl, DAG.getEntryNode(), CPIdx, | |||
323 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); | |||
324 | return Result; | |||
325 | } | |||
326 | ||||
327 | /// Expands the Constant node to a load from the constant pool. | |||
328 | SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) { | |||
329 | SDLoc dl(CP); | |||
330 | EVT VT = CP->getValueType(0); | |||
331 | SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(), | |||
332 | TLI.getPointerTy(DAG.getDataLayout())); | |||
333 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |||
334 | SDValue Result = DAG.getLoad( | |||
335 | VT, dl, DAG.getEntryNode(), CPIdx, | |||
336 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment); | |||
337 | return Result; | |||
338 | } | |||
339 | ||||
340 | /// Some target cannot handle a variable insertion index for the | |||
341 | /// INSERT_VECTOR_ELT instruction. In this case, it | |||
342 | /// is necessary to spill the vector being inserted into to memory, perform | |||
343 | /// the insert there, and then read the result back. | |||
344 | SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec, | |||
345 | SDValue Val, | |||
346 | SDValue Idx, | |||
347 | const SDLoc &dl) { | |||
348 | SDValue Tmp1 = Vec; | |||
349 | SDValue Tmp2 = Val; | |||
350 | SDValue Tmp3 = Idx; | |||
351 | ||||
352 | // If the target doesn't support this, we have to spill the input vector | |||
353 | // to a temporary stack slot, update the element, then reload it. This is | |||
354 | // badness. We could also load the value into a vector register (either | |||
355 | // with a "move to register" or "extload into register" instruction, then | |||
356 | // permute it into place, if the idx is a constant and if the idx is | |||
357 | // supported by the target. | |||
358 | EVT VT = Tmp1.getValueType(); | |||
359 | EVT EltVT = VT.getVectorElementType(); | |||
360 | SDValue StackPtr = DAG.CreateStackTemporary(VT); | |||
361 | ||||
362 | int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); | |||
363 | ||||
364 | // Store the vector. | |||
365 | SDValue Ch = DAG.getStore( | |||
366 | DAG.getEntryNode(), dl, Tmp1, StackPtr, | |||
367 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); | |||
368 | ||||
369 | SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3); | |||
370 | ||||
371 | // Store the scalar value. | |||
372 | Ch = DAG.getTruncStore(Ch, dl, Tmp2, StackPtr2, MachinePointerInfo(), EltVT); | |||
373 | // Load the updated vector. | |||
374 | return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack( | |||
375 | DAG.getMachineFunction(), SPFI)); | |||
376 | } | |||
377 | ||||
378 | SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, | |||
379 | SDValue Idx, | |||
380 | const SDLoc &dl) { | |||
381 | if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) { | |||
382 | // SCALAR_TO_VECTOR requires that the type of the value being inserted | |||
383 | // match the element type of the vector being created, except for | |||
384 | // integers in which case the inserted value can be over width. | |||
385 | EVT EltVT = Vec.getValueType().getVectorElementType(); | |||
386 | if (Val.getValueType() == EltVT || | |||
387 | (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) { | |||
388 | SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, | |||
389 | Vec.getValueType(), Val); | |||
390 | ||||
391 | unsigned NumElts = Vec.getValueType().getVectorNumElements(); | |||
392 | // We generate a shuffle of InVec and ScVec, so the shuffle mask | |||
393 | // should be 0,1,2,3,4,5... with the appropriate element replaced with | |||
394 | // elt 0 of the RHS. | |||
395 | SmallVector<int, 8> ShufOps; | |||
396 | for (unsigned i = 0; i != NumElts; ++i) | |||
397 | ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts); | |||
398 | ||||
399 | return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps); | |||
400 | } | |||
401 | } | |||
402 | return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl); | |||
403 | } | |||
404 | ||||
405 | SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { | |||
406 | LLVM_DEBUG(dbgs() << "Optimizing float store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Optimizing float store operations\n" ; } } while (false); | |||
407 | // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr' | |||
408 | // FIXME: We shouldn't do this for TargetConstantFP's. | |||
409 | // FIXME: move this to the DAG Combiner! Note that we can't regress due | |||
410 | // to phase ordering between legalized code and the dag combiner. This | |||
411 | // probably means that we need to integrate dag combiner and legalizer | |||
412 | // together. | |||
413 | // We generally can't do this one for long doubles. | |||
414 | SDValue Chain = ST->getChain(); | |||
415 | SDValue Ptr = ST->getBasePtr(); | |||
416 | unsigned Alignment = ST->getAlignment(); | |||
417 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); | |||
418 | AAMDNodes AAInfo = ST->getAAInfo(); | |||
419 | SDLoc dl(ST); | |||
420 | if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(ST->getValue())) { | |||
421 | if (CFP->getValueType(0) == MVT::f32 && | |||
422 | TLI.isTypeLegal(MVT::i32)) { | |||
423 | SDValue Con = DAG.getConstant(CFP->getValueAPF(). | |||
424 | bitcastToAPInt().zextOrTrunc(32), | |||
425 | SDLoc(CFP), MVT::i32); | |||
426 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), Alignment, | |||
427 | MMOFlags, AAInfo); | |||
428 | } | |||
429 | ||||
430 | if (CFP->getValueType(0) == MVT::f64) { | |||
431 | // If this target supports 64-bit registers, do a single 64-bit store. | |||
432 | if (TLI.isTypeLegal(MVT::i64)) { | |||
433 | SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). | |||
434 | zextOrTrunc(64), SDLoc(CFP), MVT::i64); | |||
435 | return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(), | |||
436 | Alignment, MMOFlags, AAInfo); | |||
437 | } | |||
438 | ||||
439 | if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) { | |||
440 | // Otherwise, if the target supports 32-bit registers, use 2 32-bit | |||
441 | // stores. If the target supports neither 32- nor 64-bits, this | |||
442 | // xform is certainly not worth it. | |||
443 | const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt(); | |||
444 | SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32); | |||
445 | SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32); | |||
446 | if (DAG.getDataLayout().isBigEndian()) | |||
447 | std::swap(Lo, Hi); | |||
448 | ||||
449 | Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(), Alignment, | |||
450 | MMOFlags, AAInfo); | |||
451 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |||
452 | DAG.getConstant(4, dl, Ptr.getValueType())); | |||
453 | Hi = DAG.getStore(Chain, dl, Hi, Ptr, | |||
454 | ST->getPointerInfo().getWithOffset(4), | |||
455 | MinAlign(Alignment, 4U), MMOFlags, AAInfo); | |||
456 | ||||
457 | return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); | |||
458 | } | |||
459 | } | |||
460 | } | |||
461 | return SDValue(nullptr, 0); | |||
462 | } | |||
463 | ||||
464 | void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { | |||
465 | StoreSDNode *ST = cast<StoreSDNode>(Node); | |||
466 | SDValue Chain = ST->getChain(); | |||
467 | SDValue Ptr = ST->getBasePtr(); | |||
468 | SDLoc dl(Node); | |||
469 | ||||
470 | unsigned Alignment = ST->getAlignment(); | |||
471 | MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags(); | |||
472 | AAMDNodes AAInfo = ST->getAAInfo(); | |||
473 | ||||
474 | if (!ST->isTruncatingStore()) { | |||
475 | LLVM_DEBUG(dbgs() << "Legalizing store operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legalizing store operation\n" ; } } while (false); | |||
476 | if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) { | |||
477 | ReplaceNode(ST, OptStore); | |||
478 | return; | |||
479 | } | |||
480 | ||||
481 | SDValue Value = ST->getValue(); | |||
482 | MVT VT = Value.getSimpleValueType(); | |||
483 | switch (TLI.getOperationAction(ISD::STORE, VT)) { | |||
484 | default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 484); | |||
485 | case TargetLowering::Legal: { | |||
486 | // If this is an unaligned store and the target doesn't support it, | |||
487 | // expand it. | |||
488 | EVT MemVT = ST->getMemoryVT(); | |||
489 | unsigned AS = ST->getAddressSpace(); | |||
490 | unsigned Align = ST->getAlignment(); | |||
491 | const DataLayout &DL = DAG.getDataLayout(); | |||
492 | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { | |||
493 | LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Expanding unsupported unaligned store\n" ; } } while (false); | |||
494 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); | |||
495 | ReplaceNode(SDValue(ST, 0), Result); | |||
496 | } else | |||
497 | LLVM_DEBUG(dbgs() << "Legal store\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legal store\n"; } } while (false); | |||
498 | break; | |||
499 | } | |||
500 | case TargetLowering::Custom: { | |||
501 | LLVM_DEBUG(dbgs() << "Trying custom lowering\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Trying custom lowering\n" ; } } while (false); | |||
502 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); | |||
503 | if (Res && Res != SDValue(Node, 0)) | |||
504 | ReplaceNode(SDValue(Node, 0), Res); | |||
505 | return; | |||
506 | } | |||
507 | case TargetLowering::Promote: { | |||
508 | MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT); | |||
509 | assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type" ) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 510, __PRETTY_FUNCTION__)) | |||
510 | "Can only promote stores to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote stores to same size type" ) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote stores to same size type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 510, __PRETTY_FUNCTION__)); | |||
511 | Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value); | |||
512 | SDValue Result = | |||
513 | DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |||
514 | Alignment, MMOFlags, AAInfo); | |||
515 | ReplaceNode(SDValue(Node, 0), Result); | |||
516 | break; | |||
517 | } | |||
518 | } | |||
519 | return; | |||
520 | } | |||
521 | ||||
522 | LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legalizing truncating store operations\n" ; } } while (false); | |||
523 | SDValue Value = ST->getValue(); | |||
524 | EVT StVT = ST->getMemoryVT(); | |||
525 | unsigned StWidth = StVT.getSizeInBits(); | |||
526 | auto &DL = DAG.getDataLayout(); | |||
527 | ||||
528 | if (StWidth != StVT.getStoreSizeInBits()) { | |||
529 | // Promote to a byte-sized store with upper bits zero if not | |||
530 | // storing an integral number of bytes. For example, promote | |||
531 | // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1) | |||
532 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), | |||
533 | StVT.getStoreSizeInBits()); | |||
534 | Value = DAG.getZeroExtendInReg(Value, dl, StVT); | |||
535 | SDValue Result = | |||
536 | DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT, | |||
537 | Alignment, MMOFlags, AAInfo); | |||
538 | ReplaceNode(SDValue(Node, 0), Result); | |||
539 | } else if (StWidth & (StWidth - 1)) { | |||
540 | // If not storing a power-of-2 number of bits, expand as two stores. | |||
541 | assert(!StVT.isVector() && "Unsupported truncstore!")((!StVT.isVector() && "Unsupported truncstore!") ? static_cast <void> (0) : __assert_fail ("!StVT.isVector() && \"Unsupported truncstore!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 541, __PRETTY_FUNCTION__)); | |||
542 | unsigned RoundWidth = 1 << Log2_32(StWidth); | |||
543 | assert(RoundWidth < StWidth)((RoundWidth < StWidth) ? static_cast<void> (0) : __assert_fail ("RoundWidth < StWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 543, __PRETTY_FUNCTION__)); | |||
544 | unsigned ExtraWidth = StWidth - RoundWidth; | |||
545 | assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 545, __PRETTY_FUNCTION__)); | |||
546 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!" ) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 547, __PRETTY_FUNCTION__)) | |||
547 | "Store size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Store size not an integral number of bytes!" ) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Store size not an integral number of bytes!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 547, __PRETTY_FUNCTION__)); | |||
548 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); | |||
549 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); | |||
550 | SDValue Lo, Hi; | |||
551 | unsigned IncrementSize; | |||
552 | ||||
553 | if (DL.isLittleEndian()) { | |||
554 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16) | |||
555 | // Store the bottom RoundWidth bits. | |||
556 | Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |||
557 | RoundVT, Alignment, MMOFlags, AAInfo); | |||
558 | ||||
559 | // Store the remaining ExtraWidth bits. | |||
560 | IncrementSize = RoundWidth / 8; | |||
561 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |||
562 | DAG.getConstant(IncrementSize, dl, | |||
563 | Ptr.getValueType())); | |||
564 | Hi = DAG.getNode( | |||
565 | ISD::SRL, dl, Value.getValueType(), Value, | |||
566 | DAG.getConstant(RoundWidth, dl, | |||
567 | TLI.getShiftAmountTy(Value.getValueType(), DL))); | |||
568 | Hi = DAG.getTruncStore( | |||
569 | Chain, dl, Hi, Ptr, | |||
570 | ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, | |||
571 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); | |||
572 | } else { | |||
573 | // Big endian - avoid unaligned stores. | |||
574 | // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X | |||
575 | // Store the top RoundWidth bits. | |||
576 | Hi = DAG.getNode( | |||
577 | ISD::SRL, dl, Value.getValueType(), Value, | |||
578 | DAG.getConstant(ExtraWidth, dl, | |||
579 | TLI.getShiftAmountTy(Value.getValueType(), DL))); | |||
580 | Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), | |||
581 | RoundVT, Alignment, MMOFlags, AAInfo); | |||
582 | ||||
583 | // Store the remaining ExtraWidth bits. | |||
584 | IncrementSize = RoundWidth / 8; | |||
585 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |||
586 | DAG.getConstant(IncrementSize, dl, | |||
587 | Ptr.getValueType())); | |||
588 | Lo = DAG.getTruncStore( | |||
589 | Chain, dl, Value, Ptr, | |||
590 | ST->getPointerInfo().getWithOffset(IncrementSize), ExtraVT, | |||
591 | MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo); | |||
592 | } | |||
593 | ||||
594 | // The order of the stores doesn't matter. | |||
595 | SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi); | |||
596 | ReplaceNode(SDValue(Node, 0), Result); | |||
597 | } else { | |||
598 | switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) { | |||
599 | default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 599); | |||
600 | case TargetLowering::Legal: { | |||
601 | EVT MemVT = ST->getMemoryVT(); | |||
602 | unsigned AS = ST->getAddressSpace(); | |||
603 | unsigned Align = ST->getAlignment(); | |||
604 | // If this is an unaligned store and the target doesn't support it, | |||
605 | // expand it. | |||
606 | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { | |||
607 | SDValue Result = TLI.expandUnalignedStore(ST, DAG); | |||
608 | ReplaceNode(SDValue(ST, 0), Result); | |||
609 | } | |||
610 | break; | |||
611 | } | |||
612 | case TargetLowering::Custom: { | |||
613 | SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG); | |||
614 | if (Res && Res != SDValue(Node, 0)) | |||
615 | ReplaceNode(SDValue(Node, 0), Res); | |||
616 | return; | |||
617 | } | |||
618 | case TargetLowering::Expand: | |||
619 | assert(!StVT.isVector() &&((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps" ) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 620, __PRETTY_FUNCTION__)) | |||
620 | "Vector Stores are handled in LegalizeVectorOps")((!StVT.isVector() && "Vector Stores are handled in LegalizeVectorOps" ) ? static_cast<void> (0) : __assert_fail ("!StVT.isVector() && \"Vector Stores are handled in LegalizeVectorOps\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 620, __PRETTY_FUNCTION__)); | |||
621 | ||||
622 | SDValue Result; | |||
623 | ||||
624 | // TRUNCSTORE:i16 i32 -> STORE i16 | |||
625 | if (TLI.isTypeLegal(StVT)) { | |||
626 | Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value); | |||
627 | Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |||
628 | Alignment, MMOFlags, AAInfo); | |||
629 | } else { | |||
630 | // The in-memory type isn't legal. Truncate to the type it would promote | |||
631 | // to, and then do a truncstore. | |||
632 | Value = DAG.getNode(ISD::TRUNCATE, dl, | |||
633 | TLI.getTypeToTransformTo(*DAG.getContext(), StVT), | |||
634 | Value); | |||
635 | Result = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), | |||
636 | StVT, Alignment, MMOFlags, AAInfo); | |||
637 | } | |||
638 | ||||
639 | ReplaceNode(SDValue(Node, 0), Result); | |||
640 | break; | |||
641 | } | |||
642 | } | |||
643 | } | |||
644 | ||||
645 | void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { | |||
646 | LoadSDNode *LD = cast<LoadSDNode>(Node); | |||
647 | SDValue Chain = LD->getChain(); // The chain. | |||
648 | SDValue Ptr = LD->getBasePtr(); // The base pointer. | |||
649 | SDValue Value; // The value returned by the load op. | |||
650 | SDLoc dl(Node); | |||
651 | ||||
652 | ISD::LoadExtType ExtType = LD->getExtensionType(); | |||
653 | if (ExtType == ISD::NON_EXTLOAD) { | |||
| ||||
654 | LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legalizing non-extending load operation\n" ; } } while (false); | |||
655 | MVT VT = Node->getSimpleValueType(0); | |||
656 | SDValue RVal = SDValue(Node, 0); | |||
657 | SDValue RChain = SDValue(Node, 1); | |||
658 | ||||
659 | switch (TLI.getOperationAction(Node->getOpcode(), VT)) { | |||
660 | default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 660); | |||
661 | case TargetLowering::Legal: { | |||
662 | EVT MemVT = LD->getMemoryVT(); | |||
663 | unsigned AS = LD->getAddressSpace(); | |||
664 | unsigned Align = LD->getAlignment(); | |||
665 | const DataLayout &DL = DAG.getDataLayout(); | |||
666 | // If this is an unaligned load and the target doesn't support it, | |||
667 | // expand it. | |||
668 | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { | |||
669 | std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); | |||
670 | } | |||
671 | break; | |||
672 | } | |||
673 | case TargetLowering::Custom: | |||
674 | if (SDValue Res = TLI.LowerOperation(RVal, DAG)) { | |||
675 | RVal = Res; | |||
676 | RChain = Res.getValue(1); | |||
677 | } | |||
678 | break; | |||
679 | ||||
680 | case TargetLowering::Promote: { | |||
681 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT); | |||
682 | assert(NVT.getSizeInBits() == VT.getSizeInBits() &&((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type" ) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 683, __PRETTY_FUNCTION__)) | |||
683 | "Can only promote loads to same size type")((NVT.getSizeInBits() == VT.getSizeInBits() && "Can only promote loads to same size type" ) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == VT.getSizeInBits() && \"Can only promote loads to same size type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 683, __PRETTY_FUNCTION__)); | |||
684 | ||||
685 | SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand()); | |||
686 | RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res); | |||
687 | RChain = Res.getValue(1); | |||
688 | break; | |||
689 | } | |||
690 | } | |||
691 | if (RChain.getNode() != Node) { | |||
692 | assert(RVal.getNode() != Node && "Load must be completely replaced")((RVal.getNode() != Node && "Load must be completely replaced" ) ? static_cast<void> (0) : __assert_fail ("RVal.getNode() != Node && \"Load must be completely replaced\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 692, __PRETTY_FUNCTION__)); | |||
693 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal); | |||
694 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain); | |||
695 | if (UpdatedNodes) { | |||
696 | UpdatedNodes->insert(RVal.getNode()); | |||
697 | UpdatedNodes->insert(RChain.getNode()); | |||
698 | } | |||
699 | ReplacedNode(Node); | |||
700 | } | |||
701 | return; | |||
702 | } | |||
703 | ||||
704 | LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legalizing extending load operation\n" ; } } while (false); | |||
705 | EVT SrcVT = LD->getMemoryVT(); | |||
706 | unsigned SrcWidth = SrcVT.getSizeInBits(); | |||
707 | unsigned Alignment = LD->getAlignment(); | |||
708 | MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags(); | |||
709 | AAMDNodes AAInfo = LD->getAAInfo(); | |||
710 | ||||
711 | if (SrcWidth != SrcVT.getStoreSizeInBits() && | |||
712 | // Some targets pretend to have an i1 loading operation, and actually | |||
713 | // load an i8. This trick is correct for ZEXTLOAD because the top 7 | |||
714 | // bits are guaranteed to be zero; it helps the optimizers understand | |||
715 | // that these bits are zero. It is also useful for EXTLOAD, since it | |||
716 | // tells the optimizers that those bits are undefined. It would be | |||
717 | // nice to have an effective generic way of getting these benefits... | |||
718 | // Until such a way is found, don't insist on promoting i1 here. | |||
719 | (SrcVT != MVT::i1 || | |||
720 | TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) == | |||
721 | TargetLowering::Promote)) { | |||
722 | // Promote to a byte-sized load if not loading an integral number of | |||
723 | // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24. | |||
724 | unsigned NewWidth = SrcVT.getStoreSizeInBits(); | |||
725 | EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth); | |||
726 | SDValue Ch; | |||
727 | ||||
728 | // The extra bits are guaranteed to be zero, since we stored them that | |||
729 | // way. A zext load from NVT thus automatically gives zext from SrcVT. | |||
730 | ||||
731 | ISD::LoadExtType NewExtType = | |||
732 | ExtType == ISD::ZEXTLOAD ? ISD::ZEXTLOAD : ISD::EXTLOAD; | |||
733 | ||||
734 | SDValue Result = | |||
735 | DAG.getExtLoad(NewExtType, dl, Node->getValueType(0), Chain, Ptr, | |||
736 | LD->getPointerInfo(), NVT, Alignment, MMOFlags, AAInfo); | |||
737 | ||||
738 | Ch = Result.getValue(1); // The chain. | |||
739 | ||||
740 | if (ExtType == ISD::SEXTLOAD) | |||
741 | // Having the top bits zero doesn't help when sign extending. | |||
742 | Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, | |||
743 | Result.getValueType(), | |||
744 | Result, DAG.getValueType(SrcVT)); | |||
745 | else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType()) | |||
746 | // All the top bits are guaranteed to be zero - inform the optimizers. | |||
747 | Result = DAG.getNode(ISD::AssertZext, dl, | |||
748 | Result.getValueType(), Result, | |||
749 | DAG.getValueType(SrcVT)); | |||
750 | ||||
751 | Value = Result; | |||
752 | Chain = Ch; | |||
753 | } else if (SrcWidth & (SrcWidth - 1)) { | |||
754 | // If not loading a power-of-2 number of bits, expand as two loads. | |||
755 | assert(!SrcVT.isVector() && "Unsupported extload!")((!SrcVT.isVector() && "Unsupported extload!") ? static_cast <void> (0) : __assert_fail ("!SrcVT.isVector() && \"Unsupported extload!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 755, __PRETTY_FUNCTION__)); | |||
756 | unsigned RoundWidth = 1 << Log2_32(SrcWidth); | |||
| ||||
757 | assert(RoundWidth < SrcWidth)((RoundWidth < SrcWidth) ? static_cast<void> (0) : __assert_fail ("RoundWidth < SrcWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 757, __PRETTY_FUNCTION__)); | |||
758 | unsigned ExtraWidth = SrcWidth - RoundWidth; | |||
759 | assert(ExtraWidth < RoundWidth)((ExtraWidth < RoundWidth) ? static_cast<void> (0) : __assert_fail ("ExtraWidth < RoundWidth", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 759, __PRETTY_FUNCTION__)); | |||
760 | assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!" ) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 761, __PRETTY_FUNCTION__)) | |||
761 | "Load size not an integral number of bytes!")((!(RoundWidth % 8) && !(ExtraWidth % 8) && "Load size not an integral number of bytes!" ) ? static_cast<void> (0) : __assert_fail ("!(RoundWidth % 8) && !(ExtraWidth % 8) && \"Load size not an integral number of bytes!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 761, __PRETTY_FUNCTION__)); | |||
762 | EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth); | |||
763 | EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth); | |||
764 | SDValue Lo, Hi, Ch; | |||
765 | unsigned IncrementSize; | |||
766 | auto &DL = DAG.getDataLayout(); | |||
767 | ||||
768 | if (DL.isLittleEndian()) { | |||
769 | // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16) | |||
770 | // Load the bottom RoundWidth bits. | |||
771 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, | |||
772 | LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, | |||
773 | AAInfo); | |||
774 | ||||
775 | // Load the remaining ExtraWidth bits. | |||
776 | IncrementSize = RoundWidth / 8; | |||
777 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |||
778 | DAG.getConstant(IncrementSize, dl, | |||
779 | Ptr.getValueType())); | |||
780 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, | |||
781 | LD->getPointerInfo().getWithOffset(IncrementSize), | |||
782 | ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, | |||
783 | AAInfo); | |||
784 | ||||
785 | // Build a factor node to remember that this load is independent of | |||
786 | // the other one. | |||
787 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), | |||
788 | Hi.getValue(1)); | |||
789 | ||||
790 | // Move the top bits to the right place. | |||
791 | Hi = DAG.getNode( | |||
792 | ISD::SHL, dl, Hi.getValueType(), Hi, | |||
793 | DAG.getConstant(RoundWidth, dl, | |||
794 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); | |||
795 | ||||
796 | // Join the hi and lo parts. | |||
797 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); | |||
798 | } else { | |||
799 | // Big endian - avoid unaligned loads. | |||
800 | // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8 | |||
801 | // Load the top RoundWidth bits. | |||
802 | Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr, | |||
803 | LD->getPointerInfo(), RoundVT, Alignment, MMOFlags, | |||
804 | AAInfo); | |||
805 | ||||
806 | // Load the remaining ExtraWidth bits. | |||
807 | IncrementSize = RoundWidth / 8; | |||
808 | Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr, | |||
809 | DAG.getConstant(IncrementSize, dl, | |||
810 | Ptr.getValueType())); | |||
811 | Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr, | |||
812 | LD->getPointerInfo().getWithOffset(IncrementSize), | |||
813 | ExtraVT, MinAlign(Alignment, IncrementSize), MMOFlags, | |||
814 | AAInfo); | |||
815 | ||||
816 | // Build a factor node to remember that this load is independent of | |||
817 | // the other one. | |||
818 | Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), | |||
819 | Hi.getValue(1)); | |||
820 | ||||
821 | // Move the top bits to the right place. | |||
822 | Hi = DAG.getNode( | |||
823 | ISD::SHL, dl, Hi.getValueType(), Hi, | |||
824 | DAG.getConstant(ExtraWidth, dl, | |||
825 | TLI.getShiftAmountTy(Hi.getValueType(), DL))); | |||
826 | ||||
827 | // Join the hi and lo parts. | |||
828 | Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi); | |||
829 | } | |||
830 | ||||
831 | Chain = Ch; | |||
832 | } else { | |||
833 | bool isCustom = false; | |||
834 | switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0), | |||
835 | SrcVT.getSimpleVT())) { | |||
836 | default: llvm_unreachable("This action is not supported yet!")::llvm::llvm_unreachable_internal("This action is not supported yet!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 836); | |||
837 | case TargetLowering::Custom: | |||
838 | isCustom = true; | |||
839 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
840 | case TargetLowering::Legal: | |||
841 | Value = SDValue(Node, 0); | |||
842 | Chain = SDValue(Node, 1); | |||
843 | ||||
844 | if (isCustom) { | |||
845 | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { | |||
846 | Value = Res; | |||
847 | Chain = Res.getValue(1); | |||
848 | } | |||
849 | } else { | |||
850 | // If this is an unaligned load and the target doesn't support it, | |||
851 | // expand it. | |||
852 | EVT MemVT = LD->getMemoryVT(); | |||
853 | unsigned AS = LD->getAddressSpace(); | |||
854 | unsigned Align = LD->getAlignment(); | |||
855 | const DataLayout &DL = DAG.getDataLayout(); | |||
856 | if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { | |||
857 | std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); | |||
858 | } | |||
859 | } | |||
860 | break; | |||
861 | ||||
862 | case TargetLowering::Expand: { | |||
863 | EVT DestVT = Node->getValueType(0); | |||
864 | if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) { | |||
865 | // If the source type is not legal, see if there is a legal extload to | |||
866 | // an intermediate type that we can then extend further. | |||
867 | EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT()); | |||
868 | if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT? | |||
869 | TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) { | |||
870 | // If we are loading a legal type, this is a non-extload followed by a | |||
871 | // full extend. | |||
872 | ISD::LoadExtType MidExtType = | |||
873 | (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType; | |||
874 | ||||
875 | SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr, | |||
876 | SrcVT, LD->getMemOperand()); | |||
877 | unsigned ExtendOp = | |||
878 | ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType); | |||
879 | Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load); | |||
880 | Chain = Load.getValue(1); | |||
881 | break; | |||
882 | } | |||
883 | ||||
884 | // Handle the special case of fp16 extloads. EXTLOAD doesn't have the | |||
885 | // normal undefined upper bits behavior to allow using an in-reg extend | |||
886 | // with the illegal FP type, so load as an integer and do the | |||
887 | // from-integer conversion. | |||
888 | if (SrcVT.getScalarType() == MVT::f16) { | |||
889 | EVT ISrcVT = SrcVT.changeTypeToInteger(); | |||
890 | EVT IDestVT = DestVT.changeTypeToInteger(); | |||
891 | EVT LoadVT = TLI.getRegisterType(IDestVT.getSimpleVT()); | |||
892 | ||||
893 | SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, LoadVT, | |||
894 | Chain, Ptr, ISrcVT, | |||
895 | LD->getMemOperand()); | |||
896 | Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result); | |||
897 | Chain = Result.getValue(1); | |||
898 | break; | |||
899 | } | |||
900 | } | |||
901 | ||||
902 | assert(!SrcVT.isVector() &&((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps" ) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 903, __PRETTY_FUNCTION__)) | |||
903 | "Vector Loads are handled in LegalizeVectorOps")((!SrcVT.isVector() && "Vector Loads are handled in LegalizeVectorOps" ) ? static_cast<void> (0) : __assert_fail ("!SrcVT.isVector() && \"Vector Loads are handled in LegalizeVectorOps\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 903, __PRETTY_FUNCTION__)); | |||
904 | ||||
905 | // FIXME: This does not work for vectors on most targets. Sign- | |||
906 | // and zero-extend operations are currently folded into extending | |||
907 | // loads, whether they are legal or not, and then we end up here | |||
908 | // without any support for legalizing them. | |||
909 | assert(ExtType != ISD::EXTLOAD &&((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!" ) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 910, __PRETTY_FUNCTION__)) | |||
910 | "EXTLOAD should always be supported!")((ExtType != ISD::EXTLOAD && "EXTLOAD should always be supported!" ) ? static_cast<void> (0) : __assert_fail ("ExtType != ISD::EXTLOAD && \"EXTLOAD should always be supported!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 910, __PRETTY_FUNCTION__)); | |||
911 | // Turn the unsupported load into an EXTLOAD followed by an | |||
912 | // explicit zero/sign extend inreg. | |||
913 | SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl, | |||
914 | Node->getValueType(0), | |||
915 | Chain, Ptr, SrcVT, | |||
916 | LD->getMemOperand()); | |||
917 | SDValue ValRes; | |||
918 | if (ExtType == ISD::SEXTLOAD) | |||
919 | ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, | |||
920 | Result.getValueType(), | |||
921 | Result, DAG.getValueType(SrcVT)); | |||
922 | else | |||
923 | ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT.getScalarType()); | |||
924 | Value = ValRes; | |||
925 | Chain = Result.getValue(1); | |||
926 | break; | |||
927 | } | |||
928 | } | |||
929 | } | |||
930 | ||||
931 | // Since loads produce two values, make sure to remember that we legalized | |||
932 | // both of them. | |||
933 | if (Chain.getNode() != Node) { | |||
934 | assert(Value.getNode() != Node && "Load must be completely replaced")((Value.getNode() != Node && "Load must be completely replaced" ) ? static_cast<void> (0) : __assert_fail ("Value.getNode() != Node && \"Load must be completely replaced\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 934, __PRETTY_FUNCTION__)); | |||
935 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value); | |||
936 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); | |||
937 | if (UpdatedNodes) { | |||
938 | UpdatedNodes->insert(Value.getNode()); | |||
939 | UpdatedNodes->insert(Chain.getNode()); | |||
940 | } | |||
941 | ReplacedNode(Node); | |||
942 | } | |||
943 | } | |||
944 | ||||
945 | /// Return a legal replacement for the given operation, with all legal operands. | |||
946 | void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { | |||
947 | LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "\nLegalizing: "; Node-> dump(&DAG); } } while (false); | |||
948 | ||||
949 | // Allow illegal target nodes and illegal registers. | |||
950 | if (Node->getOpcode() == ISD::TargetConstant || | |||
951 | Node->getOpcode() == ISD::Register) | |||
952 | return; | |||
953 | ||||
954 | #ifndef NDEBUG | |||
955 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) | |||
956 | assert((TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType (i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node-> getValueType(i))) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 959, __PRETTY_FUNCTION__)) | |||
957 | TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType (i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node-> getValueType(i))) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 959, __PRETTY_FUNCTION__)) | |||
958 | TLI.isTypeLegal(Node->getValueType(i))) &&(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType (i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node-> getValueType(i))) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 959, __PRETTY_FUNCTION__)) | |||
959 | "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Node->getValueType (i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node-> getValueType(i))) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) == TargetLowering::TypeLegal || TLI.isTypeLegal(Node->getValueType(i))) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 959, __PRETTY_FUNCTION__)); | |||
960 | ||||
961 | for (const SDValue &Op : Node->op_values()) | |||
962 | assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType ()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode( ) == ISD::Register) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 967, __PRETTY_FUNCTION__)) | |||
963 | TargetLowering::TypeLegal ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType ()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode( ) == ISD::Register) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 967, __PRETTY_FUNCTION__)) | |||
964 | TLI.isTypeLegal(Op.getValueType()) ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType ()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode( ) == ISD::Register) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 967, __PRETTY_FUNCTION__)) | |||
965 | Op.getOpcode() == ISD::TargetConstant ||(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType ()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode( ) == ISD::Register) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 967, __PRETTY_FUNCTION__)) | |||
966 | Op.getOpcode() == ISD::Register) &&(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType ()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode( ) == ISD::Register) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 967, __PRETTY_FUNCTION__)) | |||
967 | "Unexpected illegal type!")(((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType ()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode( ) == ISD::Register) && "Unexpected illegal type!") ? static_cast <void> (0) : __assert_fail ("(TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) == TargetLowering::TypeLegal || TLI.isTypeLegal(Op.getValueType()) || Op.getOpcode() == ISD::TargetConstant || Op.getOpcode() == ISD::Register) && \"Unexpected illegal type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 967, __PRETTY_FUNCTION__)); | |||
968 | #endif | |||
969 | ||||
970 | // Figure out the correct action; the way to query this varies by opcode | |||
971 | TargetLowering::LegalizeAction Action = TargetLowering::Legal; | |||
972 | bool SimpleFinishLegalizing = true; | |||
973 | switch (Node->getOpcode()) { | |||
974 | case ISD::INTRINSIC_W_CHAIN: | |||
975 | case ISD::INTRINSIC_WO_CHAIN: | |||
976 | case ISD::INTRINSIC_VOID: | |||
977 | case ISD::STACKSAVE: | |||
978 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); | |||
979 | break; | |||
980 | case ISD::GET_DYNAMIC_AREA_OFFSET: | |||
981 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
982 | Node->getValueType(0)); | |||
983 | break; | |||
984 | case ISD::VAARG: | |||
985 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
986 | Node->getValueType(0)); | |||
987 | if (Action != TargetLowering::Promote) | |||
988 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other); | |||
989 | break; | |||
990 | case ISD::FP_TO_FP16: | |||
991 | case ISD::SINT_TO_FP: | |||
992 | case ISD::UINT_TO_FP: | |||
993 | case ISD::EXTRACT_VECTOR_ELT: | |||
994 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
995 | Node->getOperand(0).getValueType()); | |||
996 | break; | |||
997 | case ISD::FP_ROUND_INREG: | |||
998 | case ISD::SIGN_EXTEND_INREG: { | |||
999 | EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT(); | |||
1000 | Action = TLI.getOperationAction(Node->getOpcode(), InnerType); | |||
1001 | break; | |||
1002 | } | |||
1003 | case ISD::ATOMIC_STORE: | |||
1004 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
1005 | Node->getOperand(2).getValueType()); | |||
1006 | break; | |||
1007 | case ISD::SELECT_CC: | |||
1008 | case ISD::SETCC: | |||
1009 | case ISD::BR_CC: { | |||
1010 | unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 : | |||
1011 | Node->getOpcode() == ISD::SETCC ? 2 : 1; | |||
1012 | unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 : 0; | |||
1013 | MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType(); | |||
1014 | ISD::CondCode CCCode = | |||
1015 | cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get(); | |||
1016 | Action = TLI.getCondCodeAction(CCCode, OpVT); | |||
1017 | if (Action == TargetLowering::Legal) { | |||
1018 | if (Node->getOpcode() == ISD::SELECT_CC) | |||
1019 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
1020 | Node->getValueType(0)); | |||
1021 | else | |||
1022 | Action = TLI.getOperationAction(Node->getOpcode(), OpVT); | |||
1023 | } | |||
1024 | break; | |||
1025 | } | |||
1026 | case ISD::LOAD: | |||
1027 | case ISD::STORE: | |||
1028 | // FIXME: Model these properly. LOAD and STORE are complicated, and | |||
1029 | // STORE expects the unlegalized operand in some cases. | |||
1030 | SimpleFinishLegalizing = false; | |||
1031 | break; | |||
1032 | case ISD::CALLSEQ_START: | |||
1033 | case ISD::CALLSEQ_END: | |||
1034 | // FIXME: This shouldn't be necessary. These nodes have special properties | |||
1035 | // dealing with the recursive nature of legalization. Removing this | |||
1036 | // special case should be done as part of making LegalizeDAG non-recursive. | |||
1037 | SimpleFinishLegalizing = false; | |||
1038 | break; | |||
1039 | case ISD::EXTRACT_ELEMENT: | |||
1040 | case ISD::FLT_ROUNDS_: | |||
1041 | case ISD::MERGE_VALUES: | |||
1042 | case ISD::EH_RETURN: | |||
1043 | case ISD::FRAME_TO_ARGS_OFFSET: | |||
1044 | case ISD::EH_DWARF_CFA: | |||
1045 | case ISD::EH_SJLJ_SETJMP: | |||
1046 | case ISD::EH_SJLJ_LONGJMP: | |||
1047 | case ISD::EH_SJLJ_SETUP_DISPATCH: | |||
1048 | // These operations lie about being legal: when they claim to be legal, | |||
1049 | // they should actually be expanded. | |||
1050 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |||
1051 | if (Action == TargetLowering::Legal) | |||
1052 | Action = TargetLowering::Expand; | |||
1053 | break; | |||
1054 | case ISD::INIT_TRAMPOLINE: | |||
1055 | case ISD::ADJUST_TRAMPOLINE: | |||
1056 | case ISD::FRAMEADDR: | |||
1057 | case ISD::RETURNADDR: | |||
1058 | case ISD::ADDROFRETURNADDR: | |||
1059 | case ISD::SPONENTRY: | |||
1060 | // These operations lie about being legal: when they claim to be legal, | |||
1061 | // they should actually be custom-lowered. | |||
1062 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |||
1063 | if (Action == TargetLowering::Legal) | |||
1064 | Action = TargetLowering::Custom; | |||
1065 | break; | |||
1066 | case ISD::READCYCLECOUNTER: | |||
1067 | // READCYCLECOUNTER returns an i64, even if type legalization might have | |||
1068 | // expanded that to several smaller types. | |||
1069 | Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64); | |||
1070 | break; | |||
1071 | case ISD::READ_REGISTER: | |||
1072 | case ISD::WRITE_REGISTER: | |||
1073 | // Named register is legal in the DAG, but blocked by register name | |||
1074 | // selection if not implemented by target (to chose the correct register) | |||
1075 | // They'll be converted to Copy(To/From)Reg. | |||
1076 | Action = TargetLowering::Legal; | |||
1077 | break; | |||
1078 | case ISD::DEBUGTRAP: | |||
1079 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |||
1080 | if (Action == TargetLowering::Expand) { | |||
1081 | // replace ISD::DEBUGTRAP with ISD::TRAP | |||
1082 | SDValue NewVal; | |||
1083 | NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(), | |||
1084 | Node->getOperand(0)); | |||
1085 | ReplaceNode(Node, NewVal.getNode()); | |||
1086 | LegalizeOp(NewVal.getNode()); | |||
1087 | return; | |||
1088 | } | |||
1089 | break; | |||
1090 | case ISD::STRICT_FADD: | |||
1091 | case ISD::STRICT_FSUB: | |||
1092 | case ISD::STRICT_FMUL: | |||
1093 | case ISD::STRICT_FDIV: | |||
1094 | case ISD::STRICT_FREM: | |||
1095 | case ISD::STRICT_FSQRT: | |||
1096 | case ISD::STRICT_FMA: | |||
1097 | case ISD::STRICT_FPOW: | |||
1098 | case ISD::STRICT_FPOWI: | |||
1099 | case ISD::STRICT_FSIN: | |||
1100 | case ISD::STRICT_FCOS: | |||
1101 | case ISD::STRICT_FEXP: | |||
1102 | case ISD::STRICT_FEXP2: | |||
1103 | case ISD::STRICT_FLOG: | |||
1104 | case ISD::STRICT_FLOG10: | |||
1105 | case ISD::STRICT_FLOG2: | |||
1106 | case ISD::STRICT_FRINT: | |||
1107 | case ISD::STRICT_FNEARBYINT: | |||
1108 | case ISD::STRICT_FMAXNUM: | |||
1109 | case ISD::STRICT_FMINNUM: | |||
1110 | case ISD::STRICT_FCEIL: | |||
1111 | case ISD::STRICT_FFLOOR: | |||
1112 | case ISD::STRICT_FROUND: | |||
1113 | case ISD::STRICT_FTRUNC: | |||
1114 | // These pseudo-ops get legalized as if they were their non-strict | |||
1115 | // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT | |||
1116 | // is also legal, but if ISD::FSQRT requires expansion then so does | |||
1117 | // ISD::STRICT_FSQRT. | |||
1118 | Action = TLI.getStrictFPOperationAction(Node->getOpcode(), | |||
1119 | Node->getValueType(0)); | |||
1120 | break; | |||
1121 | case ISD::SADDSAT: | |||
1122 | case ISD::UADDSAT: | |||
1123 | case ISD::SSUBSAT: | |||
1124 | case ISD::USUBSAT: { | |||
1125 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |||
1126 | break; | |||
1127 | } | |||
1128 | case ISD::SMULFIX: | |||
1129 | case ISD::UMULFIX: { | |||
1130 | unsigned Scale = Node->getConstantOperandVal(2); | |||
1131 | Action = TLI.getFixedPointOperationAction(Node->getOpcode(), | |||
1132 | Node->getValueType(0), Scale); | |||
1133 | break; | |||
1134 | } | |||
1135 | case ISD::MSCATTER: | |||
1136 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
1137 | cast<MaskedScatterSDNode>(Node)->getValue().getValueType()); | |||
1138 | break; | |||
1139 | case ISD::MSTORE: | |||
1140 | Action = TLI.getOperationAction(Node->getOpcode(), | |||
1141 | cast<MaskedStoreSDNode>(Node)->getValue().getValueType()); | |||
1142 | break; | |||
1143 | case ISD::VECREDUCE_FADD: | |||
1144 | case ISD::VECREDUCE_FMUL: | |||
1145 | case ISD::VECREDUCE_ADD: | |||
1146 | case ISD::VECREDUCE_MUL: | |||
1147 | case ISD::VECREDUCE_AND: | |||
1148 | case ISD::VECREDUCE_OR: | |||
1149 | case ISD::VECREDUCE_XOR: | |||
1150 | case ISD::VECREDUCE_SMAX: | |||
1151 | case ISD::VECREDUCE_SMIN: | |||
1152 | case ISD::VECREDUCE_UMAX: | |||
1153 | case ISD::VECREDUCE_UMIN: | |||
1154 | case ISD::VECREDUCE_FMAX: | |||
1155 | case ISD::VECREDUCE_FMIN: | |||
1156 | Action = TLI.getOperationAction( | |||
1157 | Node->getOpcode(), Node->getOperand(0).getValueType()); | |||
1158 | break; | |||
1159 | default: | |||
1160 | if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { | |||
1161 | Action = TargetLowering::Legal; | |||
1162 | } else { | |||
1163 | Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); | |||
1164 | } | |||
1165 | break; | |||
1166 | } | |||
1167 | ||||
1168 | if (SimpleFinishLegalizing) { | |||
1169 | SDNode *NewNode = Node; | |||
1170 | switch (Node->getOpcode()) { | |||
1171 | default: break; | |||
1172 | case ISD::SHL: | |||
1173 | case ISD::SRL: | |||
1174 | case ISD::SRA: | |||
1175 | case ISD::ROTL: | |||
1176 | case ISD::ROTR: { | |||
1177 | // Legalizing shifts/rotates requires adjusting the shift amount | |||
1178 | // to the appropriate width. | |||
1179 | SDValue Op0 = Node->getOperand(0); | |||
1180 | SDValue Op1 = Node->getOperand(1); | |||
1181 | if (!Op1.getValueType().isVector()) { | |||
1182 | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1); | |||
1183 | // The getShiftAmountOperand() may create a new operand node or | |||
1184 | // return the existing one. If new operand is created we need | |||
1185 | // to update the parent node. | |||
1186 | // Do not try to legalize SAO here! It will be automatically legalized | |||
1187 | // in the next round. | |||
1188 | if (SAO != Op1) | |||
1189 | NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO); | |||
1190 | } | |||
1191 | } | |||
1192 | break; | |||
1193 | case ISD::FSHL: | |||
1194 | case ISD::FSHR: | |||
1195 | case ISD::SRL_PARTS: | |||
1196 | case ISD::SRA_PARTS: | |||
1197 | case ISD::SHL_PARTS: { | |||
1198 | // Legalizing shifts/rotates requires adjusting the shift amount | |||
1199 | // to the appropriate width. | |||
1200 | SDValue Op0 = Node->getOperand(0); | |||
1201 | SDValue Op1 = Node->getOperand(1); | |||
1202 | SDValue Op2 = Node->getOperand(2); | |||
1203 | if (!Op2.getValueType().isVector()) { | |||
1204 | SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2); | |||
1205 | // The getShiftAmountOperand() may create a new operand node or | |||
1206 | // return the existing one. If new operand is created we need | |||
1207 | // to update the parent node. | |||
1208 | if (SAO != Op2) | |||
1209 | NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO); | |||
1210 | } | |||
1211 | break; | |||
1212 | } | |||
1213 | } | |||
1214 | ||||
1215 | if (NewNode != Node) { | |||
1216 | ReplaceNode(Node, NewNode); | |||
1217 | Node = NewNode; | |||
1218 | } | |||
1219 | switch (Action) { | |||
1220 | case TargetLowering::Legal: | |||
1221 | LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legal node: nothing to do\n" ; } } while (false); | |||
1222 | return; | |||
1223 | case TargetLowering::Custom: | |||
1224 | LLVM_DEBUG(dbgs() << "Trying custom legalization\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Trying custom legalization\n" ; } } while (false); | |||
1225 | // FIXME: The handling for custom lowering with multiple results is | |||
1226 | // a complete mess. | |||
1227 | if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) { | |||
1228 | if (!(Res.getNode() != Node || Res.getResNo() != 0)) | |||
1229 | return; | |||
1230 | ||||
1231 | if (Node->getNumValues() == 1) { | |||
1232 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Successfully custom legalized node\n" ; } } while (false); | |||
1233 | // We can just directly replace this node with the lowered value. | |||
1234 | ReplaceNode(SDValue(Node, 0), Res); | |||
1235 | return; | |||
1236 | } | |||
1237 | ||||
1238 | SmallVector<SDValue, 8> ResultVals; | |||
1239 | for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) | |||
1240 | ResultVals.push_back(Res.getValue(i)); | |||
1241 | LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Successfully custom legalized node\n" ; } } while (false); | |||
1242 | ReplaceNode(Node, ResultVals.data()); | |||
1243 | return; | |||
1244 | } | |||
1245 | LLVM_DEBUG(dbgs() << "Could not custom legalize node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Could not custom legalize node\n" ; } } while (false); | |||
1246 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
1247 | case TargetLowering::Expand: | |||
1248 | if (ExpandNode(Node)) | |||
1249 | return; | |||
1250 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
1251 | case TargetLowering::LibCall: | |||
1252 | ConvertNodeToLibcall(Node); | |||
1253 | return; | |||
1254 | case TargetLowering::Promote: | |||
1255 | PromoteNode(Node); | |||
1256 | return; | |||
1257 | } | |||
1258 | } | |||
1259 | ||||
1260 | switch (Node->getOpcode()) { | |||
1261 | default: | |||
1262 | #ifndef NDEBUG | |||
1263 | dbgs() << "NODE: "; | |||
1264 | Node->dump( &DAG); | |||
1265 | dbgs() << "\n"; | |||
1266 | #endif | |||
1267 | llvm_unreachable("Do not know how to legalize this operator!")::llvm::llvm_unreachable_internal("Do not know how to legalize this operator!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1267); | |||
1268 | ||||
1269 | case ISD::CALLSEQ_START: | |||
1270 | case ISD::CALLSEQ_END: | |||
1271 | break; | |||
1272 | case ISD::LOAD: | |||
1273 | return LegalizeLoadOps(Node); | |||
1274 | case ISD::STORE: | |||
1275 | return LegalizeStoreOps(Node); | |||
1276 | } | |||
1277 | } | |||
1278 | ||||
1279 | SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) { | |||
1280 | SDValue Vec = Op.getOperand(0); | |||
1281 | SDValue Idx = Op.getOperand(1); | |||
1282 | SDLoc dl(Op); | |||
1283 | ||||
1284 | // Before we generate a new store to a temporary stack slot, see if there is | |||
1285 | // already one that we can use. There often is because when we scalarize | |||
1286 | // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole | |||
1287 | // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in | |||
1288 | // the vector. If all are expanded here, we don't want one store per vector | |||
1289 | // element. | |||
1290 | ||||
1291 | // Caches for hasPredecessorHelper | |||
1292 | SmallPtrSet<const SDNode *, 32> Visited; | |||
1293 | SmallVector<const SDNode *, 16> Worklist; | |||
1294 | Visited.insert(Op.getNode()); | |||
1295 | Worklist.push_back(Idx.getNode()); | |||
1296 | SDValue StackPtr, Ch; | |||
1297 | for (SDNode::use_iterator UI = Vec.getNode()->use_begin(), | |||
1298 | UE = Vec.getNode()->use_end(); UI != UE; ++UI) { | |||
1299 | SDNode *User = *UI; | |||
1300 | if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) { | |||
1301 | if (ST->isIndexed() || ST->isTruncatingStore() || | |||
1302 | ST->getValue() != Vec) | |||
1303 | continue; | |||
1304 | ||||
1305 | // Make sure that nothing else could have stored into the destination of | |||
1306 | // this store. | |||
1307 | if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode())) | |||
1308 | continue; | |||
1309 | ||||
1310 | // If the index is dependent on the store we will introduce a cycle when | |||
1311 | // creating the load (the load uses the index, and by replacing the chain | |||
1312 | // we will make the index dependent on the load). Also, the store might be | |||
1313 | // dependent on the extractelement and introduce a cycle when creating | |||
1314 | // the load. | |||
1315 | if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) || | |||
1316 | ST->hasPredecessor(Op.getNode())) | |||
1317 | continue; | |||
1318 | ||||
1319 | StackPtr = ST->getBasePtr(); | |||
1320 | Ch = SDValue(ST, 0); | |||
1321 | break; | |||
1322 | } | |||
1323 | } | |||
1324 | ||||
1325 | EVT VecVT = Vec.getValueType(); | |||
1326 | ||||
1327 | if (!Ch.getNode()) { | |||
1328 | // Store the value to a temporary stack slot, then LOAD the returned part. | |||
1329 | StackPtr = DAG.CreateStackTemporary(VecVT); | |||
1330 | Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, | |||
1331 | MachinePointerInfo()); | |||
1332 | } | |||
1333 | ||||
1334 | StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); | |||
1335 | ||||
1336 | SDValue NewLoad; | |||
1337 | ||||
1338 | if (Op.getValueType().isVector()) | |||
1339 | NewLoad = | |||
1340 | DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo()); | |||
1341 | else | |||
1342 | NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr, | |||
1343 | MachinePointerInfo(), | |||
1344 | VecVT.getVectorElementType()); | |||
1345 | ||||
1346 | // Replace the chain going out of the store, by the one out of the load. | |||
1347 | DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1)); | |||
1348 | ||||
1349 | // We introduced a cycle though, so update the loads operands, making sure | |||
1350 | // to use the original store's chain as an incoming chain. | |||
1351 | SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(), | |||
1352 | NewLoad->op_end()); | |||
1353 | NewLoadOperands[0] = Ch; | |||
1354 | NewLoad = | |||
1355 | SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0); | |||
1356 | return NewLoad; | |||
1357 | } | |||
1358 | ||||
1359 | SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) { | |||
1360 | assert(Op.getValueType().isVector() && "Non-vector insert subvector!")((Op.getValueType().isVector() && "Non-vector insert subvector!" ) ? static_cast<void> (0) : __assert_fail ("Op.getValueType().isVector() && \"Non-vector insert subvector!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1360, __PRETTY_FUNCTION__)); | |||
1361 | ||||
1362 | SDValue Vec = Op.getOperand(0); | |||
1363 | SDValue Part = Op.getOperand(1); | |||
1364 | SDValue Idx = Op.getOperand(2); | |||
1365 | SDLoc dl(Op); | |||
1366 | ||||
1367 | // Store the value to a temporary stack slot, then LOAD the returned part. | |||
1368 | EVT VecVT = Vec.getValueType(); | |||
1369 | SDValue StackPtr = DAG.CreateStackTemporary(VecVT); | |||
1370 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); | |||
1371 | MachinePointerInfo PtrInfo = | |||
1372 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); | |||
1373 | ||||
1374 | // First store the whole vector. | |||
1375 | SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo); | |||
1376 | ||||
1377 | // Then store the inserted part. | |||
1378 | SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx); | |||
1379 | ||||
1380 | // Store the subvector. | |||
1381 | Ch = DAG.getStore(Ch, dl, Part, SubStackPtr, MachinePointerInfo()); | |||
1382 | ||||
1383 | // Finally, load the updated vector. | |||
1384 | return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo); | |||
1385 | } | |||
1386 | ||||
1387 | SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { | |||
1388 | // We can't handle this case efficiently. Allocate a sufficiently | |||
1389 | // aligned object on the stack, store each element into it, then load | |||
1390 | // the result as a vector. | |||
1391 | // Create the stack frame object. | |||
1392 | EVT VT = Node->getValueType(0); | |||
1393 | EVT EltVT = VT.getVectorElementType(); | |||
1394 | SDLoc dl(Node); | |||
1395 | SDValue FIPtr = DAG.CreateStackTemporary(VT); | |||
1396 | int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex(); | |||
1397 | MachinePointerInfo PtrInfo = | |||
1398 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI); | |||
1399 | ||||
1400 | // Emit a store of each element to the stack slot. | |||
1401 | SmallVector<SDValue, 8> Stores; | |||
1402 | unsigned TypeByteSize = EltVT.getSizeInBits() / 8; | |||
1403 | // Store (in the right endianness) the elements to memory. | |||
1404 | for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { | |||
1405 | // Ignore undef elements. | |||
1406 | if (Node->getOperand(i).isUndef()) continue; | |||
1407 | ||||
1408 | unsigned Offset = TypeByteSize*i; | |||
1409 | ||||
1410 | SDValue Idx = DAG.getConstant(Offset, dl, FIPtr.getValueType()); | |||
1411 | Idx = DAG.getNode(ISD::ADD, dl, FIPtr.getValueType(), FIPtr, Idx); | |||
1412 | ||||
1413 | // If the destination vector element type is narrower than the source | |||
1414 | // element type, only store the bits necessary. | |||
1415 | if (EltVT.bitsLT(Node->getOperand(i).getValueType().getScalarType())) { | |||
1416 | Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl, | |||
1417 | Node->getOperand(i), Idx, | |||
1418 | PtrInfo.getWithOffset(Offset), EltVT)); | |||
1419 | } else | |||
1420 | Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i), | |||
1421 | Idx, PtrInfo.getWithOffset(Offset))); | |||
1422 | } | |||
1423 | ||||
1424 | SDValue StoreChain; | |||
1425 | if (!Stores.empty()) // Not all undef elements? | |||
1426 | StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores); | |||
1427 | else | |||
1428 | StoreChain = DAG.getEntryNode(); | |||
1429 | ||||
1430 | // Result is a load from the stack slot. | |||
1431 | return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo); | |||
1432 | } | |||
1433 | ||||
1434 | /// Bitcast a floating-point value to an integer value. Only bitcast the part | |||
1435 | /// containing the sign bit if the target has no integer value capable of | |||
1436 | /// holding all bits of the floating-point value. | |||
1437 | void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State, | |||
1438 | const SDLoc &DL, | |||
1439 | SDValue Value) const { | |||
1440 | EVT FloatVT = Value.getValueType(); | |||
1441 | unsigned NumBits = FloatVT.getSizeInBits(); | |||
1442 | State.FloatVT = FloatVT; | |||
1443 | EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits); | |||
1444 | // Convert to an integer of the same size. | |||
1445 | if (TLI.isTypeLegal(IVT)) { | |||
1446 | State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value); | |||
1447 | State.SignMask = APInt::getSignMask(NumBits); | |||
1448 | State.SignBit = NumBits - 1; | |||
1449 | return; | |||
1450 | } | |||
1451 | ||||
1452 | auto &DataLayout = DAG.getDataLayout(); | |||
1453 | // Store the float to memory, then load the sign part out as an integer. | |||
1454 | MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8); | |||
1455 | // First create a temporary that is aligned for both the load and store. | |||
1456 | SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy); | |||
1457 | int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex(); | |||
1458 | // Then store the float to it. | |||
1459 | State.FloatPtr = StackPtr; | |||
1460 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1461 | State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI); | |||
1462 | State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr, | |||
1463 | State.FloatPointerInfo); | |||
1464 | ||||
1465 | SDValue IntPtr; | |||
1466 | if (DataLayout.isBigEndian()) { | |||
1467 | assert(FloatVT.isByteSized() && "Unsupported floating point type!")((FloatVT.isByteSized() && "Unsupported floating point type!" ) ? static_cast<void> (0) : __assert_fail ("FloatVT.isByteSized() && \"Unsupported floating point type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1467, __PRETTY_FUNCTION__)); | |||
1468 | // Load out a legal integer with the same sign bit as the float. | |||
1469 | IntPtr = StackPtr; | |||
1470 | State.IntPointerInfo = State.FloatPointerInfo; | |||
1471 | } else { | |||
1472 | // Advance the pointer so that the loaded byte will contain the sign bit. | |||
1473 | unsigned ByteOffset = (FloatVT.getSizeInBits() / 8) - 1; | |||
1474 | IntPtr = DAG.getNode(ISD::ADD, DL, StackPtr.getValueType(), StackPtr, | |||
1475 | DAG.getConstant(ByteOffset, DL, StackPtr.getValueType())); | |||
1476 | State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI, | |||
1477 | ByteOffset); | |||
1478 | } | |||
1479 | ||||
1480 | State.IntPtr = IntPtr; | |||
1481 | State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr, | |||
1482 | State.IntPointerInfo, MVT::i8); | |||
1483 | State.SignMask = APInt::getOneBitSet(LoadTy.getSizeInBits(), 7); | |||
1484 | State.SignBit = 7; | |||
1485 | } | |||
1486 | ||||
1487 | /// Replace the integer value produced by getSignAsIntValue() with a new value | |||
1488 | /// and cast the result back to a floating-point type. | |||
1489 | SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State, | |||
1490 | const SDLoc &DL, | |||
1491 | SDValue NewIntValue) const { | |||
1492 | if (!State.Chain) | |||
1493 | return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue); | |||
1494 | ||||
1495 | // Override the part containing the sign bit in the value stored on the stack. | |||
1496 | SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr, | |||
1497 | State.IntPointerInfo, MVT::i8); | |||
1498 | return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr, | |||
1499 | State.FloatPointerInfo); | |||
1500 | } | |||
1501 | ||||
1502 | SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const { | |||
1503 | SDLoc DL(Node); | |||
1504 | SDValue Mag = Node->getOperand(0); | |||
1505 | SDValue Sign = Node->getOperand(1); | |||
1506 | ||||
1507 | // Get sign bit into an integer value. | |||
1508 | FloatSignAsInt SignAsInt; | |||
1509 | getSignAsIntValue(SignAsInt, DL, Sign); | |||
1510 | ||||
1511 | EVT IntVT = SignAsInt.IntValue.getValueType(); | |||
1512 | SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT); | |||
1513 | SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue, | |||
1514 | SignMask); | |||
1515 | ||||
1516 | // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X) | |||
1517 | EVT FloatVT = Mag.getValueType(); | |||
1518 | if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) && | |||
1519 | TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) { | |||
1520 | SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag); | |||
1521 | SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue); | |||
1522 | SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit, | |||
1523 | DAG.getConstant(0, DL, IntVT), ISD::SETNE); | |||
1524 | return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue); | |||
1525 | } | |||
1526 | ||||
1527 | // Transform Mag value to integer, and clear the sign bit. | |||
1528 | FloatSignAsInt MagAsInt; | |||
1529 | getSignAsIntValue(MagAsInt, DL, Mag); | |||
1530 | EVT MagVT = MagAsInt.IntValue.getValueType(); | |||
1531 | SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT); | |||
1532 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue, | |||
1533 | ClearSignMask); | |||
1534 | ||||
1535 | // Get the signbit at the right position for MagAsInt. | |||
1536 | int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit; | |||
1537 | EVT ShiftVT = IntVT; | |||
1538 | if (SignBit.getValueSizeInBits() < ClearedSign.getValueSizeInBits()) { | |||
1539 | SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit); | |||
1540 | ShiftVT = MagVT; | |||
1541 | } | |||
1542 | if (ShiftAmount > 0) { | |||
1543 | SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT); | |||
1544 | SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst); | |||
1545 | } else if (ShiftAmount < 0) { | |||
1546 | SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT); | |||
1547 | SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst); | |||
1548 | } | |||
1549 | if (SignBit.getValueSizeInBits() > ClearedSign.getValueSizeInBits()) { | |||
1550 | SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit); | |||
1551 | } | |||
1552 | ||||
1553 | // Store the part with the modified sign and convert back to float. | |||
1554 | SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit); | |||
1555 | return modifySignAsInt(MagAsInt, DL, CopiedSign); | |||
1556 | } | |||
1557 | ||||
1558 | SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const { | |||
1559 | SDLoc DL(Node); | |||
1560 | SDValue Value = Node->getOperand(0); | |||
1561 | ||||
1562 | // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal. | |||
1563 | EVT FloatVT = Value.getValueType(); | |||
1564 | if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) { | |||
1565 | SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT); | |||
1566 | return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero); | |||
1567 | } | |||
1568 | ||||
1569 | // Transform value to integer, clear the sign bit and transform back. | |||
1570 | FloatSignAsInt ValueAsInt; | |||
1571 | getSignAsIntValue(ValueAsInt, DL, Value); | |||
1572 | EVT IntVT = ValueAsInt.IntValue.getValueType(); | |||
1573 | SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT); | |||
1574 | SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue, | |||
1575 | ClearSignMask); | |||
1576 | return modifySignAsInt(ValueAsInt, DL, ClearedSign); | |||
1577 | } | |||
1578 | ||||
1579 | void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node, | |||
1580 | SmallVectorImpl<SDValue> &Results) { | |||
1581 | unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); | |||
1582 | assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" " not tell us which reg is the stack pointer!") ? static_cast <void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1583, __PRETTY_FUNCTION__)) | |||
1583 | " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and" " not tell us which reg is the stack pointer!") ? static_cast <void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1583, __PRETTY_FUNCTION__)); | |||
1584 | SDLoc dl(Node); | |||
1585 | EVT VT = Node->getValueType(0); | |||
1586 | SDValue Tmp1 = SDValue(Node, 0); | |||
1587 | SDValue Tmp2 = SDValue(Node, 1); | |||
1588 | SDValue Tmp3 = Node->getOperand(2); | |||
1589 | SDValue Chain = Tmp1.getOperand(0); | |||
1590 | ||||
1591 | // Chain the dynamic stack allocation so that it doesn't modify the stack | |||
1592 | // pointer when other instructions are using the stack. | |||
1593 | Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl); | |||
1594 | ||||
1595 | SDValue Size = Tmp2.getOperand(1); | |||
1596 | SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT); | |||
1597 | Chain = SP.getValue(1); | |||
1598 | unsigned Align = cast<ConstantSDNode>(Tmp3)->getZExtValue(); | |||
1599 | unsigned StackAlign = | |||
1600 | DAG.getSubtarget().getFrameLowering()->getStackAlignment(); | |||
1601 | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value | |||
1602 | if (Align > StackAlign) | |||
1603 | Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1, | |||
1604 | DAG.getConstant(-(uint64_t)Align, dl, VT)); | |||
1605 | Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain | |||
1606 | ||||
1607 | Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true), | |||
1608 | DAG.getIntPtrConstant(0, dl, true), SDValue(), dl); | |||
1609 | ||||
1610 | Results.push_back(Tmp1); | |||
1611 | Results.push_back(Tmp2); | |||
1612 | } | |||
1613 | ||||
1614 | /// Legalize a SETCC with given LHS and RHS and condition code CC on the current | |||
1615 | /// target. | |||
1616 | /// | |||
1617 | /// If the SETCC has been legalized using AND / OR, then the legalized node | |||
1618 | /// will be stored in LHS. RHS and CC will be set to SDValue(). NeedInvert | |||
1619 | /// will be set to false. | |||
1620 | /// | |||
1621 | /// If the SETCC has been legalized by using getSetCCSwappedOperands(), | |||
1622 | /// then the values of LHS and RHS will be swapped, CC will be set to the | |||
1623 | /// new condition, and NeedInvert will be set to false. | |||
1624 | /// | |||
1625 | /// If the SETCC has been legalized using the inverse condcode, then LHS and | |||
1626 | /// RHS will be unchanged, CC will set to the inverted condcode, and NeedInvert | |||
1627 | /// will be set to true. The caller must invert the result of the SETCC with | |||
1628 | /// SelectionDAG::getLogicalNOT() or take equivalent action to swap the effect | |||
1629 | /// of a true/false result. | |||
1630 | /// | |||
1631 | /// \returns true if the SetCC has been legalized, false if it hasn't. | |||
1632 | bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, | |||
1633 | SDValue &RHS, SDValue &CC, | |||
1634 | bool &NeedInvert, | |||
1635 | const SDLoc &dl) { | |||
1636 | MVT OpVT = LHS.getSimpleValueType(); | |||
1637 | ISD::CondCode CCCode = cast<CondCodeSDNode>(CC)->get(); | |||
1638 | NeedInvert = false; | |||
1639 | bool NeedSwap = false; | |||
1640 | switch (TLI.getCondCodeAction(CCCode, OpVT)) { | |||
1641 | default: llvm_unreachable("Unknown condition code action!")::llvm::llvm_unreachable_internal("Unknown condition code action!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1641); | |||
1642 | case TargetLowering::Legal: | |||
1643 | // Nothing to do. | |||
1644 | break; | |||
1645 | case TargetLowering::Expand: { | |||
1646 | ISD::CondCode InvCC = ISD::getSetCCSwappedOperands(CCCode); | |||
1647 | if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { | |||
1648 | std::swap(LHS, RHS); | |||
1649 | CC = DAG.getCondCode(InvCC); | |||
1650 | return true; | |||
1651 | } | |||
1652 | // Swapping operands didn't work. Try inverting the condition. | |||
1653 | InvCC = getSetCCInverse(CCCode, OpVT.isInteger()); | |||
1654 | if (!TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { | |||
1655 | // If inverting the condition is not enough, try swapping operands | |||
1656 | // on top of it. | |||
1657 | InvCC = ISD::getSetCCSwappedOperands(InvCC); | |||
1658 | NeedSwap = true; | |||
1659 | } | |||
1660 | if (TLI.isCondCodeLegalOrCustom(InvCC, OpVT)) { | |||
1661 | CC = DAG.getCondCode(InvCC); | |||
1662 | NeedInvert = true; | |||
1663 | if (NeedSwap) | |||
1664 | std::swap(LHS, RHS); | |||
1665 | return true; | |||
1666 | } | |||
1667 | ||||
1668 | ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID; | |||
1669 | unsigned Opc = 0; | |||
1670 | switch (CCCode) { | |||
1671 | default: llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1671); | |||
1672 | case ISD::SETO: | |||
1673 | assert(TLI.isCondCodeLegal(ISD::SETOEQ, OpVT)((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1674, __PRETTY_FUNCTION__)) | |||
1674 | && "If SETO is expanded, SETOEQ must be legal!")((TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && "If SETO is expanded, SETOEQ must be legal!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETOEQ, OpVT) && \"If SETO is expanded, SETOEQ must be legal!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1674, __PRETTY_FUNCTION__)); | |||
1675 | CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break; | |||
1676 | case ISD::SETUO: | |||
1677 | assert(TLI.isCondCodeLegal(ISD::SETUNE, OpVT)((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1678, __PRETTY_FUNCTION__)) | |||
1678 | && "If SETUO is expanded, SETUNE must be legal!")((TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && "If SETUO is expanded, SETUNE must be legal!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isCondCodeLegal(ISD::SETUNE, OpVT) && \"If SETUO is expanded, SETUNE must be legal!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1678, __PRETTY_FUNCTION__)); | |||
1679 | CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR; break; | |||
1680 | case ISD::SETOEQ: | |||
1681 | case ISD::SETOGT: | |||
1682 | case ISD::SETOGE: | |||
1683 | case ISD::SETOLT: | |||
1684 | case ISD::SETOLE: | |||
1685 | case ISD::SETONE: | |||
1686 | case ISD::SETUEQ: | |||
1687 | case ISD::SETUNE: | |||
1688 | case ISD::SETUGT: | |||
1689 | case ISD::SETUGE: | |||
1690 | case ISD::SETULT: | |||
1691 | case ISD::SETULE: | |||
1692 | // If we are floating point, assign and break, otherwise fall through. | |||
1693 | if (!OpVT.isInteger()) { | |||
1694 | // We can use the 4th bit to tell if we are the unordered | |||
1695 | // or ordered version of the opcode. | |||
1696 | CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO; | |||
1697 | Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND; | |||
1698 | CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10); | |||
1699 | break; | |||
1700 | } | |||
1701 | // Fallthrough if we are unsigned integer. | |||
1702 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
1703 | case ISD::SETLE: | |||
1704 | case ISD::SETGT: | |||
1705 | case ISD::SETGE: | |||
1706 | case ISD::SETLT: | |||
1707 | case ISD::SETNE: | |||
1708 | case ISD::SETEQ: | |||
1709 | // If all combinations of inverting the condition and swapping operands | |||
1710 | // didn't work then we have no means to expand the condition. | |||
1711 | llvm_unreachable("Don't know how to expand this condition!")::llvm::llvm_unreachable_internal("Don't know how to expand this condition!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1711); | |||
1712 | } | |||
1713 | ||||
1714 | SDValue SetCC1, SetCC2; | |||
1715 | if (CCCode != ISD::SETO && CCCode != ISD::SETUO) { | |||
1716 | // If we aren't the ordered or unorder operation, | |||
1717 | // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS). | |||
1718 | SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1); | |||
1719 | SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2); | |||
1720 | } else { | |||
1721 | // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS) | |||
1722 | SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1); | |||
1723 | SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2); | |||
1724 | } | |||
1725 | LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2); | |||
1726 | RHS = SDValue(); | |||
1727 | CC = SDValue(); | |||
1728 | return true; | |||
1729 | } | |||
1730 | } | |||
1731 | return false; | |||
1732 | } | |||
1733 | ||||
1734 | /// Emit a store/load combination to the stack. This stores | |||
1735 | /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does | |||
1736 | /// a load from the stack slot to DestVT, extending it if needed. | |||
1737 | /// The resultant code need not be legal. | |||
1738 | SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, | |||
1739 | EVT DestVT, const SDLoc &dl) { | |||
1740 | // Create the stack frame object. | |||
1741 | unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment( | |||
1742 | SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); | |||
1743 | SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign); | |||
1744 | ||||
1745 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr); | |||
1746 | int SPFI = StackPtrFI->getIndex(); | |||
1747 | MachinePointerInfo PtrInfo = | |||
1748 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI); | |||
1749 | ||||
1750 | unsigned SrcSize = SrcOp.getValueSizeInBits(); | |||
1751 | unsigned SlotSize = SlotVT.getSizeInBits(); | |||
1752 | unsigned DestSize = DestVT.getSizeInBits(); | |||
1753 | Type *DestType = DestVT.getTypeForEVT(*DAG.getContext()); | |||
1754 | unsigned DestAlign = DAG.getDataLayout().getPrefTypeAlignment(DestType); | |||
1755 | ||||
1756 | // Emit a store to the stack slot. Use a truncstore if the input value is | |||
1757 | // later than DestVT. | |||
1758 | SDValue Store; | |||
1759 | ||||
1760 | if (SrcSize > SlotSize) | |||
1761 | Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, | |||
1762 | SlotVT, SrcAlign); | |||
1763 | else { | |||
1764 | assert(SrcSize == SlotSize && "Invalid store")((SrcSize == SlotSize && "Invalid store") ? static_cast <void> (0) : __assert_fail ("SrcSize == SlotSize && \"Invalid store\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1764, __PRETTY_FUNCTION__)); | |||
1765 | Store = | |||
1766 | DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign); | |||
1767 | } | |||
1768 | ||||
1769 | // Result is a load from the stack slot. | |||
1770 | if (SlotSize == DestSize) | |||
1771 | return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); | |||
1772 | ||||
1773 | assert(SlotSize < DestSize && "Unknown extension!")((SlotSize < DestSize && "Unknown extension!") ? static_cast <void> (0) : __assert_fail ("SlotSize < DestSize && \"Unknown extension!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1773, __PRETTY_FUNCTION__)); | |||
1774 | return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, | |||
1775 | DestAlign); | |||
1776 | } | |||
1777 | ||||
1778 | SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) { | |||
1779 | SDLoc dl(Node); | |||
1780 | // Create a vector sized/aligned stack slot, store the value to element #0, | |||
1781 | // then load the whole vector back out. | |||
1782 | SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0)); | |||
1783 | ||||
1784 | FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr); | |||
1785 | int SPFI = StackPtrFI->getIndex(); | |||
1786 | ||||
1787 | SDValue Ch = DAG.getTruncStore( | |||
1788 | DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr, | |||
1789 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI), | |||
1790 | Node->getValueType(0).getVectorElementType()); | |||
1791 | return DAG.getLoad( | |||
1792 | Node->getValueType(0), dl, Ch, StackPtr, | |||
1793 | MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI)); | |||
1794 | } | |||
1795 | ||||
1796 | static bool | |||
1797 | ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, | |||
1798 | const TargetLowering &TLI, SDValue &Res) { | |||
1799 | unsigned NumElems = Node->getNumOperands(); | |||
1800 | SDLoc dl(Node); | |||
1801 | EVT VT = Node->getValueType(0); | |||
1802 | ||||
1803 | // Try to group the scalars into pairs, shuffle the pairs together, then | |||
1804 | // shuffle the pairs of pairs together, etc. until the vector has | |||
1805 | // been built. This will work only if all of the necessary shuffle masks | |||
1806 | // are legal. | |||
1807 | ||||
1808 | // We do this in two phases; first to check the legality of the shuffles, | |||
1809 | // and next, assuming that all shuffles are legal, to create the new nodes. | |||
1810 | for (int Phase = 0; Phase < 2; ++Phase) { | |||
1811 | SmallVector<std::pair<SDValue, SmallVector<int, 16>>, 16> IntermedVals, | |||
1812 | NewIntermedVals; | |||
1813 | for (unsigned i = 0; i < NumElems; ++i) { | |||
1814 | SDValue V = Node->getOperand(i); | |||
1815 | if (V.isUndef()) | |||
1816 | continue; | |||
1817 | ||||
1818 | SDValue Vec; | |||
1819 | if (Phase) | |||
1820 | Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V); | |||
1821 | IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i))); | |||
1822 | } | |||
1823 | ||||
1824 | while (IntermedVals.size() > 2) { | |||
1825 | NewIntermedVals.clear(); | |||
1826 | for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) { | |||
1827 | // This vector and the next vector are shuffled together (simply to | |||
1828 | // append the one to the other). | |||
1829 | SmallVector<int, 16> ShuffleVec(NumElems, -1); | |||
1830 | ||||
1831 | SmallVector<int, 16> FinalIndices; | |||
1832 | FinalIndices.reserve(IntermedVals[i].second.size() + | |||
1833 | IntermedVals[i+1].second.size()); | |||
1834 | ||||
1835 | int k = 0; | |||
1836 | for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f; | |||
1837 | ++j, ++k) { | |||
1838 | ShuffleVec[k] = j; | |||
1839 | FinalIndices.push_back(IntermedVals[i].second[j]); | |||
1840 | } | |||
1841 | for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f; | |||
1842 | ++j, ++k) { | |||
1843 | ShuffleVec[k] = NumElems + j; | |||
1844 | FinalIndices.push_back(IntermedVals[i+1].second[j]); | |||
1845 | } | |||
1846 | ||||
1847 | SDValue Shuffle; | |||
1848 | if (Phase) | |||
1849 | Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first, | |||
1850 | IntermedVals[i+1].first, | |||
1851 | ShuffleVec); | |||
1852 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) | |||
1853 | return false; | |||
1854 | NewIntermedVals.push_back( | |||
1855 | std::make_pair(Shuffle, std::move(FinalIndices))); | |||
1856 | } | |||
1857 | ||||
1858 | // If we had an odd number of defined values, then append the last | |||
1859 | // element to the array of new vectors. | |||
1860 | if ((IntermedVals.size() & 1) != 0) | |||
1861 | NewIntermedVals.push_back(IntermedVals.back()); | |||
1862 | ||||
1863 | IntermedVals.swap(NewIntermedVals); | |||
1864 | } | |||
1865 | ||||
1866 | assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&((IntermedVals.size() <= 2 && IntermedVals.size() > 0 && "Invalid number of intermediate vectors") ? static_cast <void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1867, __PRETTY_FUNCTION__)) | |||
1867 | "Invalid number of intermediate vectors")((IntermedVals.size() <= 2 && IntermedVals.size() > 0 && "Invalid number of intermediate vectors") ? static_cast <void> (0) : __assert_fail ("IntermedVals.size() <= 2 && IntermedVals.size() > 0 && \"Invalid number of intermediate vectors\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1867, __PRETTY_FUNCTION__)); | |||
1868 | SDValue Vec1 = IntermedVals[0].first; | |||
1869 | SDValue Vec2; | |||
1870 | if (IntermedVals.size() > 1) | |||
1871 | Vec2 = IntermedVals[1].first; | |||
1872 | else if (Phase) | |||
1873 | Vec2 = DAG.getUNDEF(VT); | |||
1874 | ||||
1875 | SmallVector<int, 16> ShuffleVec(NumElems, -1); | |||
1876 | for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i) | |||
1877 | ShuffleVec[IntermedVals[0].second[i]] = i; | |||
1878 | for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i) | |||
1879 | ShuffleVec[IntermedVals[1].second[i]] = NumElems + i; | |||
1880 | ||||
1881 | if (Phase) | |||
1882 | Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); | |||
1883 | else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT)) | |||
1884 | return false; | |||
1885 | } | |||
1886 | ||||
1887 | return true; | |||
1888 | } | |||
1889 | ||||
1890 | /// Expand a BUILD_VECTOR node on targets that don't | |||
1891 | /// support the operation, but do support the resultant vector type. | |||
1892 | SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) { | |||
1893 | unsigned NumElems = Node->getNumOperands(); | |||
1894 | SDValue Value1, Value2; | |||
1895 | SDLoc dl(Node); | |||
1896 | EVT VT = Node->getValueType(0); | |||
1897 | EVT OpVT = Node->getOperand(0).getValueType(); | |||
1898 | EVT EltVT = VT.getVectorElementType(); | |||
1899 | ||||
1900 | // If the only non-undef value is the low element, turn this into a | |||
1901 | // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X. | |||
1902 | bool isOnlyLowElement = true; | |||
1903 | bool MoreThanTwoValues = false; | |||
1904 | bool isConstant = true; | |||
1905 | for (unsigned i = 0; i < NumElems; ++i) { | |||
1906 | SDValue V = Node->getOperand(i); | |||
1907 | if (V.isUndef()) | |||
1908 | continue; | |||
1909 | if (i > 0) | |||
1910 | isOnlyLowElement = false; | |||
1911 | if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V)) | |||
1912 | isConstant = false; | |||
1913 | ||||
1914 | if (!Value1.getNode()) { | |||
1915 | Value1 = V; | |||
1916 | } else if (!Value2.getNode()) { | |||
1917 | if (V != Value1) | |||
1918 | Value2 = V; | |||
1919 | } else if (V != Value1 && V != Value2) { | |||
1920 | MoreThanTwoValues = true; | |||
1921 | } | |||
1922 | } | |||
1923 | ||||
1924 | if (!Value1.getNode()) | |||
1925 | return DAG.getUNDEF(VT); | |||
1926 | ||||
1927 | if (isOnlyLowElement) | |||
1928 | return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0)); | |||
1929 | ||||
1930 | // If all elements are constants, create a load from the constant pool. | |||
1931 | if (isConstant) { | |||
1932 | SmallVector<Constant*, 16> CV; | |||
1933 | for (unsigned i = 0, e = NumElems; i != e; ++i) { | |||
1934 | if (ConstantFPSDNode *V = | |||
1935 | dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) { | |||
1936 | CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue())); | |||
1937 | } else if (ConstantSDNode *V = | |||
1938 | dyn_cast<ConstantSDNode>(Node->getOperand(i))) { | |||
1939 | if (OpVT==EltVT) | |||
1940 | CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue())); | |||
1941 | else { | |||
1942 | // If OpVT and EltVT don't match, EltVT is not legal and the | |||
1943 | // element values have been promoted/truncated earlier. Undo this; | |||
1944 | // we don't want a v16i8 to become a v16i32 for example. | |||
1945 | const ConstantInt *CI = V->getConstantIntValue(); | |||
1946 | CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()), | |||
1947 | CI->getZExtValue())); | |||
1948 | } | |||
1949 | } else { | |||
1950 | assert(Node->getOperand(i).isUndef())((Node->getOperand(i).isUndef()) ? static_cast<void> (0) : __assert_fail ("Node->getOperand(i).isUndef()", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 1950, __PRETTY_FUNCTION__)); | |||
1951 | Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext()); | |||
1952 | CV.push_back(UndefValue::get(OpNTy)); | |||
1953 | } | |||
1954 | } | |||
1955 | Constant *CP = ConstantVector::get(CV); | |||
1956 | SDValue CPIdx = | |||
1957 | DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout())); | |||
1958 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |||
1959 | return DAG.getLoad( | |||
1960 | VT, dl, DAG.getEntryNode(), CPIdx, | |||
1961 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), | |||
1962 | Alignment); | |||
1963 | } | |||
1964 | ||||
1965 | SmallSet<SDValue, 16> DefinedValues; | |||
1966 | for (unsigned i = 0; i < NumElems; ++i) { | |||
1967 | if (Node->getOperand(i).isUndef()) | |||
1968 | continue; | |||
1969 | DefinedValues.insert(Node->getOperand(i)); | |||
1970 | } | |||
1971 | ||||
1972 | if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) { | |||
1973 | if (!MoreThanTwoValues) { | |||
1974 | SmallVector<int, 8> ShuffleVec(NumElems, -1); | |||
1975 | for (unsigned i = 0; i < NumElems; ++i) { | |||
1976 | SDValue V = Node->getOperand(i); | |||
1977 | if (V.isUndef()) | |||
1978 | continue; | |||
1979 | ShuffleVec[i] = V == Value1 ? 0 : NumElems; | |||
1980 | } | |||
1981 | if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) { | |||
1982 | // Get the splatted value into the low element of a vector register. | |||
1983 | SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1); | |||
1984 | SDValue Vec2; | |||
1985 | if (Value2.getNode()) | |||
1986 | Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2); | |||
1987 | else | |||
1988 | Vec2 = DAG.getUNDEF(VT); | |||
1989 | ||||
1990 | // Return shuffle(LowValVec, undef, <0,0,0,0>) | |||
1991 | return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec); | |||
1992 | } | |||
1993 | } else { | |||
1994 | SDValue Res; | |||
1995 | if (ExpandBVWithShuffles(Node, DAG, TLI, Res)) | |||
1996 | return Res; | |||
1997 | } | |||
1998 | } | |||
1999 | ||||
2000 | // Otherwise, we can't handle this case efficiently. | |||
2001 | return ExpandVectorBuildThroughStack(Node); | |||
2002 | } | |||
2003 | ||||
2004 | // Expand a node into a call to a libcall. If the result value | |||
2005 | // does not fit into a register, return the lo part and set the hi part to the | |||
2006 | // by-reg argument. If it does fit into a single register, return the result | |||
2007 | // and leave the Hi part unset. | |||
2008 | SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, | |||
2009 | bool isSigned) { | |||
2010 | TargetLowering::ArgListTy Args; | |||
2011 | TargetLowering::ArgListEntry Entry; | |||
2012 | for (const SDValue &Op : Node->op_values()) { | |||
2013 | EVT ArgVT = Op.getValueType(); | |||
2014 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); | |||
2015 | Entry.Node = Op; | |||
2016 | Entry.Ty = ArgTy; | |||
2017 | Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); | |||
2018 | Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned); | |||
2019 | Args.push_back(Entry); | |||
2020 | } | |||
2021 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |||
2022 | TLI.getPointerTy(DAG.getDataLayout())); | |||
2023 | ||||
2024 | EVT RetVT = Node->getValueType(0); | |||
2025 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); | |||
2026 | ||||
2027 | // By default, the input chain to this libcall is the entry node of the | |||
2028 | // function. If the libcall is going to be emitted as a tail call then | |||
2029 | // TLI.isUsedByReturnOnly will change it to the right chain if the return | |||
2030 | // node which is being folded has a non-entry input chain. | |||
2031 | SDValue InChain = DAG.getEntryNode(); | |||
2032 | ||||
2033 | // isTailCall may be true since the callee does not reference caller stack | |||
2034 | // frame. Check if it's in the right position and that the return types match. | |||
2035 | SDValue TCChain = InChain; | |||
2036 | const Function &F = DAG.getMachineFunction().getFunction(); | |||
2037 | bool isTailCall = | |||
2038 | TLI.isInTailCallPosition(DAG, Node, TCChain) && | |||
2039 | (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy()); | |||
2040 | if (isTailCall) | |||
2041 | InChain = TCChain; | |||
2042 | ||||
2043 | TargetLowering::CallLoweringInfo CLI(DAG); | |||
2044 | bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned); | |||
2045 | CLI.setDebugLoc(SDLoc(Node)) | |||
2046 | .setChain(InChain) | |||
2047 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, | |||
2048 | std::move(Args)) | |||
2049 | .setTailCall(isTailCall) | |||
2050 | .setSExtResult(signExtend) | |||
2051 | .setZExtResult(!signExtend) | |||
2052 | .setIsPostTypeLegalization(true); | |||
2053 | ||||
2054 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); | |||
2055 | ||||
2056 | if (!CallInfo.second.getNode()) { | |||
2057 | LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Created tailcall: "; DAG. getRoot().dump(); } } while (false); | |||
2058 | // It's a tailcall, return the chain (which is the DAG root). | |||
2059 | return DAG.getRoot(); | |||
2060 | } | |||
2061 | ||||
2062 | LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump())do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Created libcall: "; CallInfo .first.dump(); } } while (false); | |||
2063 | return CallInfo.first; | |||
2064 | } | |||
2065 | ||||
2066 | // Expand a node into a call to a libcall. Similar to | |||
2067 | // ExpandLibCall except that the first operand is the in-chain. | |||
2068 | std::pair<SDValue, SDValue> | |||
2069 | SelectionDAGLegalize::ExpandChainLibCall(RTLIB::Libcall LC, | |||
2070 | SDNode *Node, | |||
2071 | bool isSigned) { | |||
2072 | SDValue InChain = Node->getOperand(0); | |||
2073 | ||||
2074 | TargetLowering::ArgListTy Args; | |||
2075 | TargetLowering::ArgListEntry Entry; | |||
2076 | for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) { | |||
2077 | EVT ArgVT = Node->getOperand(i).getValueType(); | |||
2078 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); | |||
2079 | Entry.Node = Node->getOperand(i); | |||
2080 | Entry.Ty = ArgTy; | |||
2081 | Entry.IsSExt = isSigned; | |||
2082 | Entry.IsZExt = !isSigned; | |||
2083 | Args.push_back(Entry); | |||
2084 | } | |||
2085 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |||
2086 | TLI.getPointerTy(DAG.getDataLayout())); | |||
2087 | ||||
2088 | Type *RetTy = Node->getValueType(0).getTypeForEVT(*DAG.getContext()); | |||
2089 | ||||
2090 | TargetLowering::CallLoweringInfo CLI(DAG); | |||
2091 | CLI.setDebugLoc(SDLoc(Node)) | |||
2092 | .setChain(InChain) | |||
2093 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, | |||
2094 | std::move(Args)) | |||
2095 | .setSExtResult(isSigned) | |||
2096 | .setZExtResult(!isSigned); | |||
2097 | ||||
2098 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); | |||
2099 | ||||
2100 | return CallInfo; | |||
2101 | } | |||
2102 | ||||
2103 | SDValue SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node, | |||
2104 | RTLIB::Libcall Call_F32, | |||
2105 | RTLIB::Libcall Call_F64, | |||
2106 | RTLIB::Libcall Call_F80, | |||
2107 | RTLIB::Libcall Call_F128, | |||
2108 | RTLIB::Libcall Call_PPCF128) { | |||
2109 | if (Node->isStrictFPOpcode()) | |||
2110 | Node = DAG.mutateStrictFPToFP(Node); | |||
2111 | ||||
2112 | RTLIB::Libcall LC; | |||
2113 | switch (Node->getSimpleValueType(0).SimpleTy) { | |||
2114 | default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2114); | |||
2115 | case MVT::f32: LC = Call_F32; break; | |||
2116 | case MVT::f64: LC = Call_F64; break; | |||
2117 | case MVT::f80: LC = Call_F80; break; | |||
2118 | case MVT::f128: LC = Call_F128; break; | |||
2119 | case MVT::ppcf128: LC = Call_PPCF128; break; | |||
2120 | } | |||
2121 | return ExpandLibCall(LC, Node, false); | |||
2122 | } | |||
2123 | ||||
2124 | SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, | |||
2125 | RTLIB::Libcall Call_I8, | |||
2126 | RTLIB::Libcall Call_I16, | |||
2127 | RTLIB::Libcall Call_I32, | |||
2128 | RTLIB::Libcall Call_I64, | |||
2129 | RTLIB::Libcall Call_I128) { | |||
2130 | RTLIB::Libcall LC; | |||
2131 | switch (Node->getSimpleValueType(0).SimpleTy) { | |||
2132 | default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2132); | |||
2133 | case MVT::i8: LC = Call_I8; break; | |||
2134 | case MVT::i16: LC = Call_I16; break; | |||
2135 | case MVT::i32: LC = Call_I32; break; | |||
2136 | case MVT::i64: LC = Call_I64; break; | |||
2137 | case MVT::i128: LC = Call_I128; break; | |||
2138 | } | |||
2139 | return ExpandLibCall(LC, Node, isSigned); | |||
2140 | } | |||
2141 | ||||
2142 | /// Issue libcalls to __{u}divmod to compute div / rem pairs. | |||
2143 | void | |||
2144 | SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, | |||
2145 | SmallVectorImpl<SDValue> &Results) { | |||
2146 | unsigned Opcode = Node->getOpcode(); | |||
2147 | bool isSigned = Opcode == ISD::SDIVREM; | |||
2148 | ||||
2149 | RTLIB::Libcall LC; | |||
2150 | switch (Node->getSimpleValueType(0).SimpleTy) { | |||
2151 | default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2151); | |||
2152 | case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break; | |||
2153 | case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break; | |||
2154 | case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break; | |||
2155 | case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break; | |||
2156 | case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break; | |||
2157 | } | |||
2158 | ||||
2159 | // The input chain to this libcall is the entry node of the function. | |||
2160 | // Legalizing the call will automatically add the previous call to the | |||
2161 | // dependence. | |||
2162 | SDValue InChain = DAG.getEntryNode(); | |||
2163 | ||||
2164 | EVT RetVT = Node->getValueType(0); | |||
2165 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); | |||
2166 | ||||
2167 | TargetLowering::ArgListTy Args; | |||
2168 | TargetLowering::ArgListEntry Entry; | |||
2169 | for (const SDValue &Op : Node->op_values()) { | |||
2170 | EVT ArgVT = Op.getValueType(); | |||
2171 | Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext()); | |||
2172 | Entry.Node = Op; | |||
2173 | Entry.Ty = ArgTy; | |||
2174 | Entry.IsSExt = isSigned; | |||
2175 | Entry.IsZExt = !isSigned; | |||
2176 | Args.push_back(Entry); | |||
2177 | } | |||
2178 | ||||
2179 | // Also pass the return address of the remainder. | |||
2180 | SDValue FIPtr = DAG.CreateStackTemporary(RetVT); | |||
2181 | Entry.Node = FIPtr; | |||
2182 | Entry.Ty = RetTy->getPointerTo(); | |||
2183 | Entry.IsSExt = isSigned; | |||
2184 | Entry.IsZExt = !isSigned; | |||
2185 | Args.push_back(Entry); | |||
2186 | ||||
2187 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |||
2188 | TLI.getPointerTy(DAG.getDataLayout())); | |||
2189 | ||||
2190 | SDLoc dl(Node); | |||
2191 | TargetLowering::CallLoweringInfo CLI(DAG); | |||
2192 | CLI.setDebugLoc(dl) | |||
2193 | .setChain(InChain) | |||
2194 | .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, | |||
2195 | std::move(Args)) | |||
2196 | .setSExtResult(isSigned) | |||
2197 | .setZExtResult(!isSigned); | |||
2198 | ||||
2199 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); | |||
2200 | ||||
2201 | // Remainder is loaded back from the stack frame. | |||
2202 | SDValue Rem = | |||
2203 | DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo()); | |||
2204 | Results.push_back(CallInfo.first); | |||
2205 | Results.push_back(Rem); | |||
2206 | } | |||
2207 | ||||
2208 | /// Return true if sincos libcall is available. | |||
2209 | static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) { | |||
2210 | RTLIB::Libcall LC; | |||
2211 | switch (Node->getSimpleValueType(0).SimpleTy) { | |||
2212 | default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2212); | |||
2213 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; | |||
2214 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; | |||
2215 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; | |||
2216 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; | |||
2217 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; | |||
2218 | } | |||
2219 | return TLI.getLibcallName(LC) != nullptr; | |||
2220 | } | |||
2221 | ||||
2222 | /// Only issue sincos libcall if both sin and cos are needed. | |||
2223 | static bool useSinCos(SDNode *Node) { | |||
2224 | unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN | |||
2225 | ? ISD::FCOS : ISD::FSIN; | |||
2226 | ||||
2227 | SDValue Op0 = Node->getOperand(0); | |||
2228 | for (SDNode::use_iterator UI = Op0.getNode()->use_begin(), | |||
2229 | UE = Op0.getNode()->use_end(); UI != UE; ++UI) { | |||
2230 | SDNode *User = *UI; | |||
2231 | if (User == Node) | |||
2232 | continue; | |||
2233 | // The other user might have been turned into sincos already. | |||
2234 | if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS) | |||
2235 | return true; | |||
2236 | } | |||
2237 | return false; | |||
2238 | } | |||
2239 | ||||
2240 | /// Issue libcalls to sincos to compute sin / cos pairs. | |||
2241 | void | |||
2242 | SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node, | |||
2243 | SmallVectorImpl<SDValue> &Results) { | |||
2244 | RTLIB::Libcall LC; | |||
2245 | switch (Node->getSimpleValueType(0).SimpleTy) { | |||
2246 | default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2246); | |||
2247 | case MVT::f32: LC = RTLIB::SINCOS_F32; break; | |||
2248 | case MVT::f64: LC = RTLIB::SINCOS_F64; break; | |||
2249 | case MVT::f80: LC = RTLIB::SINCOS_F80; break; | |||
2250 | case MVT::f128: LC = RTLIB::SINCOS_F128; break; | |||
2251 | case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break; | |||
2252 | } | |||
2253 | ||||
2254 | // The input chain to this libcall is the entry node of the function. | |||
2255 | // Legalizing the call will automatically add the previous call to the | |||
2256 | // dependence. | |||
2257 | SDValue InChain = DAG.getEntryNode(); | |||
2258 | ||||
2259 | EVT RetVT = Node->getValueType(0); | |||
2260 | Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); | |||
2261 | ||||
2262 | TargetLowering::ArgListTy Args; | |||
2263 | TargetLowering::ArgListEntry Entry; | |||
2264 | ||||
2265 | // Pass the argument. | |||
2266 | Entry.Node = Node->getOperand(0); | |||
2267 | Entry.Ty = RetTy; | |||
2268 | Entry.IsSExt = false; | |||
2269 | Entry.IsZExt = false; | |||
2270 | Args.push_back(Entry); | |||
2271 | ||||
2272 | // Pass the return address of sin. | |||
2273 | SDValue SinPtr = DAG.CreateStackTemporary(RetVT); | |||
2274 | Entry.Node = SinPtr; | |||
2275 | Entry.Ty = RetTy->getPointerTo(); | |||
2276 | Entry.IsSExt = false; | |||
2277 | Entry.IsZExt = false; | |||
2278 | Args.push_back(Entry); | |||
2279 | ||||
2280 | // Also pass the return address of the cos. | |||
2281 | SDValue CosPtr = DAG.CreateStackTemporary(RetVT); | |||
2282 | Entry.Node = CosPtr; | |||
2283 | Entry.Ty = RetTy->getPointerTo(); | |||
2284 | Entry.IsSExt = false; | |||
2285 | Entry.IsZExt = false; | |||
2286 | Args.push_back(Entry); | |||
2287 | ||||
2288 | SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), | |||
2289 | TLI.getPointerTy(DAG.getDataLayout())); | |||
2290 | ||||
2291 | SDLoc dl(Node); | |||
2292 | TargetLowering::CallLoweringInfo CLI(DAG); | |||
2293 | CLI.setDebugLoc(dl).setChain(InChain).setLibCallee( | |||
2294 | TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee, | |||
2295 | std::move(Args)); | |||
2296 | ||||
2297 | std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI); | |||
2298 | ||||
2299 | Results.push_back( | |||
2300 | DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo())); | |||
2301 | Results.push_back( | |||
2302 | DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo())); | |||
2303 | } | |||
2304 | ||||
2305 | /// This function is responsible for legalizing a | |||
2306 | /// INT_TO_FP operation of the specified operand when the target requests that | |||
2307 | /// we expand it. At this point, we know that the result and operand types are | |||
2308 | /// legal for the target. | |||
2309 | SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned, SDValue Op0, | |||
2310 | EVT DestVT, | |||
2311 | const SDLoc &dl) { | |||
2312 | EVT SrcVT = Op0.getValueType(); | |||
2313 | ||||
2314 | // TODO: Should any fast-math-flags be set for the created nodes? | |||
2315 | LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legalizing INT_TO_FP\n"; } } while (false); | |||
2316 | if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64)) { | |||
2317 | LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double " "expansion\n"; } } while (false) | |||
2318 | "expansion\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "32-bit [signed|unsigned] integer to float/double " "expansion\n"; } } while (false); | |||
2319 | ||||
2320 | // Get the stack frame index of a 8 byte buffer. | |||
2321 | SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64); | |||
2322 | ||||
2323 | // word offset constant for Hi/Lo address computation | |||
2324 | SDValue WordOff = DAG.getConstant(sizeof(int), dl, | |||
2325 | StackSlot.getValueType()); | |||
2326 | // set up Hi and Lo (into buffer) address based on endian | |||
2327 | SDValue Hi = StackSlot; | |||
2328 | SDValue Lo = DAG.getNode(ISD::ADD, dl, StackSlot.getValueType(), | |||
2329 | StackSlot, WordOff); | |||
2330 | if (DAG.getDataLayout().isLittleEndian()) | |||
2331 | std::swap(Hi, Lo); | |||
2332 | ||||
2333 | // if signed map to unsigned space | |||
2334 | SDValue Op0Mapped; | |||
2335 | if (isSigned) { | |||
2336 | // constant used to invert sign bit (signed to unsigned mapping) | |||
2337 | SDValue SignBit = DAG.getConstant(0x80000000u, dl, MVT::i32); | |||
2338 | Op0Mapped = DAG.getNode(ISD::XOR, dl, MVT::i32, Op0, SignBit); | |||
2339 | } else { | |||
2340 | Op0Mapped = Op0; | |||
2341 | } | |||
2342 | // store the lo of the constructed double - based on integer input | |||
2343 | SDValue Store1 = DAG.getStore(DAG.getEntryNode(), dl, Op0Mapped, Lo, | |||
2344 | MachinePointerInfo()); | |||
2345 | // initial hi portion of constructed double | |||
2346 | SDValue InitialHi = DAG.getConstant(0x43300000u, dl, MVT::i32); | |||
2347 | // store the hi of the constructed double - biased exponent | |||
2348 | SDValue Store2 = | |||
2349 | DAG.getStore(Store1, dl, InitialHi, Hi, MachinePointerInfo()); | |||
2350 | // load the constructed double | |||
2351 | SDValue Load = | |||
2352 | DAG.getLoad(MVT::f64, dl, Store2, StackSlot, MachinePointerInfo()); | |||
2353 | // FP constant to bias correct the final result | |||
2354 | SDValue Bias = DAG.getConstantFP(isSigned ? | |||
2355 | BitsToDouble(0x4330000080000000ULL) : | |||
2356 | BitsToDouble(0x4330000000000000ULL), | |||
2357 | dl, MVT::f64); | |||
2358 | // subtract the bias | |||
2359 | SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias); | |||
2360 | // final result | |||
2361 | SDValue Result = DAG.getFPExtendOrRound(Sub, dl, DestVT); | |||
2362 | return Result; | |||
2363 | } | |||
2364 | assert(!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet")((!isSigned && "Legalize cannot Expand SINT_TO_FP for i64 yet" ) ? static_cast<void> (0) : __assert_fail ("!isSigned && \"Legalize cannot Expand SINT_TO_FP for i64 yet\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2364, __PRETTY_FUNCTION__)); | |||
2365 | // Code below here assumes !isSigned without checking again. | |||
2366 | ||||
2367 | SDValue Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0); | |||
2368 | ||||
2369 | SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0, | |||
2370 | DAG.getConstant(0, dl, SrcVT), ISD::SETLT); | |||
2371 | SDValue Zero = DAG.getIntPtrConstant(0, dl), | |||
2372 | Four = DAG.getIntPtrConstant(4, dl); | |||
2373 | SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(), | |||
2374 | SignSet, Four, Zero); | |||
2375 | ||||
2376 | // If the sign bit of the integer is set, the large number will be treated | |||
2377 | // as a negative number. To counteract this, the dynamic code adds an | |||
2378 | // offset depending on the data type. | |||
2379 | uint64_t FF; | |||
2380 | switch (SrcVT.getSimpleVT().SimpleTy) { | |||
2381 | default: llvm_unreachable("Unsupported integer type!")::llvm::llvm_unreachable_internal("Unsupported integer type!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2381); | |||
2382 | case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float) | |||
2383 | case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float) | |||
2384 | case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float) | |||
2385 | case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float) | |||
2386 | } | |||
2387 | if (DAG.getDataLayout().isLittleEndian()) | |||
2388 | FF <<= 32; | |||
2389 | Constant *FudgeFactor = ConstantInt::get( | |||
2390 | Type::getInt64Ty(*DAG.getContext()), FF); | |||
2391 | ||||
2392 | SDValue CPIdx = | |||
2393 | DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout())); | |||
2394 | unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment(); | |||
2395 | CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset); | |||
2396 | Alignment = std::min(Alignment, 4u); | |||
2397 | SDValue FudgeInReg; | |||
2398 | if (DestVT == MVT::f32) | |||
2399 | FudgeInReg = DAG.getLoad( | |||
2400 | MVT::f32, dl, DAG.getEntryNode(), CPIdx, | |||
2401 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), | |||
2402 | Alignment); | |||
2403 | else { | |||
2404 | SDValue Load = DAG.getExtLoad( | |||
2405 | ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx, | |||
2406 | MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32, | |||
2407 | Alignment); | |||
2408 | HandleSDNode Handle(Load); | |||
2409 | LegalizeOp(Load.getNode()); | |||
2410 | FudgeInReg = Handle.getValue(); | |||
2411 | } | |||
2412 | ||||
2413 | return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg); | |||
2414 | } | |||
2415 | ||||
2416 | /// This function is responsible for legalizing a | |||
2417 | /// *INT_TO_FP operation of the specified operand when the target requests that | |||
2418 | /// we promote it. At this point, we know that the result and operand types are | |||
2419 | /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP | |||
2420 | /// operation that takes a larger input. | |||
2421 | SDValue SelectionDAGLegalize::PromoteLegalINT_TO_FP(SDValue LegalOp, EVT DestVT, | |||
2422 | bool isSigned, | |||
2423 | const SDLoc &dl) { | |||
2424 | // First step, figure out the appropriate *INT_TO_FP operation to use. | |||
2425 | EVT NewInTy = LegalOp.getValueType(); | |||
2426 | ||||
2427 | unsigned OpToUse = 0; | |||
2428 | ||||
2429 | // Scan for the appropriate larger type to use. | |||
2430 | while (true) { | |||
2431 | NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1); | |||
2432 | assert(NewInTy.isInteger() && "Ran out of possibilities!")((NewInTy.isInteger() && "Ran out of possibilities!") ? static_cast<void> (0) : __assert_fail ("NewInTy.isInteger() && \"Ran out of possibilities!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2432, __PRETTY_FUNCTION__)); | |||
2433 | ||||
2434 | // If the target supports SINT_TO_FP of this type, use it. | |||
2435 | if (TLI.isOperationLegalOrCustom(ISD::SINT_TO_FP, NewInTy)) { | |||
2436 | OpToUse = ISD::SINT_TO_FP; | |||
2437 | break; | |||
2438 | } | |||
2439 | if (isSigned) continue; | |||
2440 | ||||
2441 | // If the target supports UINT_TO_FP of this type, use it. | |||
2442 | if (TLI.isOperationLegalOrCustom(ISD::UINT_TO_FP, NewInTy)) { | |||
2443 | OpToUse = ISD::UINT_TO_FP; | |||
2444 | break; | |||
2445 | } | |||
2446 | ||||
2447 | // Otherwise, try a larger type. | |||
2448 | } | |||
2449 | ||||
2450 | // Okay, we found the operation and type to use. Zero extend our input to the | |||
2451 | // desired type then run the operation on it. | |||
2452 | return DAG.getNode(OpToUse, dl, DestVT, | |||
2453 | DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, | |||
2454 | dl, NewInTy, LegalOp)); | |||
2455 | } | |||
2456 | ||||
2457 | /// This function is responsible for legalizing a | |||
2458 | /// FP_TO_*INT operation of the specified operand when the target requests that | |||
2459 | /// we promote it. At this point, we know that the result and operand types are | |||
2460 | /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT | |||
2461 | /// operation that returns a larger result. | |||
2462 | SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDValue LegalOp, EVT DestVT, | |||
2463 | bool isSigned, | |||
2464 | const SDLoc &dl) { | |||
2465 | // First step, figure out the appropriate FP_TO*INT operation to use. | |||
2466 | EVT NewOutTy = DestVT; | |||
2467 | ||||
2468 | unsigned OpToUse = 0; | |||
2469 | ||||
2470 | // Scan for the appropriate larger type to use. | |||
2471 | while (true) { | |||
2472 | NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1); | |||
2473 | assert(NewOutTy.isInteger() && "Ran out of possibilities!")((NewOutTy.isInteger() && "Ran out of possibilities!" ) ? static_cast<void> (0) : __assert_fail ("NewOutTy.isInteger() && \"Ran out of possibilities!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2473, __PRETTY_FUNCTION__)); | |||
2474 | ||||
2475 | // A larger signed type can hold all unsigned values of the requested type, | |||
2476 | // so using FP_TO_SINT is valid | |||
2477 | if (TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NewOutTy)) { | |||
2478 | OpToUse = ISD::FP_TO_SINT; | |||
2479 | break; | |||
2480 | } | |||
2481 | ||||
2482 | // However, if the value may be < 0.0, we *must* use some FP_TO_SINT. | |||
2483 | if (!isSigned && TLI.isOperationLegalOrCustom(ISD::FP_TO_UINT, NewOutTy)) { | |||
2484 | OpToUse = ISD::FP_TO_UINT; | |||
2485 | break; | |||
2486 | } | |||
2487 | ||||
2488 | // Otherwise, try a larger type. | |||
2489 | } | |||
2490 | ||||
2491 | // Okay, we found the operation and type to use. | |||
2492 | SDValue Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp); | |||
2493 | ||||
2494 | // Truncate the result of the extended FP_TO_*INT operation to the desired | |||
2495 | // size. | |||
2496 | return DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation); | |||
2497 | } | |||
2498 | ||||
2499 | /// Legalize a BITREVERSE scalar/vector operation as a series of mask + shifts. | |||
2500 | SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { | |||
2501 | EVT VT = Op.getValueType(); | |||
2502 | EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); | |||
2503 | unsigned Sz = VT.getScalarSizeInBits(); | |||
2504 | ||||
2505 | SDValue Tmp, Tmp2, Tmp3; | |||
2506 | ||||
2507 | // If we can, perform BSWAP first and then the mask+swap the i4, then i2 | |||
2508 | // and finally the i1 pairs. | |||
2509 | // TODO: We can easily support i4/i2 legal types if any target ever does. | |||
2510 | if (Sz >= 8 && isPowerOf2_32(Sz)) { | |||
2511 | // Create the masks - repeating the pattern every byte. | |||
2512 | APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0)); | |||
2513 | APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC)); | |||
2514 | APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA)); | |||
2515 | APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F)); | |||
2516 | APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33)); | |||
2517 | APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55)); | |||
2518 | ||||
2519 | // BSWAP if the type is wider than a single byte. | |||
2520 | Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); | |||
2521 | ||||
2522 | // swap i4: ((V & 0xF0) >> 4) | ((V & 0x0F) << 4) | |||
2523 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi4, dl, VT)); | |||
2524 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo4, dl, VT)); | |||
2525 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(4, dl, SHVT)); | |||
2526 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(4, dl, SHVT)); | |||
2527 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); | |||
2528 | ||||
2529 | // swap i2: ((V & 0xCC) >> 2) | ((V & 0x33) << 2) | |||
2530 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi2, dl, VT)); | |||
2531 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo2, dl, VT)); | |||
2532 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(2, dl, SHVT)); | |||
2533 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(2, dl, SHVT)); | |||
2534 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); | |||
2535 | ||||
2536 | // swap i1: ((V & 0xAA) >> 1) | ((V & 0x55) << 1) | |||
2537 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskHi1, dl, VT)); | |||
2538 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp, DAG.getConstant(MaskLo1, dl, VT)); | |||
2539 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Tmp2, DAG.getConstant(1, dl, SHVT)); | |||
2540 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Tmp3, DAG.getConstant(1, dl, SHVT)); | |||
2541 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp3); | |||
2542 | return Tmp; | |||
2543 | } | |||
2544 | ||||
2545 | Tmp = DAG.getConstant(0, dl, VT); | |||
2546 | for (unsigned I = 0, J = Sz-1; I < Sz; ++I, --J) { | |||
2547 | if (I < J) | |||
2548 | Tmp2 = | |||
2549 | DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(J - I, dl, SHVT)); | |||
2550 | else | |||
2551 | Tmp2 = | |||
2552 | DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(I - J, dl, SHVT)); | |||
2553 | ||||
2554 | APInt Shift(Sz, 1); | |||
2555 | Shift <<= J; | |||
2556 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(Shift, dl, VT)); | |||
2557 | Tmp = DAG.getNode(ISD::OR, dl, VT, Tmp, Tmp2); | |||
2558 | } | |||
2559 | ||||
2560 | return Tmp; | |||
2561 | } | |||
2562 | ||||
2563 | /// Open code the operations for BSWAP of the specified operation. | |||
2564 | SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { | |||
2565 | EVT VT = Op.getValueType(); | |||
2566 | EVT SHVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); | |||
2567 | SDValue Tmp1, Tmp2, Tmp3, Tmp4, Tmp5, Tmp6, Tmp7, Tmp8; | |||
2568 | switch (VT.getSimpleVT().getScalarType().SimpleTy) { | |||
2569 | default: llvm_unreachable("Unhandled Expand type in BSWAP!")::llvm::llvm_unreachable_internal("Unhandled Expand type in BSWAP!" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2569); | |||
2570 | case MVT::i16: | |||
2571 | // Use a rotate by 8. This can be further expanded if necessary. | |||
2572 | return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); | |||
2573 | case MVT::i32: | |||
2574 | Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); | |||
2575 | Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); | |||
2576 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); | |||
2577 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); | |||
2578 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, | |||
2579 | DAG.getConstant(0xFF0000, dl, VT)); | |||
2580 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, DAG.getConstant(0xFF00, dl, VT)); | |||
2581 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); | |||
2582 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); | |||
2583 | return DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); | |||
2584 | case MVT::i64: | |||
2585 | Tmp8 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); | |||
2586 | Tmp7 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); | |||
2587 | Tmp6 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); | |||
2588 | Tmp5 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); | |||
2589 | Tmp4 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); | |||
2590 | Tmp3 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); | |||
2591 | Tmp2 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(40, dl, SHVT)); | |||
2592 | Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(56, dl, SHVT)); | |||
2593 | Tmp7 = DAG.getNode(ISD::AND, dl, VT, Tmp7, | |||
2594 | DAG.getConstant(255ULL<<48, dl, VT)); | |||
2595 | Tmp6 = DAG.getNode(ISD::AND, dl, VT, Tmp6, | |||
2596 | DAG.getConstant(255ULL<<40, dl, VT)); | |||
2597 | Tmp5 = DAG.getNode(ISD::AND, dl, VT, Tmp5, | |||
2598 | DAG.getConstant(255ULL<<32, dl, VT)); | |||
2599 | Tmp4 = DAG.getNode(ISD::AND, dl, VT, Tmp4, | |||
2600 | DAG.getConstant(255ULL<<24, dl, VT)); | |||
2601 | Tmp3 = DAG.getNode(ISD::AND, dl, VT, Tmp3, | |||
2602 | DAG.getConstant(255ULL<<16, dl, VT)); | |||
2603 | Tmp2 = DAG.getNode(ISD::AND, dl, VT, Tmp2, | |||
2604 | DAG.getConstant(255ULL<<8 , dl, VT)); | |||
2605 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp7); | |||
2606 | Tmp6 = DAG.getNode(ISD::OR, dl, VT, Tmp6, Tmp5); | |||
2607 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp3); | |||
2608 | Tmp2 = DAG.getNode(ISD::OR, dl, VT, Tmp2, Tmp1); | |||
2609 | Tmp8 = DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp6); | |||
2610 | Tmp4 = DAG.getNode(ISD::OR, dl, VT, Tmp4, Tmp2); | |||
2611 | return DAG.getNode(ISD::OR, dl, VT, Tmp8, Tmp4); | |||
2612 | } | |||
2613 | } | |||
2614 | ||||
2615 | bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { | |||
2616 | LLVM_DEBUG(dbgs() << "Trying to expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Trying to expand node\n"; } } while (false); | |||
2617 | SmallVector<SDValue, 8> Results; | |||
2618 | SDLoc dl(Node); | |||
2619 | SDValue Tmp1, Tmp2, Tmp3, Tmp4; | |||
2620 | bool NeedInvert; | |||
2621 | switch (Node->getOpcode()) { | |||
2622 | case ISD::ABS: | |||
2623 | if (TLI.expandABS(Node, Tmp1, DAG)) | |||
2624 | Results.push_back(Tmp1); | |||
2625 | break; | |||
2626 | case ISD::CTPOP: | |||
2627 | if (TLI.expandCTPOP(Node, Tmp1, DAG)) | |||
2628 | Results.push_back(Tmp1); | |||
2629 | break; | |||
2630 | case ISD::CTLZ: | |||
2631 | case ISD::CTLZ_ZERO_UNDEF: | |||
2632 | if (TLI.expandCTLZ(Node, Tmp1, DAG)) | |||
2633 | Results.push_back(Tmp1); | |||
2634 | break; | |||
2635 | case ISD::CTTZ: | |||
2636 | case ISD::CTTZ_ZERO_UNDEF: | |||
2637 | if (TLI.expandCTTZ(Node, Tmp1, DAG)) | |||
2638 | Results.push_back(Tmp1); | |||
2639 | break; | |||
2640 | case ISD::BITREVERSE: | |||
2641 | Results.push_back(ExpandBITREVERSE(Node->getOperand(0), dl)); | |||
2642 | break; | |||
2643 | case ISD::BSWAP: | |||
2644 | Results.push_back(ExpandBSWAP(Node->getOperand(0), dl)); | |||
2645 | break; | |||
2646 | case ISD::FRAMEADDR: | |||
2647 | case ISD::RETURNADDR: | |||
2648 | case ISD::FRAME_TO_ARGS_OFFSET: | |||
2649 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); | |||
2650 | break; | |||
2651 | case ISD::EH_DWARF_CFA: { | |||
2652 | SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl, | |||
2653 | TLI.getPointerTy(DAG.getDataLayout())); | |||
2654 | SDValue Offset = DAG.getNode(ISD::ADD, dl, | |||
2655 | CfaArg.getValueType(), | |||
2656 | DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl, | |||
2657 | CfaArg.getValueType()), | |||
2658 | CfaArg); | |||
2659 | SDValue FA = DAG.getNode( | |||
2660 | ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()), | |||
2661 | DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout()))); | |||
2662 | Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(), | |||
2663 | FA, Offset)); | |||
2664 | break; | |||
2665 | } | |||
2666 | case ISD::FLT_ROUNDS_: | |||
2667 | Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0))); | |||
2668 | break; | |||
2669 | case ISD::EH_RETURN: | |||
2670 | case ISD::EH_LABEL: | |||
2671 | case ISD::PREFETCH: | |||
2672 | case ISD::VAEND: | |||
2673 | case ISD::EH_SJLJ_LONGJMP: | |||
2674 | // If the target didn't expand these, there's nothing to do, so just | |||
2675 | // preserve the chain and be done. | |||
2676 | Results.push_back(Node->getOperand(0)); | |||
2677 | break; | |||
2678 | case ISD::READCYCLECOUNTER: | |||
2679 | // If the target didn't expand this, just return 'zero' and preserve the | |||
2680 | // chain. | |||
2681 | Results.append(Node->getNumValues() - 1, | |||
2682 | DAG.getConstant(0, dl, Node->getValueType(0))); | |||
2683 | Results.push_back(Node->getOperand(0)); | |||
2684 | break; | |||
2685 | case ISD::EH_SJLJ_SETJMP: | |||
2686 | // If the target didn't expand this, just return 'zero' and preserve the | |||
2687 | // chain. | |||
2688 | Results.push_back(DAG.getConstant(0, dl, MVT::i32)); | |||
2689 | Results.push_back(Node->getOperand(0)); | |||
2690 | break; | |||
2691 | case ISD::ATOMIC_LOAD: { | |||
2692 | // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP. | |||
2693 | SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0)); | |||
2694 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); | |||
2695 | SDValue Swap = DAG.getAtomicCmpSwap( | |||
2696 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, | |||
2697 | Node->getOperand(0), Node->getOperand(1), Zero, Zero, | |||
2698 | cast<AtomicSDNode>(Node)->getMemOperand()); | |||
2699 | Results.push_back(Swap.getValue(0)); | |||
2700 | Results.push_back(Swap.getValue(1)); | |||
2701 | break; | |||
2702 | } | |||
2703 | case ISD::ATOMIC_STORE: { | |||
2704 | // There is no libcall for atomic store; fake it with ATOMIC_SWAP. | |||
2705 | SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl, | |||
2706 | cast<AtomicSDNode>(Node)->getMemoryVT(), | |||
2707 | Node->getOperand(0), | |||
2708 | Node->getOperand(1), Node->getOperand(2), | |||
2709 | cast<AtomicSDNode>(Node)->getMemOperand()); | |||
2710 | Results.push_back(Swap.getValue(1)); | |||
2711 | break; | |||
2712 | } | |||
2713 | case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: { | |||
2714 | // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and | |||
2715 | // splits out the success value as a comparison. Expanding the resulting | |||
2716 | // ATOMIC_CMP_SWAP will produce a libcall. | |||
2717 | SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other); | |||
2718 | SDValue Res = DAG.getAtomicCmpSwap( | |||
2719 | ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs, | |||
2720 | Node->getOperand(0), Node->getOperand(1), Node->getOperand(2), | |||
2721 | Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand()); | |||
2722 | ||||
2723 | SDValue ExtRes = Res; | |||
2724 | SDValue LHS = Res; | |||
2725 | SDValue RHS = Node->getOperand(1); | |||
2726 | ||||
2727 | EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT(); | |||
2728 | EVT OuterType = Node->getValueType(0); | |||
2729 | switch (TLI.getExtendForAtomicOps()) { | |||
2730 | case ISD::SIGN_EXTEND: | |||
2731 | LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res, | |||
2732 | DAG.getValueType(AtomicType)); | |||
2733 | RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType, | |||
2734 | Node->getOperand(2), DAG.getValueType(AtomicType)); | |||
2735 | ExtRes = LHS; | |||
2736 | break; | |||
2737 | case ISD::ZERO_EXTEND: | |||
2738 | LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res, | |||
2739 | DAG.getValueType(AtomicType)); | |||
2740 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); | |||
2741 | ExtRes = LHS; | |||
2742 | break; | |||
2743 | case ISD::ANY_EXTEND: | |||
2744 | LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType); | |||
2745 | RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType); | |||
2746 | break; | |||
2747 | default: | |||
2748 | llvm_unreachable("Invalid atomic op extension")::llvm::llvm_unreachable_internal("Invalid atomic op extension" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2748); | |||
2749 | } | |||
2750 | ||||
2751 | SDValue Success = | |||
2752 | DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ); | |||
2753 | ||||
2754 | Results.push_back(ExtRes.getValue(0)); | |||
2755 | Results.push_back(Success); | |||
2756 | Results.push_back(Res.getValue(1)); | |||
2757 | break; | |||
2758 | } | |||
2759 | case ISD::DYNAMIC_STACKALLOC: | |||
2760 | ExpandDYNAMIC_STACKALLOC(Node, Results); | |||
2761 | break; | |||
2762 | case ISD::MERGE_VALUES: | |||
2763 | for (unsigned i = 0; i < Node->getNumValues(); i++) | |||
2764 | Results.push_back(Node->getOperand(i)); | |||
2765 | break; | |||
2766 | case ISD::UNDEF: { | |||
2767 | EVT VT = Node->getValueType(0); | |||
2768 | if (VT.isInteger()) | |||
2769 | Results.push_back(DAG.getConstant(0, dl, VT)); | |||
2770 | else { | |||
2771 | assert(VT.isFloatingPoint() && "Unknown value type!")((VT.isFloatingPoint() && "Unknown value type!") ? static_cast <void> (0) : __assert_fail ("VT.isFloatingPoint() && \"Unknown value type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2771, __PRETTY_FUNCTION__)); | |||
2772 | Results.push_back(DAG.getConstantFP(0, dl, VT)); | |||
2773 | } | |||
2774 | break; | |||
2775 | } | |||
2776 | case ISD::FP_ROUND: | |||
2777 | case ISD::BITCAST: | |||
2778 | Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), | |||
2779 | Node->getValueType(0), dl); | |||
2780 | Results.push_back(Tmp1); | |||
2781 | break; | |||
2782 | case ISD::FP_EXTEND: | |||
2783 | Tmp1 = EmitStackConvert(Node->getOperand(0), | |||
2784 | Node->getOperand(0).getValueType(), | |||
2785 | Node->getValueType(0), dl); | |||
2786 | Results.push_back(Tmp1); | |||
2787 | break; | |||
2788 | case ISD::SIGN_EXTEND_INREG: { | |||
2789 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); | |||
2790 | EVT VT = Node->getValueType(0); | |||
2791 | ||||
2792 | // An in-register sign-extend of a boolean is a negation: | |||
2793 | // 'true' (1) sign-extended is -1. | |||
2794 | // 'false' (0) sign-extended is 0. | |||
2795 | // However, we must mask the high bits of the source operand because the | |||
2796 | // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero. | |||
2797 | ||||
2798 | // TODO: Do this for vectors too? | |||
2799 | if (ExtraVT.getSizeInBits() == 1) { | |||
2800 | SDValue One = DAG.getConstant(1, dl, VT); | |||
2801 | SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One); | |||
2802 | SDValue Zero = DAG.getConstant(0, dl, VT); | |||
2803 | SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And); | |||
2804 | Results.push_back(Neg); | |||
2805 | break; | |||
2806 | } | |||
2807 | ||||
2808 | // NOTE: we could fall back on load/store here too for targets without | |||
2809 | // SRA. However, it is doubtful that any exist. | |||
2810 | EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout()); | |||
2811 | unsigned BitsDiff = VT.getScalarSizeInBits() - | |||
2812 | ExtraVT.getScalarSizeInBits(); | |||
2813 | SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy); | |||
2814 | Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0), | |||
2815 | Node->getOperand(0), ShiftCst); | |||
2816 | Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst); | |||
2817 | Results.push_back(Tmp1); | |||
2818 | break; | |||
2819 | } | |||
2820 | case ISD::FP_ROUND_INREG: { | |||
2821 | // The only way we can lower this is to turn it into a TRUNCSTORE, | |||
2822 | // EXTLOAD pair, targeting a temporary location (a stack slot). | |||
2823 | ||||
2824 | // NOTE: there is a choice here between constantly creating new stack | |||
2825 | // slots and always reusing the same one. We currently always create | |||
2826 | // new ones, as reuse may inhibit scheduling. | |||
2827 | EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT(); | |||
2828 | Tmp1 = EmitStackConvert(Node->getOperand(0), ExtraVT, | |||
2829 | Node->getValueType(0), dl); | |||
2830 | Results.push_back(Tmp1); | |||
2831 | break; | |||
2832 | } | |||
2833 | case ISD::UINT_TO_FP: | |||
2834 | if (TLI.expandUINT_TO_FP(Node, Tmp1, DAG)) { | |||
2835 | Results.push_back(Tmp1); | |||
2836 | break; | |||
2837 | } | |||
2838 | LLVM_FALLTHROUGH[[clang::fallthrough]]; | |||
2839 | case ISD::SINT_TO_FP: | |||
2840 | Tmp1 = ExpandLegalINT_TO_FP(Node->getOpcode() == ISD::SINT_TO_FP, | |||
2841 | Node->getOperand(0), Node->getValueType(0), dl); | |||
2842 | Results.push_back(Tmp1); | |||
2843 | break; | |||
2844 | case ISD::FP_TO_SINT: | |||
2845 | if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) | |||
2846 | Results.push_back(Tmp1); | |||
2847 | break; | |||
2848 | case ISD::FP_TO_UINT: | |||
2849 | if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG)) | |||
2850 | Results.push_back(Tmp1); | |||
2851 | break; | |||
2852 | case ISD::VAARG: | |||
2853 | Results.push_back(DAG.expandVAArg(Node)); | |||
2854 | Results.push_back(Results[0].getValue(1)); | |||
2855 | break; | |||
2856 | case ISD::VACOPY: | |||
2857 | Results.push_back(DAG.expandVACopy(Node)); | |||
2858 | break; | |||
2859 | case ISD::EXTRACT_VECTOR_ELT: | |||
2860 | if (Node->getOperand(0).getValueType().getVectorNumElements() == 1) | |||
2861 | // This must be an access of the only element. Return it. | |||
2862 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), | |||
2863 | Node->getOperand(0)); | |||
2864 | else | |||
2865 | Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0)); | |||
2866 | Results.push_back(Tmp1); | |||
2867 | break; | |||
2868 | case ISD::EXTRACT_SUBVECTOR: | |||
2869 | Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0))); | |||
2870 | break; | |||
2871 | case ISD::INSERT_SUBVECTOR: | |||
2872 | Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0))); | |||
2873 | break; | |||
2874 | case ISD::CONCAT_VECTORS: | |||
2875 | Results.push_back(ExpandVectorBuildThroughStack(Node)); | |||
2876 | break; | |||
2877 | case ISD::SCALAR_TO_VECTOR: | |||
2878 | Results.push_back(ExpandSCALAR_TO_VECTOR(Node)); | |||
2879 | break; | |||
2880 | case ISD::INSERT_VECTOR_ELT: | |||
2881 | Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0), | |||
2882 | Node->getOperand(1), | |||
2883 | Node->getOperand(2), dl)); | |||
2884 | break; | |||
2885 | case ISD::VECTOR_SHUFFLE: { | |||
2886 | SmallVector<int, 32> NewMask; | |||
2887 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); | |||
2888 | ||||
2889 | EVT VT = Node->getValueType(0); | |||
2890 | EVT EltVT = VT.getVectorElementType(); | |||
2891 | SDValue Op0 = Node->getOperand(0); | |||
2892 | SDValue Op1 = Node->getOperand(1); | |||
2893 | if (!TLI.isTypeLegal(EltVT)) { | |||
2894 | EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT); | |||
2895 | ||||
2896 | // BUILD_VECTOR operands are allowed to be wider than the element type. | |||
2897 | // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept | |||
2898 | // it. | |||
2899 | if (NewEltVT.bitsLT(EltVT)) { | |||
2900 | // Convert shuffle node. | |||
2901 | // If original node was v4i64 and the new EltVT is i32, | |||
2902 | // cast operands to v8i32 and re-build the mask. | |||
2903 | ||||
2904 | // Calculate new VT, the size of the new VT should be equal to original. | |||
2905 | EVT NewVT = | |||
2906 | EVT::getVectorVT(*DAG.getContext(), NewEltVT, | |||
2907 | VT.getSizeInBits() / NewEltVT.getSizeInBits()); | |||
2908 | assert(NewVT.bitsEq(VT))((NewVT.bitsEq(VT)) ? static_cast<void> (0) : __assert_fail ("NewVT.bitsEq(VT)", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2908, __PRETTY_FUNCTION__)); | |||
2909 | ||||
2910 | // cast operands to new VT | |||
2911 | Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0); | |||
2912 | Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1); | |||
2913 | ||||
2914 | // Convert the shuffle mask | |||
2915 | unsigned int factor = | |||
2916 | NewVT.getVectorNumElements()/VT.getVectorNumElements(); | |||
2917 | ||||
2918 | // EltVT gets smaller | |||
2919 | assert(factor > 0)((factor > 0) ? static_cast<void> (0) : __assert_fail ("factor > 0", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 2919, __PRETTY_FUNCTION__)); | |||
2920 | ||||
2921 | for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { | |||
2922 | if (Mask[i] < 0) { | |||
2923 | for (unsigned fi = 0; fi < factor; ++fi) | |||
2924 | NewMask.push_back(Mask[i]); | |||
2925 | } | |||
2926 | else { | |||
2927 | for (unsigned fi = 0; fi < factor; ++fi) | |||
2928 | NewMask.push_back(Mask[i]*factor+fi); | |||
2929 | } | |||
2930 | } | |||
2931 | Mask = NewMask; | |||
2932 | VT = NewVT; | |||
2933 | } | |||
2934 | EltVT = NewEltVT; | |||
2935 | } | |||
2936 | unsigned NumElems = VT.getVectorNumElements(); | |||
2937 | SmallVector<SDValue, 16> Ops; | |||
2938 | for (unsigned i = 0; i != NumElems; ++i) { | |||
2939 | if (Mask[i] < 0) { | |||
2940 | Ops.push_back(DAG.getUNDEF(EltVT)); | |||
2941 | continue; | |||
2942 | } | |||
2943 | unsigned Idx = Mask[i]; | |||
2944 | if (Idx < NumElems) | |||
2945 | Ops.push_back(DAG.getNode( | |||
2946 | ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0, | |||
2947 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); | |||
2948 | else | |||
2949 | Ops.push_back(DAG.getNode( | |||
2950 | ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1, | |||
2951 | DAG.getConstant(Idx - NumElems, dl, | |||
2952 | TLI.getVectorIdxTy(DAG.getDataLayout())))); | |||
2953 | } | |||
2954 | ||||
2955 | Tmp1 = DAG.getBuildVector(VT, dl, Ops); | |||
2956 | // We may have changed the BUILD_VECTOR type. Cast it back to the Node type. | |||
2957 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1); | |||
2958 | Results.push_back(Tmp1); | |||
2959 | break; | |||
2960 | } | |||
2961 | case ISD::EXTRACT_ELEMENT: { | |||
2962 | EVT OpTy = Node->getOperand(0).getValueType(); | |||
2963 | if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) { | |||
2964 | // 1 -> Hi | |||
2965 | Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0), | |||
2966 | DAG.getConstant(OpTy.getSizeInBits() / 2, dl, | |||
2967 | TLI.getShiftAmountTy( | |||
2968 | Node->getOperand(0).getValueType(), | |||
2969 | DAG.getDataLayout()))); | |||
2970 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1); | |||
2971 | } else { | |||
2972 | // 0 -> Lo | |||
2973 | Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), | |||
2974 | Node->getOperand(0)); | |||
2975 | } | |||
2976 | Results.push_back(Tmp1); | |||
2977 | break; | |||
2978 | } | |||
2979 | case ISD::STACKSAVE: | |||
2980 | // Expand to CopyFromReg if the target set | |||
2981 | // StackPointerRegisterToSaveRestore. | |||
2982 | if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { | |||
2983 | Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP, | |||
2984 | Node->getValueType(0))); | |||
2985 | Results.push_back(Results[0].getValue(1)); | |||
2986 | } else { | |||
2987 | Results.push_back(DAG.getUNDEF(Node->getValueType(0))); | |||
2988 | Results.push_back(Node->getOperand(0)); | |||
2989 | } | |||
2990 | break; | |||
2991 | case ISD::STACKRESTORE: | |||
2992 | // Expand to CopyToReg if the target set | |||
2993 | // StackPointerRegisterToSaveRestore. | |||
2994 | if (unsigned SP = TLI.getStackPointerRegisterToSaveRestore()) { | |||
2995 | Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP, | |||
2996 | Node->getOperand(1))); | |||
2997 | } else { | |||
2998 | Results.push_back(Node->getOperand(0)); | |||
2999 | } | |||
3000 | break; | |||
3001 | case ISD::GET_DYNAMIC_AREA_OFFSET: | |||
3002 | Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0))); | |||
3003 | Results.push_back(Results[0].getValue(0)); | |||
3004 | break; | |||
3005 | case ISD::FCOPYSIGN: | |||
3006 | Results.push_back(ExpandFCOPYSIGN(Node)); | |||
3007 | break; | |||
3008 | case ISD::FNEG: | |||
3009 | // Expand Y = FNEG(X) -> Y = SUB -0.0, X | |||
3010 | Tmp1 = DAG.getConstantFP(-0.0, dl, Node->getValueType(0)); | |||
3011 | // TODO: If FNEG has fast-math-flags, propagate them to the FSUB. | |||
3012 | Tmp1 = DAG.getNode(ISD::FSUB, dl, Node->getValueType(0), Tmp1, | |||
3013 | Node->getOperand(0)); | |||
3014 | Results.push_back(Tmp1); | |||
3015 | break; | |||
3016 | case ISD::FABS: | |||
3017 | Results.push_back(ExpandFABS(Node)); | |||
3018 | break; | |||
3019 | case ISD::SMIN: | |||
3020 | case ISD::SMAX: | |||
3021 | case ISD::UMIN: | |||
3022 | case ISD::UMAX: { | |||
3023 | // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B | |||
3024 | ISD::CondCode Pred; | |||
3025 | switch (Node->getOpcode()) { | |||
3026 | default: llvm_unreachable("How did we get here?")::llvm::llvm_unreachable_internal("How did we get here?", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3026); | |||
3027 | case ISD::SMAX: Pred = ISD::SETGT; break; | |||
3028 | case ISD::SMIN: Pred = ISD::SETLT; break; | |||
3029 | case ISD::UMAX: Pred = ISD::SETUGT; break; | |||
3030 | case ISD::UMIN: Pred = ISD::SETULT; break; | |||
3031 | } | |||
3032 | Tmp1 = Node->getOperand(0); | |||
3033 | Tmp2 = Node->getOperand(1); | |||
3034 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred); | |||
3035 | Results.push_back(Tmp1); | |||
3036 | break; | |||
3037 | } | |||
3038 | case ISD::FMINNUM: | |||
3039 | case ISD::FMAXNUM: { | |||
3040 | if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) | |||
3041 | Results.push_back(Expanded); | |||
3042 | break; | |||
3043 | } | |||
3044 | case ISD::FSIN: | |||
3045 | case ISD::FCOS: { | |||
3046 | EVT VT = Node->getValueType(0); | |||
3047 | // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin / | |||
3048 | // fcos which share the same operand and both are used. | |||
3049 | if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) || | |||
3050 | isSinCosLibcallAvailable(Node, TLI)) | |||
3051 | && useSinCos(Node)) { | |||
3052 | SDVTList VTs = DAG.getVTList(VT, VT); | |||
3053 | Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0)); | |||
3054 | if (Node->getOpcode() == ISD::FCOS) | |||
3055 | Tmp1 = Tmp1.getValue(1); | |||
3056 | Results.push_back(Tmp1); | |||
3057 | } | |||
3058 | break; | |||
3059 | } | |||
3060 | case ISD::FMAD: | |||
3061 | llvm_unreachable("Illegal fmad should never be formed")::llvm::llvm_unreachable_internal("Illegal fmad should never be formed" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3061); | |||
3062 | ||||
3063 | case ISD::FP16_TO_FP: | |||
3064 | if (Node->getValueType(0) != MVT::f32) { | |||
3065 | // We can extend to types bigger than f32 in two steps without changing | |||
3066 | // the result. Since "f16 -> f32" is much more commonly available, give | |||
3067 | // CodeGen the option of emitting that before resorting to a libcall. | |||
3068 | SDValue Res = | |||
3069 | DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0)); | |||
3070 | Results.push_back( | |||
3071 | DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res)); | |||
3072 | } | |||
3073 | break; | |||
3074 | case ISD::FP_TO_FP16: | |||
3075 | LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Legalizing FP_TO_FP16\n"; } } while (false); | |||
3076 | if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) { | |||
3077 | SDValue Op = Node->getOperand(0); | |||
3078 | MVT SVT = Op.getSimpleValueType(); | |||
3079 | if ((SVT == MVT::f64 || SVT == MVT::f80) && | |||
3080 | TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) { | |||
3081 | // Under fastmath, we can expand this node into a fround followed by | |||
3082 | // a float-half conversion. | |||
3083 | SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op, | |||
3084 | DAG.getIntPtrConstant(0, dl)); | |||
3085 | Results.push_back( | |||
3086 | DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal)); | |||
3087 | } | |||
3088 | } | |||
3089 | break; | |||
3090 | case ISD::ConstantFP: { | |||
3091 | ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node); | |||
3092 | // Check to see if this FP immediate is already legal. | |||
3093 | // If this is a legal constant, turn it into a TargetConstantFP node. | |||
3094 | if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), | |||
3095 | DAG.getMachineFunction().getFunction().hasOptSize())) | |||
3096 | Results.push_back(ExpandConstantFP(CFP, true)); | |||
3097 | break; | |||
3098 | } | |||
3099 | case ISD::Constant: { | |||
3100 | ConstantSDNode *CP = cast<ConstantSDNode>(Node); | |||
3101 | Results.push_back(ExpandConstant(CP)); | |||
3102 | break; | |||
3103 | } | |||
3104 | case ISD::FSUB: { | |||
3105 | EVT VT = Node->getValueType(0); | |||
3106 | if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) && | |||
3107 | TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) { | |||
3108 | const SDNodeFlags Flags = Node->getFlags(); | |||
3109 | Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1)); | |||
3110 | Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags); | |||
3111 | Results.push_back(Tmp1); | |||
3112 | } | |||
3113 | break; | |||
3114 | } | |||
3115 | case ISD::SUB: { | |||
3116 | EVT VT = Node->getValueType(0); | |||
3117 | assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom (ISD::XOR, VT) && "Don't know how to expand this subtraction!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3119, __PRETTY_FUNCTION__)) | |||
3118 | TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom (ISD::XOR, VT) && "Don't know how to expand this subtraction!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3119, __PRETTY_FUNCTION__)) | |||
3119 | "Don't know how to expand this subtraction!")((TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom (ISD::XOR, VT) && "Don't know how to expand this subtraction!" ) ? static_cast<void> (0) : __assert_fail ("TLI.isOperationLegalOrCustom(ISD::ADD, VT) && TLI.isOperationLegalOrCustom(ISD::XOR, VT) && \"Don't know how to expand this subtraction!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3119, __PRETTY_FUNCTION__)); | |||
3120 | Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1), | |||
3121 | DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl, | |||
3122 | VT)); | |||
3123 | Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT)); | |||
3124 | Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1)); | |||
3125 | break; | |||
3126 | } | |||
3127 | case ISD::UREM: | |||
3128 | case ISD::SREM: { | |||
3129 | EVT VT = Node->getValueType(0); | |||
3130 | bool isSigned = Node->getOpcode() == ISD::SREM; | |||
3131 | unsigned DivOpc = isSigned ? ISD::SDIV : ISD::UDIV; | |||
3132 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; | |||
3133 | Tmp2 = Node->getOperand(0); | |||
3134 | Tmp3 = Node->getOperand(1); | |||
3135 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { | |||
3136 | SDVTList VTs = DAG.getVTList(VT, VT); | |||
3137 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1); | |||
3138 | Results.push_back(Tmp1); | |||
3139 | } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) { | |||
3140 | // X % Y -> X-X/Y*Y | |||
3141 | Tmp1 = DAG.getNode(DivOpc, dl, VT, Tmp2, Tmp3); | |||
3142 | Tmp1 = DAG.getNode(ISD::MUL, dl, VT, Tmp1, Tmp3); | |||
3143 | Tmp1 = DAG.getNode(ISD::SUB, dl, VT, Tmp2, Tmp1); | |||
3144 | Results.push_back(Tmp1); | |||
3145 | } | |||
3146 | break; | |||
3147 | } | |||
3148 | case ISD::UDIV: | |||
3149 | case ISD::SDIV: { | |||
3150 | bool isSigned = Node->getOpcode() == ISD::SDIV; | |||
3151 | unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM; | |||
3152 | EVT VT = Node->getValueType(0); | |||
3153 | if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) { | |||
3154 | SDVTList VTs = DAG.getVTList(VT, VT); | |||
3155 | Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0), | |||
3156 | Node->getOperand(1)); | |||
3157 | Results.push_back(Tmp1); | |||
3158 | } | |||
3159 | break; | |||
3160 | } | |||
3161 | case ISD::MULHU: | |||
3162 | case ISD::MULHS: { | |||
3163 | unsigned ExpandOpcode = | |||
3164 | Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI; | |||
3165 | EVT VT = Node->getValueType(0); | |||
3166 | SDVTList VTs = DAG.getVTList(VT, VT); | |||
3167 | ||||
3168 | Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0), | |||
3169 | Node->getOperand(1)); | |||
3170 | Results.push_back(Tmp1.getValue(1)); | |||
3171 | break; | |||
3172 | } | |||
3173 | case ISD::UMUL_LOHI: | |||
3174 | case ISD::SMUL_LOHI: { | |||
3175 | SDValue LHS = Node->getOperand(0); | |||
3176 | SDValue RHS = Node->getOperand(1); | |||
3177 | MVT VT = LHS.getSimpleValueType(); | |||
3178 | unsigned MULHOpcode = | |||
3179 | Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS; | |||
3180 | ||||
3181 | if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) { | |||
3182 | Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS)); | |||
3183 | Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS)); | |||
3184 | break; | |||
3185 | } | |||
3186 | ||||
3187 | SmallVector<SDValue, 4> Halves; | |||
3188 | EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext()); | |||
3189 | assert(TLI.isTypeLegal(HalfType))((TLI.isTypeLegal(HalfType)) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(HalfType)", "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3189, __PRETTY_FUNCTION__)); | |||
3190 | if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, Node, LHS, RHS, Halves, | |||
3191 | HalfType, DAG, | |||
3192 | TargetLowering::MulExpansionKind::Always)) { | |||
3193 | for (unsigned i = 0; i < 2; ++i) { | |||
3194 | SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]); | |||
3195 | SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]); | |||
3196 | SDValue Shift = DAG.getConstant( | |||
3197 | HalfType.getScalarSizeInBits(), dl, | |||
3198 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); | |||
3199 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); | |||
3200 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); | |||
3201 | } | |||
3202 | break; | |||
3203 | } | |||
3204 | break; | |||
3205 | } | |||
3206 | case ISD::MUL: { | |||
3207 | EVT VT = Node->getValueType(0); | |||
3208 | SDVTList VTs = DAG.getVTList(VT, VT); | |||
3209 | // See if multiply or divide can be lowered using two-result operations. | |||
3210 | // We just need the low half of the multiply; try both the signed | |||
3211 | // and unsigned forms. If the target supports both SMUL_LOHI and | |||
3212 | // UMUL_LOHI, form a preference by checking which forms of plain | |||
3213 | // MULH it supports. | |||
3214 | bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT); | |||
3215 | bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT); | |||
3216 | bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT); | |||
3217 | bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT); | |||
3218 | unsigned OpToUse = 0; | |||
3219 | if (HasSMUL_LOHI && !HasMULHS) { | |||
3220 | OpToUse = ISD::SMUL_LOHI; | |||
3221 | } else if (HasUMUL_LOHI && !HasMULHU) { | |||
3222 | OpToUse = ISD::UMUL_LOHI; | |||
3223 | } else if (HasSMUL_LOHI) { | |||
3224 | OpToUse = ISD::SMUL_LOHI; | |||
3225 | } else if (HasUMUL_LOHI) { | |||
3226 | OpToUse = ISD::UMUL_LOHI; | |||
3227 | } | |||
3228 | if (OpToUse) { | |||
3229 | Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0), | |||
3230 | Node->getOperand(1))); | |||
3231 | break; | |||
3232 | } | |||
3233 | ||||
3234 | SDValue Lo, Hi; | |||
3235 | EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext()); | |||
3236 | if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) && | |||
3237 | TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) && | |||
3238 | TLI.isOperationLegalOrCustom(ISD::SHL, VT) && | |||
3239 | TLI.isOperationLegalOrCustom(ISD::OR, VT) && | |||
3240 | TLI.expandMUL(Node, Lo, Hi, HalfType, DAG, | |||
3241 | TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) { | |||
3242 | Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo); | |||
3243 | Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi); | |||
3244 | SDValue Shift = | |||
3245 | DAG.getConstant(HalfType.getSizeInBits(), dl, | |||
3246 | TLI.getShiftAmountTy(HalfType, DAG.getDataLayout())); | |||
3247 | Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift); | |||
3248 | Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi)); | |||
3249 | } | |||
3250 | break; | |||
3251 | } | |||
3252 | case ISD::FSHL: | |||
3253 | case ISD::FSHR: | |||
3254 | if (TLI.expandFunnelShift(Node, Tmp1, DAG)) | |||
3255 | Results.push_back(Tmp1); | |||
3256 | break; | |||
3257 | case ISD::ROTL: | |||
3258 | case ISD::ROTR: | |||
3259 | if (TLI.expandROT(Node, Tmp1, DAG)) | |||
3260 | Results.push_back(Tmp1); | |||
3261 | break; | |||
3262 | case ISD::SADDSAT: | |||
3263 | case ISD::UADDSAT: | |||
3264 | case ISD::SSUBSAT: | |||
3265 | case ISD::USUBSAT: | |||
3266 | Results.push_back(TLI.expandAddSubSat(Node, DAG)); | |||
3267 | break; | |||
3268 | case ISD::SMULFIX: | |||
3269 | case ISD::UMULFIX: | |||
3270 | Results.push_back(TLI.expandFixedPointMul(Node, DAG)); | |||
3271 | break; | |||
3272 | case ISD::SADDO: | |||
3273 | case ISD::SSUBO: { | |||
3274 | SDValue LHS = Node->getOperand(0); | |||
3275 | SDValue RHS = Node->getOperand(1); | |||
3276 | bool IsAdd = Node->getOpcode() == ISD::SADDO; | |||
3277 | ||||
3278 | SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, | |||
3279 | LHS.getValueType(), LHS, RHS); | |||
3280 | Results.push_back(Sum); | |||
3281 | ||||
3282 | EVT ResultType = Node->getValueType(1); | |||
3283 | EVT OType = getSetCCResultType(Node->getValueType(0)); | |||
3284 | ||||
3285 | // If SADDSAT/SSUBSAT is legal, compare results to detect overflow. | |||
3286 | unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT; | |||
3287 | if (TLI.isOperationLegalOrCustom(OpcSat, LHS.getValueType())) { | |||
3288 | SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS); | |||
3289 | SDValue SetCC = DAG.getSetCC(dl, OType, Sum, Sat, ISD::SETNE); | |||
3290 | Results.push_back( | |||
3291 | DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType)); | |||
3292 | break; | |||
3293 | } | |||
3294 | ||||
3295 | SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); | |||
3296 | ||||
3297 | // LHSSign -> LHS >= 0 | |||
3298 | // RHSSign -> RHS >= 0 | |||
3299 | // SumSign -> Sum >= 0 | |||
3300 | // | |||
3301 | // Add: | |||
3302 | // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) | |||
3303 | // Sub: | |||
3304 | // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) | |||
3305 | SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); | |||
3306 | SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); | |||
3307 | SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, | |||
3308 | IsAdd ? ISD::SETEQ : ISD::SETNE); | |||
3309 | ||||
3310 | SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); | |||
3311 | SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); | |||
3312 | ||||
3313 | SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); | |||
3314 | Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType)); | |||
3315 | break; | |||
3316 | } | |||
3317 | case ISD::UADDO: | |||
3318 | case ISD::USUBO: { | |||
3319 | SDValue LHS = Node->getOperand(0); | |||
3320 | SDValue RHS = Node->getOperand(1); | |||
3321 | bool IsAdd = Node->getOpcode() == ISD::UADDO; | |||
3322 | ||||
3323 | // If ADD/SUBCARRY is legal, use that instead. | |||
3324 | unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY; | |||
3325 | if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) { | |||
3326 | SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1)); | |||
3327 | SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(), | |||
3328 | { LHS, RHS, CarryIn }); | |||
3329 | Results.push_back(SDValue(NodeCarry.getNode(), 0)); | |||
3330 | Results.push_back(SDValue(NodeCarry.getNode(), 1)); | |||
3331 | break; | |||
3332 | } | |||
3333 | ||||
3334 | SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, | |||
3335 | LHS.getValueType(), LHS, RHS); | |||
3336 | Results.push_back(Sum); | |||
3337 | ||||
3338 | EVT ResultType = Node->getValueType(1); | |||
3339 | EVT SetCCType = getSetCCResultType(Node->getValueType(0)); | |||
3340 | ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; | |||
3341 | SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); | |||
3342 | ||||
3343 | Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType)); | |||
3344 | break; | |||
3345 | } | |||
3346 | case ISD::UMULO: | |||
3347 | case ISD::SMULO: { | |||
3348 | SDValue Result, Overflow; | |||
3349 | if (TLI.expandMULO(Node, Result, Overflow, DAG)) { | |||
3350 | Results.push_back(Result); | |||
3351 | Results.push_back(Overflow); | |||
3352 | } | |||
3353 | break; | |||
3354 | } | |||
3355 | case ISD::BUILD_PAIR: { | |||
3356 | EVT PairTy = Node->getValueType(0); | |||
3357 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0)); | |||
3358 | Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1)); | |||
3359 | Tmp2 = DAG.getNode( | |||
3360 | ISD::SHL, dl, PairTy, Tmp2, | |||
3361 | DAG.getConstant(PairTy.getSizeInBits() / 2, dl, | |||
3362 | TLI.getShiftAmountTy(PairTy, DAG.getDataLayout()))); | |||
3363 | Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2)); | |||
3364 | break; | |||
3365 | } | |||
3366 | case ISD::SELECT: | |||
3367 | Tmp1 = Node->getOperand(0); | |||
3368 | Tmp2 = Node->getOperand(1); | |||
3369 | Tmp3 = Node->getOperand(2); | |||
3370 | if (Tmp1.getOpcode() == ISD::SETCC) { | |||
3371 | Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1), | |||
3372 | Tmp2, Tmp3, | |||
3373 | cast<CondCodeSDNode>(Tmp1.getOperand(2))->get()); | |||
3374 | } else { | |||
3375 | Tmp1 = DAG.getSelectCC(dl, Tmp1, | |||
3376 | DAG.getConstant(0, dl, Tmp1.getValueType()), | |||
3377 | Tmp2, Tmp3, ISD::SETNE); | |||
3378 | } | |||
3379 | Results.push_back(Tmp1); | |||
3380 | break; | |||
3381 | case ISD::BR_JT: { | |||
3382 | SDValue Chain = Node->getOperand(0); | |||
3383 | SDValue Table = Node->getOperand(1); | |||
3384 | SDValue Index = Node->getOperand(2); | |||
3385 | ||||
3386 | const DataLayout &TD = DAG.getDataLayout(); | |||
3387 | EVT PTy = TLI.getPointerTy(TD); | |||
3388 | ||||
3389 | unsigned EntrySize = | |||
3390 | DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD); | |||
3391 | ||||
3392 | // For power-of-two jumptable entry sizes convert multiplication to a shift. | |||
3393 | // This transformation needs to be done here since otherwise the MIPS | |||
3394 | // backend will end up emitting a three instruction multiply sequence | |||
3395 | // instead of a single shift and MSP430 will call a runtime function. | |||
3396 | if (llvm::isPowerOf2_32(EntrySize)) | |||
3397 | Index = DAG.getNode( | |||
3398 | ISD::SHL, dl, Index.getValueType(), Index, | |||
3399 | DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType())); | |||
3400 | else | |||
3401 | Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index, | |||
3402 | DAG.getConstant(EntrySize, dl, Index.getValueType())); | |||
3403 | SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(), | |||
3404 | Index, Table); | |||
3405 | ||||
3406 | EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8); | |||
3407 | SDValue LD = DAG.getExtLoad( | |||
3408 | ISD::SEXTLOAD, dl, PTy, Chain, Addr, | |||
3409 | MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT); | |||
3410 | Addr = LD; | |||
3411 | if (TLI.isJumpTableRelative()) { | |||
3412 | // For PIC, the sequence is: | |||
3413 | // BRIND(load(Jumptable + index) + RelocBase) | |||
3414 | // RelocBase can be JumpTable, GOT or some sort of global base. | |||
3415 | Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr, | |||
3416 | TLI.getPICJumpTableRelocBase(Table, DAG)); | |||
3417 | } | |||
3418 | ||||
3419 | Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG); | |||
3420 | Results.push_back(Tmp1); | |||
3421 | break; | |||
3422 | } | |||
3423 | case ISD::BRCOND: | |||
3424 | // Expand brcond's setcc into its constituent parts and create a BR_CC | |||
3425 | // Node. | |||
3426 | Tmp1 = Node->getOperand(0); | |||
3427 | Tmp2 = Node->getOperand(1); | |||
3428 | if (Tmp2.getOpcode() == ISD::SETCC) { | |||
3429 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, | |||
3430 | Tmp1, Tmp2.getOperand(2), | |||
3431 | Tmp2.getOperand(0), Tmp2.getOperand(1), | |||
3432 | Node->getOperand(2)); | |||
3433 | } else { | |||
3434 | // We test only the i1 bit. Skip the AND if UNDEF or another AND. | |||
3435 | if (Tmp2.isUndef() || | |||
3436 | (Tmp2.getOpcode() == ISD::AND && | |||
3437 | isa<ConstantSDNode>(Tmp2.getOperand(1)) && | |||
3438 | cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1)) | |||
3439 | Tmp3 = Tmp2; | |||
3440 | else | |||
3441 | Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2, | |||
3442 | DAG.getConstant(1, dl, Tmp2.getValueType())); | |||
3443 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, | |||
3444 | DAG.getCondCode(ISD::SETNE), Tmp3, | |||
3445 | DAG.getConstant(0, dl, Tmp3.getValueType()), | |||
3446 | Node->getOperand(2)); | |||
3447 | } | |||
3448 | Results.push_back(Tmp1); | |||
3449 | break; | |||
3450 | case ISD::SETCC: { | |||
3451 | Tmp1 = Node->getOperand(0); | |||
3452 | Tmp2 = Node->getOperand(1); | |||
3453 | Tmp3 = Node->getOperand(2); | |||
3454 | bool Legalized = LegalizeSetCCCondCode(Node->getValueType(0), Tmp1, Tmp2, | |||
3455 | Tmp3, NeedInvert, dl); | |||
3456 | ||||
3457 | if (Legalized) { | |||
3458 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the | |||
3459 | // condition code, create a new SETCC node. | |||
3460 | if (Tmp3.getNode()) | |||
3461 | Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), | |||
3462 | Tmp1, Tmp2, Tmp3); | |||
3463 | ||||
3464 | // If we expanded the SETCC by inverting the condition code, then wrap | |||
3465 | // the existing SETCC in a NOT to restore the intended condition. | |||
3466 | if (NeedInvert) | |||
3467 | Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0)); | |||
3468 | ||||
3469 | Results.push_back(Tmp1); | |||
3470 | break; | |||
3471 | } | |||
3472 | ||||
3473 | // Otherwise, SETCC for the given comparison type must be completely | |||
3474 | // illegal; expand it into a SELECT_CC. | |||
3475 | EVT VT = Node->getValueType(0); | |||
3476 | int TrueValue; | |||
3477 | switch (TLI.getBooleanContents(Tmp1.getValueType())) { | |||
3478 | case TargetLowering::ZeroOrOneBooleanContent: | |||
3479 | case TargetLowering::UndefinedBooleanContent: | |||
3480 | TrueValue = 1; | |||
3481 | break; | |||
3482 | case TargetLowering::ZeroOrNegativeOneBooleanContent: | |||
3483 | TrueValue = -1; | |||
3484 | break; | |||
3485 | } | |||
3486 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2, | |||
3487 | DAG.getConstant(TrueValue, dl, VT), | |||
3488 | DAG.getConstant(0, dl, VT), | |||
3489 | Tmp3); | |||
3490 | Results.push_back(Tmp1); | |||
3491 | break; | |||
3492 | } | |||
3493 | case ISD::SELECT_CC: { | |||
3494 | Tmp1 = Node->getOperand(0); // LHS | |||
3495 | Tmp2 = Node->getOperand(1); // RHS | |||
3496 | Tmp3 = Node->getOperand(2); // True | |||
3497 | Tmp4 = Node->getOperand(3); // False | |||
3498 | EVT VT = Node->getValueType(0); | |||
3499 | SDValue CC = Node->getOperand(4); | |||
3500 | ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get(); | |||
3501 | ||||
3502 | if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) { | |||
3503 | // If the condition code is legal, then we need to expand this | |||
3504 | // node using SETCC and SELECT. | |||
3505 | EVT CmpVT = Tmp1.getValueType(); | |||
3506 | assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " "expanded.") ? static_cast<void> (0) : __assert_fail ( "!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3508, __PRETTY_FUNCTION__)) | |||
3507 | "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " "expanded.") ? static_cast<void> (0) : __assert_fail ( "!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3508, __PRETTY_FUNCTION__)) | |||
3508 | "expanded.")((!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " "expanded.") ? static_cast<void> (0) : __assert_fail ( "!TLI.isOperationExpand(ISD::SELECT, VT) && \"Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be \" \"expanded.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3508, __PRETTY_FUNCTION__)); | |||
3509 | EVT CCVT = | |||
3510 | TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT); | |||
3511 | SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC); | |||
3512 | Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); | |||
3513 | break; | |||
3514 | } | |||
3515 | ||||
3516 | // SELECT_CC is legal, so the condition code must not be. | |||
3517 | bool Legalized = false; | |||
3518 | // Try to legalize by inverting the condition. This is for targets that | |||
3519 | // might support an ordered version of a condition, but not the unordered | |||
3520 | // version (or vice versa). | |||
3521 | ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, | |||
3522 | Tmp1.getValueType().isInteger()); | |||
3523 | if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) { | |||
3524 | // Use the new condition code and swap true and false | |||
3525 | Legalized = true; | |||
3526 | Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); | |||
3527 | } else { | |||
3528 | // If The inverse is not legal, then try to swap the arguments using | |||
3529 | // the inverse condition code. | |||
3530 | ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC); | |||
3531 | if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) { | |||
3532 | // The swapped inverse condition is legal, so swap true and false, | |||
3533 | // lhs and rhs. | |||
3534 | Legalized = true; | |||
3535 | Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); | |||
3536 | } | |||
3537 | } | |||
3538 | ||||
3539 | if (!Legalized) { | |||
3540 | Legalized = LegalizeSetCCCondCode( | |||
3541 | getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC, NeedInvert, | |||
3542 | dl); | |||
3543 | ||||
3544 | assert(Legalized && "Can't legalize SELECT_CC with legal condition!")((Legalized && "Can't legalize SELECT_CC with legal condition!" ) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize SELECT_CC with legal condition!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3544, __PRETTY_FUNCTION__)); | |||
3545 | ||||
3546 | // If we expanded the SETCC by inverting the condition code, then swap | |||
3547 | // the True/False operands to match. | |||
3548 | if (NeedInvert) | |||
3549 | std::swap(Tmp3, Tmp4); | |||
3550 | ||||
3551 | // If we expanded the SETCC by swapping LHS and RHS, or by inverting the | |||
3552 | // condition code, create a new SELECT_CC node. | |||
3553 | if (CC.getNode()) { | |||
3554 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), | |||
3555 | Tmp1, Tmp2, Tmp3, Tmp4, CC); | |||
3556 | } else { | |||
3557 | Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType()); | |||
3558 | CC = DAG.getCondCode(ISD::SETNE); | |||
3559 | Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, | |||
3560 | Tmp2, Tmp3, Tmp4, CC); | |||
3561 | } | |||
3562 | } | |||
3563 | Results.push_back(Tmp1); | |||
3564 | break; | |||
3565 | } | |||
3566 | case ISD::BR_CC: { | |||
3567 | Tmp1 = Node->getOperand(0); // Chain | |||
3568 | Tmp2 = Node->getOperand(2); // LHS | |||
3569 | Tmp3 = Node->getOperand(3); // RHS | |||
3570 | Tmp4 = Node->getOperand(1); // CC | |||
3571 | ||||
3572 | bool Legalized = LegalizeSetCCCondCode(getSetCCResultType( | |||
3573 | Tmp2.getValueType()), Tmp2, Tmp3, Tmp4, NeedInvert, dl); | |||
3574 | (void)Legalized; | |||
3575 | assert(Legalized && "Can't legalize BR_CC with legal condition!")((Legalized && "Can't legalize BR_CC with legal condition!" ) ? static_cast<void> (0) : __assert_fail ("Legalized && \"Can't legalize BR_CC with legal condition!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3575, __PRETTY_FUNCTION__)); | |||
3576 | ||||
3577 | assert(!NeedInvert && "Don't know how to invert BR_CC!")((!NeedInvert && "Don't know how to invert BR_CC!") ? static_cast<void> (0) : __assert_fail ("!NeedInvert && \"Don't know how to invert BR_CC!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3577, __PRETTY_FUNCTION__)); | |||
3578 | ||||
3579 | // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC | |||
3580 | // node. | |||
3581 | if (Tmp4.getNode()) { | |||
3582 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, | |||
3583 | Tmp4, Tmp2, Tmp3, Node->getOperand(4)); | |||
3584 | } else { | |||
3585 | Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType()); | |||
3586 | Tmp4 = DAG.getCondCode(ISD::SETNE); | |||
3587 | Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4, | |||
3588 | Tmp2, Tmp3, Node->getOperand(4)); | |||
3589 | } | |||
3590 | Results.push_back(Tmp1); | |||
3591 | break; | |||
3592 | } | |||
3593 | case ISD::BUILD_VECTOR: | |||
3594 | Results.push_back(ExpandBUILD_VECTOR(Node)); | |||
3595 | break; | |||
3596 | case ISD::SRA: | |||
3597 | case ISD::SRL: | |||
3598 | case ISD::SHL: { | |||
3599 | // Scalarize vector SRA/SRL/SHL. | |||
3600 | EVT VT = Node->getValueType(0); | |||
3601 | assert(VT.isVector() && "Unable to legalize non-vector shift")((VT.isVector() && "Unable to legalize non-vector shift" ) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Unable to legalize non-vector shift\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3601, __PRETTY_FUNCTION__)); | |||
3602 | assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal")((TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal" ) ? static_cast<void> (0) : __assert_fail ("TLI.isTypeLegal(VT.getScalarType())&& \"Element type must be legal\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3602, __PRETTY_FUNCTION__)); | |||
3603 | unsigned NumElem = VT.getVectorNumElements(); | |||
3604 | ||||
3605 | SmallVector<SDValue, 8> Scalars; | |||
3606 | for (unsigned Idx = 0; Idx < NumElem; Idx++) { | |||
3607 | SDValue Ex = DAG.getNode( | |||
3608 | ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(0), | |||
3609 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); | |||
3610 | SDValue Sh = DAG.getNode( | |||
3611 | ISD::EXTRACT_VECTOR_ELT, dl, VT.getScalarType(), Node->getOperand(1), | |||
3612 | DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); | |||
3613 | Scalars.push_back(DAG.getNode(Node->getOpcode(), dl, | |||
3614 | VT.getScalarType(), Ex, Sh)); | |||
3615 | } | |||
3616 | ||||
3617 | SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars); | |||
3618 | ReplaceNode(SDValue(Node, 0), Result); | |||
3619 | break; | |||
3620 | } | |||
3621 | case ISD::VECREDUCE_FADD: | |||
3622 | case ISD::VECREDUCE_FMUL: | |||
3623 | case ISD::VECREDUCE_ADD: | |||
3624 | case ISD::VECREDUCE_MUL: | |||
3625 | case ISD::VECREDUCE_AND: | |||
3626 | case ISD::VECREDUCE_OR: | |||
3627 | case ISD::VECREDUCE_XOR: | |||
3628 | case ISD::VECREDUCE_SMAX: | |||
3629 | case ISD::VECREDUCE_SMIN: | |||
3630 | case ISD::VECREDUCE_UMAX: | |||
3631 | case ISD::VECREDUCE_UMIN: | |||
3632 | case ISD::VECREDUCE_FMAX: | |||
3633 | case ISD::VECREDUCE_FMIN: | |||
3634 | Results.push_back(TLI.expandVecReduce(Node, DAG)); | |||
3635 | break; | |||
3636 | case ISD::GLOBAL_OFFSET_TABLE: | |||
3637 | case ISD::GlobalAddress: | |||
3638 | case ISD::GlobalTLSAddress: | |||
3639 | case ISD::ExternalSymbol: | |||
3640 | case ISD::ConstantPool: | |||
3641 | case ISD::JumpTable: | |||
3642 | case ISD::INTRINSIC_W_CHAIN: | |||
3643 | case ISD::INTRINSIC_WO_CHAIN: | |||
3644 | case ISD::INTRINSIC_VOID: | |||
3645 | // FIXME: Custom lowering for these operations shouldn't return null! | |||
3646 | break; | |||
3647 | } | |||
3648 | ||||
3649 | // Replace the original node with the legalized result. | |||
3650 | if (Results.empty()) { | |||
3651 | LLVM_DEBUG(dbgs() << "Cannot expand node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Cannot expand node\n"; } } while (false); | |||
3652 | return false; | |||
3653 | } | |||
3654 | ||||
3655 | LLVM_DEBUG(dbgs() << "Successfully expanded node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Successfully expanded node\n" ; } } while (false); | |||
3656 | ReplaceNode(Node, Results.data()); | |||
3657 | return true; | |||
3658 | } | |||
3659 | ||||
3660 | void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) { | |||
3661 | LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Trying to convert node to libcall\n" ; } } while (false); | |||
3662 | SmallVector<SDValue, 8> Results; | |||
3663 | SDLoc dl(Node); | |||
3664 | // FIXME: Check flags on the node to see if we can use a finite call. | |||
3665 | bool CanUseFiniteLibCall = TM.Options.NoInfsFPMath && TM.Options.NoNaNsFPMath; | |||
3666 | unsigned Opc = Node->getOpcode(); | |||
3667 | switch (Opc) { | |||
3668 | case ISD::ATOMIC_FENCE: { | |||
3669 | // If the target didn't lower this, lower it to '__sync_synchronize()' call | |||
3670 | // FIXME: handle "fence singlethread" more efficiently. | |||
3671 | TargetLowering::ArgListTy Args; | |||
3672 | ||||
3673 | TargetLowering::CallLoweringInfo CLI(DAG); | |||
3674 | CLI.setDebugLoc(dl) | |||
3675 | .setChain(Node->getOperand(0)) | |||
3676 | .setLibCallee( | |||
3677 | CallingConv::C, Type::getVoidTy(*DAG.getContext()), | |||
3678 | DAG.getExternalSymbol("__sync_synchronize", | |||
3679 | TLI.getPointerTy(DAG.getDataLayout())), | |||
3680 | std::move(Args)); | |||
3681 | ||||
3682 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); | |||
3683 | ||||
3684 | Results.push_back(CallResult.second); | |||
3685 | break; | |||
3686 | } | |||
3687 | // By default, atomic intrinsics are marked Legal and lowered. Targets | |||
3688 | // which don't support them directly, however, may want libcalls, in which | |||
3689 | // case they mark them Expand, and we get here. | |||
3690 | case ISD::ATOMIC_SWAP: | |||
3691 | case ISD::ATOMIC_LOAD_ADD: | |||
3692 | case ISD::ATOMIC_LOAD_SUB: | |||
3693 | case ISD::ATOMIC_LOAD_AND: | |||
3694 | case ISD::ATOMIC_LOAD_CLR: | |||
3695 | case ISD::ATOMIC_LOAD_OR: | |||
3696 | case ISD::ATOMIC_LOAD_XOR: | |||
3697 | case ISD::ATOMIC_LOAD_NAND: | |||
3698 | case ISD::ATOMIC_LOAD_MIN: | |||
3699 | case ISD::ATOMIC_LOAD_MAX: | |||
3700 | case ISD::ATOMIC_LOAD_UMIN: | |||
3701 | case ISD::ATOMIC_LOAD_UMAX: | |||
3702 | case ISD::ATOMIC_CMP_SWAP: { | |||
3703 | MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT(); | |||
3704 | RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT); | |||
3705 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!")((LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!" ) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unexpected atomic op or value type!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3705, __PRETTY_FUNCTION__)); | |||
3706 | ||||
3707 | std::pair<SDValue, SDValue> Tmp = ExpandChainLibCall(LC, Node, false); | |||
3708 | Results.push_back(Tmp.first); | |||
3709 | Results.push_back(Tmp.second); | |||
3710 | break; | |||
3711 | } | |||
3712 | case ISD::TRAP: { | |||
3713 | // If this operation is not supported, lower it to 'abort()' call | |||
3714 | TargetLowering::ArgListTy Args; | |||
3715 | TargetLowering::CallLoweringInfo CLI(DAG); | |||
3716 | CLI.setDebugLoc(dl) | |||
3717 | .setChain(Node->getOperand(0)) | |||
3718 | .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()), | |||
3719 | DAG.getExternalSymbol( | |||
3720 | "abort", TLI.getPointerTy(DAG.getDataLayout())), | |||
3721 | std::move(Args)); | |||
3722 | std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI); | |||
3723 | ||||
3724 | Results.push_back(CallResult.second); | |||
3725 | break; | |||
3726 | } | |||
3727 | case ISD::FMINNUM: | |||
3728 | case ISD::STRICT_FMINNUM: | |||
3729 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64, | |||
3730 | RTLIB::FMIN_F80, RTLIB::FMIN_F128, | |||
3731 | RTLIB::FMIN_PPCF128)); | |||
3732 | break; | |||
3733 | case ISD::FMAXNUM: | |||
3734 | case ISD::STRICT_FMAXNUM: | |||
3735 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64, | |||
3736 | RTLIB::FMAX_F80, RTLIB::FMAX_F128, | |||
3737 | RTLIB::FMAX_PPCF128)); | |||
3738 | break; | |||
3739 | case ISD::FSQRT: | |||
3740 | case ISD::STRICT_FSQRT: | |||
3741 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64, | |||
3742 | RTLIB::SQRT_F80, RTLIB::SQRT_F128, | |||
3743 | RTLIB::SQRT_PPCF128)); | |||
3744 | break; | |||
3745 | case ISD::FCBRT: | |||
3746 | Results.push_back(ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64, | |||
3747 | RTLIB::CBRT_F80, RTLIB::CBRT_F128, | |||
3748 | RTLIB::CBRT_PPCF128)); | |||
3749 | break; | |||
3750 | case ISD::FSIN: | |||
3751 | case ISD::STRICT_FSIN: | |||
3752 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64, | |||
3753 | RTLIB::SIN_F80, RTLIB::SIN_F128, | |||
3754 | RTLIB::SIN_PPCF128)); | |||
3755 | break; | |||
3756 | case ISD::FCOS: | |||
3757 | case ISD::STRICT_FCOS: | |||
3758 | Results.push_back(ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64, | |||
3759 | RTLIB::COS_F80, RTLIB::COS_F128, | |||
3760 | RTLIB::COS_PPCF128)); | |||
3761 | break; | |||
3762 | case ISD::FSINCOS: | |||
3763 | // Expand into sincos libcall. | |||
3764 | ExpandSinCosLibCall(Node, Results); | |||
3765 | break; | |||
3766 | case ISD::FLOG: | |||
3767 | case ISD::STRICT_FLOG: | |||
3768 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log_finite)) | |||
3769 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_FINITE_F32, | |||
3770 | RTLIB::LOG_FINITE_F64, | |||
3771 | RTLIB::LOG_FINITE_F80, | |||
3772 | RTLIB::LOG_FINITE_F128, | |||
3773 | RTLIB::LOG_FINITE_PPCF128)); | |||
3774 | else | |||
3775 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, | |||
3776 | RTLIB::LOG_F80, RTLIB::LOG_F128, | |||
3777 | RTLIB::LOG_PPCF128)); | |||
3778 | break; | |||
3779 | case ISD::FLOG2: | |||
3780 | case ISD::STRICT_FLOG2: | |||
3781 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log2_finite)) | |||
3782 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_FINITE_F32, | |||
3783 | RTLIB::LOG2_FINITE_F64, | |||
3784 | RTLIB::LOG2_FINITE_F80, | |||
3785 | RTLIB::LOG2_FINITE_F128, | |||
3786 | RTLIB::LOG2_FINITE_PPCF128)); | |||
3787 | else | |||
3788 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, | |||
3789 | RTLIB::LOG2_F80, RTLIB::LOG2_F128, | |||
3790 | RTLIB::LOG2_PPCF128)); | |||
3791 | break; | |||
3792 | case ISD::FLOG10: | |||
3793 | case ISD::STRICT_FLOG10: | |||
3794 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_log10_finite)) | |||
3795 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_FINITE_F32, | |||
3796 | RTLIB::LOG10_FINITE_F64, | |||
3797 | RTLIB::LOG10_FINITE_F80, | |||
3798 | RTLIB::LOG10_FINITE_F128, | |||
3799 | RTLIB::LOG10_FINITE_PPCF128)); | |||
3800 | else | |||
3801 | Results.push_back(ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, | |||
3802 | RTLIB::LOG10_F80, RTLIB::LOG10_F128, | |||
3803 | RTLIB::LOG10_PPCF128)); | |||
3804 | break; | |||
3805 | case ISD::FEXP: | |||
3806 | case ISD::STRICT_FEXP: | |||
3807 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp_finite)) | |||
3808 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_FINITE_F32, | |||
3809 | RTLIB::EXP_FINITE_F64, | |||
3810 | RTLIB::EXP_FINITE_F80, | |||
3811 | RTLIB::EXP_FINITE_F128, | |||
3812 | RTLIB::EXP_FINITE_PPCF128)); | |||
3813 | else | |||
3814 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, | |||
3815 | RTLIB::EXP_F80, RTLIB::EXP_F128, | |||
3816 | RTLIB::EXP_PPCF128)); | |||
3817 | break; | |||
3818 | case ISD::FEXP2: | |||
3819 | case ISD::STRICT_FEXP2: | |||
3820 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_exp2_finite)) | |||
3821 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_FINITE_F32, | |||
3822 | RTLIB::EXP2_FINITE_F64, | |||
3823 | RTLIB::EXP2_FINITE_F80, | |||
3824 | RTLIB::EXP2_FINITE_F128, | |||
3825 | RTLIB::EXP2_FINITE_PPCF128)); | |||
3826 | else | |||
3827 | Results.push_back(ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, | |||
3828 | RTLIB::EXP2_F80, RTLIB::EXP2_F128, | |||
3829 | RTLIB::EXP2_PPCF128)); | |||
3830 | break; | |||
3831 | case ISD::FTRUNC: | |||
3832 | case ISD::STRICT_FTRUNC: | |||
3833 | Results.push_back(ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64, | |||
3834 | RTLIB::TRUNC_F80, RTLIB::TRUNC_F128, | |||
3835 | RTLIB::TRUNC_PPCF128)); | |||
3836 | break; | |||
3837 | case ISD::FFLOOR: | |||
3838 | case ISD::STRICT_FFLOOR: | |||
3839 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64, | |||
3840 | RTLIB::FLOOR_F80, RTLIB::FLOOR_F128, | |||
3841 | RTLIB::FLOOR_PPCF128)); | |||
3842 | break; | |||
3843 | case ISD::FCEIL: | |||
3844 | case ISD::STRICT_FCEIL: | |||
3845 | Results.push_back(ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64, | |||
3846 | RTLIB::CEIL_F80, RTLIB::CEIL_F128, | |||
3847 | RTLIB::CEIL_PPCF128)); | |||
3848 | break; | |||
3849 | case ISD::FRINT: | |||
3850 | case ISD::STRICT_FRINT: | |||
3851 | Results.push_back(ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64, | |||
3852 | RTLIB::RINT_F80, RTLIB::RINT_F128, | |||
3853 | RTLIB::RINT_PPCF128)); | |||
3854 | break; | |||
3855 | case ISD::FNEARBYINT: | |||
3856 | case ISD::STRICT_FNEARBYINT: | |||
3857 | Results.push_back(ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32, | |||
3858 | RTLIB::NEARBYINT_F64, | |||
3859 | RTLIB::NEARBYINT_F80, | |||
3860 | RTLIB::NEARBYINT_F128, | |||
3861 | RTLIB::NEARBYINT_PPCF128)); | |||
3862 | break; | |||
3863 | case ISD::FROUND: | |||
3864 | case ISD::STRICT_FROUND: | |||
3865 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ROUND_F32, | |||
3866 | RTLIB::ROUND_F64, | |||
3867 | RTLIB::ROUND_F80, | |||
3868 | RTLIB::ROUND_F128, | |||
3869 | RTLIB::ROUND_PPCF128)); | |||
3870 | break; | |||
3871 | case ISD::FPOWI: | |||
3872 | case ISD::STRICT_FPOWI: | |||
3873 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64, | |||
3874 | RTLIB::POWI_F80, RTLIB::POWI_F128, | |||
3875 | RTLIB::POWI_PPCF128)); | |||
3876 | break; | |||
3877 | case ISD::FPOW: | |||
3878 | case ISD::STRICT_FPOW: | |||
3879 | if (CanUseFiniteLibCall && DAG.getLibInfo().has(LibFunc_pow_finite)) | |||
3880 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_FINITE_F32, | |||
3881 | RTLIB::POW_FINITE_F64, | |||
3882 | RTLIB::POW_FINITE_F80, | |||
3883 | RTLIB::POW_FINITE_F128, | |||
3884 | RTLIB::POW_FINITE_PPCF128)); | |||
3885 | else | |||
3886 | Results.push_back(ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, | |||
3887 | RTLIB::POW_F80, RTLIB::POW_F128, | |||
3888 | RTLIB::POW_PPCF128)); | |||
3889 | break; | |||
3890 | case ISD::FDIV: | |||
3891 | Results.push_back(ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64, | |||
3892 | RTLIB::DIV_F80, RTLIB::DIV_F128, | |||
3893 | RTLIB::DIV_PPCF128)); | |||
3894 | break; | |||
3895 | case ISD::FREM: | |||
3896 | case ISD::STRICT_FREM: | |||
3897 | Results.push_back(ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64, | |||
3898 | RTLIB::REM_F80, RTLIB::REM_F128, | |||
3899 | RTLIB::REM_PPCF128)); | |||
3900 | break; | |||
3901 | case ISD::FMA: | |||
3902 | case ISD::STRICT_FMA: | |||
3903 | Results.push_back(ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64, | |||
3904 | RTLIB::FMA_F80, RTLIB::FMA_F128, | |||
3905 | RTLIB::FMA_PPCF128)); | |||
3906 | break; | |||
3907 | case ISD::FADD: | |||
3908 | Results.push_back(ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64, | |||
3909 | RTLIB::ADD_F80, RTLIB::ADD_F128, | |||
3910 | RTLIB::ADD_PPCF128)); | |||
3911 | break; | |||
3912 | case ISD::FMUL: | |||
3913 | Results.push_back(ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64, | |||
3914 | RTLIB::MUL_F80, RTLIB::MUL_F128, | |||
3915 | RTLIB::MUL_PPCF128)); | |||
3916 | break; | |||
3917 | case ISD::FP16_TO_FP: | |||
3918 | if (Node->getValueType(0) == MVT::f32) { | |||
3919 | Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false)); | |||
3920 | } | |||
3921 | break; | |||
3922 | case ISD::FP_TO_FP16: { | |||
3923 | RTLIB::Libcall LC = | |||
3924 | RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16); | |||
3925 | assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16")((LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16" ) ? static_cast<void> (0) : __assert_fail ("LC != RTLIB::UNKNOWN_LIBCALL && \"Unable to expand fp_to_fp16\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3925, __PRETTY_FUNCTION__)); | |||
3926 | Results.push_back(ExpandLibCall(LC, Node, false)); | |||
3927 | break; | |||
3928 | } | |||
3929 | case ISD::FSUB: | |||
3930 | Results.push_back(ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64, | |||
3931 | RTLIB::SUB_F80, RTLIB::SUB_F128, | |||
3932 | RTLIB::SUB_PPCF128)); | |||
3933 | break; | |||
3934 | case ISD::SREM: | |||
3935 | Results.push_back(ExpandIntLibCall(Node, true, | |||
3936 | RTLIB::SREM_I8, | |||
3937 | RTLIB::SREM_I16, RTLIB::SREM_I32, | |||
3938 | RTLIB::SREM_I64, RTLIB::SREM_I128)); | |||
3939 | break; | |||
3940 | case ISD::UREM: | |||
3941 | Results.push_back(ExpandIntLibCall(Node, false, | |||
3942 | RTLIB::UREM_I8, | |||
3943 | RTLIB::UREM_I16, RTLIB::UREM_I32, | |||
3944 | RTLIB::UREM_I64, RTLIB::UREM_I128)); | |||
3945 | break; | |||
3946 | case ISD::SDIV: | |||
3947 | Results.push_back(ExpandIntLibCall(Node, true, | |||
3948 | RTLIB::SDIV_I8, | |||
3949 | RTLIB::SDIV_I16, RTLIB::SDIV_I32, | |||
3950 | RTLIB::SDIV_I64, RTLIB::SDIV_I128)); | |||
3951 | break; | |||
3952 | case ISD::UDIV: | |||
3953 | Results.push_back(ExpandIntLibCall(Node, false, | |||
3954 | RTLIB::UDIV_I8, | |||
3955 | RTLIB::UDIV_I16, RTLIB::UDIV_I32, | |||
3956 | RTLIB::UDIV_I64, RTLIB::UDIV_I128)); | |||
3957 | break; | |||
3958 | case ISD::SDIVREM: | |||
3959 | case ISD::UDIVREM: | |||
3960 | // Expand into divrem libcall | |||
3961 | ExpandDivRemLibCall(Node, Results); | |||
3962 | break; | |||
3963 | case ISD::MUL: | |||
3964 | Results.push_back(ExpandIntLibCall(Node, false, | |||
3965 | RTLIB::MUL_I8, | |||
3966 | RTLIB::MUL_I16, RTLIB::MUL_I32, | |||
3967 | RTLIB::MUL_I64, RTLIB::MUL_I128)); | |||
3968 | break; | |||
3969 | case ISD::CTLZ_ZERO_UNDEF: | |||
3970 | switch (Node->getSimpleValueType(0).SimpleTy) { | |||
3971 | default: | |||
3972 | llvm_unreachable("LibCall explicitly requested, but not available")::llvm::llvm_unreachable_internal("LibCall explicitly requested, but not available" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 3972); | |||
3973 | case MVT::i32: | |||
3974 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false)); | |||
3975 | break; | |||
3976 | case MVT::i64: | |||
3977 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false)); | |||
3978 | break; | |||
3979 | case MVT::i128: | |||
3980 | Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false)); | |||
3981 | break; | |||
3982 | } | |||
3983 | break; | |||
3984 | } | |||
3985 | ||||
3986 | // Replace the original node with the legalized result. | |||
3987 | if (!Results.empty()) { | |||
3988 | LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Successfully converted node to libcall\n" ; } } while (false); | |||
3989 | ReplaceNode(Node, Results.data()); | |||
3990 | } else | |||
3991 | LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Could not convert node to libcall\n" ; } } while (false); | |||
3992 | } | |||
3993 | ||||
3994 | // Determine the vector type to use in place of an original scalar element when | |||
3995 | // promoting equally sized vectors. | |||
3996 | static MVT getPromotedVectorElementType(const TargetLowering &TLI, | |||
3997 | MVT EltVT, MVT NewEltVT) { | |||
3998 | unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits(); | |||
3999 | MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt); | |||
4000 | assert(TLI.isTypeLegal(MidVT) && "unexpected")((TLI.isTypeLegal(MidVT) && "unexpected") ? static_cast <void> (0) : __assert_fail ("TLI.isTypeLegal(MidVT) && \"unexpected\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4000, __PRETTY_FUNCTION__)); | |||
4001 | return MidVT; | |||
4002 | } | |||
4003 | ||||
4004 | void SelectionDAGLegalize::PromoteNode(SDNode *Node) { | |||
4005 | LLVM_DEBUG(dbgs() << "Trying to promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Trying to promote node\n" ; } } while (false); | |||
4006 | SmallVector<SDValue, 8> Results; | |||
4007 | MVT OVT = Node->getSimpleValueType(0); | |||
4008 | if (Node->getOpcode() == ISD::UINT_TO_FP || | |||
4009 | Node->getOpcode() == ISD::SINT_TO_FP || | |||
4010 | Node->getOpcode() == ISD::SETCC || | |||
4011 | Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT || | |||
4012 | Node->getOpcode() == ISD::INSERT_VECTOR_ELT) { | |||
4013 | OVT = Node->getOperand(0).getSimpleValueType(); | |||
4014 | } | |||
4015 | if (Node->getOpcode() == ISD::BR_CC) | |||
4016 | OVT = Node->getOperand(2).getSimpleValueType(); | |||
4017 | MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT); | |||
4018 | SDLoc dl(Node); | |||
4019 | SDValue Tmp1, Tmp2, Tmp3; | |||
4020 | switch (Node->getOpcode()) { | |||
4021 | case ISD::CTTZ: | |||
4022 | case ISD::CTTZ_ZERO_UNDEF: | |||
4023 | case ISD::CTLZ: | |||
4024 | case ISD::CTLZ_ZERO_UNDEF: | |||
4025 | case ISD::CTPOP: | |||
4026 | // Zero extend the argument. | |||
4027 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); | |||
4028 | if (Node->getOpcode() == ISD::CTTZ) { | |||
4029 | // The count is the same in the promoted type except if the original | |||
4030 | // value was zero. This can be handled by setting the bit just off | |||
4031 | // the top of the original type. | |||
4032 | auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(), | |||
4033 | OVT.getSizeInBits()); | |||
4034 | Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1, | |||
4035 | DAG.getConstant(TopBit, dl, NVT)); | |||
4036 | } | |||
4037 | // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is | |||
4038 | // already the correct result. | |||
4039 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); | |||
4040 | if (Node->getOpcode() == ISD::CTLZ || | |||
4041 | Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) { | |||
4042 | // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT)) | |||
4043 | Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1, | |||
4044 | DAG.getConstant(NVT.getSizeInBits() - | |||
4045 | OVT.getSizeInBits(), dl, NVT)); | |||
4046 | } | |||
4047 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); | |||
4048 | break; | |||
4049 | case ISD::BITREVERSE: | |||
4050 | case ISD::BSWAP: { | |||
4051 | unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits(); | |||
4052 | Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0)); | |||
4053 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); | |||
4054 | Tmp1 = DAG.getNode( | |||
4055 | ISD::SRL, dl, NVT, Tmp1, | |||
4056 | DAG.getConstant(DiffBits, dl, | |||
4057 | TLI.getShiftAmountTy(NVT, DAG.getDataLayout()))); | |||
4058 | ||||
4059 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); | |||
4060 | break; | |||
4061 | } | |||
4062 | case ISD::FP_TO_UINT: | |||
4063 | case ISD::FP_TO_SINT: | |||
4064 | Tmp1 = PromoteLegalFP_TO_INT(Node->getOperand(0), Node->getValueType(0), | |||
4065 | Node->getOpcode() == ISD::FP_TO_SINT, dl); | |||
4066 | Results.push_back(Tmp1); | |||
4067 | break; | |||
4068 | case ISD::UINT_TO_FP: | |||
4069 | case ISD::SINT_TO_FP: | |||
4070 | Tmp1 = PromoteLegalINT_TO_FP(Node->getOperand(0), Node->getValueType(0), | |||
4071 | Node->getOpcode() == ISD::SINT_TO_FP, dl); | |||
4072 | Results.push_back(Tmp1); | |||
4073 | break; | |||
4074 | case ISD::VAARG: { | |||
4075 | SDValue Chain = Node->getOperand(0); // Get the chain. | |||
4076 | SDValue Ptr = Node->getOperand(1); // Get the pointer. | |||
4077 | ||||
4078 | unsigned TruncOp; | |||
4079 | if (OVT.isVector()) { | |||
4080 | TruncOp = ISD::BITCAST; | |||
4081 | } else { | |||
4082 | assert(OVT.isInteger()((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types" ) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4083, __PRETTY_FUNCTION__)) | |||
4083 | && "VAARG promotion is supported only for vectors or integer types")((OVT.isInteger() && "VAARG promotion is supported only for vectors or integer types" ) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"VAARG promotion is supported only for vectors or integer types\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4083, __PRETTY_FUNCTION__)); | |||
4084 | TruncOp = ISD::TRUNCATE; | |||
4085 | } | |||
4086 | ||||
4087 | // Perform the larger operation, then convert back | |||
4088 | Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2), | |||
4089 | Node->getConstantOperandVal(3)); | |||
4090 | Chain = Tmp1.getValue(1); | |||
4091 | ||||
4092 | Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1); | |||
4093 | ||||
4094 | // Modified the chain result - switch anything that used the old chain to | |||
4095 | // use the new one. | |||
4096 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2); | |||
4097 | DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain); | |||
4098 | if (UpdatedNodes) { | |||
4099 | UpdatedNodes->insert(Tmp2.getNode()); | |||
4100 | UpdatedNodes->insert(Chain.getNode()); | |||
4101 | } | |||
4102 | ReplacedNode(Node); | |||
4103 | break; | |||
4104 | } | |||
4105 | case ISD::MUL: | |||
4106 | case ISD::SDIV: | |||
4107 | case ISD::SREM: | |||
4108 | case ISD::UDIV: | |||
4109 | case ISD::UREM: | |||
4110 | case ISD::AND: | |||
4111 | case ISD::OR: | |||
4112 | case ISD::XOR: { | |||
4113 | unsigned ExtOp, TruncOp; | |||
4114 | if (OVT.isVector()) { | |||
4115 | ExtOp = ISD::BITCAST; | |||
4116 | TruncOp = ISD::BITCAST; | |||
4117 | } else { | |||
4118 | assert(OVT.isInteger() && "Cannot promote logic operation")((OVT.isInteger() && "Cannot promote logic operation" ) ? static_cast<void> (0) : __assert_fail ("OVT.isInteger() && \"Cannot promote logic operation\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4118, __PRETTY_FUNCTION__)); | |||
4119 | ||||
4120 | switch (Node->getOpcode()) { | |||
4121 | default: | |||
4122 | ExtOp = ISD::ANY_EXTEND; | |||
4123 | break; | |||
4124 | case ISD::SDIV: | |||
4125 | case ISD::SREM: | |||
4126 | ExtOp = ISD::SIGN_EXTEND; | |||
4127 | break; | |||
4128 | case ISD::UDIV: | |||
4129 | case ISD::UREM: | |||
4130 | ExtOp = ISD::ZERO_EXTEND; | |||
4131 | break; | |||
4132 | } | |||
4133 | TruncOp = ISD::TRUNCATE; | |||
4134 | } | |||
4135 | // Promote each of the values to the new type. | |||
4136 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); | |||
4137 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |||
4138 | // Perform the larger operation, then convert back | |||
4139 | Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); | |||
4140 | Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1)); | |||
4141 | break; | |||
4142 | } | |||
4143 | case ISD::UMUL_LOHI: | |||
4144 | case ISD::SMUL_LOHI: { | |||
4145 | // Promote to a multiply in a wider integer type. | |||
4146 | unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND | |||
4147 | : ISD::SIGN_EXTEND; | |||
4148 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); | |||
4149 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |||
4150 | Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2); | |||
4151 | ||||
4152 | auto &DL = DAG.getDataLayout(); | |||
4153 | unsigned OriginalSize = OVT.getScalarSizeInBits(); | |||
4154 | Tmp2 = DAG.getNode( | |||
4155 | ISD::SRL, dl, NVT, Tmp1, | |||
4156 | DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT))); | |||
4157 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1)); | |||
4158 | Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2)); | |||
4159 | break; | |||
4160 | } | |||
4161 | case ISD::SELECT: { | |||
4162 | unsigned ExtOp, TruncOp; | |||
4163 | if (Node->getValueType(0).isVector() || | |||
4164 | Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) { | |||
4165 | ExtOp = ISD::BITCAST; | |||
4166 | TruncOp = ISD::BITCAST; | |||
4167 | } else if (Node->getValueType(0).isInteger()) { | |||
4168 | ExtOp = ISD::ANY_EXTEND; | |||
4169 | TruncOp = ISD::TRUNCATE; | |||
4170 | } else { | |||
4171 | ExtOp = ISD::FP_EXTEND; | |||
4172 | TruncOp = ISD::FP_ROUND; | |||
4173 | } | |||
4174 | Tmp1 = Node->getOperand(0); | |||
4175 | // Promote each of the values to the new type. | |||
4176 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |||
4177 | Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); | |||
4178 | // Perform the larger operation, then round down. | |||
4179 | Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); | |||
4180 | if (TruncOp != ISD::FP_ROUND) | |||
4181 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); | |||
4182 | else | |||
4183 | Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1, | |||
4184 | DAG.getIntPtrConstant(0, dl)); | |||
4185 | Results.push_back(Tmp1); | |||
4186 | break; | |||
4187 | } | |||
4188 | case ISD::VECTOR_SHUFFLE: { | |||
4189 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask(); | |||
4190 | ||||
4191 | // Cast the two input vectors. | |||
4192 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0)); | |||
4193 | Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1)); | |||
4194 | ||||
4195 | // Convert the shuffle mask to the right # elements. | |||
4196 | Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask); | |||
4197 | Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1); | |||
4198 | Results.push_back(Tmp1); | |||
4199 | break; | |||
4200 | } | |||
4201 | case ISD::SETCC: { | |||
4202 | unsigned ExtOp = ISD::FP_EXTEND; | |||
4203 | if (NVT.isInteger()) { | |||
4204 | ISD::CondCode CCCode = | |||
4205 | cast<CondCodeSDNode>(Node->getOperand(2))->get(); | |||
4206 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; | |||
4207 | } | |||
4208 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); | |||
4209 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); | |||
4210 | Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), | |||
4211 | Tmp1, Tmp2, Node->getOperand(2))); | |||
4212 | break; | |||
4213 | } | |||
4214 | case ISD::BR_CC: { | |||
4215 | unsigned ExtOp = ISD::FP_EXTEND; | |||
4216 | if (NVT.isInteger()) { | |||
4217 | ISD::CondCode CCCode = | |||
4218 | cast<CondCodeSDNode>(Node->getOperand(1))->get(); | |||
4219 | ExtOp = isSignedIntSetCC(CCCode) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND; | |||
4220 | } | |||
4221 | Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); | |||
4222 | Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3)); | |||
4223 | Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), | |||
4224 | Node->getOperand(0), Node->getOperand(1), | |||
4225 | Tmp1, Tmp2, Node->getOperand(4))); | |||
4226 | break; | |||
4227 | } | |||
4228 | case ISD::FADD: | |||
4229 | case ISD::FSUB: | |||
4230 | case ISD::FMUL: | |||
4231 | case ISD::FDIV: | |||
4232 | case ISD::FREM: | |||
4233 | case ISD::FMINNUM: | |||
4234 | case ISD::FMAXNUM: | |||
4235 | case ISD::FPOW: | |||
4236 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); | |||
4237 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); | |||
4238 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, | |||
4239 | Node->getFlags()); | |||
4240 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, | |||
4241 | Tmp3, DAG.getIntPtrConstant(0, dl))); | |||
4242 | break; | |||
4243 | case ISD::FMA: | |||
4244 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); | |||
4245 | Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1)); | |||
4246 | Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2)); | |||
4247 | Results.push_back( | |||
4248 | DAG.getNode(ISD::FP_ROUND, dl, OVT, | |||
4249 | DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3), | |||
4250 | DAG.getIntPtrConstant(0, dl))); | |||
4251 | break; | |||
4252 | case ISD::FCOPYSIGN: | |||
4253 | case ISD::FPOWI: { | |||
4254 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); | |||
4255 | Tmp2 = Node->getOperand(1); | |||
4256 | Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2); | |||
4257 | ||||
4258 | // fcopysign doesn't change anything but the sign bit, so | |||
4259 | // (fp_round (fcopysign (fpext a), b)) | |||
4260 | // is as precise as | |||
4261 | // (fp_round (fpext a)) | |||
4262 | // which is a no-op. Mark it as a TRUNCating FP_ROUND. | |||
4263 | const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN); | |||
4264 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, | |||
4265 | Tmp3, DAG.getIntPtrConstant(isTrunc, dl))); | |||
4266 | break; | |||
4267 | } | |||
4268 | case ISD::FFLOOR: | |||
4269 | case ISD::FCEIL: | |||
4270 | case ISD::FRINT: | |||
4271 | case ISD::FNEARBYINT: | |||
4272 | case ISD::FROUND: | |||
4273 | case ISD::FTRUNC: | |||
4274 | case ISD::FNEG: | |||
4275 | case ISD::FSQRT: | |||
4276 | case ISD::FSIN: | |||
4277 | case ISD::FCOS: | |||
4278 | case ISD::FLOG: | |||
4279 | case ISD::FLOG2: | |||
4280 | case ISD::FLOG10: | |||
4281 | case ISD::FABS: | |||
4282 | case ISD::FEXP: | |||
4283 | case ISD::FEXP2: | |||
4284 | Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0)); | |||
4285 | Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1); | |||
4286 | Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT, | |||
4287 | Tmp2, DAG.getIntPtrConstant(0, dl))); | |||
4288 | break; | |||
4289 | case ISD::BUILD_VECTOR: { | |||
4290 | MVT EltVT = OVT.getVectorElementType(); | |||
4291 | MVT NewEltVT = NVT.getVectorElementType(); | |||
4292 | ||||
4293 | // Handle bitcasts to a different vector type with the same total bit size | |||
4294 | // | |||
4295 | // e.g. v2i64 = build_vector i64:x, i64:y => v4i32 | |||
4296 | // => | |||
4297 | // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y)) | |||
4298 | ||||
4299 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits () && "Invalid promote type for build_vector") ? static_cast <void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4300, __PRETTY_FUNCTION__)) | |||
4300 | "Invalid promote type for build_vector")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits () && "Invalid promote type for build_vector") ? static_cast <void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for build_vector\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4300, __PRETTY_FUNCTION__)); | |||
4301 | assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast <void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4301, __PRETTY_FUNCTION__)); | |||
4302 | ||||
4303 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); | |||
4304 | ||||
4305 | SmallVector<SDValue, 8> NewOps; | |||
4306 | for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) { | |||
4307 | SDValue Op = Node->getOperand(I); | |||
4308 | NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op)); | |||
4309 | } | |||
4310 | ||||
4311 | SDLoc SL(Node); | |||
4312 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps); | |||
4313 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); | |||
4314 | Results.push_back(CvtVec); | |||
4315 | break; | |||
4316 | } | |||
4317 | case ISD::EXTRACT_VECTOR_ELT: { | |||
4318 | MVT EltVT = OVT.getVectorElementType(); | |||
4319 | MVT NewEltVT = NVT.getVectorElementType(); | |||
4320 | ||||
4321 | // Handle bitcasts to a different vector type with the same total bit size. | |||
4322 | // | |||
4323 | // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32 | |||
4324 | // => | |||
4325 | // v4i32:castx = bitcast x:v2i64 | |||
4326 | // | |||
4327 | // i64 = bitcast | |||
4328 | // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))), | |||
4329 | // (i32 (extract_vector_elt castx, (2 * y + 1))) | |||
4330 | // | |||
4331 | ||||
4332 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits () && "Invalid promote type for extract_vector_elt") ? static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4333, __PRETTY_FUNCTION__)) | |||
4333 | "Invalid promote type for extract_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits () && "Invalid promote type for extract_vector_elt") ? static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for extract_vector_elt\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4333, __PRETTY_FUNCTION__)); | |||
4334 | assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast <void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4334, __PRETTY_FUNCTION__)); | |||
4335 | ||||
4336 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); | |||
4337 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); | |||
4338 | ||||
4339 | SDValue Idx = Node->getOperand(1); | |||
4340 | EVT IdxVT = Idx.getValueType(); | |||
4341 | SDLoc SL(Node); | |||
4342 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT); | |||
4343 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); | |||
4344 | ||||
4345 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); | |||
4346 | ||||
4347 | SmallVector<SDValue, 8> NewOps; | |||
4348 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { | |||
4349 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); | |||
4350 | SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); | |||
4351 | ||||
4352 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, | |||
4353 | CastVec, TmpIdx); | |||
4354 | NewOps.push_back(Elt); | |||
4355 | } | |||
4356 | ||||
4357 | SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps); | |||
4358 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec)); | |||
4359 | break; | |||
4360 | } | |||
4361 | case ISD::INSERT_VECTOR_ELT: { | |||
4362 | MVT EltVT = OVT.getVectorElementType(); | |||
4363 | MVT NewEltVT = NVT.getVectorElementType(); | |||
4364 | ||||
4365 | // Handle bitcasts to a different vector type with the same total bit size | |||
4366 | // | |||
4367 | // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32 | |||
4368 | // => | |||
4369 | // v4i32:castx = bitcast x:v2i64 | |||
4370 | // v2i32:casty = bitcast y:i64 | |||
4371 | // | |||
4372 | // v2i64 = bitcast | |||
4373 | // (v4i32 insert_vector_elt | |||
4374 | // (v4i32 insert_vector_elt v4i32:castx, | |||
4375 | // (extract_vector_elt casty, 0), 2 * z), | |||
4376 | // (extract_vector_elt casty, 1), (2 * z + 1)) | |||
4377 | ||||
4378 | assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits () && "Invalid promote type for insert_vector_elt") ? static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4379, __PRETTY_FUNCTION__)) | |||
4379 | "Invalid promote type for insert_vector_elt")((NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits () && "Invalid promote type for insert_vector_elt") ? static_cast<void> (0) : __assert_fail ("NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() && \"Invalid promote type for insert_vector_elt\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4379, __PRETTY_FUNCTION__)); | |||
4380 | assert(NewEltVT.bitsLT(EltVT) && "not handled")((NewEltVT.bitsLT(EltVT) && "not handled") ? static_cast <void> (0) : __assert_fail ("NewEltVT.bitsLT(EltVT) && \"not handled\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4380, __PRETTY_FUNCTION__)); | |||
4381 | ||||
4382 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); | |||
4383 | unsigned NewEltsPerOldElt = MidVT.getVectorNumElements(); | |||
4384 | ||||
4385 | SDValue Val = Node->getOperand(1); | |||
4386 | SDValue Idx = Node->getOperand(2); | |||
4387 | EVT IdxVT = Idx.getValueType(); | |||
4388 | SDLoc SL(Node); | |||
4389 | ||||
4390 | SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT); | |||
4391 | SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor); | |||
4392 | ||||
4393 | SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0)); | |||
4394 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); | |||
4395 | ||||
4396 | SDValue NewVec = CastVec; | |||
4397 | for (unsigned I = 0; I < NewEltsPerOldElt; ++I) { | |||
4398 | SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT); | |||
4399 | SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset); | |||
4400 | ||||
4401 | SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT, | |||
4402 | CastVal, IdxOffset); | |||
4403 | ||||
4404 | NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT, | |||
4405 | NewVec, Elt, InEltIdx); | |||
4406 | } | |||
4407 | ||||
4408 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec)); | |||
4409 | break; | |||
4410 | } | |||
4411 | case ISD::SCALAR_TO_VECTOR: { | |||
4412 | MVT EltVT = OVT.getVectorElementType(); | |||
4413 | MVT NewEltVT = NVT.getVectorElementType(); | |||
4414 | ||||
4415 | // Handle bitcasts to different vector type with the same total bit size. | |||
4416 | // | |||
4417 | // e.g. v2i64 = scalar_to_vector x:i64 | |||
4418 | // => | |||
4419 | // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef) | |||
4420 | // | |||
4421 | ||||
4422 | MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT); | |||
4423 | SDValue Val = Node->getOperand(0); | |||
4424 | SDLoc SL(Node); | |||
4425 | ||||
4426 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val); | |||
4427 | SDValue Undef = DAG.getUNDEF(MidVT); | |||
4428 | ||||
4429 | SmallVector<SDValue, 8> NewElts; | |||
4430 | NewElts.push_back(CastVal); | |||
4431 | for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I) | |||
4432 | NewElts.push_back(Undef); | |||
4433 | ||||
4434 | SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts); | |||
4435 | SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat); | |||
4436 | Results.push_back(CvtVec); | |||
4437 | break; | |||
4438 | } | |||
4439 | case ISD::ATOMIC_SWAP: { | |||
4440 | AtomicSDNode *AM = cast<AtomicSDNode>(Node); | |||
4441 | SDLoc SL(Node); | |||
4442 | SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); | |||
4443 | assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&((NVT.getSizeInBits() == OVT.getSizeInBits() && "unexpected promotion type" ) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == OVT.getSizeInBits() && \"unexpected promotion type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4444, __PRETTY_FUNCTION__)) | |||
4444 | "unexpected promotion type")((NVT.getSizeInBits() == OVT.getSizeInBits() && "unexpected promotion type" ) ? static_cast<void> (0) : __assert_fail ("NVT.getSizeInBits() == OVT.getSizeInBits() && \"unexpected promotion type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4444, __PRETTY_FUNCTION__)); | |||
4445 | assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&((AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && "unexpected atomic_swap with illegal type") ? static_cast <void> (0) : __assert_fail ("AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && \"unexpected atomic_swap with illegal type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4446, __PRETTY_FUNCTION__)) | |||
4446 | "unexpected atomic_swap with illegal type")((AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && "unexpected atomic_swap with illegal type") ? static_cast <void> (0) : __assert_fail ("AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && \"unexpected atomic_swap with illegal type\"" , "/build/llvm-toolchain-snapshot-9~svn359999/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp" , 4446, __PRETTY_FUNCTION__)); | |||
4447 | ||||
4448 | SDValue NewAtomic | |||
4449 | = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, | |||
4450 | DAG.getVTList(NVT, MVT::Other), | |||
4451 | { AM->getChain(), AM->getBasePtr(), CastVal }, | |||
4452 | AM->getMemOperand()); | |||
4453 | Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); | |||
4454 | Results.push_back(NewAtomic.getValue(1)); | |||
4455 | break; | |||
4456 | } | |||
4457 | } | |||
4458 | ||||
4459 | // Replace the original node with the legalized result. | |||
4460 | if (!Results.empty()) { | |||
4461 | LLVM_DEBUG(dbgs() << "Successfully promoted node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Successfully promoted node\n" ; } } while (false); | |||
4462 | ReplaceNode(Node, Results.data()); | |||
4463 | } else | |||
4464 | LLVM_DEBUG(dbgs() << "Could not promote node\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType ("legalizedag")) { dbgs() << "Could not promote node\n" ; } } while (false); | |||
4465 | } | |||
4466 | ||||
4467 | /// This is the entry point for the file. | |||
4468 | void SelectionDAG::Legalize() { | |||
4469 | AssignTopologicalOrder(); | |||
4470 | ||||
4471 | SmallPtrSet<SDNode *, 16> LegalizedNodes; | |||
4472 | // Use a delete listener to remove nodes which were deleted during | |||
4473 | // legalization from LegalizeNodes. This is needed to handle the situation | |||
4474 | // where a new node is allocated by the object pool to the same address of a | |||
4475 | // previously deleted node. | |||
4476 | DAGNodeDeletedListener DeleteListener( | |||
4477 | *this, | |||
4478 | [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); }); | |||
4479 | ||||
4480 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes); | |||
4481 | ||||
4482 | // Visit all the nodes. We start in topological order, so that we see | |||
4483 | // nodes with their original operands intact. Legalization can produce | |||
4484 | // new nodes which may themselves need to be legalized. Iterate until all | |||
4485 | // nodes have been legalized. | |||
4486 | while (true) { | |||
4487 | bool AnyLegalized = false; | |||
4488 | for (auto NI = allnodes_end(); NI != allnodes_begin();) { | |||
4489 | --NI; | |||
4490 | ||||
4491 | SDNode *N = &*NI; | |||
4492 | if (N->use_empty() && N != getRoot().getNode()) { | |||
4493 | ++NI; | |||
4494 | DeleteNode(N); | |||
4495 | continue; | |||
4496 | } | |||
4497 | ||||
4498 | if (LegalizedNodes.insert(N).second) { | |||
4499 | AnyLegalized = true; | |||
4500 | Legalizer.LegalizeOp(N); | |||
4501 | ||||
4502 | if (N->use_empty() && N != getRoot().getNode()) { | |||
4503 | ++NI; | |||
4504 | DeleteNode(N); | |||
4505 | } | |||
4506 | } | |||
4507 | } | |||
4508 | if (!AnyLegalized) | |||
4509 | break; | |||
4510 | ||||
4511 | } | |||
4512 | ||||
4513 | // Remove dead nodes now. | |||
4514 | RemoveDeadNodes(); | |||
4515 | } | |||
4516 | ||||
4517 | bool SelectionDAG::LegalizeOp(SDNode *N, | |||
4518 | SmallSetVector<SDNode *, 16> &UpdatedNodes) { | |||
4519 | SmallPtrSet<SDNode *, 16> LegalizedNodes; | |||
4520 | SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes); | |||
4521 | ||||
4522 | // Directly insert the node in question, and legalize it. This will recurse | |||
4523 | // as needed through operands. | |||
4524 | LegalizedNodes.insert(N); | |||
4525 | Legalizer.LegalizeOp(N); | |||
4526 | ||||
4527 | return LegalizedNodes.count(N); | |||
4528 | } |
1 | //===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===// |
2 | // |
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
4 | // See https://llvm.org/LICENSE.txt for license information. |
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
6 | // |
7 | //===----------------------------------------------------------------------===// |
8 | // |
9 | // This file contains some functions that are useful for math stuff. |
10 | // |
11 | //===----------------------------------------------------------------------===// |
12 | |
13 | #ifndef LLVM_SUPPORT_MATHEXTRAS_H |
14 | #define LLVM_SUPPORT_MATHEXTRAS_H |
15 | |
16 | #include "llvm/Support/Compiler.h" |
17 | #include "llvm/Support/SwapByteOrder.h" |
18 | #include <algorithm> |
19 | #include <cassert> |
20 | #include <climits> |
21 | #include <cstring> |
22 | #include <limits> |
23 | #include <type_traits> |
24 | |
25 | #ifdef __ANDROID_NDK__ |
26 | #include <android/api-level.h> |
27 | #endif |
28 | |
29 | #ifdef _MSC_VER |
30 | // Declare these intrinsics manually rather including intrin.h. It's very |
31 | // expensive, and MathExtras.h is popular. |
32 | // #include <intrin.h> |
33 | extern "C" { |
34 | unsigned char _BitScanForward(unsigned long *_Index, unsigned long _Mask); |
35 | unsigned char _BitScanForward64(unsigned long *_Index, unsigned __int64 _Mask); |
36 | unsigned char _BitScanReverse(unsigned long *_Index, unsigned long _Mask); |
37 | unsigned char _BitScanReverse64(unsigned long *_Index, unsigned __int64 _Mask); |
38 | } |
39 | #endif |
40 | |
41 | namespace llvm { |
42 | /// The behavior an operation has on an input of 0. |
43 | enum ZeroBehavior { |
44 | /// The returned value is undefined. |
45 | ZB_Undefined, |
46 | /// The returned value is numeric_limits<T>::max() |
47 | ZB_Max, |
48 | /// The returned value is numeric_limits<T>::digits |
49 | ZB_Width |
50 | }; |
51 | |
52 | namespace detail { |
53 | template <typename T, std::size_t SizeOfT> struct TrailingZerosCounter { |
54 | static std::size_t count(T Val, ZeroBehavior) { |
55 | if (!Val) |
56 | return std::numeric_limits<T>::digits; |
57 | if (Val & 0x1) |
58 | return 0; |
59 | |
60 | // Bisection method. |
61 | std::size_t ZeroBits = 0; |
62 | T Shift = std::numeric_limits<T>::digits >> 1; |
63 | T Mask = std::numeric_limits<T>::max() >> Shift; |
64 | while (Shift) { |
65 | if ((Val & Mask) == 0) { |
66 | Val >>= Shift; |
67 | ZeroBits |= Shift; |
68 | } |
69 | Shift >>= 1; |
70 | Mask >>= Shift; |
71 | } |
72 | return ZeroBits; |
73 | } |
74 | }; |
75 | |
76 | #if __GNUC__4 >= 4 || defined(_MSC_VER) |
77 | template <typename T> struct TrailingZerosCounter<T, 4> { |
78 | static std::size_t count(T Val, ZeroBehavior ZB) { |
79 | if (ZB != ZB_Undefined && Val == 0) |
80 | return 32; |
81 | |
82 | #if __has_builtin(__builtin_ctz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20 ) + ((0) << 10) + (0)) |
83 | return __builtin_ctz(Val); |
84 | #elif defined(_MSC_VER) |
85 | unsigned long Index; |
86 | _BitScanForward(&Index, Val); |
87 | return Index; |
88 | #endif |
89 | } |
90 | }; |
91 | |
92 | #if !defined(_MSC_VER) || defined(_M_X64) |
93 | template <typename T> struct TrailingZerosCounter<T, 8> { |
94 | static std::size_t count(T Val, ZeroBehavior ZB) { |
95 | if (ZB != ZB_Undefined && Val == 0) |
96 | return 64; |
97 | |
98 | #if __has_builtin(__builtin_ctzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20 ) + ((0) << 10) + (0)) |
99 | return __builtin_ctzll(Val); |
100 | #elif defined(_MSC_VER) |
101 | unsigned long Index; |
102 | _BitScanForward64(&Index, Val); |
103 | return Index; |
104 | #endif |
105 | } |
106 | }; |
107 | #endif |
108 | #endif |
109 | } // namespace detail |
110 | |
111 | /// Count number of 0's from the least significant bit to the most |
112 | /// stopping at the first 1. |
113 | /// |
114 | /// Only unsigned integral types are allowed. |
115 | /// |
116 | /// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are |
117 | /// valid arguments. |
118 | template <typename T> |
119 | std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { |
120 | static_assert(std::numeric_limits<T>::is_integer && |
121 | !std::numeric_limits<T>::is_signed, |
122 | "Only unsigned integral types are allowed."); |
123 | return llvm::detail::TrailingZerosCounter<T, sizeof(T)>::count(Val, ZB); |
124 | } |
125 | |
126 | namespace detail { |
127 | template <typename T, std::size_t SizeOfT> struct LeadingZerosCounter { |
128 | static std::size_t count(T Val, ZeroBehavior) { |
129 | if (!Val) |
130 | return std::numeric_limits<T>::digits; |
131 | |
132 | // Bisection method. |
133 | std::size_t ZeroBits = 0; |
134 | for (T Shift = std::numeric_limits<T>::digits >> 1; Shift; Shift >>= 1) { |
135 | T Tmp = Val >> Shift; |
136 | if (Tmp) |
137 | Val = Tmp; |
138 | else |
139 | ZeroBits |= Shift; |
140 | } |
141 | return ZeroBits; |
142 | } |
143 | }; |
144 | |
145 | #if __GNUC__4 >= 4 || defined(_MSC_VER) |
146 | template <typename T> struct LeadingZerosCounter<T, 4> { |
147 | static std::size_t count(T Val, ZeroBehavior ZB) { |
148 | if (ZB != ZB_Undefined && Val == 0) |
149 | return 32; |
150 | |
151 | #if __has_builtin(__builtin_clz)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20 ) + ((0) << 10) + (0)) |
152 | return __builtin_clz(Val); |
153 | #elif defined(_MSC_VER) |
154 | unsigned long Index; |
155 | _BitScanReverse(&Index, Val); |
156 | return Index ^ 31; |
157 | #endif |
158 | } |
159 | }; |
160 | |
161 | #if !defined(_MSC_VER) || defined(_M_X64) |
162 | template <typename T> struct LeadingZerosCounter<T, 8> { |
163 | static std::size_t count(T Val, ZeroBehavior ZB) { |
164 | if (ZB != ZB_Undefined && Val == 0) |
165 | return 64; |
166 | |
167 | #if __has_builtin(__builtin_clzll)1 || LLVM_GNUC_PREREQ(4, 0, 0)((4 << 20) + (2 << 10) + 1 >= ((4) << 20 ) + ((0) << 10) + (0)) |
168 | return __builtin_clzll(Val); |
169 | #elif defined(_MSC_VER) |
170 | unsigned long Index; |
171 | _BitScanReverse64(&Index, Val); |
172 | return Index ^ 63; |
173 | #endif |
174 | } |
175 | }; |
176 | #endif |
177 | #endif |
178 | } // namespace detail |
179 | |
180 | /// Count number of 0's from the most significant bit to the least |
181 | /// stopping at the first 1. |
182 | /// |
183 | /// Only unsigned integral types are allowed. |
184 | /// |
185 | /// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are |
186 | /// valid arguments. |
187 | template <typename T> |
188 | std::size_t countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) { |
189 | static_assert(std::numeric_limits<T>::is_integer && |
190 | !std::numeric_limits<T>::is_signed, |
191 | "Only unsigned integral types are allowed."); |
192 | return llvm::detail::LeadingZerosCounter<T, sizeof(T)>::count(Val, ZB); |
193 | } |
194 | |
195 | /// Get the index of the first set bit starting from the least |
196 | /// significant bit. |
197 | /// |
198 | /// Only unsigned integral types are allowed. |
199 | /// |
200 | /// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are |
201 | /// valid arguments. |
202 | template <typename T> T findFirstSet(T Val, ZeroBehavior ZB = ZB_Max) { |
203 | if (ZB == ZB_Max && Val == 0) |
204 | return std::numeric_limits<T>::max(); |
205 | |
206 | return countTrailingZeros(Val, ZB_Undefined); |
207 | } |
208 | |
209 | /// Create a bitmask with the N right-most bits set to 1, and all other |
210 | /// bits set to 0. Only unsigned types are allowed. |
211 | template <typename T> T maskTrailingOnes(unsigned N) { |
212 | static_assert(std::is_unsigned<T>::value, "Invalid type!"); |
213 | const unsigned Bits = CHAR_BIT8 * sizeof(T); |
214 | assert(N <= Bits && "Invalid bit index")((N <= Bits && "Invalid bit index") ? static_cast< void> (0) : __assert_fail ("N <= Bits && \"Invalid bit index\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 214, __PRETTY_FUNCTION__)); |
215 | return N == 0 ? 0 : (T(-1) >> (Bits - N)); |
216 | } |
217 | |
218 | /// Create a bitmask with the N left-most bits set to 1, and all other |
219 | /// bits set to 0. Only unsigned types are allowed. |
220 | template <typename T> T maskLeadingOnes(unsigned N) { |
221 | return ~maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N); |
222 | } |
223 | |
224 | /// Create a bitmask with the N right-most bits set to 0, and all other |
225 | /// bits set to 1. Only unsigned types are allowed. |
226 | template <typename T> T maskTrailingZeros(unsigned N) { |
227 | return maskLeadingOnes<T>(CHAR_BIT8 * sizeof(T) - N); |
228 | } |
229 | |
230 | /// Create a bitmask with the N left-most bits set to 0, and all other |
231 | /// bits set to 1. Only unsigned types are allowed. |
232 | template <typename T> T maskLeadingZeros(unsigned N) { |
233 | return maskTrailingOnes<T>(CHAR_BIT8 * sizeof(T) - N); |
234 | } |
235 | |
236 | /// Get the index of the last set bit starting from the least |
237 | /// significant bit. |
238 | /// |
239 | /// Only unsigned integral types are allowed. |
240 | /// |
241 | /// \param ZB the behavior on an input of 0. Only ZB_Max and ZB_Undefined are |
242 | /// valid arguments. |
243 | template <typename T> T findLastSet(T Val, ZeroBehavior ZB = ZB_Max) { |
244 | if (ZB == ZB_Max && Val == 0) |
245 | return std::numeric_limits<T>::max(); |
246 | |
247 | // Use ^ instead of - because both gcc and llvm can remove the associated ^ |
248 | // in the __builtin_clz intrinsic on x86. |
249 | return countLeadingZeros(Val, ZB_Undefined) ^ |
250 | (std::numeric_limits<T>::digits - 1); |
251 | } |
252 | |
253 | /// Macro compressed bit reversal table for 256 bits. |
254 | /// |
255 | /// http://graphics.stanford.edu/~seander/bithacks.html#BitReverseTable |
256 | static const unsigned char BitReverseTable256[256] = { |
257 | #define R2(n) n, n + 2 * 64, n + 1 * 64, n + 3 * 64 |
258 | #define R4(n) R2(n), R2(n + 2 * 16), R2(n + 1 * 16), R2(n + 3 * 16) |
259 | #define R6(n) R4(n), R4(n + 2 * 4), R4(n + 1 * 4), R4(n + 3 * 4) |
260 | R6(0), R6(2), R6(1), R6(3) |
261 | #undef R2 |
262 | #undef R4 |
263 | #undef R6 |
264 | }; |
265 | |
266 | /// Reverse the bits in \p Val. |
267 | template <typename T> |
268 | T reverseBits(T Val) { |
269 | unsigned char in[sizeof(Val)]; |
270 | unsigned char out[sizeof(Val)]; |
271 | std::memcpy(in, &Val, sizeof(Val)); |
272 | for (unsigned i = 0; i < sizeof(Val); ++i) |
273 | out[(sizeof(Val) - i) - 1] = BitReverseTable256[in[i]]; |
274 | std::memcpy(&Val, out, sizeof(Val)); |
275 | return Val; |
276 | } |
277 | |
278 | // NOTE: The following support functions use the _32/_64 extensions instead of |
279 | // type overloading so that signed and unsigned integers can be used without |
280 | // ambiguity. |
281 | |
282 | /// Return the high 32 bits of a 64 bit value. |
283 | constexpr inline uint32_t Hi_32(uint64_t Value) { |
284 | return static_cast<uint32_t>(Value >> 32); |
285 | } |
286 | |
287 | /// Return the low 32 bits of a 64 bit value. |
288 | constexpr inline uint32_t Lo_32(uint64_t Value) { |
289 | return static_cast<uint32_t>(Value); |
290 | } |
291 | |
292 | /// Make a 64-bit integer from a high / low pair of 32-bit integers. |
293 | constexpr inline uint64_t Make_64(uint32_t High, uint32_t Low) { |
294 | return ((uint64_t)High << 32) | (uint64_t)Low; |
295 | } |
296 | |
297 | /// Checks if an integer fits into the given bit width. |
298 | template <unsigned N> constexpr inline bool isInt(int64_t x) { |
299 | return N >= 64 || (-(INT64_C(1)1L<<(N-1)) <= x && x < (INT64_C(1)1L<<(N-1))); |
300 | } |
301 | // Template specializations to get better code for common cases. |
302 | template <> constexpr inline bool isInt<8>(int64_t x) { |
303 | return static_cast<int8_t>(x) == x; |
304 | } |
305 | template <> constexpr inline bool isInt<16>(int64_t x) { |
306 | return static_cast<int16_t>(x) == x; |
307 | } |
308 | template <> constexpr inline bool isInt<32>(int64_t x) { |
309 | return static_cast<int32_t>(x) == x; |
310 | } |
311 | |
312 | /// Checks if a signed integer is an N bit number shifted left by S. |
313 | template <unsigned N, unsigned S> |
314 | constexpr inline bool isShiftedInt(int64_t x) { |
315 | static_assert( |
316 | N > 0, "isShiftedInt<0> doesn't make sense (refers to a 0-bit number."); |
317 | static_assert(N + S <= 64, "isShiftedInt<N, S> with N + S > 64 is too wide."); |
318 | return isInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0); |
319 | } |
320 | |
321 | /// Checks if an unsigned integer fits into the given bit width. |
322 | /// |
323 | /// This is written as two functions rather than as simply |
324 | /// |
325 | /// return N >= 64 || X < (UINT64_C(1) << N); |
326 | /// |
327 | /// to keep MSVC from (incorrectly) warning on isUInt<64> that we're shifting |
328 | /// left too many places. |
329 | template <unsigned N> |
330 | constexpr inline typename std::enable_if<(N < 64), bool>::type |
331 | isUInt(uint64_t X) { |
332 | static_assert(N > 0, "isUInt<0> doesn't make sense"); |
333 | return X < (UINT64_C(1)1UL << (N)); |
334 | } |
335 | template <unsigned N> |
336 | constexpr inline typename std::enable_if<N >= 64, bool>::type |
337 | isUInt(uint64_t X) { |
338 | return true; |
339 | } |
340 | |
341 | // Template specializations to get better code for common cases. |
342 | template <> constexpr inline bool isUInt<8>(uint64_t x) { |
343 | return static_cast<uint8_t>(x) == x; |
344 | } |
345 | template <> constexpr inline bool isUInt<16>(uint64_t x) { |
346 | return static_cast<uint16_t>(x) == x; |
347 | } |
348 | template <> constexpr inline bool isUInt<32>(uint64_t x) { |
349 | return static_cast<uint32_t>(x) == x; |
350 | } |
351 | |
352 | /// Checks if a unsigned integer is an N bit number shifted left by S. |
353 | template <unsigned N, unsigned S> |
354 | constexpr inline bool isShiftedUInt(uint64_t x) { |
355 | static_assert( |
356 | N > 0, "isShiftedUInt<0> doesn't make sense (refers to a 0-bit number)"); |
357 | static_assert(N + S <= 64, |
358 | "isShiftedUInt<N, S> with N + S > 64 is too wide."); |
359 | // Per the two static_asserts above, S must be strictly less than 64. So |
360 | // 1 << S is not undefined behavior. |
361 | return isUInt<N + S>(x) && (x % (UINT64_C(1)1UL << S) == 0); |
362 | } |
363 | |
364 | /// Gets the maximum value for a N-bit unsigned integer. |
365 | inline uint64_t maxUIntN(uint64_t N) { |
366 | assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range" ) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 366, __PRETTY_FUNCTION__)); |
367 | |
368 | // uint64_t(1) << 64 is undefined behavior, so we can't do |
369 | // (uint64_t(1) << N) - 1 |
370 | // without checking first that N != 64. But this works and doesn't have a |
371 | // branch. |
372 | return UINT64_MAX(18446744073709551615UL) >> (64 - N); |
373 | } |
374 | |
375 | /// Gets the minimum value for a N-bit signed integer. |
376 | inline int64_t minIntN(int64_t N) { |
377 | assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range" ) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 377, __PRETTY_FUNCTION__)); |
378 | |
379 | return -(UINT64_C(1)1UL<<(N-1)); |
380 | } |
381 | |
382 | /// Gets the maximum value for a N-bit signed integer. |
383 | inline int64_t maxIntN(int64_t N) { |
384 | assert(N > 0 && N <= 64 && "integer width out of range")((N > 0 && N <= 64 && "integer width out of range" ) ? static_cast<void> (0) : __assert_fail ("N > 0 && N <= 64 && \"integer width out of range\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 384, __PRETTY_FUNCTION__)); |
385 | |
386 | // This relies on two's complement wraparound when N == 64, so we convert to |
387 | // int64_t only at the very end to avoid UB. |
388 | return (UINT64_C(1)1UL << (N - 1)) - 1; |
389 | } |
390 | |
391 | /// Checks if an unsigned integer fits into the given (dynamic) bit width. |
392 | inline bool isUIntN(unsigned N, uint64_t x) { |
393 | return N >= 64 || x <= maxUIntN(N); |
394 | } |
395 | |
396 | /// Checks if an signed integer fits into the given (dynamic) bit width. |
397 | inline bool isIntN(unsigned N, int64_t x) { |
398 | return N >= 64 || (minIntN(N) <= x && x <= maxIntN(N)); |
399 | } |
400 | |
401 | /// Return true if the argument is a non-empty sequence of ones starting at the |
402 | /// least significant bit with the remainder zero (32 bit version). |
403 | /// Ex. isMask_32(0x0000FFFFU) == true. |
404 | constexpr inline bool isMask_32(uint32_t Value) { |
405 | return Value && ((Value + 1) & Value) == 0; |
406 | } |
407 | |
408 | /// Return true if the argument is a non-empty sequence of ones starting at the |
409 | /// least significant bit with the remainder zero (64 bit version). |
410 | constexpr inline bool isMask_64(uint64_t Value) { |
411 | return Value && ((Value + 1) & Value) == 0; |
412 | } |
413 | |
414 | /// Return true if the argument contains a non-empty sequence of ones with the |
415 | /// remainder zero (32 bit version.) Ex. isShiftedMask_32(0x0000FF00U) == true. |
416 | constexpr inline bool isShiftedMask_32(uint32_t Value) { |
417 | return Value && isMask_32((Value - 1) | Value); |
418 | } |
419 | |
420 | /// Return true if the argument contains a non-empty sequence of ones with the |
421 | /// remainder zero (64 bit version.) |
422 | constexpr inline bool isShiftedMask_64(uint64_t Value) { |
423 | return Value && isMask_64((Value - 1) | Value); |
424 | } |
425 | |
426 | /// Return true if the argument is a power of two > 0. |
427 | /// Ex. isPowerOf2_32(0x00100000U) == true (32 bit edition.) |
428 | constexpr inline bool isPowerOf2_32(uint32_t Value) { |
429 | return Value && !(Value & (Value - 1)); |
430 | } |
431 | |
432 | /// Return true if the argument is a power of two > 0 (64 bit edition.) |
433 | constexpr inline bool isPowerOf2_64(uint64_t Value) { |
434 | return Value && !(Value & (Value - 1)); |
435 | } |
436 | |
437 | /// Return a byte-swapped representation of the 16-bit argument. |
438 | inline uint16_t ByteSwap_16(uint16_t Value) { |
439 | return sys::SwapByteOrder_16(Value); |
440 | } |
441 | |
442 | /// Return a byte-swapped representation of the 32-bit argument. |
443 | inline uint32_t ByteSwap_32(uint32_t Value) { |
444 | return sys::SwapByteOrder_32(Value); |
445 | } |
446 | |
447 | /// Return a byte-swapped representation of the 64-bit argument. |
448 | inline uint64_t ByteSwap_64(uint64_t Value) { |
449 | return sys::SwapByteOrder_64(Value); |
450 | } |
451 | |
452 | /// Count the number of ones from the most significant bit to the first |
453 | /// zero bit. |
454 | /// |
455 | /// Ex. countLeadingOnes(0xFF0FFF00) == 8. |
456 | /// Only unsigned integral types are allowed. |
457 | /// |
458 | /// \param ZB the behavior on an input of all ones. Only ZB_Width and |
459 | /// ZB_Undefined are valid arguments. |
460 | template <typename T> |
461 | std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { |
462 | static_assert(std::numeric_limits<T>::is_integer && |
463 | !std::numeric_limits<T>::is_signed, |
464 | "Only unsigned integral types are allowed."); |
465 | return countLeadingZeros<T>(~Value, ZB); |
466 | } |
467 | |
468 | /// Count the number of ones from the least significant bit to the first |
469 | /// zero bit. |
470 | /// |
471 | /// Ex. countTrailingOnes(0x00FF00FF) == 8. |
472 | /// Only unsigned integral types are allowed. |
473 | /// |
474 | /// \param ZB the behavior on an input of all ones. Only ZB_Width and |
475 | /// ZB_Undefined are valid arguments. |
476 | template <typename T> |
477 | std::size_t countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) { |
478 | static_assert(std::numeric_limits<T>::is_integer && |
479 | !std::numeric_limits<T>::is_signed, |
480 | "Only unsigned integral types are allowed."); |
481 | return countTrailingZeros<T>(~Value, ZB); |
482 | } |
483 | |
484 | namespace detail { |
485 | template <typename T, std::size_t SizeOfT> struct PopulationCounter { |
486 | static unsigned count(T Value) { |
487 | // Generic version, forward to 32 bits. |
488 | static_assert(SizeOfT <= 4, "Not implemented!"); |
489 | #if __GNUC__4 >= 4 |
490 | return __builtin_popcount(Value); |
491 | #else |
492 | uint32_t v = Value; |
493 | v = v - ((v >> 1) & 0x55555555); |
494 | v = (v & 0x33333333) + ((v >> 2) & 0x33333333); |
495 | return ((v + (v >> 4) & 0xF0F0F0F) * 0x1010101) >> 24; |
496 | #endif |
497 | } |
498 | }; |
499 | |
500 | template <typename T> struct PopulationCounter<T, 8> { |
501 | static unsigned count(T Value) { |
502 | #if __GNUC__4 >= 4 |
503 | return __builtin_popcountll(Value); |
504 | #else |
505 | uint64_t v = Value; |
506 | v = v - ((v >> 1) & 0x5555555555555555ULL); |
507 | v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); |
508 | v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; |
509 | return unsigned((uint64_t)(v * 0x0101010101010101ULL) >> 56); |
510 | #endif |
511 | } |
512 | }; |
513 | } // namespace detail |
514 | |
515 | /// Count the number of set bits in a value. |
516 | /// Ex. countPopulation(0xF000F000) = 8 |
517 | /// Returns 0 if the word is zero. |
518 | template <typename T> |
519 | inline unsigned countPopulation(T Value) { |
520 | static_assert(std::numeric_limits<T>::is_integer && |
521 | !std::numeric_limits<T>::is_signed, |
522 | "Only unsigned integral types are allowed."); |
523 | return detail::PopulationCounter<T, sizeof(T)>::count(Value); |
524 | } |
525 | |
526 | /// Return the log base 2 of the specified value. |
527 | inline double Log2(double Value) { |
528 | #if defined(__ANDROID_API__) && __ANDROID_API__ < 18 |
529 | return __builtin_log(Value) / __builtin_log(2.0); |
530 | #else |
531 | return log2(Value); |
532 | #endif |
533 | } |
534 | |
535 | /// Return the floor log base 2 of the specified value, -1 if the value is zero. |
536 | /// (32 bit edition.) |
537 | /// Ex. Log2_32(32) == 5, Log2_32(1) == 0, Log2_32(0) == -1, Log2_32(6) == 2 |
538 | inline unsigned Log2_32(uint32_t Value) { |
539 | return 31 - countLeadingZeros(Value); |
540 | } |
541 | |
542 | /// Return the floor log base 2 of the specified value, -1 if the value is zero. |
543 | /// (64 bit edition.) |
544 | inline unsigned Log2_64(uint64_t Value) { |
545 | return 63 - countLeadingZeros(Value); |
546 | } |
547 | |
548 | /// Return the ceil log base 2 of the specified value, 32 if the value is zero. |
549 | /// (32 bit edition). |
550 | /// Ex. Log2_32_Ceil(32) == 5, Log2_32_Ceil(1) == 0, Log2_32_Ceil(6) == 3 |
551 | inline unsigned Log2_32_Ceil(uint32_t Value) { |
552 | return 32 - countLeadingZeros(Value - 1); |
553 | } |
554 | |
555 | /// Return the ceil log base 2 of the specified value, 64 if the value is zero. |
556 | /// (64 bit edition.) |
557 | inline unsigned Log2_64_Ceil(uint64_t Value) { |
558 | return 64 - countLeadingZeros(Value - 1); |
559 | } |
560 | |
561 | /// Return the greatest common divisor of the values using Euclid's algorithm. |
562 | inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) { |
563 | while (B) { |
564 | uint64_t T = B; |
565 | B = A % B; |
566 | A = T; |
567 | } |
568 | return A; |
569 | } |
570 | |
571 | /// This function takes a 64-bit integer and returns the bit equivalent double. |
572 | inline double BitsToDouble(uint64_t Bits) { |
573 | double D; |
574 | static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes"); |
575 | memcpy(&D, &Bits, sizeof(Bits)); |
576 | return D; |
577 | } |
578 | |
579 | /// This function takes a 32-bit integer and returns the bit equivalent float. |
580 | inline float BitsToFloat(uint32_t Bits) { |
581 | float F; |
582 | static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes"); |
583 | memcpy(&F, &Bits, sizeof(Bits)); |
584 | return F; |
585 | } |
586 | |
587 | /// This function takes a double and returns the bit equivalent 64-bit integer. |
588 | /// Note that copying doubles around changes the bits of NaNs on some hosts, |
589 | /// notably x86, so this routine cannot be used if these bits are needed. |
590 | inline uint64_t DoubleToBits(double Double) { |
591 | uint64_t Bits; |
592 | static_assert(sizeof(uint64_t) == sizeof(double), "Unexpected type sizes"); |
593 | memcpy(&Bits, &Double, sizeof(Double)); |
594 | return Bits; |
595 | } |
596 | |
597 | /// This function takes a float and returns the bit equivalent 32-bit integer. |
598 | /// Note that copying floats around changes the bits of NaNs on some hosts, |
599 | /// notably x86, so this routine cannot be used if these bits are needed. |
600 | inline uint32_t FloatToBits(float Float) { |
601 | uint32_t Bits; |
602 | static_assert(sizeof(uint32_t) == sizeof(float), "Unexpected type sizes"); |
603 | memcpy(&Bits, &Float, sizeof(Float)); |
604 | return Bits; |
605 | } |
606 | |
607 | /// A and B are either alignments or offsets. Return the minimum alignment that |
608 | /// may be assumed after adding the two together. |
609 | constexpr inline uint64_t MinAlign(uint64_t A, uint64_t B) { |
610 | // The largest power of 2 that divides both A and B. |
611 | // |
612 | // Replace "-Value" by "1+~Value" in the following commented code to avoid |
613 | // MSVC warning C4146 |
614 | // return (A | B) & -(A | B); |
615 | return (A | B) & (1 + ~(A | B)); |
616 | } |
617 | |
618 | /// Aligns \c Addr to \c Alignment bytes, rounding up. |
619 | /// |
620 | /// Alignment should be a power of two. This method rounds up, so |
621 | /// alignAddr(7, 4) == 8 and alignAddr(8, 4) == 8. |
622 | inline uintptr_t alignAddr(const void *Addr, size_t Alignment) { |
623 | assert(Alignment && isPowerOf2_64((uint64_t)Alignment) &&((Alignment && isPowerOf2_64((uint64_t)Alignment) && "Alignment is not a power of two!") ? static_cast<void> (0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 624, __PRETTY_FUNCTION__)) |
624 | "Alignment is not a power of two!")((Alignment && isPowerOf2_64((uint64_t)Alignment) && "Alignment is not a power of two!") ? static_cast<void> (0) : __assert_fail ("Alignment && isPowerOf2_64((uint64_t)Alignment) && \"Alignment is not a power of two!\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 624, __PRETTY_FUNCTION__)); |
625 | |
626 | assert((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr)(((uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr) ? static_cast <void> (0) : __assert_fail ("(uintptr_t)Addr + Alignment - 1 >= (uintptr_t)Addr" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 626, __PRETTY_FUNCTION__)); |
627 | |
628 | return (((uintptr_t)Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1)); |
629 | } |
630 | |
631 | /// Returns the necessary adjustment for aligning \c Ptr to \c Alignment |
632 | /// bytes, rounding up. |
633 | inline size_t alignmentAdjustment(const void *Ptr, size_t Alignment) { |
634 | return alignAddr(Ptr, Alignment) - (uintptr_t)Ptr; |
635 | } |
636 | |
637 | /// Returns the next power of two (in 64-bits) that is strictly greater than A. |
638 | /// Returns zero on overflow. |
639 | inline uint64_t NextPowerOf2(uint64_t A) { |
640 | A |= (A >> 1); |
641 | A |= (A >> 2); |
642 | A |= (A >> 4); |
643 | A |= (A >> 8); |
644 | A |= (A >> 16); |
645 | A |= (A >> 32); |
646 | return A + 1; |
647 | } |
648 | |
649 | /// Returns the power of two which is less than or equal to the given value. |
650 | /// Essentially, it is a floor operation across the domain of powers of two. |
651 | inline uint64_t PowerOf2Floor(uint64_t A) { |
652 | if (!A) return 0; |
653 | return 1ull << (63 - countLeadingZeros(A, ZB_Undefined)); |
654 | } |
655 | |
656 | /// Returns the power of two which is greater than or equal to the given value. |
657 | /// Essentially, it is a ceil operation across the domain of powers of two. |
658 | inline uint64_t PowerOf2Ceil(uint64_t A) { |
659 | if (!A) |
660 | return 0; |
661 | return NextPowerOf2(A - 1); |
662 | } |
663 | |
664 | /// Returns the next integer (mod 2**64) that is greater than or equal to |
665 | /// \p Value and is a multiple of \p Align. \p Align must be non-zero. |
666 | /// |
667 | /// If non-zero \p Skew is specified, the return value will be a minimal |
668 | /// integer that is greater than or equal to \p Value and equal to |
669 | /// \p Align * N + \p Skew for some integer N. If \p Skew is larger than |
670 | /// \p Align, its value is adjusted to '\p Skew mod \p Align'. |
671 | /// |
672 | /// Examples: |
673 | /// \code |
674 | /// alignTo(5, 8) = 8 |
675 | /// alignTo(17, 8) = 24 |
676 | /// alignTo(~0LL, 8) = 0 |
677 | /// alignTo(321, 255) = 510 |
678 | /// |
679 | /// alignTo(5, 8, 7) = 7 |
680 | /// alignTo(17, 8, 1) = 17 |
681 | /// alignTo(~0LL, 8, 3) = 3 |
682 | /// alignTo(321, 255, 42) = 552 |
683 | /// \endcode |
684 | inline uint64_t alignTo(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { |
685 | assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast< void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 685, __PRETTY_FUNCTION__)); |
686 | Skew %= Align; |
687 | return (Value + Align - 1 - Skew) / Align * Align + Skew; |
688 | } |
689 | |
690 | /// Returns the next integer (mod 2**64) that is greater than or equal to |
691 | /// \p Value and is a multiple of \c Align. \c Align must be non-zero. |
692 | template <uint64_t Align> constexpr inline uint64_t alignTo(uint64_t Value) { |
693 | static_assert(Align != 0u, "Align must be non-zero"); |
694 | return (Value + Align - 1) / Align * Align; |
695 | } |
696 | |
697 | /// Returns the integer ceil(Numerator / Denominator). |
698 | inline uint64_t divideCeil(uint64_t Numerator, uint64_t Denominator) { |
699 | return alignTo(Numerator, Denominator) / Denominator; |
700 | } |
701 | |
702 | /// \c alignTo for contexts where a constant expression is required. |
703 | /// \sa alignTo |
704 | /// |
705 | /// \todo FIXME: remove when \c constexpr becomes really \c constexpr |
706 | template <uint64_t Align> |
707 | struct AlignTo { |
708 | static_assert(Align != 0u, "Align must be non-zero"); |
709 | template <uint64_t Value> |
710 | struct from_value { |
711 | static const uint64_t value = (Value + Align - 1) / Align * Align; |
712 | }; |
713 | }; |
714 | |
715 | /// Returns the largest uint64_t less than or equal to \p Value and is |
716 | /// \p Skew mod \p Align. \p Align must be non-zero |
717 | inline uint64_t alignDown(uint64_t Value, uint64_t Align, uint64_t Skew = 0) { |
718 | assert(Align != 0u && "Align can't be 0.")((Align != 0u && "Align can't be 0.") ? static_cast< void> (0) : __assert_fail ("Align != 0u && \"Align can't be 0.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 718, __PRETTY_FUNCTION__)); |
719 | Skew %= Align; |
720 | return (Value - Skew) / Align * Align + Skew; |
721 | } |
722 | |
723 | /// Returns the offset to the next integer (mod 2**64) that is greater than |
724 | /// or equal to \p Value and is a multiple of \p Align. \p Align must be |
725 | /// non-zero. |
726 | inline uint64_t OffsetToAlignment(uint64_t Value, uint64_t Align) { |
727 | return alignTo(Value, Align) - Value; |
728 | } |
729 | |
730 | /// Sign-extend the number in the bottom B bits of X to a 32-bit integer. |
731 | /// Requires 0 < B <= 32. |
732 | template <unsigned B> constexpr inline int32_t SignExtend32(uint32_t X) { |
733 | static_assert(B > 0, "Bit width can't be 0."); |
734 | static_assert(B <= 32, "Bit width out of range."); |
735 | return int32_t(X << (32 - B)) >> (32 - B); |
736 | } |
737 | |
738 | /// Sign-extend the number in the bottom B bits of X to a 32-bit integer. |
739 | /// Requires 0 < B < 32. |
740 | inline int32_t SignExtend32(uint32_t X, unsigned B) { |
741 | assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast< void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 741, __PRETTY_FUNCTION__)); |
742 | assert(B <= 32 && "Bit width out of range.")((B <= 32 && "Bit width out of range.") ? static_cast <void> (0) : __assert_fail ("B <= 32 && \"Bit width out of range.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 742, __PRETTY_FUNCTION__)); |
743 | return int32_t(X << (32 - B)) >> (32 - B); |
744 | } |
745 | |
746 | /// Sign-extend the number in the bottom B bits of X to a 64-bit integer. |
747 | /// Requires 0 < B < 64. |
748 | template <unsigned B> constexpr inline int64_t SignExtend64(uint64_t x) { |
749 | static_assert(B > 0, "Bit width can't be 0."); |
750 | static_assert(B <= 64, "Bit width out of range."); |
751 | return int64_t(x << (64 - B)) >> (64 - B); |
752 | } |
753 | |
754 | /// Sign-extend the number in the bottom B bits of X to a 64-bit integer. |
755 | /// Requires 0 < B < 64. |
756 | inline int64_t SignExtend64(uint64_t X, unsigned B) { |
757 | assert(B > 0 && "Bit width can't be 0.")((B > 0 && "Bit width can't be 0.") ? static_cast< void> (0) : __assert_fail ("B > 0 && \"Bit width can't be 0.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 757, __PRETTY_FUNCTION__)); |
758 | assert(B <= 64 && "Bit width out of range.")((B <= 64 && "Bit width out of range.") ? static_cast <void> (0) : __assert_fail ("B <= 64 && \"Bit width out of range.\"" , "/build/llvm-toolchain-snapshot-9~svn359999/include/llvm/Support/MathExtras.h" , 758, __PRETTY_FUNCTION__)); |
759 | return int64_t(X << (64 - B)) >> (64 - B); |
760 | } |
761 | |
762 | /// Subtract two unsigned integers, X and Y, of type T and return the absolute |
763 | /// value of the result. |
764 | template <typename T> |
765 | typename std::enable_if<std::is_unsigned<T>::value, T>::type |
766 | AbsoluteDifference(T X, T Y) { |
767 | return std::max(X, Y) - std::min(X, Y); |
768 | } |
769 | |
770 | /// Add two unsigned integers, X and Y, of type T. Clamp the result to the |
771 | /// maximum representable value of T on overflow. ResultOverflowed indicates if |
772 | /// the result is larger than the maximum representable value of type T. |
773 | template <typename T> |
774 | typename std::enable_if<std::is_unsigned<T>::value, T>::type |
775 | SaturatingAdd(T X, T Y, bool *ResultOverflowed = nullptr) { |
776 | bool Dummy; |
777 | bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; |
778 | // Hacker's Delight, p. 29 |
779 | T Z = X + Y; |
780 | Overflowed = (Z < X || Z < Y); |
781 | if (Overflowed) |
782 | return std::numeric_limits<T>::max(); |
783 | else |
784 | return Z; |
785 | } |
786 | |
787 | /// Multiply two unsigned integers, X and Y, of type T. Clamp the result to the |
788 | /// maximum representable value of T on overflow. ResultOverflowed indicates if |
789 | /// the result is larger than the maximum representable value of type T. |
790 | template <typename T> |
791 | typename std::enable_if<std::is_unsigned<T>::value, T>::type |
792 | SaturatingMultiply(T X, T Y, bool *ResultOverflowed = nullptr) { |
793 | bool Dummy; |
794 | bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; |
795 | |
796 | // Hacker's Delight, p. 30 has a different algorithm, but we don't use that |
797 | // because it fails for uint16_t (where multiplication can have undefined |
798 | // behavior due to promotion to int), and requires a division in addition |
799 | // to the multiplication. |
800 | |
801 | Overflowed = false; |
802 | |
803 | // Log2(Z) would be either Log2Z or Log2Z + 1. |
804 | // Special case: if X or Y is 0, Log2_64 gives -1, and Log2Z |
805 | // will necessarily be less than Log2Max as desired. |
806 | int Log2Z = Log2_64(X) + Log2_64(Y); |
807 | const T Max = std::numeric_limits<T>::max(); |
808 | int Log2Max = Log2_64(Max); |
809 | if (Log2Z < Log2Max) { |
810 | return X * Y; |
811 | } |
812 | if (Log2Z > Log2Max) { |
813 | Overflowed = true; |
814 | return Max; |
815 | } |
816 | |
817 | // We're going to use the top bit, and maybe overflow one |
818 | // bit past it. Multiply all but the bottom bit then add |
819 | // that on at the end. |
820 | T Z = (X >> 1) * Y; |
821 | if (Z & ~(Max >> 1)) { |
822 | Overflowed = true; |
823 | return Max; |
824 | } |
825 | Z <<= 1; |
826 | if (X & 1) |
827 | return SaturatingAdd(Z, Y, ResultOverflowed); |
828 | |
829 | return Z; |
830 | } |
831 | |
832 | /// Multiply two unsigned integers, X and Y, and add the unsigned integer, A to |
833 | /// the product. Clamp the result to the maximum representable value of T on |
834 | /// overflow. ResultOverflowed indicates if the result is larger than the |
835 | /// maximum representable value of type T. |
836 | template <typename T> |
837 | typename std::enable_if<std::is_unsigned<T>::value, T>::type |
838 | SaturatingMultiplyAdd(T X, T Y, T A, bool *ResultOverflowed = nullptr) { |
839 | bool Dummy; |
840 | bool &Overflowed = ResultOverflowed ? *ResultOverflowed : Dummy; |
841 | |
842 | T Product = SaturatingMultiply(X, Y, &Overflowed); |
843 | if (Overflowed) |
844 | return Product; |
845 | |
846 | return SaturatingAdd(A, Product, &Overflowed); |
847 | } |
848 | |
849 | /// Use this rather than HUGE_VALF; the latter causes warnings on MSVC. |
850 | extern const float huge_valf; |
851 | } // End llvm namespace |
852 | |
853 | #endif |