Line data Source code
1 : //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===//
2 : //
3 : // The LLVM Compiler Infrastructure
4 : //
5 : // This file is distributed under the University of Illinois Open Source
6 : // License. See LICENSE.TXT for details.
7 : //
8 : //===----------------------------------------------------------------------===//
9 : //
10 : // This file implements generic type expansion and splitting for LegalizeTypes.
11 : // The routines here perform legalization when the details of the type (such as
12 : // whether it is an integer or a float) do not matter.
13 : // Expansion is the act of changing a computation in an illegal type to be a
14 : // computation in two identical registers of a smaller type. The Lo/Hi part
15 : // is required to be stored first in memory on little/big-endian machines.
16 : // Splitting is the act of changing a computation in an illegal type to be a
17 : // computation in two not necessarily identical registers of a smaller type.
18 : // There are no requirements on how the type is represented in memory.
19 : //
20 : //===----------------------------------------------------------------------===//
21 :
22 : #include "LegalizeTypes.h"
23 : #include "llvm/IR/DataLayout.h"
24 : using namespace llvm;
25 :
26 : #define DEBUG_TYPE "legalize-types"
27 :
28 : //===----------------------------------------------------------------------===//
29 : // Generic Result Expansion.
30 : //===----------------------------------------------------------------------===//
31 :
32 : // These routines assume that the Lo/Hi part is stored first in memory on
33 : // little/big-endian machines, followed by the Hi/Lo part. This means that
34 : // they cannot be used as is on vectors, for which Lo is always stored first.
35 0 : void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
36 : SDValue &Lo, SDValue &Hi) {
37 0 : SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
38 0 : GetExpandedOp(Op, Lo, Hi);
39 0 : }
40 :
41 12777 : void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
42 12777 : EVT OutVT = N->getValueType(0);
43 12777 : EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
44 12777 : SDValue InOp = N->getOperand(0);
45 25554 : EVT InVT = InOp.getValueType();
46 : SDLoc dl(N);
47 :
48 : // Handle some special cases efficiently.
49 12777 : switch (getTypeAction(InVT)) {
50 : case TargetLowering::TypeLegal:
51 : case TargetLowering::TypePromoteInteger:
52 : break;
53 : case TargetLowering::TypePromoteFloat:
54 : llvm_unreachable("Bitcast of a promotion-needing float should never need"
55 : "expansion");
56 528 : case TargetLowering::TypeSoftenFloat: {
57 : // Expand the floating point operand only if it was converted to integers.
58 : // Otherwise, it is a legal type like f128 that can be saved in a register.
59 528 : auto SoftenedOp = GetSoftenedFloat(InOp);
60 1056 : if (isLegalInHWReg(SoftenedOp.getValueType()))
61 : break;
62 468 : SplitInteger(SoftenedOp, Lo, Hi);
63 936 : Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
64 936 : Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
65 468 : return;
66 : }
67 174 : case TargetLowering::TypeExpandInteger:
68 : case TargetLowering::TypeExpandFloat: {
69 174 : auto &DL = DAG.getDataLayout();
70 : // Convert the expanded pieces of the input.
71 174 : GetExpandedOp(InOp, Lo, Hi);
72 348 : if (TLI.hasBigEndianPartOrdering(InVT, DL) !=
73 : TLI.hasBigEndianPartOrdering(OutVT, DL))
74 : std::swap(Lo, Hi);
75 348 : Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
76 348 : Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
77 174 : return;
78 : }
79 1971 : case TargetLowering::TypeSplitVector:
80 1971 : GetSplitVector(InOp, Lo, Hi);
81 1971 : if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
82 : std::swap(Lo, Hi);
83 3942 : Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
84 3942 : Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
85 1971 : return;
86 2416 : case TargetLowering::TypeScalarizeVector:
87 : // Convert the element instead.
88 2416 : SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
89 4832 : Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
90 4832 : Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
91 2416 : return;
92 1 : case TargetLowering::TypeWidenVector: {
93 : assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
94 1 : InOp = GetWidenedVector(InOp);
95 1 : EVT LoVT, HiVT;
96 1 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
97 1 : std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
98 1 : if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
99 : std::swap(Lo, Hi);
100 2 : Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
101 2 : Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
102 : return;
103 : }
104 : }
105 :
106 7747 : if (InVT.isVector() && OutVT.isInteger()) {
107 : // Handle cases like i64 = BITCAST v1i64 on x86, where the operand
108 : // is legal but the result is not.
109 : unsigned NumElems = 2;
110 7310 : EVT ElemVT = NOutVT;
111 7310 : EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
112 :
113 : // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
114 13131 : while (!isTypeLegal(NVT)) {
115 5826 : unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
116 : // If the element size is smaller than byte, bail.
117 5826 : if (NewSizeInBits < 8)
118 : break;
119 5821 : NumElems *= 2;
120 5821 : ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
121 5821 : NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
122 : }
123 :
124 7310 : if (isTypeLegal(NVT)) {
125 14610 : SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
126 :
127 : SmallVector<SDValue, 8> Vals;
128 40269 : for (unsigned i = 0; i < NumElems; ++i)
129 65928 : Vals.push_back(DAG.getNode(
130 : ISD::EXTRACT_VECTOR_ELT, dl, ElemVT, CastInOp,
131 65928 : DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))));
132 :
133 : // Build Lo, Hi pair by pairing extracted elements if needed.
134 : unsigned Slot = 0;
135 25659 : for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
136 : // Each iteration will BUILD_PAIR two nodes and append the result until
137 : // there are only two nodes left, i.e. Lo and Hi.
138 18354 : SDValue LHS = Vals[Slot];
139 18354 : SDValue RHS = Vals[Slot + 1];
140 :
141 18354 : if (DAG.getDataLayout().isBigEndian())
142 : std::swap(LHS, RHS);
143 :
144 36708 : Vals.push_back(DAG.getNode(
145 : ISD::BUILD_PAIR, dl,
146 18354 : EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1),
147 36708 : LHS, RHS));
148 : }
149 7305 : Lo = Vals[Slot++];
150 7305 : Hi = Vals[Slot++];
151 :
152 7305 : if (DAG.getDataLayout().isBigEndian())
153 : std::swap(Lo, Hi);
154 :
155 : return;
156 : }
157 : }
158 :
159 : // Lower the bit-convert to a store/load from the stack.
160 : assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
161 :
162 : // Create the stack frame object. Make sure it is aligned for both
163 : // the source and expanded destination types.
164 884 : unsigned Alignment = DAG.getDataLayout().getPrefTypeAlignment(
165 442 : NOutVT.getTypeForEVT(*DAG.getContext()));
166 442 : SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
167 442 : int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
168 : MachinePointerInfo PtrInfo =
169 442 : MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
170 :
171 : // Emit a store to the stack slot.
172 884 : SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
173 :
174 : // Load the first half from the stack slot.
175 884 : Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo);
176 :
177 : // Increment the pointer to the other half.
178 442 : unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
179 442 : StackPtr = DAG.getNode(ISD::ADD, dl, StackPtr.getValueType(), StackPtr,
180 : DAG.getConstant(IncrementSize, dl,
181 442 : StackPtr.getValueType()));
182 :
183 : // Load the second half from the stack slot.
184 442 : Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
185 : PtrInfo.getWithOffset(IncrementSize),
186 442 : MinAlign(Alignment, IncrementSize));
187 :
188 : // Handle endianness of the load.
189 442 : if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
190 : std::swap(Lo, Hi);
191 : }
192 :
193 18023 : void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,
194 : SDValue &Hi) {
195 : // Return the operands.
196 18023 : Lo = N->getOperand(0);
197 18023 : Hi = N->getOperand(1);
198 18023 : }
199 :
200 2770 : void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,
201 : SDValue &Hi) {
202 5540 : GetExpandedOp(N->getOperand(0), Lo, Hi);
203 8310 : SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ?
204 2770 : Hi : Lo;
205 :
206 : assert(Part.getValueType() == N->getValueType(0) &&
207 : "Type twice as big as expanded type not itself expanded!");
208 :
209 2770 : GetPairElements(Part, Lo, Hi);
210 2770 : }
211 :
212 1987 : void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
213 : SDValue &Hi) {
214 1987 : SDValue OldVec = N->getOperand(0);
215 3974 : unsigned OldElts = OldVec.getValueType().getVectorNumElements();
216 1987 : EVT OldEltVT = OldVec.getValueType().getVectorElementType();
217 : SDLoc dl(N);
218 :
219 : // Convert to a vector of the expanded element type, for example
220 : // <3 x i64> -> <6 x i32>.
221 1987 : EVT OldVT = N->getValueType(0);
222 1987 : EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
223 :
224 2173 : if (OldVT != OldEltVT) {
225 : // The result of EXTRACT_VECTOR_ELT may be larger than the element type of
226 : // the input vector. If so, extend the elements of the input vector to the
227 : // same bitwidth as the result before expanding.
228 : assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!");
229 3 : EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts);
230 9 : OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0));
231 : }
232 :
233 1987 : SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
234 1987 : EVT::getVectorVT(*DAG.getContext(),
235 : NewVT, 2*OldElts),
236 1987 : OldVec);
237 :
238 : // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
239 1987 : SDValue Idx = N->getOperand(1);
240 :
241 3974 : Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
242 3974 : Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
243 :
244 1987 : Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
245 1987 : DAG.getConstant(1, dl, Idx.getValueType()));
246 3974 : Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
247 :
248 1987 : if (DAG.getDataLayout().isBigEndian())
249 : std::swap(Lo, Hi);
250 1987 : }
251 :
252 112556 : void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
253 : SDValue &Hi) {
254 : assert(ISD::isNormalLoad(N) && "This routine only for normal loads!");
255 : SDLoc dl(N);
256 :
257 : LoadSDNode *LD = cast<LoadSDNode>(N);
258 112556 : EVT ValueVT = LD->getValueType(0);
259 112556 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
260 112556 : SDValue Chain = LD->getChain();
261 112556 : SDValue Ptr = LD->getBasePtr();
262 112556 : unsigned Alignment = LD->getAlignment();
263 112556 : AAMDNodes AAInfo = LD->getAAInfo();
264 :
265 : assert(NVT.isByteSized() && "Expanded type not byte sized!");
266 :
267 225112 : Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(), Alignment,
268 225112 : LD->getMemOperand()->getFlags(), AAInfo);
269 :
270 : // Increment the pointer to the other half.
271 112556 : unsigned IncrementSize = NVT.getSizeInBits() / 8;
272 112556 : Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
273 112556 : DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
274 225112 : Hi = DAG.getLoad(NVT, dl, Chain, Ptr,
275 : LD->getPointerInfo().getWithOffset(IncrementSize),
276 : MinAlign(Alignment, IncrementSize),
277 337668 : LD->getMemOperand()->getFlags(), AAInfo);
278 :
279 : // Build a factor node to remember that this load is independent of the
280 : // other one.
281 112556 : Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
282 225112 : Hi.getValue(1));
283 :
284 : // Handle endianness of the load.
285 112556 : if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
286 : std::swap(Lo, Hi);
287 :
288 : // Modified the chain - switch anything that used the old chain to use
289 : // the new one.
290 112556 : ReplaceValueWith(SDValue(N, 1), Chain);
291 112556 : }
292 :
293 17 : void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
294 17 : EVT OVT = N->getValueType(0);
295 17 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
296 17 : SDValue Chain = N->getOperand(0);
297 17 : SDValue Ptr = N->getOperand(1);
298 : SDLoc dl(N);
299 17 : const unsigned Align = N->getConstantOperandVal(3);
300 :
301 34 : Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);
302 34 : Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
303 17 : Chain = Hi.getValue(1);
304 :
305 : // Handle endianness of the load.
306 17 : if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
307 : std::swap(Lo, Hi);
308 :
309 : // Modified the chain - switch anything that used the old chain to use
310 : // the new one.
311 17 : ReplaceValueWith(SDValue(N, 1), Chain);
312 17 : }
313 :
314 :
315 : //===--------------------------------------------------------------------===//
316 : // Generic Operand Expansion.
317 : //===--------------------------------------------------------------------===//
318 :
319 56721 : void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
320 : SmallVectorImpl<SDValue> &Ops,
321 : EVT EltVT) {
322 : assert(Op.getValueType().isInteger());
323 : SDLoc DL(Op);
324 56721 : SDValue Parts[2];
325 :
326 56721 : if (NumElements > 1) {
327 24373 : NumElements >>= 1;
328 24373 : SplitInteger(Op, Parts[0], Parts[1]);
329 24373 : if (DAG.getDataLayout().isBigEndian())
330 : std::swap(Parts[0], Parts[1]);
331 24373 : IntegerToVector(Parts[0], NumElements, Ops, EltVT);
332 24373 : IntegerToVector(Parts[1], NumElements, Ops, EltVT);
333 : } else {
334 64696 : Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));
335 : }
336 56721 : }
337 :
338 8609 : SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {
339 : SDLoc dl(N);
340 25827 : if (N->getValueType(0).isVector() &&
341 15958 : N->getOperand(0).getValueType().isInteger()) {
342 : // An illegal expanding type is being converted to a legal vector type.
343 : // Make a two element vector out of the expanded parts and convert that
344 : // instead, but only if the new vector type is legal (otherwise there
345 : // is no point, and it might create expansion loops). For example, on
346 : // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32.
347 : //
348 : // FIXME: I'm not sure why we are first trying to split the input into
349 : // a 2 element vector, so I'm leaving it here to maintain the current
350 : // behavior.
351 : unsigned NumElts = 2;
352 : EVT OVT = N->getOperand(0).getValueType();
353 7975 : EVT NVT = EVT::getVectorVT(*DAG.getContext(),
354 15950 : TLI.getTypeToTransformTo(*DAG.getContext(), OVT),
355 7975 : NumElts);
356 7975 : if (!isTypeLegal(NVT)) {
357 : // If we can't find a legal type by splitting the integer in half,
358 : // then we can use the node's value type.
359 7083 : NumElts = N->getValueType(0).getVectorNumElements();
360 2361 : NVT = N->getValueType(0);
361 : }
362 :
363 : SmallVector<SDValue, 8> Ops;
364 7975 : IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType());
365 :
366 : SDValue Vec =
367 7975 : DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts));
368 23925 : return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);
369 : }
370 :
371 : // Otherwise, store to a temporary and load out again as the new type.
372 1268 : return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
373 : }
374 :
375 23951 : SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
376 : // The vector type is legal but the element type needs expansion.
377 47902 : EVT VecVT = N->getValueType(0);
378 : unsigned NumElts = VecVT.getVectorNumElements();
379 23951 : EVT OldVT = N->getOperand(0).getValueType();
380 23951 : EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
381 : SDLoc dl(N);
382 :
383 : assert(OldVT == VecVT.getVectorElementType() &&
384 : "BUILD_VECTOR operand type doesn't match vector element type!");
385 :
386 : // Build a vector of twice the length out of the expanded elements.
387 : // For example <3 x i64> -> <6 x i32>.
388 : SmallVector<SDValue, 16> NewElts;
389 23951 : NewElts.reserve(NumElts*2);
390 :
391 73703 : for (unsigned i = 0; i < NumElts; ++i) {
392 49752 : SDValue Lo, Hi;
393 99504 : GetExpandedOp(N->getOperand(i), Lo, Hi);
394 49752 : if (DAG.getDataLayout().isBigEndian())
395 : std::swap(Lo, Hi);
396 49752 : NewElts.push_back(Lo);
397 49752 : NewElts.push_back(Hi);
398 : }
399 :
400 23951 : EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size());
401 47902 : SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
402 :
403 : // Convert the new vector to the old vector type.
404 47902 : return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
405 : }
406 :
407 19602 : SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {
408 19602 : SDValue Lo, Hi;
409 39204 : GetExpandedOp(N->getOperand(0), Lo, Hi);
410 58806 : return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo;
411 : }
412 :
413 32 : SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
414 : // The vector type is legal but the element type needs expansion.
415 64 : EVT VecVT = N->getValueType(0);
416 : unsigned NumElts = VecVT.getVectorNumElements();
417 : SDLoc dl(N);
418 :
419 32 : SDValue Val = N->getOperand(1);
420 : EVT OldEVT = Val.getValueType();
421 32 : EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);
422 :
423 : assert(OldEVT == VecVT.getVectorElementType() &&
424 : "Inserted element type doesn't match vector element type!");
425 :
426 : // Bitconvert to a vector of twice the length with elements of the expanded
427 : // type, insert the expanded vector elements, and then convert back.
428 32 : EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);
429 32 : SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
430 64 : NewVecVT, N->getOperand(0));
431 :
432 32 : SDValue Lo, Hi;
433 32 : GetExpandedOp(Val, Lo, Hi);
434 32 : if (DAG.getDataLayout().isBigEndian())
435 : std::swap(Lo, Hi);
436 :
437 32 : SDValue Idx = N->getOperand(2);
438 64 : Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
439 64 : NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
440 32 : Idx = DAG.getNode(ISD::ADD, dl,
441 : Idx.getValueType(), Idx,
442 32 : DAG.getConstant(1, dl, Idx.getValueType()));
443 64 : NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
444 :
445 : // Convert the new vector to the old vector type.
446 64 : return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
447 : }
448 :
449 64 : SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
450 : SDLoc dl(N);
451 128 : EVT VT = N->getValueType(0);
452 : assert(VT.getVectorElementType() == N->getOperand(0).getValueType() &&
453 : "SCALAR_TO_VECTOR operand type doesn't match vector element type!");
454 : unsigned NumElts = VT.getVectorNumElements();
455 64 : SmallVector<SDValue, 16> Ops(NumElts);
456 64 : Ops[0] = N->getOperand(0);
457 128 : SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
458 174 : for (unsigned i = 1; i < NumElts; ++i)
459 220 : Ops[i] = UndefVal;
460 128 : return DAG.getBuildVector(VT, dl, Ops);
461 : }
462 :
463 112733 : SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
464 : assert(ISD::isNormalStore(N) && "This routine only for normal stores!");
465 : assert(OpNo == 1 && "Can only expand the stored value so far");
466 : SDLoc dl(N);
467 :
468 : StoreSDNode *St = cast<StoreSDNode>(N);
469 112733 : EVT ValueVT = St->getValue().getValueType();
470 112733 : EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
471 112733 : SDValue Chain = St->getChain();
472 112733 : SDValue Ptr = St->getBasePtr();
473 112733 : unsigned Alignment = St->getAlignment();
474 112733 : AAMDNodes AAInfo = St->getAAInfo();
475 :
476 : assert(NVT.isByteSized() && "Expanded type not byte sized!");
477 112733 : unsigned IncrementSize = NVT.getSizeInBits() / 8;
478 :
479 112733 : SDValue Lo, Hi;
480 112733 : GetExpandedOp(St->getValue(), Lo, Hi);
481 :
482 112733 : if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
483 : std::swap(Lo, Hi);
484 :
485 225466 : Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(), Alignment,
486 225466 : St->getMemOperand()->getFlags(), AAInfo);
487 :
488 112733 : Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
489 225466 : Hi = DAG.getStore(Chain, dl, Hi, Ptr,
490 : St->getPointerInfo().getWithOffset(IncrementSize),
491 112733 : MinAlign(Alignment, IncrementSize),
492 338199 : St->getMemOperand()->getFlags(), AAInfo);
493 :
494 225466 : return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
495 : }
496 :
497 :
498 : //===--------------------------------------------------------------------===//
499 : // Generic Result Splitting.
500 : //===--------------------------------------------------------------------===//
501 :
502 : // Be careful to make no assumptions about which of Lo/Hi is stored first in
503 : // memory (for vectors it is always Lo first followed by Hi in the following
504 : // bytes; for integers and floats it is Lo first if and only if the machine is
505 : // little-endian).
506 :
507 53 : void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
508 : SDValue &Lo, SDValue &Hi) {
509 53 : SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
510 53 : GetSplitOp(Op, Lo, Hi);
511 53 : }
512 :
513 85 : static std::pair<SDValue, SDValue> SplitVSETCC(const SDNode *N,
514 : SelectionDAG &DAG) {
515 : SDLoc DL(N);
516 : EVT LoVT, HiVT;
517 170 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
518 :
519 : // Split the inputs.
520 : SDValue Lo, Hi, LL, LH, RL, RH;
521 85 : std::tie(LL, LH) = DAG.SplitVectorOperand(N, 0);
522 85 : std::tie(RL, RH) = DAG.SplitVectorOperand(N, 1);
523 :
524 255 : Lo = DAG.getNode(N->getOpcode(), DL, LoVT, LL, RL, N->getOperand(2));
525 255 : Hi = DAG.getNode(N->getOpcode(), DL, HiVT, LH, RH, N->getOperand(2));
526 :
527 85 : return std::make_pair(Lo, Hi);
528 : }
529 :
530 1821 : void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
531 1821 : SDValue LL, LH, RL, RH, CL, CH;
532 : SDLoc dl(N);
533 3642 : GetSplitOp(N->getOperand(1), LL, LH);
534 3642 : GetSplitOp(N->getOperand(2), RL, RH);
535 :
536 1821 : SDValue Cond = N->getOperand(0);
537 1821 : CL = CH = Cond;
538 5463 : if (Cond.getValueType().isVector()) {
539 1067 : if (SDValue Res = WidenVSELECTAndMask(N))
540 684 : std::tie(CL, CH) = DAG.SplitVector(Res->getOperand(0), dl);
541 : // It seems to improve code to generate two narrow SETCCs as opposed to
542 : // splitting a wide result vector.
543 1450 : else if (Cond.getOpcode() == ISD::SETCC)
544 85 : std::tie(CL, CH) = SplitVSETCC(Cond.getNode(), DAG);
545 : // Check if there are already splitted versions of the vector available and
546 : // use those instead of splitting the mask operand again.
547 1280 : else if (getTypeAction(Cond.getValueType()) ==
548 : TargetLowering::TypeSplitVector)
549 590 : GetSplitVector(Cond, CL, CH);
550 : else
551 50 : std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
552 : }
553 :
554 7284 : Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL);
555 7284 : Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH);
556 1821 : }
557 :
558 627 : void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
559 : SDValue &Hi) {
560 627 : SDValue LL, LH, RL, RH;
561 : SDLoc dl(N);
562 1254 : GetSplitOp(N->getOperand(2), LL, LH);
563 1254 : GetSplitOp(N->getOperand(3), RL, RH);
564 :
565 1254 : Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0),
566 1881 : N->getOperand(1), LL, RL, N->getOperand(4));
567 627 : Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0),
568 1881 : N->getOperand(1), LH, RH, N->getOperand(4));
569 627 : }
570 :
571 3647 : void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
572 : EVT LoVT, HiVT;
573 7294 : std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
574 3647 : Lo = DAG.getUNDEF(LoVT);
575 3647 : Hi = DAG.getUNDEF(HiVT);
576 3647 : }
|
