LLVM  14.0.0git
LegalizeTypesGeneric.cpp
Go to the documentation of this file.
1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements generic type expansion and splitting for LegalizeTypes.
10 // The routines here perform legalization when the details of the type (such as
11 // whether it is an integer or a float) do not matter.
12 // Expansion is the act of changing a computation in an illegal type to be a
13 // computation in two identical registers of a smaller type. The Lo/Hi part
14 // is required to be stored first in memory on little/big-endian machines.
15 // Splitting is the act of changing a computation in an illegal type to be a
16 // computation in two not necessarily identical registers of a smaller type.
17 // There are no requirements on how the type is represented in memory.
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DataLayout.h"
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "legalize-types"
26 
27 //===----------------------------------------------------------------------===//
28 // Generic Result Expansion.
29 //===----------------------------------------------------------------------===//
30 
31 // These routines assume that the Lo/Hi part is stored first in memory on
32 // little/big-endian machines, followed by the Hi/Lo part. This means that
33 // they cannot be used as is on vectors, for which Lo is always stored first.
34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
35  SDValue &Lo, SDValue &Hi) {
36  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
37  GetExpandedOp(Op, Lo, Hi);
38 }
39 
40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
41  EVT OutVT = N->getValueType(0);
42  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
43  SDValue InOp = N->getOperand(0);
44  EVT InVT = InOp.getValueType();
45  SDLoc dl(N);
46 
47  // Handle some special cases efficiently.
48  switch (getTypeAction(InVT)) {
51  break;
54  llvm_unreachable("Bitcast of a promotion-needing float should never need"
55  "expansion");
57  SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
58  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
59  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
60  return;
63  auto &DL = DAG.getDataLayout();
64  // Convert the expanded pieces of the input.
65  GetExpandedOp(InOp, Lo, Hi);
66  if (TLI.hasBigEndianPartOrdering(InVT, DL) !=
67  TLI.hasBigEndianPartOrdering(OutVT, DL))
68  std::swap(Lo, Hi);
69  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
70  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
71  return;
72  }
74  GetSplitVector(InOp, Lo, Hi);
75  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
76  std::swap(Lo, Hi);
77  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
78  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
79  return;
81  // Convert the element instead.
82  SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
83  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
84  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
85  return;
87  report_fatal_error("Scalarization of scalable vectors is not supported.");
89  assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
90  InOp = GetWidenedVector(InOp);
91  EVT LoVT, HiVT;
92  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
93  std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
94  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
95  std::swap(Lo, Hi);
96  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
97  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
98  return;
99  }
100  }
101 
102  if (InVT.isVector() && OutVT.isInteger()) {
103  // Handle cases like i64 = BITCAST v1i64 on x86, where the operand
104  // is legal but the result is not.
105  unsigned NumElems = 2;
106  EVT ElemVT = NOutVT;
107  EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
108 
109  // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
110  while (!isTypeLegal(NVT)) {
111  unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
112  // If the element size is smaller than byte, bail.
113  if (NewSizeInBits < 8)
114  break;
115  NumElems *= 2;
116  ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
117  NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
118  }
119 
120  if (isTypeLegal(NVT)) {
121  SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
122 
124  for (unsigned i = 0; i < NumElems; ++i)
125  Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
126  CastInOp, DAG.getVectorIdxConstant(i, dl)));
127 
128  // Build Lo, Hi pair by pairing extracted elements if needed.
129  unsigned Slot = 0;
130  for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
131  // Each iteration will BUILD_PAIR two nodes and append the result until
132  // there are only two nodes left, i.e. Lo and Hi.
133  SDValue LHS = Vals[Slot];
134  SDValue RHS = Vals[Slot + 1];
135 
136  if (DAG.getDataLayout().isBigEndian())
137  std::swap(LHS, RHS);
138 
139  Vals.push_back(DAG.getNode(
140  ISD::BUILD_PAIR, dl,
141  EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1),
142  LHS, RHS));
143  }
144  Lo = Vals[Slot++];
145  Hi = Vals[Slot++];
146 
147  if (DAG.getDataLayout().isBigEndian())
148  std::swap(Lo, Hi);
149 
150  return;
151  }
152  }
153 
154  // Lower the bit-convert to a store/load from the stack.
155  assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
156 
157  // Create the stack frame object. Make sure it is aligned for both
158  // the source and expanded destination types.
159 
160  // In cases where the vector is illegal it will be broken down into parts
161  // and stored in parts - we should use the alignment for the smallest part.
162  Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false);
163  Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false);
164  Align Align = std::max(InAlign, NOutAlign);
166  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
167  MachinePointerInfo PtrInfo =
169 
170  // Emit a store to the stack slot.
171  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
172 
173  // Load the first half from the stack slot.
174  Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign);
175 
176  // Increment the pointer to the other half.
177  unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
178  StackPtr =
179  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(IncrementSize), dl);
180 
181  // Load the second half from the stack slot.
182  Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
183  PtrInfo.getWithOffset(IncrementSize), NOutAlign);
184 
185  // Handle endianness of the load.
186  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
187  std::swap(Lo, Hi);
188 }
189 
190 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,
191  SDValue &Hi) {
192  // Return the operands.
193  Lo = N->getOperand(0);
194  Hi = N->getOperand(1);
195 }
196 
197 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,
198  SDValue &Hi) {
199  GetExpandedOp(N->getOperand(0), Lo, Hi);
200  SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ?
201  Hi : Lo;
202 
203  assert(Part.getValueType() == N->getValueType(0) &&
204  "Type twice as big as expanded type not itself expanded!");
205 
206  GetPairElements(Part, Lo, Hi);
207 }
208 
209 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
210  SDValue &Hi) {
211  SDValue OldVec = N->getOperand(0);
212  unsigned OldElts = OldVec.getValueType().getVectorNumElements();
213  EVT OldEltVT = OldVec.getValueType().getVectorElementType();
214  SDLoc dl(N);
215 
216  // Convert to a vector of the expanded element type, for example
217  // <3 x i64> -> <6 x i32>.
218  EVT OldVT = N->getValueType(0);
219  EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
220 
221  if (OldVT != OldEltVT) {
222  // The result of EXTRACT_VECTOR_ELT may be larger than the element type of
223  // the input vector. If so, extend the elements of the input vector to the
224  // same bitwidth as the result before expanding.
225  assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!");
226  EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts);
227  OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0));
228  }
229 
230  SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
232  NewVT, 2*OldElts),
233  OldVec);
234 
235  // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
236  SDValue Idx = N->getOperand(1);
237 
238  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
239  Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
240 
241  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
242  DAG.getConstant(1, dl, Idx.getValueType()));
243  Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
244 
245  if (DAG.getDataLayout().isBigEndian())
246  std::swap(Lo, Hi);
247 }
248 
249 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
250  SDValue &Hi) {
251  assert(ISD::isNormalLoad(N) && "This routine only for normal loads!");
252  SDLoc dl(N);
253 
254  LoadSDNode *LD = cast<LoadSDNode>(N);
255  assert(!LD->isAtomic() && "Atomics can not be split");
256  EVT ValueVT = LD->getValueType(0);
257  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
258  SDValue Chain = LD->getChain();
259  SDValue Ptr = LD->getBasePtr();
260  AAMDNodes AAInfo = LD->getAAInfo();
261 
262  assert(NVT.isByteSized() && "Expanded type not byte sized!");
263 
264  Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(),
265  LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
266  AAInfo);
267 
268  // Increment the pointer to the other half.
269  unsigned IncrementSize = NVT.getSizeInBits() / 8;
270  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
271  Hi = DAG.getLoad(
272  NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize),
273  LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo);
274 
275  // Build a factor node to remember that this load is independent of the
276  // other one.
277  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
278  Hi.getValue(1));
279 
280  // Handle endianness of the load.
281  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
282  std::swap(Lo, Hi);
283 
284  // Modified the chain - switch anything that used the old chain to use
285  // the new one.
286  ReplaceValueWith(SDValue(N, 1), Chain);
287 }
288 
289 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
290  EVT OVT = N->getValueType(0);
291  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
292  SDValue Chain = N->getOperand(0);
293  SDValue Ptr = N->getOperand(1);
294  SDLoc dl(N);
295  const unsigned Align = N->getConstantOperandVal(3);
296 
297  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);
298  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
299  Chain = Hi.getValue(1);
300 
301  // Handle endianness of the load.
302  if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
303  std::swap(Lo, Hi);
304 
305  // Modified the chain - switch anything that used the old chain to use
306  // the new one.
307  ReplaceValueWith(SDValue(N, 1), Chain);
308 }
309 
310 
311 //===--------------------------------------------------------------------===//
312 // Generic Operand Expansion.
313 //===--------------------------------------------------------------------===//
314 
315 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
317  EVT EltVT) {
318  assert(Op.getValueType().isInteger());
319  SDLoc DL(Op);
320  SDValue Parts[2];
321 
322  if (NumElements > 1) {
323  NumElements >>= 1;
324  SplitInteger(Op, Parts[0], Parts[1]);
325  if (DAG.getDataLayout().isBigEndian())
326  std::swap(Parts[0], Parts[1]);
327  IntegerToVector(Parts[0], NumElements, Ops, EltVT);
328  IntegerToVector(Parts[1], NumElements, Ops, EltVT);
329  } else {
330  Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));
331  }
332 }
333 
334 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {
335  SDLoc dl(N);
336  if (N->getValueType(0).isVector() &&
337  N->getOperand(0).getValueType().isInteger()) {
338  // An illegal expanding type is being converted to a legal vector type.
339  // Make a two element vector out of the expanded parts and convert that
340  // instead, but only if the new vector type is legal (otherwise there
341  // is no point, and it might create expansion loops). For example, on
342  // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32.
343  //
344  // FIXME: I'm not sure why we are first trying to split the input into
345  // a 2 element vector, so I'm leaving it here to maintain the current
346  // behavior.
347  unsigned NumElts = 2;
348  EVT OVT = N->getOperand(0).getValueType();
349  EVT NVT = EVT::getVectorVT(*DAG.getContext(),
350  TLI.getTypeToTransformTo(*DAG.getContext(), OVT),
351  NumElts);
352  if (!isTypeLegal(NVT)) {
353  // If we can't find a legal type by splitting the integer in half,
354  // then we can use the node's value type.
355  NumElts = N->getValueType(0).getVectorNumElements();
356  NVT = N->getValueType(0);
357  }
358 
360  IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType());
361 
362  SDValue Vec =
363  DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts));
364  return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);
365  }
366 
367  // Otherwise, store to a temporary and load out again as the new type.
368  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
369 }
370 
371 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
372  // The vector type is legal but the element type needs expansion.
373  EVT VecVT = N->getValueType(0);
374  unsigned NumElts = VecVT.getVectorNumElements();
375  EVT OldVT = N->getOperand(0).getValueType();
376  EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
377  SDLoc dl(N);
378 
379  assert(OldVT == VecVT.getVectorElementType() &&
380  "BUILD_VECTOR operand type doesn't match vector element type!");
381 
382  // Build a vector of twice the length out of the expanded elements.
383  // For example <3 x i64> -> <6 x i32>.
384  SmallVector<SDValue, 16> NewElts;
385  NewElts.reserve(NumElts*2);
386 
387  for (unsigned i = 0; i < NumElts; ++i) {
388  SDValue Lo, Hi;
389  GetExpandedOp(N->getOperand(i), Lo, Hi);
390  if (DAG.getDataLayout().isBigEndian())
391  std::swap(Lo, Hi);
392  NewElts.push_back(Lo);
393  NewElts.push_back(Hi);
394  }
395 
396  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size());
397  SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
398 
399  // Convert the new vector to the old vector type.
400  return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
401 }
402 
403 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {
404  SDValue Lo, Hi;
405  GetExpandedOp(N->getOperand(0), Lo, Hi);
406  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo;
407 }
408 
409 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
410  // The vector type is legal but the element type needs expansion.
411  EVT VecVT = N->getValueType(0);
412  unsigned NumElts = VecVT.getVectorNumElements();
413  SDLoc dl(N);
414 
415  SDValue Val = N->getOperand(1);
416  EVT OldEVT = Val.getValueType();
417  EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);
418 
419  assert(OldEVT == VecVT.getVectorElementType() &&
420  "Inserted element type doesn't match vector element type!");
421 
422  // Bitconvert to a vector of twice the length with elements of the expanded
423  // type, insert the expanded vector elements, and then convert back.
424  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);
425  SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
426  NewVecVT, N->getOperand(0));
427 
428  SDValue Lo, Hi;
429  GetExpandedOp(Val, Lo, Hi);
430  if (DAG.getDataLayout().isBigEndian())
431  std::swap(Lo, Hi);
432 
433  SDValue Idx = N->getOperand(2);
434  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
435  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
436  Idx = DAG.getNode(ISD::ADD, dl,
437  Idx.getValueType(), Idx,
438  DAG.getConstant(1, dl, Idx.getValueType()));
439  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
440 
441  // Convert the new vector to the old vector type.
442  return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
443 }
444 
445 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
446  SDLoc dl(N);
447  EVT VT = N->getValueType(0);
448  assert(VT.getVectorElementType() == N->getOperand(0).getValueType() &&
449  "SCALAR_TO_VECTOR operand type doesn't match vector element type!");
450  unsigned NumElts = VT.getVectorNumElements();
451  SmallVector<SDValue, 16> Ops(NumElts);
452  Ops[0] = N->getOperand(0);
453  SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
454  for (unsigned i = 1; i < NumElts; ++i)
455  Ops[i] = UndefVal;
456  return DAG.getBuildVector(VT, dl, Ops);
457 }
458 
459 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
460  assert(ISD::isNormalStore(N) && "This routine only for normal stores!");
461  assert(OpNo == 1 && "Can only expand the stored value so far");
462  SDLoc dl(N);
463 
464  StoreSDNode *St = cast<StoreSDNode>(N);
465  assert(!St->isAtomic() && "Atomics can not be split");
466  EVT ValueVT = St->getValue().getValueType();
467  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
468  SDValue Chain = St->getChain();
469  SDValue Ptr = St->getBasePtr();
470  AAMDNodes AAInfo = St->getAAInfo();
471 
472  assert(NVT.isByteSized() && "Expanded type not byte sized!");
473  unsigned IncrementSize = NVT.getSizeInBits() / 8;
474 
475  SDValue Lo, Hi;
476  GetExpandedOp(St->getValue(), Lo, Hi);
477 
478  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
479  std::swap(Lo, Hi);
480 
481  Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(),
482  St->getOriginalAlign(), St->getMemOperand()->getFlags(),
483  AAInfo);
484 
485  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
486  Hi = DAG.getStore(
487  Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize),
488  St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo);
489 
490  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
491 }
492 
493 
494 //===--------------------------------------------------------------------===//
495 // Generic Result Splitting.
496 //===--------------------------------------------------------------------===//
497 
498 // Be careful to make no assumptions about which of Lo/Hi is stored first in
499 // memory (for vectors it is always Lo first followed by Hi in the following
500 // bytes; for integers and floats it is Lo first if and only if the machine is
501 // little-endian).
502 
503 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
504  SDValue &Lo, SDValue &Hi) {
505  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
506  GetSplitOp(Op, Lo, Hi);
507 }
508 
509 void DAGTypeLegalizer::SplitRes_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
510  SDValue LL, LH, RL, RH, CL, CH;
511  SDLoc dl(N);
512  GetSplitOp(N->getOperand(1), LL, LH);
513  GetSplitOp(N->getOperand(2), RL, RH);
514 
515  SDValue Cond = N->getOperand(0);
516  CL = CH = Cond;
517  if (Cond.getValueType().isVector()) {
518  if (SDValue Res = WidenVSELECTMask(N))
519  std::tie(CL, CH) = DAG.SplitVector(Res, dl);
520  // Check if there are already splitted versions of the vector available and
521  // use those instead of splitting the mask operand again.
522  else if (getTypeAction(Cond.getValueType()) ==
524  GetSplitVector(Cond, CL, CH);
525  // It seems to improve code to generate two narrow SETCCs as opposed to
526  // splitting a wide result vector.
527  else if (Cond.getOpcode() == ISD::SETCC) {
528  // If the condition is a vXi1 vector, and the LHS of the setcc is a legal
529  // type and the setcc result type is the same vXi1, then leave the setcc
530  // alone.
531  EVT CondLHSVT = Cond.getOperand(0).getValueType();
532  if (Cond.getValueType().getVectorElementType() == MVT::i1 &&
533  isTypeLegal(CondLHSVT) &&
534  getSetCCResultType(CondLHSVT) == Cond.getValueType())
535  std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
536  else
537  SplitVecRes_SETCC(Cond.getNode(), CL, CH);
538  } else
539  std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
540  }
541 
542  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL);
543  Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH);
544 }
545 
546 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
547  SDValue &Hi) {
548  SDValue LL, LH, RL, RH;
549  SDLoc dl(N);
550  GetSplitOp(N->getOperand(2), LL, LH);
551  GetSplitOp(N->getOperand(3), RL, RH);
552 
553  Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0),
554  N->getOperand(1), LL, RL, N->getOperand(4));
555  Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0),
556  N->getOperand(1), LH, RH, N->getOperand(4));
557 }
558 
559 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
560  EVT LoVT, HiVT;
561  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
562  Lo = DAG.getUNDEF(LoVT);
563  Hi = DAG.getUNDEF(HiVT);
564 }
565 
566 void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) {
567  SDValue L, H;
568  SDLoc dl(N);
569  GetSplitOp(N->getOperand(0), L, H);
570 
571  Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L);
572  Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H);
573 }
i
i
Definition: README.txt:29
llvm::StoreSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2331
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1086
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:848
llvm::TargetLoweringBase::TypeScalarizeScalableVector
@ TypeScalarizeScalableVector
Definition: TargetLowering.h:217
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::TargetLoweringBase::TypeSoftPromoteHalf
@ TypeSoftPromoteHalf
Definition: TargetLowering.h:216
llvm::MipsISD::Lo
@ Lo
Definition: MipsISelLowering.h:79
CH
AMD64 Optimization Manual has some nice information about optimizing integer multiplication by a constant How much of it applies to Intel s X86 implementation There are definite trade offs to xmm0 cvttss2siq rdx jb L3 subss xmm0 rax cvttss2siq rdx xorq rdx rax ret instead of xmm1 cvttss2siq rcx movaps xmm2 subss xmm2 cvttss2siq rax rdx xorq rax ucomiss xmm0 cmovb rax ret Seems like the jb branch has high likelihood of being taken It would have saved a few instructions It s not possible to reference CH
Definition: README-X86-64.txt:44
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::SelectionDAG::getVectorIdxConstant
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1508
llvm::MemSDNode::getChain
const SDValue & getChain() const
Definition: SelectionDAGNodes.h:1359
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:732
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:455
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2281
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:6350
llvm::AAMDNodes
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:651
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:7512
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:702
llvm::EVT::getVectorVT
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
llvm::codeview::EncodedFramePtrReg::StackPtr
@ StackPtr
llvm::TargetLoweringBase::TypeExpandInteger
@ TypeExpandInteger
Definition: TargetLowering.h:209
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:447
llvm::MipsISD::Hi
@ Hi
Definition: MipsISelLowering.h:75
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:7462
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::EVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:363
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:694
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1121
llvm::SelectionDAG::getObjectPtrOffset
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
Definition: SelectionDAG.h:906
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:8310
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:951
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1373
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:309
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:423
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:145
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::TargetLoweringBase::TypeWidenVector
@ TypeWidenVector
Definition: TargetLowering.h:214
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:192
llvm::EVT::bitsLT
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:273
llvm::TargetLoweringBase::getTypeToTransformTo
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:942
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:243
llvm::SelectionDAG::SplitVector
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
Definition: SelectionDAG.cpp:10635
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:341
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1340
llvm::TargetLoweringBase::TypeSoftenFloat
@ TypeSoftenFloat
Definition: TargetLowering.h:210
llvm::TargetLoweringBase::TypePromoteInteger
@ TypePromoteInteger
Definition: TargetLowering.h:208
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:38
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:511
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8316
llvm::MemSDNode::getOriginalAlign
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
Definition: SelectionDAGNodes.h:1271
llvm::TargetLoweringBase::hasBigEndianPartOrdering
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
Definition: TargetLowering.h:1563
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2309
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:65
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:216
llvm::StoreSDNode::getValue
const SDValue & getValue() const
Definition: SelectionDAGNodes.h:2330
LegalizeTypes.h
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2206
llvm::MachinePointerInfo::getWithOffset
MachinePointerInfo getWithOffset(int64_t O) const
Definition: MachineMemOperand.h:80
llvm::MemSDNode::getAAInfo
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Definition: SelectionDAGNodes.h:1304
llvm::MemSDNode::isAtomic
bool isAtomic() const
Return true if the memory operation ordering is Unordered or higher.
Definition: SelectionDAGNodes.h:1325
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:155
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:2894
DataLayout.h
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:179
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::SelectionDAG::getReducedAlign
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
Definition: SelectionDAG.cpp:2178
llvm::SelectionDAG::getBuildVector
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:790
llvm::TargetLoweringBase::TypeLegal
@ TypeLegal
Definition: TargetLowering.h:207
H
#define H(x, y, z)
Definition: MD5.cpp:58
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:321
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:516
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:440
llvm::pdb::PDB_LocType::Slot
@ Slot
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:138
llvm::TargetLoweringBase::TypeExpandFloat
@ TypeExpandFloat
Definition: TargetLowering.h:211
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::TargetLoweringBase::TypePromoteFloat
@ TypePromoteFloat
Definition: TargetLowering.h:215
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:476
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:301
llvm::SelectionDAG::GetSplitDestVTs
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
Definition: SelectionDAG.cpp:10590
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1003
llvm::TargetLoweringBase::TypeScalarizeVector
@ TypeScalarizeVector
Definition: TargetLowering.h:212
N
#define N
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:437
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:2932
llvm::TargetLoweringBase::TypeSplitVector
@ TypeSplitVector
Definition: TargetLowering.h:213
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:624
llvm::ISD::INSERT_VECTOR_ELT
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:500
llvm::MachineMemOperand::getFlags
Flags getFlags() const
Return the raw flags of the source value,.
Definition: MachineMemOperand.h:220
llvm::MemSDNode::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition: SelectionDAGNodes.h:1342
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52