LLVM  16.0.0git
LegalizeTypesGeneric.cpp
Go to the documentation of this file.
1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements generic type expansion and splitting for LegalizeTypes.
10 // The routines here perform legalization when the details of the type (such as
11 // whether it is an integer or a float) do not matter.
12 // Expansion is the act of changing a computation in an illegal type to be a
13 // computation in two identical registers of a smaller type. The Lo/Hi part
14 // is required to be stored first in memory on little/big-endian machines.
15 // Splitting is the act of changing a computation in an illegal type to be a
16 // computation in two not necessarily identical registers of a smaller type.
17 // There are no requirements on how the type is represented in memory.
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DataLayout.h"
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "legalize-types"
26 
27 //===----------------------------------------------------------------------===//
28 // Generic Result Expansion.
29 //===----------------------------------------------------------------------===//
30 
31 // These routines assume that the Lo/Hi part is stored first in memory on
32 // little/big-endian machines, followed by the Hi/Lo part. This means that
33 // they cannot be used as is on vectors, for which Lo is always stored first.
34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
35  SDValue &Lo, SDValue &Hi) {
36  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
37  GetExpandedOp(Op, Lo, Hi);
38 }
39 
40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
41  EVT OutVT = N->getValueType(0);
42  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
43  SDValue InOp = N->getOperand(0);
44  EVT InVT = InOp.getValueType();
45  SDLoc dl(N);
46 
47  // Handle some special cases efficiently.
48  switch (getTypeAction(InVT)) {
51  break;
54  llvm_unreachable("Bitcast of a promotion-needing float should never need"
55  "expansion");
57  SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
58  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
59  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
60  return;
63  auto &DL = DAG.getDataLayout();
64  // Convert the expanded pieces of the input.
65  GetExpandedOp(InOp, Lo, Hi);
66  if (TLI.hasBigEndianPartOrdering(InVT, DL) !=
67  TLI.hasBigEndianPartOrdering(OutVT, DL))
68  std::swap(Lo, Hi);
69  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
70  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
71  return;
72  }
74  GetSplitVector(InOp, Lo, Hi);
75  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
76  std::swap(Lo, Hi);
77  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
78  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
79  return;
81  // Convert the element instead.
82  SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
83  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
84  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
85  return;
87  report_fatal_error("Scalarization of scalable vectors is not supported.");
89  assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
90  InOp = GetWidenedVector(InOp);
91  EVT LoVT, HiVT;
92  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
93  std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
94  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
95  std::swap(Lo, Hi);
96  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
97  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
98  return;
99  }
100  }
101 
102  if (InVT.isVector() && OutVT.isInteger()) {
103  // Handle cases like i64 = BITCAST v1i64 on x86, where the operand
104  // is legal but the result is not.
105  unsigned NumElems = 2;
106  EVT ElemVT = NOutVT;
107  EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
108 
109  // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
110  while (!isTypeLegal(NVT)) {
111  unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
112  // If the element size is smaller than byte, bail.
113  if (NewSizeInBits < 8)
114  break;
115  NumElems *= 2;
116  ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
117  NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
118  }
119 
120  if (isTypeLegal(NVT)) {
121  SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
122 
124  for (unsigned i = 0; i < NumElems; ++i)
125  Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
126  CastInOp, DAG.getVectorIdxConstant(i, dl)));
127 
128  // Build Lo, Hi pair by pairing extracted elements if needed.
129  unsigned Slot = 0;
130  for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
131  // Each iteration will BUILD_PAIR two nodes and append the result until
132  // there are only two nodes left, i.e. Lo and Hi.
133  SDValue LHS = Vals[Slot];
134  SDValue RHS = Vals[Slot + 1];
135 
136  if (DAG.getDataLayout().isBigEndian())
137  std::swap(LHS, RHS);
138 
139  Vals.push_back(DAG.getNode(
140  ISD::BUILD_PAIR, dl,
141  EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1),
142  LHS, RHS));
143  }
144  Lo = Vals[Slot++];
145  Hi = Vals[Slot++];
146 
147  if (DAG.getDataLayout().isBigEndian())
148  std::swap(Lo, Hi);
149 
150  return;
151  }
152  }
153 
154  // Lower the bit-convert to a store/load from the stack.
155  assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
156 
157  // Create the stack frame object. Make sure it is aligned for both
158  // the source and expanded destination types.
159 
160  // In cases where the vector is illegal it will be broken down into parts
161  // and stored in parts - we should use the alignment for the smallest part.
162  Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false);
163  Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false);
164  Align Align = std::max(InAlign, NOutAlign);
166  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
167  MachinePointerInfo PtrInfo =
169 
170  // Emit a store to the stack slot.
171  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
172 
173  // Load the first half from the stack slot.
174  Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign);
175 
176  // Increment the pointer to the other half.
177  unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
178  StackPtr =
179  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(IncrementSize), dl);
180 
181  // Load the second half from the stack slot.
182  Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
183  PtrInfo.getWithOffset(IncrementSize), NOutAlign);
184 
185  // Handle endianness of the load.
186  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
187  std::swap(Lo, Hi);
188 }
189 
190 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,
191  SDValue &Hi) {
192  // Return the operands.
193  Lo = N->getOperand(0);
194  Hi = N->getOperand(1);
195 }
196 
197 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,
198  SDValue &Hi) {
199  GetExpandedOp(N->getOperand(0), Lo, Hi);
200  SDValue Part = N->getConstantOperandVal(1) ? Hi : Lo;
201 
202  assert(Part.getValueType() == N->getValueType(0) &&
203  "Type twice as big as expanded type not itself expanded!");
204 
205  GetPairElements(Part, Lo, Hi);
206 }
207 
208 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
209  SDValue &Hi) {
210  SDValue OldVec = N->getOperand(0);
211  ElementCount OldEltCount = OldVec.getValueType().getVectorElementCount();
212  EVT OldEltVT = OldVec.getValueType().getVectorElementType();
213  SDLoc dl(N);
214 
215  // Convert to a vector of the expanded element type, for example
216  // <3 x i64> -> <6 x i32>.
217  EVT OldVT = N->getValueType(0);
218  EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
219 
220  if (OldVT != OldEltVT) {
221  // The result of EXTRACT_VECTOR_ELT may be larger than the element type of
222  // the input vector. If so, extend the elements of the input vector to the
223  // same bitwidth as the result before expanding.
224  assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!");
225  EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldEltCount);
226  OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0));
227  }
228 
229  SDValue NewVec = DAG.getNode(
230  ISD::BITCAST, dl,
231  EVT::getVectorVT(*DAG.getContext(), NewVT, OldEltCount * 2), OldVec);
232 
233  // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
234  SDValue Idx = N->getOperand(1);
235 
236  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
237  Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
238 
239  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
240  DAG.getConstant(1, dl, Idx.getValueType()));
241  Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
242 
243  if (DAG.getDataLayout().isBigEndian())
244  std::swap(Lo, Hi);
245 }
246 
247 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
248  SDValue &Hi) {
249  assert(ISD::isNormalLoad(N) && "This routine only for normal loads!");
250  SDLoc dl(N);
251 
252  LoadSDNode *LD = cast<LoadSDNode>(N);
253  assert(!LD->isAtomic() && "Atomics can not be split");
254  EVT ValueVT = LD->getValueType(0);
255  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
256  SDValue Chain = LD->getChain();
257  SDValue Ptr = LD->getBasePtr();
258  AAMDNodes AAInfo = LD->getAAInfo();
259 
260  assert(NVT.isByteSized() && "Expanded type not byte sized!");
261 
262  Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(),
263  LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
264  AAInfo);
265 
266  // Increment the pointer to the other half.
267  unsigned IncrementSize = NVT.getSizeInBits() / 8;
268  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
269  Hi = DAG.getLoad(
270  NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize),
271  LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo);
272 
273  // Build a factor node to remember that this load is independent of the
274  // other one.
275  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
276  Hi.getValue(1));
277 
278  // Handle endianness of the load.
279  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
280  std::swap(Lo, Hi);
281 
282  // Modified the chain - switch anything that used the old chain to use
283  // the new one.
284  ReplaceValueWith(SDValue(N, 1), Chain);
285 }
286 
287 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
288  EVT OVT = N->getValueType(0);
289  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
290  SDValue Chain = N->getOperand(0);
291  SDValue Ptr = N->getOperand(1);
292  SDLoc dl(N);
293  const unsigned Align = N->getConstantOperandVal(3);
294 
295  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);
296  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
297  Chain = Hi.getValue(1);
298 
299  // Handle endianness of the load.
300  if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
301  std::swap(Lo, Hi);
302 
303  // Modified the chain - switch anything that used the old chain to use
304  // the new one.
305  ReplaceValueWith(SDValue(N, 1), Chain);
306 }
307 
308 
309 //===--------------------------------------------------------------------===//
310 // Generic Operand Expansion.
311 //===--------------------------------------------------------------------===//
312 
313 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
315  EVT EltVT) {
316  assert(Op.getValueType().isInteger());
317  SDLoc DL(Op);
318  SDValue Parts[2];
319 
320  if (NumElements > 1) {
321  NumElements >>= 1;
322  SplitInteger(Op, Parts[0], Parts[1]);
323  if (DAG.getDataLayout().isBigEndian())
324  std::swap(Parts[0], Parts[1]);
325  IntegerToVector(Parts[0], NumElements, Ops, EltVT);
326  IntegerToVector(Parts[1], NumElements, Ops, EltVT);
327  } else {
328  Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));
329  }
330 }
331 
332 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {
333  SDLoc dl(N);
334  if (N->getValueType(0).isVector() &&
335  N->getOperand(0).getValueType().isInteger()) {
336  // An illegal expanding type is being converted to a legal vector type.
337  // Make a two element vector out of the expanded parts and convert that
338  // instead, but only if the new vector type is legal (otherwise there
339  // is no point, and it might create expansion loops). For example, on
340  // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32.
341  //
342  // FIXME: I'm not sure why we are first trying to split the input into
343  // a 2 element vector, so I'm leaving it here to maintain the current
344  // behavior.
345  unsigned NumElts = 2;
346  EVT OVT = N->getOperand(0).getValueType();
347  EVT NVT = EVT::getVectorVT(*DAG.getContext(),
348  TLI.getTypeToTransformTo(*DAG.getContext(), OVT),
349  NumElts);
350  if (!isTypeLegal(NVT)) {
351  // If we can't find a legal type by splitting the integer in half,
352  // then we can use the node's value type.
353  NumElts = N->getValueType(0).getVectorNumElements();
354  NVT = N->getValueType(0);
355  }
356 
358  IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType());
359 
360  SDValue Vec =
361  DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts));
362  return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);
363  }
364 
365  // Otherwise, store to a temporary and load out again as the new type.
366  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
367 }
368 
369 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
370  // The vector type is legal but the element type needs expansion.
371  EVT VecVT = N->getValueType(0);
372  unsigned NumElts = VecVT.getVectorNumElements();
373  EVT OldVT = N->getOperand(0).getValueType();
374  EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
375  SDLoc dl(N);
376 
377  assert(OldVT == VecVT.getVectorElementType() &&
378  "BUILD_VECTOR operand type doesn't match vector element type!");
379 
380  // Build a vector of twice the length out of the expanded elements.
381  // For example <3 x i64> -> <6 x i32>.
382  SmallVector<SDValue, 16> NewElts;
383  NewElts.reserve(NumElts*2);
384 
385  for (unsigned i = 0; i < NumElts; ++i) {
386  SDValue Lo, Hi;
387  GetExpandedOp(N->getOperand(i), Lo, Hi);
388  if (DAG.getDataLayout().isBigEndian())
389  std::swap(Lo, Hi);
390  NewElts.push_back(Lo);
391  NewElts.push_back(Hi);
392  }
393 
394  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size());
395  SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
396 
397  // Convert the new vector to the old vector type.
398  return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
399 }
400 
401 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {
402  SDValue Lo, Hi;
403  GetExpandedOp(N->getOperand(0), Lo, Hi);
404  return N->getConstantOperandVal(1) ? Hi : Lo;
405 }
406 
407 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
408  // The vector type is legal but the element type needs expansion.
409  EVT VecVT = N->getValueType(0);
410  unsigned NumElts = VecVT.getVectorNumElements();
411  SDLoc dl(N);
412 
413  SDValue Val = N->getOperand(1);
414  EVT OldEVT = Val.getValueType();
415  EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);
416 
417  assert(OldEVT == VecVT.getVectorElementType() &&
418  "Inserted element type doesn't match vector element type!");
419 
420  // Bitconvert to a vector of twice the length with elements of the expanded
421  // type, insert the expanded vector elements, and then convert back.
422  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);
423  SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
424  NewVecVT, N->getOperand(0));
425 
426  SDValue Lo, Hi;
427  GetExpandedOp(Val, Lo, Hi);
428  if (DAG.getDataLayout().isBigEndian())
429  std::swap(Lo, Hi);
430 
431  SDValue Idx = N->getOperand(2);
432  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
433  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
434  Idx = DAG.getNode(ISD::ADD, dl,
435  Idx.getValueType(), Idx,
436  DAG.getConstant(1, dl, Idx.getValueType()));
437  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
438 
439  // Convert the new vector to the old vector type.
440  return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
441 }
442 
443 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
444  SDLoc dl(N);
445  EVT VT = N->getValueType(0);
446  assert(VT.getVectorElementType() == N->getOperand(0).getValueType() &&
447  "SCALAR_TO_VECTOR operand type doesn't match vector element type!");
448  unsigned NumElts = VT.getVectorNumElements();
449  SmallVector<SDValue, 16> Ops(NumElts);
450  Ops[0] = N->getOperand(0);
451  SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
452  for (unsigned i = 1; i < NumElts; ++i)
453  Ops[i] = UndefVal;
454  return DAG.getBuildVector(VT, dl, Ops);
455 }
456 
457 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
458  assert(ISD::isNormalStore(N) && "This routine only for normal stores!");
459  assert(OpNo == 1 && "Can only expand the stored value so far");
460  SDLoc dl(N);
461 
462  StoreSDNode *St = cast<StoreSDNode>(N);
463  assert(!St->isAtomic() && "Atomics can not be split");
464  EVT ValueVT = St->getValue().getValueType();
465  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
466  SDValue Chain = St->getChain();
467  SDValue Ptr = St->getBasePtr();
468  AAMDNodes AAInfo = St->getAAInfo();
469 
470  assert(NVT.isByteSized() && "Expanded type not byte sized!");
471  unsigned IncrementSize = NVT.getSizeInBits() / 8;
472 
473  SDValue Lo, Hi;
474  GetExpandedOp(St->getValue(), Lo, Hi);
475 
476  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
477  std::swap(Lo, Hi);
478 
479  Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(),
480  St->getOriginalAlign(), St->getMemOperand()->getFlags(),
481  AAInfo);
482 
483  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
484  Hi = DAG.getStore(
485  Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize),
486  St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo);
487 
488  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
489 }
490 
491 
492 //===--------------------------------------------------------------------===//
493 // Generic Result Splitting.
494 //===--------------------------------------------------------------------===//
495 
496 // Be careful to make no assumptions about which of Lo/Hi is stored first in
497 // memory (for vectors it is always Lo first followed by Hi in the following
498 // bytes; for integers and floats it is Lo first if and only if the machine is
499 // little-endian).
500 
501 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
502  SDValue &Lo, SDValue &Hi) {
503  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
504  GetSplitOp(Op, Lo, Hi);
505 }
506 
507 void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
508  SDValue LL, LH, RL, RH, CL, CH;
509  SDLoc dl(N);
510  unsigned Opcode = N->getOpcode();
511  GetSplitOp(N->getOperand(1), LL, LH);
512  GetSplitOp(N->getOperand(2), RL, RH);
513 
514  SDValue Cond = N->getOperand(0);
515  CL = CH = Cond;
516  if (Cond.getValueType().isVector()) {
517  if (SDValue Res = WidenVSELECTMask(N))
518  std::tie(CL, CH) = DAG.SplitVector(Res, dl);
519  // Check if there are already splitted versions of the vector available and
520  // use those instead of splitting the mask operand again.
521  else if (getTypeAction(Cond.getValueType()) ==
523  GetSplitVector(Cond, CL, CH);
524  // It seems to improve code to generate two narrow SETCCs as opposed to
525  // splitting a wide result vector.
526  else if (Cond.getOpcode() == ISD::SETCC) {
527  // If the condition is a vXi1 vector, and the LHS of the setcc is a legal
528  // type and the setcc result type is the same vXi1, then leave the setcc
529  // alone.
530  EVT CondLHSVT = Cond.getOperand(0).getValueType();
531  if (Cond.getValueType().getVectorElementType() == MVT::i1 &&
532  isTypeLegal(CondLHSVT) &&
533  getSetCCResultType(CondLHSVT) == Cond.getValueType())
534  std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
535  else
536  SplitVecRes_SETCC(Cond.getNode(), CL, CH);
537  } else
538  std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
539  }
540 
541  if (Opcode != ISD::VP_SELECT && Opcode != ISD::VP_MERGE) {
542  Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL);
543  Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH);
544  return;
545  }
546 
547  SDValue EVLLo, EVLHi;
548  std::tie(EVLLo, EVLHi) =
549  DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
550 
551  Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL, EVLLo);
552  Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);
553 }
554 
555 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
556  SDValue &Hi) {
557  SDValue LL, LH, RL, RH;
558  SDLoc dl(N);
559  GetSplitOp(N->getOperand(2), LL, LH);
560  GetSplitOp(N->getOperand(3), RL, RH);
561 
562  Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0),
563  N->getOperand(1), LL, RL, N->getOperand(4));
564  Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0),
565  N->getOperand(1), LH, RH, N->getOperand(4));
566 }
567 
568 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
569  EVT LoVT, HiVT;
570  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
571  Lo = DAG.getUNDEF(LoVT);
572  Hi = DAG.getUNDEF(HiVT);
573 }
574 
575 void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) {
576  SDValue L, H;
577  SDLoc dl(N);
578  GetSplitOp(N->getOperand(0), L, H);
579 
580  Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L);
581  Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H);
582 }
583 
584 void DAGTypeLegalizer::SplitRes_ARITH_FENCE(SDNode *N, SDValue &Lo,
585  SDValue &Hi) {
586  SDValue L, H;
587  SDLoc DL(N);
588  GetSplitOp(N->getOperand(0), L, H);
589 
590  Lo = DAG.getNode(ISD::ARITH_FENCE, DL, L.getValueType(), L);
591  Hi = DAG.getNode(ISD::ARITH_FENCE, DL, H.getValueType(), H);
592 }
i
i
Definition: README.txt:29
llvm::StoreSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2394
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:886
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1002
llvm::EVT::getVectorElementCount
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:322
llvm::ElementCount
Definition: TypeSize.h:404
llvm::TargetLoweringBase::TypeScalarizeScalableVector
@ TypeScalarizeScalableVector
Definition: TargetLowering.h:216
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::TargetLoweringBase::TypeSoftPromoteHalf
@ TypeSoftPromoteHalf
Definition: TargetLowering.h:215
llvm::MipsISD::Lo
@ Lo
Definition: MipsISelLowering.h:79
CH
AMD64 Optimization Manual has some nice information about optimizing integer multiplication by a constant How much of it applies to Intel s X86 implementation There are definite trade offs to xmm0 cvttss2siq rdx jb L3 subss xmm0 rax cvttss2siq rdx xorq rdx rax ret instead of xmm1 cvttss2siq rcx movaps xmm2 subss xmm2 cvttss2siq rax rdx xorq rax ucomiss xmm0 cmovb rax ret Seems like the jb branch has high likelihood of being taken It would have saved a few instructions It s not possible to reference CH
Definition: README-X86-64.txt:44
llvm::SelectionDAG::getVectorIdxConstant
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1647
llvm::MemSDNode::getChain
const SDValue & getChain() const
Definition: SelectionDAGNodes.h:1378
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2344
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:6874
llvm::AAMDNodes
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:652
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8099
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
llvm::EVT::getVectorVT
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::codeview::EncodedFramePtrReg::StackPtr
@ StackPtr
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::TargetLoweringBase::TypeExpandInteger
@ TypeExpandInteger
Definition: TargetLowering.h:208
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:479
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1150
llvm::MipsISD::Hi
@ Hi
Definition: MipsISelLowering.h:75
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8049
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::EVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:362
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
llvm::SelectionDAG::getObjectPtrOffset
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
Definition: SelectionDAG.h:971
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:9127
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1023
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1515
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:441
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::TargetLoweringBase::TypeWidenVector
@ TypeWidenVector
Definition: TargetLowering.h:213
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::EVT::bitsLT
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:272
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:245
llvm::SelectionDAG::SplitVector
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
Definition: SelectionDAG.cpp:11597
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1359
llvm::TargetLoweringBase::TypeSoftenFloat
@ TypeSoftenFloat
Definition: TargetLowering.h:209
llvm::TargetLoweringBase::TypePromoteInteger
@ TypePromoteInteger
Definition: TargetLowering.h:207
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:39
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9133
llvm::MemSDNode::getOriginalAlign
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
Definition: SelectionDAGNodes.h:1292
llvm::TargetLoweringBase::hasBigEndianPartOrdering
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
Definition: TargetLowering.h:1651
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2372
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:215
llvm::StoreSDNode::getValue
const SDValue & getValue() const
Definition: SelectionDAGNodes.h:2393
LegalizeTypes.h
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2343
llvm::MachinePointerInfo::getWithOffset
MachinePointerInfo getWithOffset(int64_t O) const
Definition: MachineMemOperand.h:79
llvm::MemSDNode::getAAInfo
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Definition: SelectionDAGNodes.h:1323
llvm::MemSDNode::isAtomic
bool isAtomic() const
Return true if the memory operation ordering is Unordered or higher.
Definition: SelectionDAGNodes.h:1344
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:154
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3045
DataLayout.h
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::SelectionDAG::getReducedAlign
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
Definition: SelectionDAG.cpp:2315
llvm::SelectionDAG::getBuildVector
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:822
llvm::TargetLoweringBase::TypeLegal
@ TypeLegal
Definition: TargetLowering.h:206
H
#define H(x, y, z)
Definition: MD5.cpp:57
llvm::SelectionDAG::SplitEVL
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
Definition: SelectionDAG.cpp:11618
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:351
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:548
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:469
llvm::pdb::PDB_LocType::Slot
@ Slot
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::TargetLoweringBase::TypeExpandFloat
@ TypeExpandFloat
Definition: TargetLowering.h:210
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::TargetLoweringBase::TypePromoteFloat
@ TypePromoteFloat
Definition: TargetLowering.h:214
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:475
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
llvm::SelectionDAG::GetSplitDestVTs
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
Definition: SelectionDAG.cpp:11552
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1019
llvm::TargetLoweringBase::TypeScalarizeVector
@ TypeScalarizeVector
Definition: TargetLowering.h:211
N
#define N
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:466
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3083
llvm::TargetLoweringBase::TypeSplitVector
@ TypeSplitVector
Definition: TargetLowering.h:212
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:667
llvm::ISD::INSERT_VECTOR_ELT
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:523
llvm::MachineMemOperand::getFlags
Flags getFlags() const
Return the raw flags of the source value,.
Definition: MachineMemOperand.h:219
llvm::MemSDNode::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition: SelectionDAGNodes.h:1361
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52