LLVM  15.0.0git
LegalizeTypesGeneric.cpp
Go to the documentation of this file.
1 //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements generic type expansion and splitting for LegalizeTypes.
10 // The routines here perform legalization when the details of the type (such as
11 // whether it is an integer or a float) do not matter.
12 // Expansion is the act of changing a computation in an illegal type to be a
13 // computation in two identical registers of a smaller type. The Lo/Hi part
14 // is required to be stored first in memory on little/big-endian machines.
15 // Splitting is the act of changing a computation in an illegal type to be a
16 // computation in two not necessarily identical registers of a smaller type.
17 // There are no requirements on how the type is represented in memory.
18 //
19 //===----------------------------------------------------------------------===//
20 
21 #include "LegalizeTypes.h"
22 #include "llvm/IR/DataLayout.h"
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "legalize-types"
26 
27 //===----------------------------------------------------------------------===//
28 // Generic Result Expansion.
29 //===----------------------------------------------------------------------===//
30 
31 // These routines assume that the Lo/Hi part is stored first in memory on
32 // little/big-endian machines, followed by the Hi/Lo part. This means that
33 // they cannot be used as is on vectors, for which Lo is always stored first.
34 void DAGTypeLegalizer::ExpandRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
35  SDValue &Lo, SDValue &Hi) {
36  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
37  GetExpandedOp(Op, Lo, Hi);
38 }
39 
40 void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
41  EVT OutVT = N->getValueType(0);
42  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
43  SDValue InOp = N->getOperand(0);
44  EVT InVT = InOp.getValueType();
45  SDLoc dl(N);
46 
47  // Handle some special cases efficiently.
48  switch (getTypeAction(InVT)) {
51  break;
54  llvm_unreachable("Bitcast of a promotion-needing float should never need"
55  "expansion");
57  SplitInteger(GetSoftenedFloat(InOp), Lo, Hi);
58  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
59  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
60  return;
63  auto &DL = DAG.getDataLayout();
64  // Convert the expanded pieces of the input.
65  GetExpandedOp(InOp, Lo, Hi);
66  if (TLI.hasBigEndianPartOrdering(InVT, DL) !=
67  TLI.hasBigEndianPartOrdering(OutVT, DL))
68  std::swap(Lo, Hi);
69  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
70  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
71  return;
72  }
74  GetSplitVector(InOp, Lo, Hi);
75  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
76  std::swap(Lo, Hi);
77  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
78  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
79  return;
81  // Convert the element instead.
82  SplitInteger(BitConvertToInteger(GetScalarizedVector(InOp)), Lo, Hi);
83  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
84  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
85  return;
87  report_fatal_error("Scalarization of scalable vectors is not supported.");
89  assert(!(InVT.getVectorNumElements() & 1) && "Unsupported BITCAST");
90  InOp = GetWidenedVector(InOp);
91  EVT LoVT, HiVT;
92  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(InVT);
93  std::tie(Lo, Hi) = DAG.SplitVector(InOp, dl, LoVT, HiVT);
94  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
95  std::swap(Lo, Hi);
96  Lo = DAG.getNode(ISD::BITCAST, dl, NOutVT, Lo);
97  Hi = DAG.getNode(ISD::BITCAST, dl, NOutVT, Hi);
98  return;
99  }
100  }
101 
102  if (InVT.isVector() && OutVT.isInteger()) {
103  // Handle cases like i64 = BITCAST v1i64 on x86, where the operand
104  // is legal but the result is not.
105  unsigned NumElems = 2;
106  EVT ElemVT = NOutVT;
107  EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
108 
109  // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
110  while (!isTypeLegal(NVT)) {
111  unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
112  // If the element size is smaller than byte, bail.
113  if (NewSizeInBits < 8)
114  break;
115  NumElems *= 2;
116  ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
117  NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
118  }
119 
120  if (isTypeLegal(NVT)) {
121  SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
122 
124  for (unsigned i = 0; i < NumElems; ++i)
125  Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
126  CastInOp, DAG.getVectorIdxConstant(i, dl)));
127 
128  // Build Lo, Hi pair by pairing extracted elements if needed.
129  unsigned Slot = 0;
130  for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
131  // Each iteration will BUILD_PAIR two nodes and append the result until
132  // there are only two nodes left, i.e. Lo and Hi.
133  SDValue LHS = Vals[Slot];
134  SDValue RHS = Vals[Slot + 1];
135 
136  if (DAG.getDataLayout().isBigEndian())
137  std::swap(LHS, RHS);
138 
139  Vals.push_back(DAG.getNode(
140  ISD::BUILD_PAIR, dl,
141  EVT::getIntegerVT(*DAG.getContext(), LHS.getValueSizeInBits() << 1),
142  LHS, RHS));
143  }
144  Lo = Vals[Slot++];
145  Hi = Vals[Slot++];
146 
147  if (DAG.getDataLayout().isBigEndian())
148  std::swap(Lo, Hi);
149 
150  return;
151  }
152  }
153 
154  // Lower the bit-convert to a store/load from the stack.
155  assert(NOutVT.isByteSized() && "Expanded type not byte sized!");
156 
157  // Create the stack frame object. Make sure it is aligned for both
158  // the source and expanded destination types.
159 
160  // In cases where the vector is illegal it will be broken down into parts
161  // and stored in parts - we should use the alignment for the smallest part.
162  Align InAlign = DAG.getReducedAlign(InVT, /*UseABI=*/false);
163  Align NOutAlign = DAG.getReducedAlign(NOutVT, /*UseABI=*/false);
164  Align Align = std::max(InAlign, NOutAlign);
166  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
167  MachinePointerInfo PtrInfo =
169 
170  // Emit a store to the stack slot.
171  SDValue Store = DAG.getStore(DAG.getEntryNode(), dl, InOp, StackPtr, PtrInfo);
172 
173  // Load the first half from the stack slot.
174  Lo = DAG.getLoad(NOutVT, dl, Store, StackPtr, PtrInfo, NOutAlign);
175 
176  // Increment the pointer to the other half.
177  unsigned IncrementSize = NOutVT.getSizeInBits() / 8;
178  StackPtr =
179  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(IncrementSize), dl);
180 
181  // Load the second half from the stack slot.
182  Hi = DAG.getLoad(NOutVT, dl, Store, StackPtr,
183  PtrInfo.getWithOffset(IncrementSize), NOutAlign);
184 
185  // Handle endianness of the load.
186  if (TLI.hasBigEndianPartOrdering(OutVT, DAG.getDataLayout()))
187  std::swap(Lo, Hi);
188 }
189 
190 void DAGTypeLegalizer::ExpandRes_BUILD_PAIR(SDNode *N, SDValue &Lo,
191  SDValue &Hi) {
192  // Return the operands.
193  Lo = N->getOperand(0);
194  Hi = N->getOperand(1);
195 }
196 
197 void DAGTypeLegalizer::ExpandRes_EXTRACT_ELEMENT(SDNode *N, SDValue &Lo,
198  SDValue &Hi) {
199  GetExpandedOp(N->getOperand(0), Lo, Hi);
200  SDValue Part = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ?
201  Hi : Lo;
202 
203  assert(Part.getValueType() == N->getValueType(0) &&
204  "Type twice as big as expanded type not itself expanded!");
205 
206  GetPairElements(Part, Lo, Hi);
207 }
208 
209 void DAGTypeLegalizer::ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo,
210  SDValue &Hi) {
211  SDValue OldVec = N->getOperand(0);
212  unsigned OldElts = OldVec.getValueType().getVectorNumElements();
213  EVT OldEltVT = OldVec.getValueType().getVectorElementType();
214  SDLoc dl(N);
215 
216  // Convert to a vector of the expanded element type, for example
217  // <3 x i64> -> <6 x i32>.
218  EVT OldVT = N->getValueType(0);
219  EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
220 
221  if (OldVT != OldEltVT) {
222  // The result of EXTRACT_VECTOR_ELT may be larger than the element type of
223  // the input vector. If so, extend the elements of the input vector to the
224  // same bitwidth as the result before expanding.
225  assert(OldEltVT.bitsLT(OldVT) && "Result type smaller then element type!");
226  EVT NVecVT = EVT::getVectorVT(*DAG.getContext(), OldVT, OldElts);
227  OldVec = DAG.getNode(ISD::ANY_EXTEND, dl, NVecVT, N->getOperand(0));
228  }
229 
230  SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
232  NewVT, 2*OldElts),
233  OldVec);
234 
235  // Extract the elements at 2 * Idx and 2 * Idx + 1 from the new vector.
236  SDValue Idx = N->getOperand(1);
237 
238  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
239  Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
240 
241  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx,
242  DAG.getConstant(1, dl, Idx.getValueType()));
243  Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NewVT, NewVec, Idx);
244 
245  if (DAG.getDataLayout().isBigEndian())
246  std::swap(Lo, Hi);
247 }
248 
249 void DAGTypeLegalizer::ExpandRes_NormalLoad(SDNode *N, SDValue &Lo,
250  SDValue &Hi) {
251  assert(ISD::isNormalLoad(N) && "This routine only for normal loads!");
252  SDLoc dl(N);
253 
254  LoadSDNode *LD = cast<LoadSDNode>(N);
255  assert(!LD->isAtomic() && "Atomics can not be split");
256  EVT ValueVT = LD->getValueType(0);
257  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
258  SDValue Chain = LD->getChain();
259  SDValue Ptr = LD->getBasePtr();
260  AAMDNodes AAInfo = LD->getAAInfo();
261 
262  assert(NVT.isByteSized() && "Expanded type not byte sized!");
263 
264  Lo = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getPointerInfo(),
265  LD->getOriginalAlign(), LD->getMemOperand()->getFlags(),
266  AAInfo);
267 
268  // Increment the pointer to the other half.
269  unsigned IncrementSize = NVT.getSizeInBits() / 8;
270  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
271  Hi = DAG.getLoad(
272  NVT, dl, Chain, Ptr, LD->getPointerInfo().getWithOffset(IncrementSize),
273  LD->getOriginalAlign(), LD->getMemOperand()->getFlags(), AAInfo);
274 
275  // Build a factor node to remember that this load is independent of the
276  // other one.
277  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
278  Hi.getValue(1));
279 
280  // Handle endianness of the load.
281  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
282  std::swap(Lo, Hi);
283 
284  // Modified the chain - switch anything that used the old chain to use
285  // the new one.
286  ReplaceValueWith(SDValue(N, 1), Chain);
287 }
288 
289 void DAGTypeLegalizer::ExpandRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) {
290  EVT OVT = N->getValueType(0);
291  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
292  SDValue Chain = N->getOperand(0);
293  SDValue Ptr = N->getOperand(1);
294  SDLoc dl(N);
295  const unsigned Align = N->getConstantOperandVal(3);
296 
297  Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2), Align);
298  Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, N->getOperand(2), 0);
299  Chain = Hi.getValue(1);
300 
301  // Handle endianness of the load.
302  if (TLI.hasBigEndianPartOrdering(OVT, DAG.getDataLayout()))
303  std::swap(Lo, Hi);
304 
305  // Modified the chain - switch anything that used the old chain to use
306  // the new one.
307  ReplaceValueWith(SDValue(N, 1), Chain);
308 }
309 
310 
311 //===--------------------------------------------------------------------===//
312 // Generic Operand Expansion.
313 //===--------------------------------------------------------------------===//
314 
315 void DAGTypeLegalizer::IntegerToVector(SDValue Op, unsigned NumElements,
317  EVT EltVT) {
318  assert(Op.getValueType().isInteger());
319  SDLoc DL(Op);
320  SDValue Parts[2];
321 
322  if (NumElements > 1) {
323  NumElements >>= 1;
324  SplitInteger(Op, Parts[0], Parts[1]);
325  if (DAG.getDataLayout().isBigEndian())
326  std::swap(Parts[0], Parts[1]);
327  IntegerToVector(Parts[0], NumElements, Ops, EltVT);
328  IntegerToVector(Parts[1], NumElements, Ops, EltVT);
329  } else {
330  Ops.push_back(DAG.getNode(ISD::BITCAST, DL, EltVT, Op));
331  }
332 }
333 
334 SDValue DAGTypeLegalizer::ExpandOp_BITCAST(SDNode *N) {
335  SDLoc dl(N);
336  if (N->getValueType(0).isVector() &&
337  N->getOperand(0).getValueType().isInteger()) {
338  // An illegal expanding type is being converted to a legal vector type.
339  // Make a two element vector out of the expanded parts and convert that
340  // instead, but only if the new vector type is legal (otherwise there
341  // is no point, and it might create expansion loops). For example, on
342  // x86 this turns v1i64 = BITCAST i64 into v1i64 = BITCAST v2i32.
343  //
344  // FIXME: I'm not sure why we are first trying to split the input into
345  // a 2 element vector, so I'm leaving it here to maintain the current
346  // behavior.
347  unsigned NumElts = 2;
348  EVT OVT = N->getOperand(0).getValueType();
349  EVT NVT = EVT::getVectorVT(*DAG.getContext(),
350  TLI.getTypeToTransformTo(*DAG.getContext(), OVT),
351  NumElts);
352  if (!isTypeLegal(NVT)) {
353  // If we can't find a legal type by splitting the integer in half,
354  // then we can use the node's value type.
355  NumElts = N->getValueType(0).getVectorNumElements();
356  NVT = N->getValueType(0);
357  }
358 
360  IntegerToVector(N->getOperand(0), NumElts, Ops, NVT.getVectorElementType());
361 
362  SDValue Vec =
363  DAG.getBuildVector(NVT, dl, makeArrayRef(Ops.data(), NumElts));
364  return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), Vec);
365  }
366 
367  // Otherwise, store to a temporary and load out again as the new type.
368  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
369 }
370 
371 SDValue DAGTypeLegalizer::ExpandOp_BUILD_VECTOR(SDNode *N) {
372  // The vector type is legal but the element type needs expansion.
373  EVT VecVT = N->getValueType(0);
374  unsigned NumElts = VecVT.getVectorNumElements();
375  EVT OldVT = N->getOperand(0).getValueType();
376  EVT NewVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldVT);
377  SDLoc dl(N);
378 
379  assert(OldVT == VecVT.getVectorElementType() &&
380  "BUILD_VECTOR operand type doesn't match vector element type!");
381 
382  // Build a vector of twice the length out of the expanded elements.
383  // For example <3 x i64> -> <6 x i32>.
384  SmallVector<SDValue, 16> NewElts;
385  NewElts.reserve(NumElts*2);
386 
387  for (unsigned i = 0; i < NumElts; ++i) {
388  SDValue Lo, Hi;
389  GetExpandedOp(N->getOperand(i), Lo, Hi);
390  if (DAG.getDataLayout().isBigEndian())
391  std::swap(Lo, Hi);
392  NewElts.push_back(Lo);
393  NewElts.push_back(Hi);
394  }
395 
396  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewVT, NewElts.size());
397  SDValue NewVec = DAG.getBuildVector(NewVecVT, dl, NewElts);
398 
399  // Convert the new vector to the old vector type.
400  return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
401 }
402 
403 SDValue DAGTypeLegalizer::ExpandOp_EXTRACT_ELEMENT(SDNode *N) {
404  SDValue Lo, Hi;
405  GetExpandedOp(N->getOperand(0), Lo, Hi);
406  return cast<ConstantSDNode>(N->getOperand(1))->getZExtValue() ? Hi : Lo;
407 }
408 
409 SDValue DAGTypeLegalizer::ExpandOp_INSERT_VECTOR_ELT(SDNode *N) {
410  // The vector type is legal but the element type needs expansion.
411  EVT VecVT = N->getValueType(0);
412  unsigned NumElts = VecVT.getVectorNumElements();
413  SDLoc dl(N);
414 
415  SDValue Val = N->getOperand(1);
416  EVT OldEVT = Val.getValueType();
417  EVT NewEVT = TLI.getTypeToTransformTo(*DAG.getContext(), OldEVT);
418 
419  assert(OldEVT == VecVT.getVectorElementType() &&
420  "Inserted element type doesn't match vector element type!");
421 
422  // Bitconvert to a vector of twice the length with elements of the expanded
423  // type, insert the expanded vector elements, and then convert back.
424  EVT NewVecVT = EVT::getVectorVT(*DAG.getContext(), NewEVT, NumElts*2);
425  SDValue NewVec = DAG.getNode(ISD::BITCAST, dl,
426  NewVecVT, N->getOperand(0));
427 
428  SDValue Lo, Hi;
429  GetExpandedOp(Val, Lo, Hi);
430  if (DAG.getDataLayout().isBigEndian())
431  std::swap(Lo, Hi);
432 
433  SDValue Idx = N->getOperand(2);
434  Idx = DAG.getNode(ISD::ADD, dl, Idx.getValueType(), Idx, Idx);
435  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Lo, Idx);
436  Idx = DAG.getNode(ISD::ADD, dl,
437  Idx.getValueType(), Idx,
438  DAG.getConstant(1, dl, Idx.getValueType()));
439  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NewVecVT, NewVec, Hi, Idx);
440 
441  // Convert the new vector to the old vector type.
442  return DAG.getNode(ISD::BITCAST, dl, VecVT, NewVec);
443 }
444 
445 SDValue DAGTypeLegalizer::ExpandOp_SCALAR_TO_VECTOR(SDNode *N) {
446  SDLoc dl(N);
447  EVT VT = N->getValueType(0);
448  assert(VT.getVectorElementType() == N->getOperand(0).getValueType() &&
449  "SCALAR_TO_VECTOR operand type doesn't match vector element type!");
450  unsigned NumElts = VT.getVectorNumElements();
451  SmallVector<SDValue, 16> Ops(NumElts);
452  Ops[0] = N->getOperand(0);
453  SDValue UndefVal = DAG.getUNDEF(Ops[0].getValueType());
454  for (unsigned i = 1; i < NumElts; ++i)
455  Ops[i] = UndefVal;
456  return DAG.getBuildVector(VT, dl, Ops);
457 }
458 
459 SDValue DAGTypeLegalizer::ExpandOp_NormalStore(SDNode *N, unsigned OpNo) {
460  assert(ISD::isNormalStore(N) && "This routine only for normal stores!");
461  assert(OpNo == 1 && "Can only expand the stored value so far");
462  SDLoc dl(N);
463 
464  StoreSDNode *St = cast<StoreSDNode>(N);
465  assert(!St->isAtomic() && "Atomics can not be split");
466  EVT ValueVT = St->getValue().getValueType();
467  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), ValueVT);
468  SDValue Chain = St->getChain();
469  SDValue Ptr = St->getBasePtr();
470  AAMDNodes AAInfo = St->getAAInfo();
471 
472  assert(NVT.isByteSized() && "Expanded type not byte sized!");
473  unsigned IncrementSize = NVT.getSizeInBits() / 8;
474 
475  SDValue Lo, Hi;
476  GetExpandedOp(St->getValue(), Lo, Hi);
477 
478  if (TLI.hasBigEndianPartOrdering(ValueVT, DAG.getDataLayout()))
479  std::swap(Lo, Hi);
480 
481  Lo = DAG.getStore(Chain, dl, Lo, Ptr, St->getPointerInfo(),
482  St->getOriginalAlign(), St->getMemOperand()->getFlags(),
483  AAInfo);
484 
485  Ptr = DAG.getObjectPtrOffset(dl, Ptr, TypeSize::Fixed(IncrementSize));
486  Hi = DAG.getStore(
487  Chain, dl, Hi, Ptr, St->getPointerInfo().getWithOffset(IncrementSize),
488  St->getOriginalAlign(), St->getMemOperand()->getFlags(), AAInfo);
489 
490  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
491 }
492 
493 
494 //===--------------------------------------------------------------------===//
495 // Generic Result Splitting.
496 //===--------------------------------------------------------------------===//
497 
498 // Be careful to make no assumptions about which of Lo/Hi is stored first in
499 // memory (for vectors it is always Lo first followed by Hi in the following
500 // bytes; for integers and floats it is Lo first if and only if the machine is
501 // little-endian).
502 
503 void DAGTypeLegalizer::SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
504  SDValue &Lo, SDValue &Hi) {
505  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
506  GetSplitOp(Op, Lo, Hi);
507 }
508 
509 void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
510  SDValue LL, LH, RL, RH, CL, CH;
511  SDLoc dl(N);
512  unsigned Opcode = N->getOpcode();
513  GetSplitOp(N->getOperand(1), LL, LH);
514  GetSplitOp(N->getOperand(2), RL, RH);
515 
516  SDValue Cond = N->getOperand(0);
517  CL = CH = Cond;
518  if (Cond.getValueType().isVector()) {
519  if (SDValue Res = WidenVSELECTMask(N))
520  std::tie(CL, CH) = DAG.SplitVector(Res, dl);
521  // Check if there are already splitted versions of the vector available and
522  // use those instead of splitting the mask operand again.
523  else if (getTypeAction(Cond.getValueType()) ==
525  GetSplitVector(Cond, CL, CH);
526  // It seems to improve code to generate two narrow SETCCs as opposed to
527  // splitting a wide result vector.
528  else if (Cond.getOpcode() == ISD::SETCC) {
529  // If the condition is a vXi1 vector, and the LHS of the setcc is a legal
530  // type and the setcc result type is the same vXi1, then leave the setcc
531  // alone.
532  EVT CondLHSVT = Cond.getOperand(0).getValueType();
533  if (Cond.getValueType().getVectorElementType() == MVT::i1 &&
534  isTypeLegal(CondLHSVT) &&
535  getSetCCResultType(CondLHSVT) == Cond.getValueType())
536  std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
537  else
538  SplitVecRes_SETCC(Cond.getNode(), CL, CH);
539  } else
540  std::tie(CL, CH) = DAG.SplitVector(Cond, dl);
541  }
542 
543  if (Opcode != ISD::VP_SELECT && Opcode != ISD::VP_MERGE) {
544  Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL);
545  Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH);
546  return;
547  }
548 
549  SDValue EVLLo, EVLHi;
550  std::tie(EVLLo, EVLHi) =
551  DAG.SplitEVL(N->getOperand(3), N->getValueType(0), dl);
552 
553  Lo = DAG.getNode(Opcode, dl, LL.getValueType(), CL, LL, RL, EVLLo);
554  Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);
555 }
556 
557 void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
558  SDValue &Hi) {
559  SDValue LL, LH, RL, RH;
560  SDLoc dl(N);
561  GetSplitOp(N->getOperand(2), LL, LH);
562  GetSplitOp(N->getOperand(3), RL, RH);
563 
564  Lo = DAG.getNode(ISD::SELECT_CC, dl, LL.getValueType(), N->getOperand(0),
565  N->getOperand(1), LL, RL, N->getOperand(4));
566  Hi = DAG.getNode(ISD::SELECT_CC, dl, LH.getValueType(), N->getOperand(0),
567  N->getOperand(1), LH, RH, N->getOperand(4));
568 }
569 
570 void DAGTypeLegalizer::SplitRes_UNDEF(SDNode *N, SDValue &Lo, SDValue &Hi) {
571  EVT LoVT, HiVT;
572  std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
573  Lo = DAG.getUNDEF(LoVT);
574  Hi = DAG.getUNDEF(HiVT);
575 }
576 
577 void DAGTypeLegalizer::SplitRes_FREEZE(SDNode *N, SDValue &Lo, SDValue &Hi) {
578  SDValue L, H;
579  SDLoc dl(N);
580  GetSplitOp(N->getOperand(0), L, H);
581 
582  Lo = DAG.getNode(ISD::FREEZE, dl, L.getValueType(), L);
583  Hi = DAG.getNode(ISD::FREEZE, dl, H.getValueType(), H);
584 }
585 
586 void DAGTypeLegalizer::SplitRes_ARITH_FENCE(SDNode *N, SDValue &Lo,
587  SDValue &Hi) {
588  SDValue L, H;
589  SDLoc DL(N);
590  GetSplitOp(N->getOperand(0), L, H);
591 
592  Lo = DAG.getNode(ISD::ARITH_FENCE, DL, L.getValueType(), L);
593  Hi = DAG.getNode(ISD::ARITH_FENCE, DL, H.getValueType(), H);
594 }
i
i
Definition: README.txt:29
llvm::StoreSDNode::getBasePtr
const SDValue & getBasePtr() const
Definition: SelectionDAGNodes.h:2364
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1090
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:886
llvm::TargetLoweringBase::TypeScalarizeScalableVector
@ TypeScalarizeScalableVector
Definition: TargetLowering.h:216
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::TargetLoweringBase::TypeSoftPromoteHalf
@ TypeSoftPromoteHalf
Definition: TargetLowering.h:215
llvm::MipsISD::Lo
@ Lo
Definition: MipsISelLowering.h:79
CH
AMD64 Optimization Manual has some nice information about optimizing integer multiplication by a constant How much of it applies to Intel s X86 implementation There are definite trade offs to xmm0 cvttss2siq rdx jb L3 subss xmm0 rax cvttss2siq rdx xorq rdx rax ret instead of xmm1 cvttss2siq rcx movaps xmm2 subss xmm2 cvttss2siq rax rdx xorq rax ucomiss xmm0 cmovb rax ret Seems like the jb branch has high likelihood of being taken It would have saved a few instructions It s not possible to reference CH
Definition: README-X86-64.txt:44
llvm::SelectionDAG::getVectorIdxConstant
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1584
llvm::MemSDNode::getChain
const SDValue & getChain() const
Definition: SelectionDAGNodes.h:1364
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:454
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2314
llvm::SelectionDAG::getMemBasePlusOffset
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
Definition: SelectionDAG.cpp:6608
llvm::AAMDNodes
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:652
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:7804
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
llvm::EVT::getVectorVT
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::codeview::EncodedFramePtrReg::StackPtr
@ StackPtr
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
llvm::TargetLoweringBase::TypeExpandInteger
@ TypeExpandInteger
Definition: TargetLowering.h:208
llvm::ISD::FREEZE
@ FREEZE
Definition: ISDOpcodes.h:216
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:462
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1150
llvm::MipsISD::Hi
@ Hi
Definition: MipsISelLowering.h:75
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:7754
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::EVT::getStoreSize
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:362
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1125
llvm::SelectionDAG::getObjectPtrOffset
SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, TypeSize Offset)
Create an add instruction with appropriate flags when used for addressing some offset of an object.
Definition: SelectionDAG.h:926
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:8832
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:971
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1449
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:308
llvm::TypeSize::Fixed
static TypeSize Fixed(ScalarTy MinVal)
Definition: TypeSize.h:441
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
llvm::TargetLoweringBase::TypeWidenVector
@ TypeWidenVector
Definition: TargetLowering.h:213
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::EVT::bitsLT
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:272
llvm::TargetLoweringBase::getTypeToTransformTo
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:981
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:245
llvm::SelectionDAG::SplitVector
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provides VTs and return the low/high part.
Definition: SelectionDAG.cpp:11214
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1345
llvm::TargetLoweringBase::TypeSoftenFloat
@ TypeSoftenFloat
Definition: TargetLowering.h:209
llvm::TargetLoweringBase::TypePromoteInteger
@ TypePromoteInteger
Definition: TargetLowering.h:207
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:39
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8838
llvm::MemSDNode::getOriginalAlign
Align getOriginalAlign() const
Returns alignment and volatility of the memory access.
Definition: SelectionDAGNodes.h:1276
llvm::TargetLoweringBase::hasBigEndianPartOrdering
bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const
When splitting a value of the specified type into parts, does the Lo or Hi part come first?...
Definition: TargetLowering.h:1613
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2342
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:215
llvm::StoreSDNode::getValue
const SDValue & getValue() const
Definition: SelectionDAGNodes.h:2363
LegalizeTypes.h
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2282
llvm::MachinePointerInfo::getWithOffset
MachinePointerInfo getWithOffset(int64_t O) const
Definition: MachineMemOperand.h:79
llvm::MemSDNode::getAAInfo
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
Definition: SelectionDAGNodes.h:1309
llvm::MemSDNode::isAtomic
bool isAtomic() const
Return true if the memory operation ordering is Unordered or higher.
Definition: SelectionDAGNodes.h:1330
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:154
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3015
DataLayout.h
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::SelectionDAG::getReducedAlign
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
Definition: SelectionDAG.cpp:2254
llvm::SelectionDAG::getBuildVector
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:805
llvm::TargetLoweringBase::TypeLegal
@ TypeLegal
Definition: TargetLowering.h:206
H
#define H(x, y, z)
Definition: MD5.cpp:57
llvm::SelectionDAG::SplitEVL
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
Definition: SelectionDAG.cpp:11235
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:345
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:531
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:452
llvm::pdb::PDB_LocType::Slot
@ Slot
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:137
llvm::TargetLoweringBase::TypeExpandFloat
@ TypeExpandFloat
Definition: TargetLowering.h:210
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::TargetLoweringBase::TypePromoteFloat
@ TypePromoteFloat
Definition: TargetLowering.h:214
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:475
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:300
llvm::SelectionDAG::GetSplitDestVTs
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
Definition: SelectionDAG.cpp:11169
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1006
llvm::TargetLoweringBase::TypeScalarizeVector
@ TypeScalarizeVector
Definition: TargetLowering.h:211
N
#define N
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:449
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3053
llvm::TargetLoweringBase::TypeSplitVector
@ TypeSplitVector
Definition: TargetLowering.h:212
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:644
llvm::ISD::INSERT_VECTOR_ELT
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:523
llvm::MachineMemOperand::getFlags
Flags getFlags() const
Return the raw flags of the source value,.
Definition: MachineMemOperand.h:219
llvm::MemSDNode::getPointerInfo
const MachinePointerInfo & getPointerInfo() const
Definition: SelectionDAGNodes.h:1347
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52