LLVM 19.0.0git
SelectionDAG.cpp
Go to the documentation of this file.
1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/Twine.h"
50#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
64#include "llvm/Support/Debug.h"
68#include "llvm/Support/Mutex.h"
74#include <algorithm>
75#include <cassert>
76#include <cstdint>
77#include <cstdlib>
78#include <limits>
79#include <set>
80#include <string>
81#include <utility>
82#include <vector>
83
84using namespace llvm;
85using namespace llvm::SDPatternMatch;
86
87/// makeVTList - Return an instance of the SDVTList struct initialized with the
88/// specified members.
89static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
90 SDVTList Res = {VTs, NumVTs};
91 return Res;
92}
93
94// Default null implementations of the callbacks.
98
99void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100void SelectionDAG::DAGNodeInsertedListener::anchor() {}
101
102#define DEBUG_TYPE "selectiondag"
103
104static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
105 cl::Hidden, cl::init(true),
106 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
107
108static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
109 cl::desc("Number limit for gluing ld/st of memcpy."),
110 cl::Hidden, cl::init(0));
111
113 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
114}
115
116//===----------------------------------------------------------------------===//
117// ConstantFPSDNode Class
118//===----------------------------------------------------------------------===//
119
120/// isExactlyValue - We don't rely on operator== working on double values, as
121/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
122/// As such, this method can be used to do an exact bit-for-bit comparison of
123/// two floating point values.
125 return getValueAPF().bitwiseIsEqual(V);
126}
127
129 const APFloat& Val) {
130 assert(VT.isFloatingPoint() && "Can only convert between FP types");
131
132 // convert modifies in place, so make a copy.
133 APFloat Val2 = APFloat(Val);
134 bool losesInfo;
137 &losesInfo);
138 return !losesInfo;
139}
140
141//===----------------------------------------------------------------------===//
142// ISD Namespace
143//===----------------------------------------------------------------------===//
144
145bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
146 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
147 unsigned EltSize =
148 N->getValueType(0).getVectorElementType().getSizeInBits();
149 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
150 SplatVal = Op0->getAPIntValue().trunc(EltSize);
151 return true;
152 }
153 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
154 SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize);
155 return true;
156 }
157 }
158
159 auto *BV = dyn_cast<BuildVectorSDNode>(N);
160 if (!BV)
161 return false;
162
163 APInt SplatUndef;
164 unsigned SplatBitSize;
165 bool HasUndefs;
166 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
167 // Endianness does not matter here. We are checking for a splat given the
168 // element size of the vector, and if we find such a splat for little endian
169 // layout, then that should be valid also for big endian (as the full vector
170 // size is known to be a multiple of the element size).
171 const bool IsBigEndian = false;
172 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
173 EltSize, IsBigEndian) &&
174 EltSize == SplatBitSize;
175}
176
177// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
178// specializations of the more general isConstantSplatVector()?
179
180bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
181 // Look through a bit convert.
182 while (N->getOpcode() == ISD::BITCAST)
183 N = N->getOperand(0).getNode();
184
185 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
186 APInt SplatVal;
187 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
188 }
189
190 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
191
192 unsigned i = 0, e = N->getNumOperands();
193
194 // Skip over all of the undef values.
195 while (i != e && N->getOperand(i).isUndef())
196 ++i;
197
198 // Do not accept an all-undef vector.
199 if (i == e) return false;
200
201 // Do not accept build_vectors that aren't all constants or which have non-~0
202 // elements. We have to be a bit careful here, as the type of the constant
203 // may not be the same as the type of the vector elements due to type
204 // legalization (the elements are promoted to a legal type for the target and
205 // a vector of a type may be legal when the base element type is not).
206 // We only want to check enough bits to cover the vector elements, because
207 // we care if the resultant vector is all ones, not whether the individual
208 // constants are.
209 SDValue NotZero = N->getOperand(i);
210 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
211 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
212 if (CN->getAPIntValue().countr_one() < EltSize)
213 return false;
214 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
215 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
216 return false;
217 } else
218 return false;
219
220 // Okay, we have at least one ~0 value, check to see if the rest match or are
221 // undefs. Even with the above element type twiddling, this should be OK, as
222 // the same type legalization should have applied to all the elements.
223 for (++i; i != e; ++i)
224 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
225 return false;
226 return true;
227}
228
229bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
230 // Look through a bit convert.
231 while (N->getOpcode() == ISD::BITCAST)
232 N = N->getOperand(0).getNode();
233
234 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
235 APInt SplatVal;
236 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
237 }
238
239 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
240
241 bool IsAllUndef = true;
242 for (const SDValue &Op : N->op_values()) {
243 if (Op.isUndef())
244 continue;
245 IsAllUndef = false;
246 // Do not accept build_vectors that aren't all constants or which have non-0
247 // elements. We have to be a bit careful here, as the type of the constant
248 // may not be the same as the type of the vector elements due to type
249 // legalization (the elements are promoted to a legal type for the target
250 // and a vector of a type may be legal when the base element type is not).
251 // We only want to check enough bits to cover the vector elements, because
252 // we care if the resultant vector is all zeros, not whether the individual
253 // constants are.
254 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
255 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
256 if (CN->getAPIntValue().countr_zero() < EltSize)
257 return false;
258 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
259 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
260 return false;
261 } else
262 return false;
263 }
264
265 // Do not accept an all-undef vector.
266 if (IsAllUndef)
267 return false;
268 return true;
269}
270
272 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
273}
274
276 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
277}
278
280 if (N->getOpcode() != ISD::BUILD_VECTOR)
281 return false;
282
283 for (const SDValue &Op : N->op_values()) {
284 if (Op.isUndef())
285 continue;
286 if (!isa<ConstantSDNode>(Op))
287 return false;
288 }
289 return true;
290}
291
293 if (N->getOpcode() != ISD::BUILD_VECTOR)
294 return false;
295
296 for (const SDValue &Op : N->op_values()) {
297 if (Op.isUndef())
298 continue;
299 if (!isa<ConstantFPSDNode>(Op))
300 return false;
301 }
302 return true;
303}
304
305bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
306 bool Signed) {
307 assert(N->getValueType(0).isVector() && "Expected a vector!");
308
309 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
310 if (EltSize <= NewEltSize)
311 return false;
312
313 if (N->getOpcode() == ISD::ZERO_EXTEND) {
314 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
315 NewEltSize) &&
316 !Signed;
317 }
318 if (N->getOpcode() == ISD::SIGN_EXTEND) {
319 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
320 NewEltSize) &&
321 Signed;
322 }
323 if (N->getOpcode() != ISD::BUILD_VECTOR)
324 return false;
325
326 for (const SDValue &Op : N->op_values()) {
327 if (Op.isUndef())
328 continue;
329 if (!isa<ConstantSDNode>(Op))
330 return false;
331
332 APInt C = Op->getAsAPIntVal().trunc(EltSize);
333 if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)
334 return false;
335 if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)
336 return false;
337 }
338
339 return true;
340}
341
343 // Return false if the node has no operands.
344 // This is "logically inconsistent" with the definition of "all" but
345 // is probably the desired behavior.
346 if (N->getNumOperands() == 0)
347 return false;
348 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
349}
350
352 return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();
353}
354
355template <typename ConstNodeType>
357 std::function<bool(ConstNodeType *)> Match,
358 bool AllowUndefs) {
359 // FIXME: Add support for scalar UNDEF cases?
360 if (auto *C = dyn_cast<ConstNodeType>(Op))
361 return Match(C);
362
363 // FIXME: Add support for vector UNDEF cases?
364 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
365 ISD::SPLAT_VECTOR != Op.getOpcode())
366 return false;
367
368 EVT SVT = Op.getValueType().getScalarType();
369 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
370 if (AllowUndefs && Op.getOperand(i).isUndef()) {
371 if (!Match(nullptr))
372 return false;
373 continue;
374 }
375
376 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
377 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
378 return false;
379 }
380 return true;
381}
382// Build used template types.
383template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
384 SDValue, std::function<bool(ConstantSDNode *)>, bool);
385template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
386 SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
387
389 SDValue LHS, SDValue RHS,
390 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
391 bool AllowUndefs, bool AllowTypeMismatch) {
392 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
393 return false;
394
395 // TODO: Add support for scalar UNDEF cases?
396 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
397 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
398 return Match(LHSCst, RHSCst);
399
400 // TODO: Add support for vector UNDEF cases?
401 if (LHS.getOpcode() != RHS.getOpcode() ||
402 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
403 LHS.getOpcode() != ISD::SPLAT_VECTOR))
404 return false;
405
406 EVT SVT = LHS.getValueType().getScalarType();
407 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
408 SDValue LHSOp = LHS.getOperand(i);
409 SDValue RHSOp = RHS.getOperand(i);
410 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
411 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
412 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
413 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
414 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
415 return false;
416 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
417 LHSOp.getValueType() != RHSOp.getValueType()))
418 return false;
419 if (!Match(LHSCst, RHSCst))
420 return false;
421 }
422 return true;
423}
424
426 switch (VecReduceOpcode) {
427 default:
428 llvm_unreachable("Expected VECREDUCE opcode");
431 case ISD::VP_REDUCE_FADD:
432 case ISD::VP_REDUCE_SEQ_FADD:
433 return ISD::FADD;
436 case ISD::VP_REDUCE_FMUL:
437 case ISD::VP_REDUCE_SEQ_FMUL:
438 return ISD::FMUL;
440 case ISD::VP_REDUCE_ADD:
441 return ISD::ADD;
443 case ISD::VP_REDUCE_MUL:
444 return ISD::MUL;
446 case ISD::VP_REDUCE_AND:
447 return ISD::AND;
449 case ISD::VP_REDUCE_OR:
450 return ISD::OR;
452 case ISD::VP_REDUCE_XOR:
453 return ISD::XOR;
455 case ISD::VP_REDUCE_SMAX:
456 return ISD::SMAX;
458 case ISD::VP_REDUCE_SMIN:
459 return ISD::SMIN;
461 case ISD::VP_REDUCE_UMAX:
462 return ISD::UMAX;
464 case ISD::VP_REDUCE_UMIN:
465 return ISD::UMIN;
467 case ISD::VP_REDUCE_FMAX:
468 return ISD::FMAXNUM;
470 case ISD::VP_REDUCE_FMIN:
471 return ISD::FMINNUM;
473 return ISD::FMAXIMUM;
475 return ISD::FMINIMUM;
476 }
477}
478
479bool ISD::isVPOpcode(unsigned Opcode) {
480 switch (Opcode) {
481 default:
482 return false;
483#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
484 case ISD::VPSD: \
485 return true;
486#include "llvm/IR/VPIntrinsics.def"
487 }
488}
489
490bool ISD::isVPBinaryOp(unsigned Opcode) {
491 switch (Opcode) {
492 default:
493 break;
494#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
495#define VP_PROPERTY_BINARYOP return true;
496#define END_REGISTER_VP_SDNODE(VPSD) break;
497#include "llvm/IR/VPIntrinsics.def"
498 }
499 return false;
500}
501
502bool ISD::isVPReduction(unsigned Opcode) {
503 switch (Opcode) {
504 default:
505 break;
506#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
507#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
508#define END_REGISTER_VP_SDNODE(VPSD) break;
509#include "llvm/IR/VPIntrinsics.def"
510 }
511 return false;
512}
513
514/// The operand position of the vector mask.
515std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
516 switch (Opcode) {
517 default:
518 return std::nullopt;
519#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
520 case ISD::VPSD: \
521 return MASKPOS;
522#include "llvm/IR/VPIntrinsics.def"
523 }
524}
525
526/// The operand position of the explicit vector length parameter.
527std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
528 switch (Opcode) {
529 default:
530 return std::nullopt;
531#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
532 case ISD::VPSD: \
533 return EVLPOS;
534#include "llvm/IR/VPIntrinsics.def"
535 }
536}
537
538std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
539 bool hasFPExcept) {
540 // FIXME: Return strict opcodes in case of fp exceptions.
541 switch (VPOpcode) {
542 default:
543 return std::nullopt;
544#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
545#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
546#define END_REGISTER_VP_SDNODE(VPOPC) break;
547#include "llvm/IR/VPIntrinsics.def"
548 }
549 return std::nullopt;
550}
551
552unsigned ISD::getVPForBaseOpcode(unsigned Opcode) {
553 switch (Opcode) {
554 default:
555 llvm_unreachable("can not translate this Opcode to VP.");
556#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
557#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
558#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
559#include "llvm/IR/VPIntrinsics.def"
560 }
561}
562
564 switch (ExtType) {
565 case ISD::EXTLOAD:
566 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
567 case ISD::SEXTLOAD:
568 return ISD::SIGN_EXTEND;
569 case ISD::ZEXTLOAD:
570 return ISD::ZERO_EXTEND;
571 default:
572 break;
573 }
574
575 llvm_unreachable("Invalid LoadExtType");
576}
577
579 // To perform this operation, we just need to swap the L and G bits of the
580 // operation.
581 unsigned OldL = (Operation >> 2) & 1;
582 unsigned OldG = (Operation >> 1) & 1;
583 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
584 (OldL << 1) | // New G bit
585 (OldG << 2)); // New L bit.
586}
587
589 unsigned Operation = Op;
590 if (isIntegerLike)
591 Operation ^= 7; // Flip L, G, E bits, but not U.
592 else
593 Operation ^= 15; // Flip all of the condition bits.
594
596 Operation &= ~8; // Don't let N and U bits get set.
597
598 return ISD::CondCode(Operation);
599}
600
602 return getSetCCInverseImpl(Op, Type.isInteger());
603}
604
606 bool isIntegerLike) {
607 return getSetCCInverseImpl(Op, isIntegerLike);
608}
609
610/// For an integer comparison, return 1 if the comparison is a signed operation
611/// and 2 if the result is an unsigned comparison. Return zero if the operation
612/// does not depend on the sign of the input (setne and seteq).
613static int isSignedOp(ISD::CondCode Opcode) {
614 switch (Opcode) {
615 default: llvm_unreachable("Illegal integer setcc operation!");
616 case ISD::SETEQ:
617 case ISD::SETNE: return 0;
618 case ISD::SETLT:
619 case ISD::SETLE:
620 case ISD::SETGT:
621 case ISD::SETGE: return 1;
622 case ISD::SETULT:
623 case ISD::SETULE:
624 case ISD::SETUGT:
625 case ISD::SETUGE: return 2;
626 }
627}
628
630 EVT Type) {
631 bool IsInteger = Type.isInteger();
632 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
633 // Cannot fold a signed integer setcc with an unsigned integer setcc.
634 return ISD::SETCC_INVALID;
635
636 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
637
638 // If the N and U bits get set, then the resultant comparison DOES suddenly
639 // care about orderedness, and it is true when ordered.
640 if (Op > ISD::SETTRUE2)
641 Op &= ~16; // Clear the U bit if the N bit is set.
642
643 // Canonicalize illegal integer setcc's.
644 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
645 Op = ISD::SETNE;
646
647 return ISD::CondCode(Op);
648}
649
651 EVT Type) {
652 bool IsInteger = Type.isInteger();
653 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
654 // Cannot fold a signed setcc with an unsigned setcc.
655 return ISD::SETCC_INVALID;
656
657 // Combine all of the condition bits.
658 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
659
660 // Canonicalize illegal integer setcc's.
661 if (IsInteger) {
662 switch (Result) {
663 default: break;
664 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
665 case ISD::SETOEQ: // SETEQ & SETU[LG]E
666 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
667 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
668 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
669 }
670 }
671
672 return Result;
673}
674
675//===----------------------------------------------------------------------===//
676// SDNode Profile Support
677//===----------------------------------------------------------------------===//
678
679/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
680static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
681 ID.AddInteger(OpC);
682}
683
684/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
685/// solely with their pointer.
687 ID.AddPointer(VTList.VTs);
688}
689
690/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
692 ArrayRef<SDValue> Ops) {
693 for (const auto &Op : Ops) {
694 ID.AddPointer(Op.getNode());
695 ID.AddInteger(Op.getResNo());
696 }
697}
698
699/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
701 ArrayRef<SDUse> Ops) {
702 for (const auto &Op : Ops) {
703 ID.AddPointer(Op.getNode());
704 ID.AddInteger(Op.getResNo());
705 }
706}
707
708static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
709 SDVTList VTList, ArrayRef<SDValue> OpList) {
710 AddNodeIDOpcode(ID, OpC);
711 AddNodeIDValueTypes(ID, VTList);
712 AddNodeIDOperands(ID, OpList);
713}
714
715/// If this is an SDNode with special info, add this info to the NodeID data.
717 switch (N->getOpcode()) {
720 case ISD::MCSymbol:
721 llvm_unreachable("Should only be used on nodes with operands");
722 default: break; // Normal nodes don't need extra info.
724 case ISD::Constant: {
725 const ConstantSDNode *C = cast<ConstantSDNode>(N);
726 ID.AddPointer(C->getConstantIntValue());
727 ID.AddBoolean(C->isOpaque());
728 break;
729 }
731 case ISD::ConstantFP:
732 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
733 break;
738 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
739 ID.AddPointer(GA->getGlobal());
740 ID.AddInteger(GA->getOffset());
741 ID.AddInteger(GA->getTargetFlags());
742 break;
743 }
744 case ISD::BasicBlock:
745 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
746 break;
747 case ISD::Register:
748 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
749 break;
751 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
752 break;
753 case ISD::SRCVALUE:
754 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
755 break;
756 case ISD::FrameIndex:
758 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
759 break;
762 if (cast<LifetimeSDNode>(N)->hasOffset()) {
763 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
764 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
765 }
766 break;
768 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
769 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
770 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
771 break;
772 case ISD::JumpTable:
774 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
775 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
776 break;
779 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
780 ID.AddInteger(CP->getAlign().value());
781 ID.AddInteger(CP->getOffset());
782 if (CP->isMachineConstantPoolEntry())
783 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
784 else
785 ID.AddPointer(CP->getConstVal());
786 ID.AddInteger(CP->getTargetFlags());
787 break;
788 }
789 case ISD::TargetIndex: {
790 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
791 ID.AddInteger(TI->getIndex());
792 ID.AddInteger(TI->getOffset());
793 ID.AddInteger(TI->getTargetFlags());
794 break;
795 }
796 case ISD::LOAD: {
797 const LoadSDNode *LD = cast<LoadSDNode>(N);
798 ID.AddInteger(LD->getMemoryVT().getRawBits());
799 ID.AddInteger(LD->getRawSubclassData());
800 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
801 ID.AddInteger(LD->getMemOperand()->getFlags());
802 break;
803 }
804 case ISD::STORE: {
805 const StoreSDNode *ST = cast<StoreSDNode>(N);
806 ID.AddInteger(ST->getMemoryVT().getRawBits());
807 ID.AddInteger(ST->getRawSubclassData());
808 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
809 ID.AddInteger(ST->getMemOperand()->getFlags());
810 break;
811 }
812 case ISD::VP_LOAD: {
813 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
814 ID.AddInteger(ELD->getMemoryVT().getRawBits());
815 ID.AddInteger(ELD->getRawSubclassData());
816 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
817 ID.AddInteger(ELD->getMemOperand()->getFlags());
818 break;
819 }
820 case ISD::VP_STORE: {
821 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
822 ID.AddInteger(EST->getMemoryVT().getRawBits());
823 ID.AddInteger(EST->getRawSubclassData());
824 ID.AddInteger(EST->getPointerInfo().getAddrSpace());
825 ID.AddInteger(EST->getMemOperand()->getFlags());
826 break;
827 }
828 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
829 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(N);
830 ID.AddInteger(SLD->getMemoryVT().getRawBits());
831 ID.AddInteger(SLD->getRawSubclassData());
832 ID.AddInteger(SLD->getPointerInfo().getAddrSpace());
833 break;
834 }
835 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
836 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
837 ID.AddInteger(SST->getMemoryVT().getRawBits());
838 ID.AddInteger(SST->getRawSubclassData());
839 ID.AddInteger(SST->getPointerInfo().getAddrSpace());
840 break;
841 }
842 case ISD::VP_GATHER: {
843 const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);
844 ID.AddInteger(EG->getMemoryVT().getRawBits());
845 ID.AddInteger(EG->getRawSubclassData());
846 ID.AddInteger(EG->getPointerInfo().getAddrSpace());
847 ID.AddInteger(EG->getMemOperand()->getFlags());
848 break;
849 }
850 case ISD::VP_SCATTER: {
851 const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);
852 ID.AddInteger(ES->getMemoryVT().getRawBits());
853 ID.AddInteger(ES->getRawSubclassData());
854 ID.AddInteger(ES->getPointerInfo().getAddrSpace());
855 ID.AddInteger(ES->getMemOperand()->getFlags());
856 break;
857 }
858 case ISD::MLOAD: {
859 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
860 ID.AddInteger(MLD->getMemoryVT().getRawBits());
861 ID.AddInteger(MLD->getRawSubclassData());
862 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
863 ID.AddInteger(MLD->getMemOperand()->getFlags());
864 break;
865 }
866 case ISD::MSTORE: {
867 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
868 ID.AddInteger(MST->getMemoryVT().getRawBits());
869 ID.AddInteger(MST->getRawSubclassData());
870 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
871 ID.AddInteger(MST->getMemOperand()->getFlags());
872 break;
873 }
874 case ISD::MGATHER: {
875 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
876 ID.AddInteger(MG->getMemoryVT().getRawBits());
877 ID.AddInteger(MG->getRawSubclassData());
878 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
879 ID.AddInteger(MG->getMemOperand()->getFlags());
880 break;
881 }
882 case ISD::MSCATTER: {
883 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
884 ID.AddInteger(MS->getMemoryVT().getRawBits());
885 ID.AddInteger(MS->getRawSubclassData());
886 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
887 ID.AddInteger(MS->getMemOperand()->getFlags());
888 break;
889 }
892 case ISD::ATOMIC_SWAP:
904 case ISD::ATOMIC_LOAD:
905 case ISD::ATOMIC_STORE: {
906 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
907 ID.AddInteger(AT->getMemoryVT().getRawBits());
908 ID.AddInteger(AT->getRawSubclassData());
909 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
910 ID.AddInteger(AT->getMemOperand()->getFlags());
911 break;
912 }
913 case ISD::VECTOR_SHUFFLE: {
914 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
915 for (int M : Mask)
916 ID.AddInteger(M);
917 break;
918 }
920 case ISD::BlockAddress: {
921 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
922 ID.AddPointer(BA->getBlockAddress());
923 ID.AddInteger(BA->getOffset());
924 ID.AddInteger(BA->getTargetFlags());
925 break;
926 }
927 case ISD::AssertAlign:
928 ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());
929 break;
930 case ISD::PREFETCH:
933 // Handled by MemIntrinsicSDNode check after the switch.
934 break;
935 } // end switch (N->getOpcode())
936
937 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
938 // to check.
939 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {
940 ID.AddInteger(MN->getRawSubclassData());
941 ID.AddInteger(MN->getPointerInfo().getAddrSpace());
942 ID.AddInteger(MN->getMemOperand()->getFlags());
943 ID.AddInteger(MN->getMemoryVT().getRawBits());
944 }
945}
946
947/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
948/// data.
949static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
950 AddNodeIDOpcode(ID, N->getOpcode());
951 // Add the return value info.
952 AddNodeIDValueTypes(ID, N->getVTList());
953 // Add the operand info.
954 AddNodeIDOperands(ID, N->ops());
955
956 // Handle SDNode leafs with special info.
958}
959
960//===----------------------------------------------------------------------===//
961// SelectionDAG Class
962//===----------------------------------------------------------------------===//
963
964/// doNotCSE - Return true if CSE should not be performed for this node.
965static bool doNotCSE(SDNode *N) {
966 if (N->getValueType(0) == MVT::Glue)
967 return true; // Never CSE anything that produces a glue result.
968
969 switch (N->getOpcode()) {
970 default: break;
971 case ISD::HANDLENODE:
972 case ISD::EH_LABEL:
973 return true; // Never CSE these nodes.
974 }
975
976 // Check that remaining values produced are not flags.
977 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
978 if (N->getValueType(i) == MVT::Glue)
979 return true; // Never CSE anything that produces a glue result.
980
981 return false;
982}
983
984/// RemoveDeadNodes - This method deletes all unreachable nodes in the
985/// SelectionDAG.
987 // Create a dummy node (which is not added to allnodes), that adds a reference
988 // to the root node, preventing it from being deleted.
989 HandleSDNode Dummy(getRoot());
990
992
993 // Add all obviously-dead nodes to the DeadNodes worklist.
994 for (SDNode &Node : allnodes())
995 if (Node.use_empty())
996 DeadNodes.push_back(&Node);
997
998 RemoveDeadNodes(DeadNodes);
999
1000 // If the root changed (e.g. it was a dead load, update the root).
1001 setRoot(Dummy.getValue());
1002}
1003
1004/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1005/// given list, and any nodes that become unreachable as a result.
1007
1008 // Process the worklist, deleting the nodes and adding their uses to the
1009 // worklist.
1010 while (!DeadNodes.empty()) {
1011 SDNode *N = DeadNodes.pop_back_val();
1012 // Skip to next node if we've already managed to delete the node. This could
1013 // happen if replacing a node causes a node previously added to the node to
1014 // be deleted.
1015 if (N->getOpcode() == ISD::DELETED_NODE)
1016 continue;
1017
1018 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1019 DUL->NodeDeleted(N, nullptr);
1020
1021 // Take the node out of the appropriate CSE map.
1022 RemoveNodeFromCSEMaps(N);
1023
1024 // Next, brutally remove the operand list. This is safe to do, as there are
1025 // no cycles in the graph.
1026 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1027 SDUse &Use = *I++;
1028 SDNode *Operand = Use.getNode();
1029 Use.set(SDValue());
1030
1031 // Now that we removed this operand, see if there are no uses of it left.
1032 if (Operand->use_empty())
1033 DeadNodes.push_back(Operand);
1034 }
1035
1036 DeallocateNode(N);
1037 }
1038}
1039
1041 SmallVector<SDNode*, 16> DeadNodes(1, N);
1042
1043 // Create a dummy node that adds a reference to the root node, preventing
1044 // it from being deleted. (This matters if the root is an operand of the
1045 // dead node.)
1046 HandleSDNode Dummy(getRoot());
1047
1048 RemoveDeadNodes(DeadNodes);
1049}
1050
1052 // First take this out of the appropriate CSE map.
1053 RemoveNodeFromCSEMaps(N);
1054
1055 // Finally, remove uses due to operands of this node, remove from the
1056 // AllNodes list, and delete the node.
1057 DeleteNodeNotInCSEMaps(N);
1058}
1059
1060void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1061 assert(N->getIterator() != AllNodes.begin() &&
1062 "Cannot delete the entry node!");
1063 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1064
1065 // Drop all of the operands and decrement used node's use counts.
1066 N->DropOperands();
1067
1068 DeallocateNode(N);
1069}
1070
1071void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1072 assert(!(V->isVariadic() && isParameter));
1073 if (isParameter)
1074 ByvalParmDbgValues.push_back(V);
1075 else
1076 DbgValues.push_back(V);
1077 for (const SDNode *Node : V->getSDNodes())
1078 if (Node)
1079 DbgValMap[Node].push_back(V);
1080}
1081
1082void SDDbgInfo::erase(const SDNode *Node) {
1083 DbgValMapType::iterator I = DbgValMap.find(Node);
1084 if (I == DbgValMap.end())
1085 return;
1086 for (auto &Val: I->second)
1087 Val->setIsInvalidated();
1088 DbgValMap.erase(I);
1089}
1090
1091void SelectionDAG::DeallocateNode(SDNode *N) {
1092 // If we have operands, deallocate them.
1093 removeOperands(N);
1094
1095 NodeAllocator.Deallocate(AllNodes.remove(N));
1096
1097 // Set the opcode to DELETED_NODE to help catch bugs when node
1098 // memory is reallocated.
1099 // FIXME: There are places in SDag that have grown a dependency on the opcode
1100 // value in the released node.
1101 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1102 N->NodeType = ISD::DELETED_NODE;
1103
1104 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1105 // them and forget about that node.
1106 DbgInfo->erase(N);
1107
1108 // Invalidate extra info.
1109 SDEI.erase(N);
1110}
1111
1112#ifndef NDEBUG
1113/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1114static void VerifySDNode(SDNode *N) {
1115 switch (N->getOpcode()) {
1116 default:
1117 break;
1118 case ISD::BUILD_PAIR: {
1119 EVT VT = N->getValueType(0);
1120 assert(N->getNumValues() == 1 && "Too many results!");
1121 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1122 "Wrong return type!");
1123 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1124 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1125 "Mismatched operand types!");
1126 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1127 "Wrong operand type!");
1128 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1129 "Wrong return type size");
1130 break;
1131 }
1132 case ISD::BUILD_VECTOR: {
1133 assert(N->getNumValues() == 1 && "Too many results!");
1134 assert(N->getValueType(0).isVector() && "Wrong return type!");
1135 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1136 "Wrong number of operands!");
1137 EVT EltVT = N->getValueType(0).getVectorElementType();
1138 for (const SDUse &Op : N->ops()) {
1139 assert((Op.getValueType() == EltVT ||
1140 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1141 EltVT.bitsLE(Op.getValueType()))) &&
1142 "Wrong operand type!");
1143 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1144 "Operands must all have the same type");
1145 }
1146 break;
1147 }
1148 }
1149}
1150#endif // NDEBUG
1151
1152/// Insert a newly allocated node into the DAG.
1153///
1154/// Handles insertion into the all nodes list and CSE map, as well as
1155/// verification and other common operations when a new node is allocated.
1156void SelectionDAG::InsertNode(SDNode *N) {
1157 AllNodes.push_back(N);
1158#ifndef NDEBUG
1159 N->PersistentId = NextPersistentId++;
1160 VerifySDNode(N);
1161#endif
1162 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1163 DUL->NodeInserted(N);
1164}
1165
1166/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1167/// correspond to it. This is useful when we're about to delete or repurpose
1168/// the node. We don't want future request for structurally identical nodes
1169/// to return N anymore.
1170bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1171 bool Erased = false;
1172 switch (N->getOpcode()) {
1173 case ISD::HANDLENODE: return false; // noop.
1174 case ISD::CONDCODE:
1175 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1176 "Cond code doesn't exist!");
1177 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1178 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1179 break;
1181 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1182 break;
1184 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1185 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1186 ESN->getSymbol(), ESN->getTargetFlags()));
1187 break;
1188 }
1189 case ISD::MCSymbol: {
1190 auto *MCSN = cast<MCSymbolSDNode>(N);
1191 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1192 break;
1193 }
1194 case ISD::VALUETYPE: {
1195 EVT VT = cast<VTSDNode>(N)->getVT();
1196 if (VT.isExtended()) {
1197 Erased = ExtendedValueTypeNodes.erase(VT);
1198 } else {
1199 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1200 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1201 }
1202 break;
1203 }
1204 default:
1205 // Remove it from the CSE Map.
1206 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1207 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1208 Erased = CSEMap.RemoveNode(N);
1209 break;
1210 }
1211#ifndef NDEBUG
1212 // Verify that the node was actually in one of the CSE maps, unless it has a
1213 // glue result (which cannot be CSE'd) or is one of the special cases that are
1214 // not subject to CSE.
1215 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1216 !N->isMachineOpcode() && !doNotCSE(N)) {
1217 N->dump(this);
1218 dbgs() << "\n";
1219 llvm_unreachable("Node is not in map!");
1220 }
1221#endif
1222 return Erased;
1223}
1224
1225/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1226/// maps and modified in place. Add it back to the CSE maps, unless an identical
1227/// node already exists, in which case transfer all its users to the existing
1228/// node. This transfer can potentially trigger recursive merging.
1229void
1230SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1231 // For node types that aren't CSE'd, just act as if no identical node
1232 // already exists.
1233 if (!doNotCSE(N)) {
1234 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1235 if (Existing != N) {
1236 // If there was already an existing matching node, use ReplaceAllUsesWith
1237 // to replace the dead one with the existing one. This can cause
1238 // recursive merging of other unrelated nodes down the line.
1239 ReplaceAllUsesWith(N, Existing);
1240
1241 // N is now dead. Inform the listeners and delete it.
1242 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1243 DUL->NodeDeleted(N, Existing);
1244 DeleteNodeNotInCSEMaps(N);
1245 return;
1246 }
1247 }
1248
1249 // If the node doesn't already exist, we updated it. Inform listeners.
1250 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1251 DUL->NodeUpdated(N);
1252}
1253
1254/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1255/// were replaced with those specified. If this node is never memoized,
1256/// return null, otherwise return a pointer to the slot it would take. If a
1257/// node already exists with these operands, the slot will be non-null.
1258SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1259 void *&InsertPos) {
1260 if (doNotCSE(N))
1261 return nullptr;
1262
1263 SDValue Ops[] = { Op };
1265 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1267 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1268 if (Node)
1269 Node->intersectFlagsWith(N->getFlags());
1270 return Node;
1271}
1272
1273/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1274/// were replaced with those specified. If this node is never memoized,
1275/// return null, otherwise return a pointer to the slot it would take. If a
1276/// node already exists with these operands, the slot will be non-null.
1277SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1278 SDValue Op1, SDValue Op2,
1279 void *&InsertPos) {
1280 if (doNotCSE(N))
1281 return nullptr;
1282
1283 SDValue Ops[] = { Op1, Op2 };
1285 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1287 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1288 if (Node)
1289 Node->intersectFlagsWith(N->getFlags());
1290 return Node;
1291}
1292
1293/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1294/// were replaced with those specified. If this node is never memoized,
1295/// return null, otherwise return a pointer to the slot it would take. If a
1296/// node already exists with these operands, the slot will be non-null.
1297SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1298 void *&InsertPos) {
1299 if (doNotCSE(N))
1300 return nullptr;
1301
1303 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1305 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1306 if (Node)
1307 Node->intersectFlagsWith(N->getFlags());
1308 return Node;
1309}
1310
1312 Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)
1313 : VT.getTypeForEVT(*getContext());
1314
1315 return getDataLayout().getABITypeAlign(Ty);
1316}
1317
1318// EntryNode could meaningfully have debug info if we can find it...
1320 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1321 getVTList(MVT::Other, MVT::Glue)),
1322 Root(getEntryNode()) {
1323 InsertNode(&EntryNode);
1324 DbgInfo = new SDDbgInfo();
1325}
1326
1328 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1329 const TargetLibraryInfo *LibraryInfo,
1330 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1331 BlockFrequencyInfo *BFIin,
1332 FunctionVarLocs const *VarLocs) {
1333 MF = &NewMF;
1334 SDAGISelPass = PassPtr;
1335 ORE = &NewORE;
1338 LibInfo = LibraryInfo;
1339 Context = &MF->getFunction().getContext();
1340 UA = NewUA;
1341 PSI = PSIin;
1342 BFI = BFIin;
1343 FnVarLocs = VarLocs;
1344}
1345
1347 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1348 allnodes_clear();
1349 OperandRecycler.clear(OperandAllocator);
1350 delete DbgInfo;
1351}
1352
1354 return MF->getFunction().hasOptSize() ||
1356}
1357
1358void SelectionDAG::allnodes_clear() {
1359 assert(&*AllNodes.begin() == &EntryNode);
1360 AllNodes.remove(AllNodes.begin());
1361 while (!AllNodes.empty())
1362 DeallocateNode(&AllNodes.front());
1363#ifndef NDEBUG
1364 NextPersistentId = 0;
1365#endif
1366}
1367
1368SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1369 void *&InsertPos) {
1370 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1371 if (N) {
1372 switch (N->getOpcode()) {
1373 default: break;
1374 case ISD::Constant:
1375 case ISD::ConstantFP:
1376 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1377 "debug location. Use another overload.");
1378 }
1379 }
1380 return N;
1381}
1382
1383SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1384 const SDLoc &DL, void *&InsertPos) {
1385 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1386 if (N) {
1387 switch (N->getOpcode()) {
1388 case ISD::Constant:
1389 case ISD::ConstantFP:
1390 // Erase debug location from the node if the node is used at several
1391 // different places. Do not propagate one location to all uses as it
1392 // will cause a worse single stepping debugging experience.
1393 if (N->getDebugLoc() != DL.getDebugLoc())
1394 N->setDebugLoc(DebugLoc());
1395 break;
1396 default:
1397 // When the node's point of use is located earlier in the instruction
1398 // sequence than its prior point of use, update its debug info to the
1399 // earlier location.
1400 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1401 N->setDebugLoc(DL.getDebugLoc());
1402 break;
1403 }
1404 }
1405 return N;
1406}
1407
1409 allnodes_clear();
1410 OperandRecycler.clear(OperandAllocator);
1411 OperandAllocator.Reset();
1412 CSEMap.clear();
1413
1414 ExtendedValueTypeNodes.clear();
1415 ExternalSymbols.clear();
1416 TargetExternalSymbols.clear();
1417 MCSymbols.clear();
1418 SDEI.clear();
1419 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1420 static_cast<CondCodeSDNode*>(nullptr));
1421 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1422 static_cast<SDNode*>(nullptr));
1423
1424 EntryNode.UseList = nullptr;
1425 InsertNode(&EntryNode);
1426 Root = getEntryNode();
1427 DbgInfo->clear();
1428}
1429
1431 return VT.bitsGT(Op.getValueType())
1432 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1433 : getNode(ISD::FP_ROUND, DL, VT, Op,
1434 getIntPtrConstant(0, DL, /*isTarget=*/true));
1435}
1436
1437std::pair<SDValue, SDValue>
1439 const SDLoc &DL, EVT VT) {
1440 assert(!VT.bitsEq(Op.getValueType()) &&
1441 "Strict no-op FP extend/round not allowed.");
1442 SDValue Res =
1443 VT.bitsGT(Op.getValueType())
1444 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1445 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1446 {Chain, Op, getIntPtrConstant(0, DL)});
1447
1448 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1449}
1450
1452 return VT.bitsGT(Op.getValueType()) ?
1453 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1454 getNode(ISD::TRUNCATE, DL, VT, Op);
1455}
1456
1458 return VT.bitsGT(Op.getValueType()) ?
1459 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1460 getNode(ISD::TRUNCATE, DL, VT, Op);
1461}
1462
1464 return VT.bitsGT(Op.getValueType()) ?
1465 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1466 getNode(ISD::TRUNCATE, DL, VT, Op);
1467}
1468
1470 EVT VT) {
1471 assert(!VT.isVector());
1472 auto Type = Op.getValueType();
1473 SDValue DestOp;
1474 if (Type == VT)
1475 return Op;
1476 auto Size = Op.getValueSizeInBits();
1477 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1478 if (DestOp.getValueType() == VT)
1479 return DestOp;
1480
1481 return getAnyExtOrTrunc(DestOp, DL, VT);
1482}
1483
1485 EVT VT) {
1486 assert(!VT.isVector());
1487 auto Type = Op.getValueType();
1488 SDValue DestOp;
1489 if (Type == VT)
1490 return Op;
1491 auto Size = Op.getValueSizeInBits();
1492 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1493 if (DestOp.getValueType() == VT)
1494 return DestOp;
1495
1496 return getSExtOrTrunc(DestOp, DL, VT);
1497}
1498
1500 EVT VT) {
1501 assert(!VT.isVector());
1502 auto Type = Op.getValueType();
1503 SDValue DestOp;
1504 if (Type == VT)
1505 return Op;
1506 auto Size = Op.getValueSizeInBits();
1507 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1508 if (DestOp.getValueType() == VT)
1509 return DestOp;
1510
1511 return getZExtOrTrunc(DestOp, DL, VT);
1512}
1513
1515 EVT OpVT) {
1516 if (VT.bitsLE(Op.getValueType()))
1517 return getNode(ISD::TRUNCATE, SL, VT, Op);
1518
1520 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1521}
1522
1524 EVT OpVT = Op.getValueType();
1525 assert(VT.isInteger() && OpVT.isInteger() &&
1526 "Cannot getZeroExtendInReg FP types");
1527 assert(VT.isVector() == OpVT.isVector() &&
1528 "getZeroExtendInReg type should be vector iff the operand "
1529 "type is vector!");
1530 assert((!VT.isVector() ||
1532 "Vector element counts must match in getZeroExtendInReg");
1533 assert(VT.bitsLE(OpVT) && "Not extending!");
1534 if (OpVT == VT)
1535 return Op;
1537 VT.getScalarSizeInBits());
1538 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1539}
1540
1542 // Only unsigned pointer semantics are supported right now. In the future this
1543 // might delegate to TLI to check pointer signedness.
1544 return getZExtOrTrunc(Op, DL, VT);
1545}
1546
1548 // Only unsigned pointer semantics are supported right now. In the future this
1549 // might delegate to TLI to check pointer signedness.
1550 return getZeroExtendInReg(Op, DL, VT);
1551}
1552
1554 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);
1555}
1556
1557/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1559 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1560}
1561
1563 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1564 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1565}
1566
1568 SDValue Mask, SDValue EVL, EVT VT) {
1569 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1570 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL);
1571}
1572
1574 SDValue Mask, SDValue EVL) {
1575 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1576}
1577
1579 SDValue Mask, SDValue EVL) {
1580 if (VT.bitsGT(Op.getValueType()))
1581 return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL);
1582 if (VT.bitsLT(Op.getValueType()))
1583 return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL);
1584 return Op;
1585}
1586
1588 EVT OpVT) {
1589 if (!V)
1590 return getConstant(0, DL, VT);
1591
1592 switch (TLI->getBooleanContents(OpVT)) {
1595 return getConstant(1, DL, VT);
1597 return getAllOnesConstant(DL, VT);
1598 }
1599 llvm_unreachable("Unexpected boolean content enum!");
1600}
1601
1603 bool isT, bool isO) {
1604 EVT EltVT = VT.getScalarType();
1605 assert((EltVT.getSizeInBits() >= 64 ||
1606 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1607 "getConstant with a uint64_t value that doesn't fit in the type!");
1608 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1609}
1610
1612 bool isT, bool isO) {
1613 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1614}
1615
1617 EVT VT, bool isT, bool isO) {
1618 assert(VT.isInteger() && "Cannot create FP integer constant!");
1619
1620 EVT EltVT = VT.getScalarType();
1621 const ConstantInt *Elt = &Val;
1622
1623 // In some cases the vector type is legal but the element type is illegal and
1624 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1625 // inserted value (the type does not need to match the vector element type).
1626 // Any extra bits introduced will be truncated away.
1627 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1629 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1630 APInt NewVal;
1631 if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))
1632 NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());
1633 else
1634 NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1635 Elt = ConstantInt::get(*getContext(), NewVal);
1636 }
1637 // In other cases the element type is illegal and needs to be expanded, for
1638 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1639 // the value into n parts and use a vector type with n-times the elements.
1640 // Then bitcast to the type requested.
1641 // Legalizing constants too early makes the DAGCombiner's job harder so we
1642 // only legalize if the DAG tells us we must produce legal types.
1643 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1644 TLI->getTypeAction(*getContext(), EltVT) ==
1646 const APInt &NewVal = Elt->getValue();
1647 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1648 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1649
1650 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1651 if (VT.isScalableVector() ||
1653 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1654 "Can only handle an even split!");
1655 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1656
1657 SmallVector<SDValue, 2> ScalarParts;
1658 for (unsigned i = 0; i != Parts; ++i)
1659 ScalarParts.push_back(getConstant(
1660 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1661 ViaEltVT, isT, isO));
1662
1663 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1664 }
1665
1666 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1667 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1668
1669 // Check the temporary vector is the correct size. If this fails then
1670 // getTypeToTransformTo() probably returned a type whose size (in bits)
1671 // isn't a power-of-2 factor of the requested type size.
1672 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1673
1674 SmallVector<SDValue, 2> EltParts;
1675 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1676 EltParts.push_back(getConstant(
1677 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1678 ViaEltVT, isT, isO));
1679
1680 // EltParts is currently in little endian order. If we actually want
1681 // big-endian order then reverse it now.
1682 if (getDataLayout().isBigEndian())
1683 std::reverse(EltParts.begin(), EltParts.end());
1684
1685 // The elements must be reversed when the element order is different
1686 // to the endianness of the elements (because the BITCAST is itself a
1687 // vector shuffle in this situation). However, we do not need any code to
1688 // perform this reversal because getConstant() is producing a vector
1689 // splat.
1690 // This situation occurs in MIPS MSA.
1691
1693 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1694 llvm::append_range(Ops, EltParts);
1695
1696 SDValue V =
1697 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1698 return V;
1699 }
1700
1701 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1702 "APInt size does not match type size!");
1703 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1705 AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1706 ID.AddPointer(Elt);
1707 ID.AddBoolean(isO);
1708 void *IP = nullptr;
1709 SDNode *N = nullptr;
1710 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1711 if (!VT.isVector())
1712 return SDValue(N, 0);
1713
1714 if (!N) {
1715 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1716 CSEMap.InsertNode(N, IP);
1717 InsertNode(N);
1718 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1719 }
1720
1721 SDValue Result(N, 0);
1722 if (VT.isVector())
1723 Result = getSplat(VT, DL, Result);
1724 return Result;
1725}
1726
1728 bool isTarget) {
1729 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1730}
1731
1733 const SDLoc &DL, bool LegalTypes) {
1734 assert(VT.isInteger() && "Shift amount is not an integer type!");
1735 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1736 return getConstant(Val, DL, ShiftVT);
1737}
1738
1740 const SDLoc &DL, bool LegalTypes) {
1741 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1742 return getShiftAmountConstant(Val.getZExtValue(), VT, DL, LegalTypes);
1743}
1744
1746 bool isTarget) {
1747 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1748}
1749
1751 bool isTarget) {
1752 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1753}
1754
1756 EVT VT, bool isTarget) {
1757 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1758
1759 EVT EltVT = VT.getScalarType();
1760
1761 // Do the map lookup using the actual bit pattern for the floating point
1762 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1763 // we don't have issues with SNANs.
1764 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1766 AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1767 ID.AddPointer(&V);
1768 void *IP = nullptr;
1769 SDNode *N = nullptr;
1770 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1771 if (!VT.isVector())
1772 return SDValue(N, 0);
1773
1774 if (!N) {
1775 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1776 CSEMap.InsertNode(N, IP);
1777 InsertNode(N);
1778 }
1779
1780 SDValue Result(N, 0);
1781 if (VT.isVector())
1782 Result = getSplat(VT, DL, Result);
1783 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1784 return Result;
1785}
1786
1788 bool isTarget) {
1789 EVT EltVT = VT.getScalarType();
1790 if (EltVT == MVT::f32)
1791 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1792 if (EltVT == MVT::f64)
1793 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1794 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1795 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1796 bool Ignored;
1797 APFloat APF = APFloat(Val);
1799 &Ignored);
1800 return getConstantFP(APF, DL, VT, isTarget);
1801 }
1802 llvm_unreachable("Unsupported type in getConstantFP");
1803}
1804
1806 EVT VT, int64_t Offset, bool isTargetGA,
1807 unsigned TargetFlags) {
1808 assert((TargetFlags == 0 || isTargetGA) &&
1809 "Cannot set target flags on target-independent globals");
1810
1811 // Truncate (with sign-extension) the offset value to the pointer size.
1813 if (BitWidth < 64)
1815
1816 unsigned Opc;
1817 if (GV->isThreadLocal())
1819 else
1820 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1821
1823 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1824 ID.AddPointer(GV);
1825 ID.AddInteger(Offset);
1826 ID.AddInteger(TargetFlags);
1827 void *IP = nullptr;
1828 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1829 return SDValue(E, 0);
1830
1831 auto *N = newSDNode<GlobalAddressSDNode>(
1832 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1833 CSEMap.InsertNode(N, IP);
1834 InsertNode(N);
1835 return SDValue(N, 0);
1836}
1837
1838SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1839 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1841 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1842 ID.AddInteger(FI);
1843 void *IP = nullptr;
1844 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1845 return SDValue(E, 0);
1846
1847 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1848 CSEMap.InsertNode(N, IP);
1849 InsertNode(N);
1850 return SDValue(N, 0);
1851}
1852
1853SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1854 unsigned TargetFlags) {
1855 assert((TargetFlags == 0 || isTarget) &&
1856 "Cannot set target flags on target-independent jump tables");
1857 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1859 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1860 ID.AddInteger(JTI);
1861 ID.AddInteger(TargetFlags);
1862 void *IP = nullptr;
1863 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1864 return SDValue(E, 0);
1865
1866 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1867 CSEMap.InsertNode(N, IP);
1868 InsertNode(N);
1869 return SDValue(N, 0);
1870}
1871
1873 const SDLoc &DL) {
1875 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1876 getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));
1877}
1878
1880 MaybeAlign Alignment, int Offset,
1881 bool isTarget, unsigned TargetFlags) {
1882 assert((TargetFlags == 0 || isTarget) &&
1883 "Cannot set target flags on target-independent globals");
1884 if (!Alignment)
1885 Alignment = shouldOptForSize()
1886 ? getDataLayout().getABITypeAlign(C->getType())
1887 : getDataLayout().getPrefTypeAlign(C->getType());
1888 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1890 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1891 ID.AddInteger(Alignment->value());
1892 ID.AddInteger(Offset);
1893 ID.AddPointer(C);
1894 ID.AddInteger(TargetFlags);
1895 void *IP = nullptr;
1896 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1897 return SDValue(E, 0);
1898
1899 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1900 TargetFlags);
1901 CSEMap.InsertNode(N, IP);
1902 InsertNode(N);
1903 SDValue V = SDValue(N, 0);
1904 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1905 return V;
1906}
1907
1909 MaybeAlign Alignment, int Offset,
1910 bool isTarget, unsigned TargetFlags) {
1911 assert((TargetFlags == 0 || isTarget) &&
1912 "Cannot set target flags on target-independent globals");
1913 if (!Alignment)
1914 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1915 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1917 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1918 ID.AddInteger(Alignment->value());
1919 ID.AddInteger(Offset);
1920 C->addSelectionDAGCSEId(ID);
1921 ID.AddInteger(TargetFlags);
1922 void *IP = nullptr;
1923 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1924 return SDValue(E, 0);
1925
1926 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1927 TargetFlags);
1928 CSEMap.InsertNode(N, IP);
1929 InsertNode(N);
1930 return SDValue(N, 0);
1931}
1932
1935 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt);
1936 ID.AddPointer(MBB);
1937 void *IP = nullptr;
1938 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1939 return SDValue(E, 0);
1940
1941 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1942 CSEMap.InsertNode(N, IP);
1943 InsertNode(N);
1944 return SDValue(N, 0);
1945}
1946
1948 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1949 ValueTypeNodes.size())
1950 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1951
1952 SDNode *&N = VT.isExtended() ?
1953 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1954
1955 if (N) return SDValue(N, 0);
1956 N = newSDNode<VTSDNode>(VT);
1957 InsertNode(N);
1958 return SDValue(N, 0);
1959}
1960
1962 SDNode *&N = ExternalSymbols[Sym];
1963 if (N) return SDValue(N, 0);
1964 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1965 InsertNode(N);
1966 return SDValue(N, 0);
1967}
1968
1970 SDNode *&N = MCSymbols[Sym];
1971 if (N)
1972 return SDValue(N, 0);
1973 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1974 InsertNode(N);
1975 return SDValue(N, 0);
1976}
1977
1979 unsigned TargetFlags) {
1980 SDNode *&N =
1981 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1982 if (N) return SDValue(N, 0);
1983 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1984 InsertNode(N);
1985 return SDValue(N, 0);
1986}
1987
1989 if ((unsigned)Cond >= CondCodeNodes.size())
1990 CondCodeNodes.resize(Cond+1);
1991
1992 if (!CondCodeNodes[Cond]) {
1993 auto *N = newSDNode<CondCodeSDNode>(Cond);
1994 CondCodeNodes[Cond] = N;
1995 InsertNode(N);
1996 }
1997
1998 return SDValue(CondCodeNodes[Cond], 0);
1999}
2000
2002 bool ConstantFold) {
2003 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2004 "APInt size does not match type size!");
2005
2006 if (MulImm == 0)
2007 return getConstant(0, DL, VT);
2008
2009 if (ConstantFold) {
2010 const MachineFunction &MF = getMachineFunction();
2011 const Function &F = MF.getFunction();
2012 ConstantRange CR = getVScaleRange(&F, 64);
2013 if (const APInt *C = CR.getSingleElement())
2014 return getConstant(MulImm * C->getZExtValue(), DL, VT);
2015 }
2016
2017 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
2018}
2019
2021 bool ConstantFold) {
2022 if (EC.isScalable())
2023 return getVScale(DL, VT,
2024 APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2025
2026 return getConstant(EC.getKnownMinValue(), DL, VT);
2027}
2028
2030 APInt One(ResVT.getScalarSizeInBits(), 1);
2031 return getStepVector(DL, ResVT, One);
2032}
2033
2035 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2036 if (ResVT.isScalableVector())
2037 return getNode(
2038 ISD::STEP_VECTOR, DL, ResVT,
2039 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
2040
2041 SmallVector<SDValue, 16> OpsStepConstants;
2042 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2043 OpsStepConstants.push_back(
2044 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
2045 return getBuildVector(ResVT, DL, OpsStepConstants);
2046}
2047
2048/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2049/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2051 std::swap(N1, N2);
2053}
2054
2056 SDValue N2, ArrayRef<int> Mask) {
2057 assert(VT.getVectorNumElements() == Mask.size() &&
2058 "Must have the same number of vector elements as mask elements!");
2059 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2060 "Invalid VECTOR_SHUFFLE");
2061
2062 // Canonicalize shuffle undef, undef -> undef
2063 if (N1.isUndef() && N2.isUndef())
2064 return getUNDEF(VT);
2065
2066 // Validate that all indices in Mask are within the range of the elements
2067 // input to the shuffle.
2068 int NElts = Mask.size();
2069 assert(llvm::all_of(Mask,
2070 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2071 "Index out of range");
2072
2073 // Copy the mask so we can do any needed cleanup.
2074 SmallVector<int, 8> MaskVec(Mask);
2075
2076 // Canonicalize shuffle v, v -> v, undef
2077 if (N1 == N2) {
2078 N2 = getUNDEF(VT);
2079 for (int i = 0; i != NElts; ++i)
2080 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2081 }
2082
2083 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2084 if (N1.isUndef())
2085 commuteShuffle(N1, N2, MaskVec);
2086
2087 if (TLI->hasVectorBlend()) {
2088 // If shuffling a splat, try to blend the splat instead. We do this here so
2089 // that even when this arises during lowering we don't have to re-handle it.
2090 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2091 BitVector UndefElements;
2092 SDValue Splat = BV->getSplatValue(&UndefElements);
2093 if (!Splat)
2094 return;
2095
2096 for (int i = 0; i < NElts; ++i) {
2097 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2098 continue;
2099
2100 // If this input comes from undef, mark it as such.
2101 if (UndefElements[MaskVec[i] - Offset]) {
2102 MaskVec[i] = -1;
2103 continue;
2104 }
2105
2106 // If we can blend a non-undef lane, use that instead.
2107 if (!UndefElements[i])
2108 MaskVec[i] = i + Offset;
2109 }
2110 };
2111 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2112 BlendSplat(N1BV, 0);
2113 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2114 BlendSplat(N2BV, NElts);
2115 }
2116
2117 // Canonicalize all index into lhs, -> shuffle lhs, undef
2118 // Canonicalize all index into rhs, -> shuffle rhs, undef
2119 bool AllLHS = true, AllRHS = true;
2120 bool N2Undef = N2.isUndef();
2121 for (int i = 0; i != NElts; ++i) {
2122 if (MaskVec[i] >= NElts) {
2123 if (N2Undef)
2124 MaskVec[i] = -1;
2125 else
2126 AllLHS = false;
2127 } else if (MaskVec[i] >= 0) {
2128 AllRHS = false;
2129 }
2130 }
2131 if (AllLHS && AllRHS)
2132 return getUNDEF(VT);
2133 if (AllLHS && !N2Undef)
2134 N2 = getUNDEF(VT);
2135 if (AllRHS) {
2136 N1 = getUNDEF(VT);
2137 commuteShuffle(N1, N2, MaskVec);
2138 }
2139 // Reset our undef status after accounting for the mask.
2140 N2Undef = N2.isUndef();
2141 // Re-check whether both sides ended up undef.
2142 if (N1.isUndef() && N2Undef)
2143 return getUNDEF(VT);
2144
2145 // If Identity shuffle return that node.
2146 bool Identity = true, AllSame = true;
2147 for (int i = 0; i != NElts; ++i) {
2148 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2149 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2150 }
2151 if (Identity && NElts)
2152 return N1;
2153
2154 // Shuffling a constant splat doesn't change the result.
2155 if (N2Undef) {
2156 SDValue V = N1;
2157
2158 // Look through any bitcasts. We check that these don't change the number
2159 // (and size) of elements and just changes their types.
2160 while (V.getOpcode() == ISD::BITCAST)
2161 V = V->getOperand(0);
2162
2163 // A splat should always show up as a build vector node.
2164 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2165 BitVector UndefElements;
2166 SDValue Splat = BV->getSplatValue(&UndefElements);
2167 // If this is a splat of an undef, shuffling it is also undef.
2168 if (Splat && Splat.isUndef())
2169 return getUNDEF(VT);
2170
2171 bool SameNumElts =
2172 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2173
2174 // We only have a splat which can skip shuffles if there is a splatted
2175 // value and no undef lanes rearranged by the shuffle.
2176 if (Splat && UndefElements.none()) {
2177 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2178 // number of elements match or the value splatted is a zero constant.
2179 if (SameNumElts || isNullConstant(Splat))
2180 return N1;
2181 }
2182
2183 // If the shuffle itself creates a splat, build the vector directly.
2184 if (AllSame && SameNumElts) {
2185 EVT BuildVT = BV->getValueType(0);
2186 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
2187 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
2188
2189 // We may have jumped through bitcasts, so the type of the
2190 // BUILD_VECTOR may not match the type of the shuffle.
2191 if (BuildVT != VT)
2192 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
2193 return NewBV;
2194 }
2195 }
2196 }
2197
2199 SDValue Ops[2] = { N1, N2 };
2201 for (int i = 0; i != NElts; ++i)
2202 ID.AddInteger(MaskVec[i]);
2203
2204 void* IP = nullptr;
2205 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2206 return SDValue(E, 0);
2207
2208 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2209 // SDNode doesn't have access to it. This memory will be "leaked" when
2210 // the node is deallocated, but recovered when the NodeAllocator is released.
2211 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
2212 llvm::copy(MaskVec, MaskAlloc);
2213
2214 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
2215 dl.getDebugLoc(), MaskAlloc);
2216 createOperands(N, Ops);
2217
2218 CSEMap.InsertNode(N, IP);
2219 InsertNode(N);
2220 SDValue V = SDValue(N, 0);
2221 NewSDValueDbgMsg(V, "Creating new node: ", this);
2222 return V;
2223}
2224
2226 EVT VT = SV.getValueType(0);
2227 SmallVector<int, 8> MaskVec(SV.getMask());
2229
2230 SDValue Op0 = SV.getOperand(0);
2231 SDValue Op1 = SV.getOperand(1);
2232 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
2233}
2234
2237 AddNodeIDNode(ID, ISD::Register, getVTList(VT), std::nullopt);
2238 ID.AddInteger(RegNo);
2239 void *IP = nullptr;
2240 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2241 return SDValue(E, 0);
2242
2243 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
2244 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2245 CSEMap.InsertNode(N, IP);
2246 InsertNode(N);
2247 return SDValue(N, 0);
2248}
2249
2252 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt);
2253 ID.AddPointer(RegMask);
2254 void *IP = nullptr;
2255 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2256 return SDValue(E, 0);
2257
2258 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2259 CSEMap.InsertNode(N, IP);
2260 InsertNode(N);
2261 return SDValue(N, 0);
2262}
2263
2265 MCSymbol *Label) {
2266 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2267}
2268
2269SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2270 SDValue Root, MCSymbol *Label) {
2272 SDValue Ops[] = { Root };
2273 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2274 ID.AddPointer(Label);
2275 void *IP = nullptr;
2276 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2277 return SDValue(E, 0);
2278
2279 auto *N =
2280 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2281 createOperands(N, Ops);
2282
2283 CSEMap.InsertNode(N, IP);
2284 InsertNode(N);
2285 return SDValue(N, 0);
2286}
2287
2289 int64_t Offset, bool isTarget,
2290 unsigned TargetFlags) {
2291 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2292
2294 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
2295 ID.AddPointer(BA);
2296 ID.AddInteger(Offset);
2297 ID.AddInteger(TargetFlags);
2298 void *IP = nullptr;
2299 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2300 return SDValue(E, 0);
2301
2302 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2303 CSEMap.InsertNode(N, IP);
2304 InsertNode(N);
2305 return SDValue(N, 0);
2306}
2307
2310 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt);
2311 ID.AddPointer(V);
2312
2313 void *IP = nullptr;
2314 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2315 return SDValue(E, 0);
2316
2317 auto *N = newSDNode<SrcValueSDNode>(V);
2318 CSEMap.InsertNode(N, IP);
2319 InsertNode(N);
2320 return SDValue(N, 0);
2321}
2322
2325 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt);
2326 ID.AddPointer(MD);
2327
2328 void *IP = nullptr;
2329 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2330 return SDValue(E, 0);
2331
2332 auto *N = newSDNode<MDNodeSDNode>(MD);
2333 CSEMap.InsertNode(N, IP);
2334 InsertNode(N);
2335 return SDValue(N, 0);
2336}
2337
2339 if (VT == V.getValueType())
2340 return V;
2341
2342 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2343}
2344
2346 unsigned SrcAS, unsigned DestAS) {
2347 SDValue Ops[] = {Ptr};
2350 ID.AddInteger(SrcAS);
2351 ID.AddInteger(DestAS);
2352
2353 void *IP = nullptr;
2354 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2355 return SDValue(E, 0);
2356
2357 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2358 VT, SrcAS, DestAS);
2359 createOperands(N, Ops);
2360
2361 CSEMap.InsertNode(N, IP);
2362 InsertNode(N);
2363 return SDValue(N, 0);
2364}
2365
2367 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2368}
2369
2370/// getShiftAmountOperand - Return the specified value casted to
2371/// the target's desired shift amount type.
2373 EVT OpTy = Op.getValueType();
2374 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2375 if (OpTy == ShTy || OpTy.isVector()) return Op;
2376
2377 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2378}
2379
2381 SDLoc dl(Node);
2383 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2384 EVT VT = Node->getValueType(0);
2385 SDValue Tmp1 = Node->getOperand(0);
2386 SDValue Tmp2 = Node->getOperand(1);
2387 const MaybeAlign MA(Node->getConstantOperandVal(3));
2388
2389 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2390 Tmp2, MachinePointerInfo(V));
2391 SDValue VAList = VAListLoad;
2392
2393 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2394 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2395 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2396
2397 VAList =
2398 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2399 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2400 }
2401
2402 // Increment the pointer, VAList, to the next vaarg
2403 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2404 getConstant(getDataLayout().getTypeAllocSize(
2405 VT.getTypeForEVT(*getContext())),
2406 dl, VAList.getValueType()));
2407 // Store the incremented VAList to the legalized pointer
2408 Tmp1 =
2409 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2410 // Load the actual argument out of the pointer VAList
2411 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2412}
2413
2415 SDLoc dl(Node);
2417 // This defaults to loading a pointer from the input and storing it to the
2418 // output, returning the chain.
2419 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2420 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2421 SDValue Tmp1 =
2422 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2423 Node->getOperand(2), MachinePointerInfo(VS));
2424 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2425 MachinePointerInfo(VD));
2426}
2427
2429 const DataLayout &DL = getDataLayout();
2430 Type *Ty = VT.getTypeForEVT(*getContext());
2431 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2432
2433 if (TLI->isTypeLegal(VT) || !VT.isVector())
2434 return RedAlign;
2435
2437 const Align StackAlign = TFI->getStackAlign();
2438
2439 // See if we can choose a smaller ABI alignment in cases where it's an
2440 // illegal vector type that will get broken down.
2441 if (RedAlign > StackAlign) {
2442 EVT IntermediateVT;
2443 MVT RegisterVT;
2444 unsigned NumIntermediates;
2445 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2446 NumIntermediates, RegisterVT);
2447 Ty = IntermediateVT.getTypeForEVT(*getContext());
2448 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2449 if (RedAlign2 < RedAlign)
2450 RedAlign = RedAlign2;
2451 }
2452
2453 return RedAlign;
2454}
2455
2457 MachineFrameInfo &MFI = MF->getFrameInfo();
2459 int StackID = 0;
2460 if (Bytes.isScalable())
2461 StackID = TFI->getStackIDForScalableVectors();
2462 // The stack id gives an indication of whether the object is scalable or
2463 // not, so it's safe to pass in the minimum size here.
2464 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,
2465 false, nullptr, StackID);
2466 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2467}
2468
2470 Type *Ty = VT.getTypeForEVT(*getContext());
2471 Align StackAlign =
2472 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2473 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2474}
2475
2477 TypeSize VT1Size = VT1.getStoreSize();
2478 TypeSize VT2Size = VT2.getStoreSize();
2479 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2480 "Don't know how to choose the maximum size when creating a stack "
2481 "temporary");
2482 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2483 ? VT1Size
2484 : VT2Size;
2485
2486 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2487 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2488 const DataLayout &DL = getDataLayout();
2489 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2490 return CreateStackTemporary(Bytes, Align);
2491}
2492
2494 ISD::CondCode Cond, const SDLoc &dl) {
2495 EVT OpVT = N1.getValueType();
2496
2497 auto GetUndefBooleanConstant = [&]() {
2498 if (VT.getScalarType() == MVT::i1 ||
2499 TLI->getBooleanContents(OpVT) ==
2501 return getUNDEF(VT);
2502 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2503 // so we cannot use getUNDEF(). Return zero instead.
2504 return getConstant(0, dl, VT);
2505 };
2506
2507 // These setcc operations always fold.
2508 switch (Cond) {
2509 default: break;
2510 case ISD::SETFALSE:
2511 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2512 case ISD::SETTRUE:
2513 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2514
2515 case ISD::SETOEQ:
2516 case ISD::SETOGT:
2517 case ISD::SETOGE:
2518 case ISD::SETOLT:
2519 case ISD::SETOLE:
2520 case ISD::SETONE:
2521 case ISD::SETO:
2522 case ISD::SETUO:
2523 case ISD::SETUEQ:
2524 case ISD::SETUNE:
2525 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2526 break;
2527 }
2528
2529 if (OpVT.isInteger()) {
2530 // For EQ and NE, we can always pick a value for the undef to make the
2531 // predicate pass or fail, so we can return undef.
2532 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2533 // icmp eq/ne X, undef -> undef.
2534 if ((N1.isUndef() || N2.isUndef()) &&
2535 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2536 return GetUndefBooleanConstant();
2537
2538 // If both operands are undef, we can return undef for int comparison.
2539 // icmp undef, undef -> undef.
2540 if (N1.isUndef() && N2.isUndef())
2541 return GetUndefBooleanConstant();
2542
2543 // icmp X, X -> true/false
2544 // icmp X, undef -> true/false because undef could be X.
2545 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2546 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2547 }
2548
2549 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2550 const APInt &C2 = N2C->getAPIntValue();
2551 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2552 const APInt &C1 = N1C->getAPIntValue();
2553
2555 dl, VT, OpVT);
2556 }
2557 }
2558
2559 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2560 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2561
2562 if (N1CFP && N2CFP) {
2563 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2564 switch (Cond) {
2565 default: break;
2566 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2567 return GetUndefBooleanConstant();
2568 [[fallthrough]];
2569 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2570 OpVT);
2571 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2572 return GetUndefBooleanConstant();
2573 [[fallthrough]];
2575 R==APFloat::cmpLessThan, dl, VT,
2576 OpVT);
2577 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2578 return GetUndefBooleanConstant();
2579 [[fallthrough]];
2580 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2581 OpVT);
2582 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2583 return GetUndefBooleanConstant();
2584 [[fallthrough]];
2586 VT, OpVT);
2587 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2588 return GetUndefBooleanConstant();
2589 [[fallthrough]];
2591 R==APFloat::cmpEqual, dl, VT,
2592 OpVT);
2593 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2594 return GetUndefBooleanConstant();
2595 [[fallthrough]];
2597 R==APFloat::cmpEqual, dl, VT, OpVT);
2598 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2599 OpVT);
2600 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2601 OpVT);
2603 R==APFloat::cmpEqual, dl, VT,
2604 OpVT);
2605 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2606 OpVT);
2608 R==APFloat::cmpLessThan, dl, VT,
2609 OpVT);
2611 R==APFloat::cmpUnordered, dl, VT,
2612 OpVT);
2614 VT, OpVT);
2615 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2616 OpVT);
2617 }
2618 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2619 // Ensure that the constant occurs on the RHS.
2621 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2622 return SDValue();
2623 return getSetCC(dl, VT, N2, N1, SwappedCond);
2624 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2625 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2626 // If an operand is known to be a nan (or undef that could be a nan), we can
2627 // fold it.
2628 // Choosing NaN for the undef will always make unordered comparison succeed
2629 // and ordered comparison fails.
2630 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2631 switch (ISD::getUnorderedFlavor(Cond)) {
2632 default:
2633 llvm_unreachable("Unknown flavor!");
2634 case 0: // Known false.
2635 return getBoolConstant(false, dl, VT, OpVT);
2636 case 1: // Known true.
2637 return getBoolConstant(true, dl, VT, OpVT);
2638 case 2: // Undefined.
2639 return GetUndefBooleanConstant();
2640 }
2641 }
2642
2643 // Could not fold it.
2644 return SDValue();
2645}
2646
2647/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2648/// use this predicate to simplify operations downstream.
2650 unsigned BitWidth = Op.getScalarValueSizeInBits();
2652}
2653
2654/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2655/// this predicate to simplify operations downstream. Mask is known to be zero
2656/// for bits that V cannot have.
2658 unsigned Depth) const {
2659 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2660}
2661
2662/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2663/// DemandedElts. We use this predicate to simplify operations downstream.
2664/// Mask is known to be zero for bits that V cannot have.
2666 const APInt &DemandedElts,
2667 unsigned Depth) const {
2668 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2669}
2670
2671/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2672/// DemandedElts. We use this predicate to simplify operations downstream.
2674 unsigned Depth /* = 0 */) const {
2675 return computeKnownBits(V, DemandedElts, Depth).isZero();
2676}
2677
2678/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2680 unsigned Depth) const {
2681 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2682}
2683
2685 const APInt &DemandedElts,
2686 unsigned Depth) const {
2687 EVT VT = Op.getValueType();
2688 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2689
2690 unsigned NumElts = VT.getVectorNumElements();
2691 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2692
2693 APInt KnownZeroElements = APInt::getZero(NumElts);
2694 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2695 if (!DemandedElts[EltIdx])
2696 continue; // Don't query elements that are not demanded.
2697 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
2698 if (MaskedVectorIsZero(Op, Mask, Depth))
2699 KnownZeroElements.setBit(EltIdx);
2700 }
2701 return KnownZeroElements;
2702}
2703
2704/// isSplatValue - Return true if the vector V has the same value
2705/// across all DemandedElts. For scalable vectors, we don't know the
2706/// number of lanes at compile time. Instead, we use a 1 bit APInt
2707/// to represent a conservative value for all lanes; that is, that
2708/// one bit value is implicitly splatted across all lanes.
2709bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2710 APInt &UndefElts, unsigned Depth) const {
2711 unsigned Opcode = V.getOpcode();
2712 EVT VT = V.getValueType();
2713 assert(VT.isVector() && "Vector type expected");
2714 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2715 "scalable demanded bits are ignored");
2716
2717 if (!DemandedElts)
2718 return false; // No demanded elts, better to assume we don't know anything.
2719
2720 if (Depth >= MaxRecursionDepth)
2721 return false; // Limit search depth.
2722
2723 // Deal with some common cases here that work for both fixed and scalable
2724 // vector types.
2725 switch (Opcode) {
2726 case ISD::SPLAT_VECTOR:
2727 UndefElts = V.getOperand(0).isUndef()
2728 ? APInt::getAllOnes(DemandedElts.getBitWidth())
2729 : APInt(DemandedElts.getBitWidth(), 0);
2730 return true;
2731 case ISD::ADD:
2732 case ISD::SUB:
2733 case ISD::AND:
2734 case ISD::XOR:
2735 case ISD::OR: {
2736 APInt UndefLHS, UndefRHS;
2737 SDValue LHS = V.getOperand(0);
2738 SDValue RHS = V.getOperand(1);
2739 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2740 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2741 UndefElts = UndefLHS | UndefRHS;
2742 return true;
2743 }
2744 return false;
2745 }
2746 case ISD::ABS:
2747 case ISD::TRUNCATE:
2748 case ISD::SIGN_EXTEND:
2749 case ISD::ZERO_EXTEND:
2750 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2751 default:
2752 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2753 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2754 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,
2755 Depth);
2756 break;
2757}
2758
2759 // We don't support other cases than those above for scalable vectors at
2760 // the moment.
2761 if (VT.isScalableVector())
2762 return false;
2763
2764 unsigned NumElts = VT.getVectorNumElements();
2765 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2766 UndefElts = APInt::getZero(NumElts);
2767
2768 switch (Opcode) {
2769 case ISD::BUILD_VECTOR: {
2770 SDValue Scl;
2771 for (unsigned i = 0; i != NumElts; ++i) {
2772 SDValue Op = V.getOperand(i);
2773 if (Op.isUndef()) {
2774 UndefElts.setBit(i);
2775 continue;
2776 }
2777 if (!DemandedElts[i])
2778 continue;
2779 if (Scl && Scl != Op)
2780 return false;
2781 Scl = Op;
2782 }
2783 return true;
2784 }
2785 case ISD::VECTOR_SHUFFLE: {
2786 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
2787 APInt DemandedLHS = APInt::getZero(NumElts);
2788 APInt DemandedRHS = APInt::getZero(NumElts);
2789 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2790 for (int i = 0; i != (int)NumElts; ++i) {
2791 int M = Mask[i];
2792 if (M < 0) {
2793 UndefElts.setBit(i);
2794 continue;
2795 }
2796 if (!DemandedElts[i])
2797 continue;
2798 if (M < (int)NumElts)
2799 DemandedLHS.setBit(M);
2800 else
2801 DemandedRHS.setBit(M - NumElts);
2802 }
2803
2804 // If we aren't demanding either op, assume there's no splat.
2805 // If we are demanding both ops, assume there's no splat.
2806 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
2807 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
2808 return false;
2809
2810 // See if the demanded elts of the source op is a splat or we only demand
2811 // one element, which should always be a splat.
2812 // TODO: Handle source ops splats with undefs.
2813 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
2814 APInt SrcUndefs;
2815 return (SrcElts.popcount() == 1) ||
2816 (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&
2817 (SrcElts & SrcUndefs).isZero());
2818 };
2819 if (!DemandedLHS.isZero())
2820 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
2821 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
2822 }
2824 // Offset the demanded elts by the subvector index.
2825 SDValue Src = V.getOperand(0);
2826 // We don't support scalable vectors at the moment.
2827 if (Src.getValueType().isScalableVector())
2828 return false;
2829 uint64_t Idx = V.getConstantOperandVal(1);
2830 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2831 APInt UndefSrcElts;
2832 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
2833 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2834 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2835 return true;
2836 }
2837 break;
2838 }
2842 // Widen the demanded elts by the src element count.
2843 SDValue Src = V.getOperand(0);
2844 // We don't support scalable vectors at the moment.
2845 if (Src.getValueType().isScalableVector())
2846 return false;
2847 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2848 APInt UndefSrcElts;
2849 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
2850 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2851 UndefElts = UndefSrcElts.trunc(NumElts);
2852 return true;
2853 }
2854 break;
2855 }
2856 case ISD::BITCAST: {
2857 SDValue Src = V.getOperand(0);
2858 EVT SrcVT = Src.getValueType();
2859 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
2860 unsigned BitWidth = VT.getScalarSizeInBits();
2861
2862 // Ignore bitcasts from unsupported types.
2863 // TODO: Add fp support?
2864 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
2865 break;
2866
2867 // Bitcast 'small element' vector to 'large element' vector.
2868 if ((BitWidth % SrcBitWidth) == 0) {
2869 // See if each sub element is a splat.
2870 unsigned Scale = BitWidth / SrcBitWidth;
2871 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2872 APInt ScaledDemandedElts =
2873 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2874 for (unsigned I = 0; I != Scale; ++I) {
2875 APInt SubUndefElts;
2876 APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);
2877 APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);
2878 SubDemandedElts &= ScaledDemandedElts;
2879 if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))
2880 return false;
2881 // TODO: Add support for merging sub undef elements.
2882 if (!SubUndefElts.isZero())
2883 return false;
2884 }
2885 return true;
2886 }
2887 break;
2888 }
2889 }
2890
2891 return false;
2892}
2893
2894/// Helper wrapper to main isSplatValue function.
2895bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
2896 EVT VT = V.getValueType();
2897 assert(VT.isVector() && "Vector type expected");
2898
2899 APInt UndefElts;
2900 // Since the number of lanes in a scalable vector is unknown at compile time,
2901 // we track one bit which is implicitly broadcast to all lanes. This means
2902 // that all lanes in a scalable vector are considered demanded.
2903 APInt DemandedElts
2905 return isSplatValue(V, DemandedElts, UndefElts) &&
2906 (AllowUndefs || !UndefElts);
2907}
2908
2911
2912 EVT VT = V.getValueType();
2913 unsigned Opcode = V.getOpcode();
2914 switch (Opcode) {
2915 default: {
2916 APInt UndefElts;
2917 // Since the number of lanes in a scalable vector is unknown at compile time,
2918 // we track one bit which is implicitly broadcast to all lanes. This means
2919 // that all lanes in a scalable vector are considered demanded.
2920 APInt DemandedElts
2922
2923 if (isSplatValue(V, DemandedElts, UndefElts)) {
2924 if (VT.isScalableVector()) {
2925 // DemandedElts and UndefElts are ignored for scalable vectors, since
2926 // the only supported cases are SPLAT_VECTOR nodes.
2927 SplatIdx = 0;
2928 } else {
2929 // Handle case where all demanded elements are UNDEF.
2930 if (DemandedElts.isSubsetOf(UndefElts)) {
2931 SplatIdx = 0;
2932 return getUNDEF(VT);
2933 }
2934 SplatIdx = (UndefElts & DemandedElts).countr_one();
2935 }
2936 return V;
2937 }
2938 break;
2939 }
2940 case ISD::SPLAT_VECTOR:
2941 SplatIdx = 0;
2942 return V;
2943 case ISD::VECTOR_SHUFFLE: {
2944 assert(!VT.isScalableVector());
2945 // Check if this is a shuffle node doing a splat.
2946 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2947 // getTargetVShiftNode currently struggles without the splat source.
2948 auto *SVN = cast<ShuffleVectorSDNode>(V);
2949 if (!SVN->isSplat())
2950 break;
2951 int Idx = SVN->getSplatIndex();
2952 int NumElts = V.getValueType().getVectorNumElements();
2953 SplatIdx = Idx % NumElts;
2954 return V.getOperand(Idx / NumElts);
2955 }
2956 }
2957
2958 return SDValue();
2959}
2960
2962 int SplatIdx;
2963 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2964 EVT SVT = SrcVector.getValueType().getScalarType();
2965 EVT LegalSVT = SVT;
2966 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2967 if (!SVT.isInteger())
2968 return SDValue();
2969 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2970 if (LegalSVT.bitsLT(SVT))
2971 return SDValue();
2972 }
2973 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2974 getVectorIdxConstant(SplatIdx, SDLoc(V)));
2975 }
2976 return SDValue();
2977}
2978
2979const APInt *
2981 const APInt &DemandedElts) const {
2982 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2983 V.getOpcode() == ISD::SRA) &&
2984 "Unknown shift node");
2985 unsigned BitWidth = V.getScalarValueSizeInBits();
2986 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2987 // Shifting more than the bitwidth is not valid.
2988 const APInt &ShAmt = SA->getAPIntValue();
2989 if (ShAmt.ult(BitWidth))
2990 return &ShAmt;
2991 }
2992 return nullptr;
2993}
2994
2996 SDValue V, const APInt &DemandedElts) const {
2997 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2998 V.getOpcode() == ISD::SRA) &&
2999 "Unknown shift node");
3000 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3001 return ValidAmt;
3002 unsigned BitWidth = V.getScalarValueSizeInBits();
3003 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3004 if (!BV)
3005 return nullptr;
3006 const APInt *MinShAmt = nullptr;
3007 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3008 if (!DemandedElts[i])
3009 continue;
3010 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3011 if (!SA)
3012 return nullptr;
3013 // Shifting more than the bitwidth is not valid.
3014 const APInt &ShAmt = SA->getAPIntValue();
3015 if (ShAmt.uge(BitWidth))
3016 return nullptr;
3017 if (MinShAmt && MinShAmt->ule(ShAmt))
3018 continue;
3019 MinShAmt = &ShAmt;
3020 }
3021 return MinShAmt;
3022}
3023
3025 SDValue V, const APInt &DemandedElts) const {
3026 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3027 V.getOpcode() == ISD::SRA) &&
3028 "Unknown shift node");
3029 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3030 return ValidAmt;
3031 unsigned BitWidth = V.getScalarValueSizeInBits();
3032 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3033 if (!BV)
3034 return nullptr;
3035 const APInt *MaxShAmt = nullptr;
3036 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3037 if (!DemandedElts[i])
3038 continue;
3039 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3040 if (!SA)
3041 return nullptr;
3042 // Shifting more than the bitwidth is not valid.
3043 const APInt &ShAmt = SA->getAPIntValue();
3044 if (ShAmt.uge(BitWidth))
3045 return nullptr;
3046 if (MaxShAmt && MaxShAmt->uge(ShAmt))
3047 continue;
3048 MaxShAmt = &ShAmt;
3049 }
3050 return MaxShAmt;
3051}
3052
3053/// Determine which bits of Op are known to be either zero or one and return
3054/// them in Known. For vectors, the known bits are those that are shared by
3055/// every vector element.
3057 EVT VT = Op.getValueType();
3058
3059 // Since the number of lanes in a scalable vector is unknown at compile time,
3060 // we track one bit which is implicitly broadcast to all lanes. This means
3061 // that all lanes in a scalable vector are considered demanded.
3062 APInt DemandedElts = VT.isFixedLengthVector()
3064 : APInt(1, 1);
3065 return computeKnownBits(Op, DemandedElts, Depth);
3066}
3067
3068/// Determine which bits of Op are known to be either zero or one and return
3069/// them in Known. The DemandedElts argument allows us to only collect the known
3070/// bits that are shared by the requested vector elements.
3072 unsigned Depth) const {
3073 unsigned BitWidth = Op.getScalarValueSizeInBits();
3074
3075 KnownBits Known(BitWidth); // Don't know anything.
3076
3077 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3078 // We know all of the bits for a constant!
3079 return KnownBits::makeConstant(C->getAPIntValue());
3080 }
3081 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
3082 // We know all of the bits for a constant fp!
3083 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
3084 }
3085
3086 if (Depth >= MaxRecursionDepth)
3087 return Known; // Limit search depth.
3088
3089 KnownBits Known2;
3090 unsigned NumElts = DemandedElts.getBitWidth();
3091 assert((!Op.getValueType().isFixedLengthVector() ||
3092 NumElts == Op.getValueType().getVectorNumElements()) &&
3093 "Unexpected vector size");
3094
3095 if (!DemandedElts)
3096 return Known; // No demanded elts, better to assume we don't know anything.
3097
3098 unsigned Opcode = Op.getOpcode();
3099 switch (Opcode) {
3100 case ISD::MERGE_VALUES:
3101 return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,
3102 Depth + 1);
3103 case ISD::SPLAT_VECTOR: {
3104 SDValue SrcOp = Op.getOperand(0);
3105 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3106 "Expected SPLAT_VECTOR implicit truncation");
3107 // Implicitly truncate the bits to match the official semantics of
3108 // SPLAT_VECTOR.
3109 Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth);
3110 break;
3111 }
3113 unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();
3114 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3115 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3116 for (auto [I, SrcOp] : enumerate(Op->ops())) {
3117 Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);
3118 }
3119 break;
3120 }
3121 case ISD::STEP_VECTOR: {
3122 const APInt &Step = Op.getConstantOperandAPInt(0);
3123
3124 if (Step.isPowerOf2())
3125 Known.Zero.setLowBits(Step.logBase2());
3126
3128
3129 if (!isUIntN(BitWidth, Op.getValueType().getVectorMinNumElements()))
3130 break;
3131 const APInt MinNumElts =
3132 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3133
3134 bool Overflow;
3135 const APInt MaxNumElts = getVScaleRange(&F, BitWidth)
3137 .umul_ov(MinNumElts, Overflow);
3138 if (Overflow)
3139 break;
3140
3141 const APInt MaxValue = (MaxNumElts - 1).umul_ov(Step, Overflow);
3142 if (Overflow)
3143 break;
3144
3145 Known.Zero.setHighBits(MaxValue.countl_zero());
3146 break;
3147 }
3148 case ISD::BUILD_VECTOR:
3149 assert(!Op.getValueType().isScalableVector());
3150 // Collect the known bits that are shared by every demanded vector element.
3151 Known.Zero.setAllBits(); Known.One.setAllBits();
3152 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3153 if (!DemandedElts[i])
3154 continue;
3155
3156 SDValue SrcOp = Op.getOperand(i);
3157 Known2 = computeKnownBits(SrcOp, Depth + 1);
3158
3159 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3160 if (SrcOp.getValueSizeInBits() != BitWidth) {
3161 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3162 "Expected BUILD_VECTOR implicit truncation");
3163 Known2 = Known2.trunc(BitWidth);
3164 }
3165
3166 // Known bits are the values that are shared by every demanded element.
3167 Known = Known.intersectWith(Known2);
3168
3169 // If we don't know any bits, early out.
3170 if (Known.isUnknown())
3171 break;
3172 }
3173 break;
3174 case ISD::VECTOR_SHUFFLE: {
3175 assert(!Op.getValueType().isScalableVector());
3176 // Collect the known bits that are shared by every vector element referenced
3177 // by the shuffle.
3178 APInt DemandedLHS, DemandedRHS;
3179 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3180 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3181 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
3182 DemandedLHS, DemandedRHS))
3183 break;
3184
3185 // Known bits are the values that are shared by every demanded element.
3186 Known.Zero.setAllBits(); Known.One.setAllBits();
3187 if (!!DemandedLHS) {
3188 SDValue LHS = Op.getOperand(0);
3189 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
3190 Known = Known.intersectWith(Known2);
3191 }
3192 // If we don't know any bits, early out.
3193 if (Known.isUnknown())
3194 break;
3195 if (!!DemandedRHS) {
3196 SDValue RHS = Op.getOperand(1);
3197 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
3198 Known = Known.intersectWith(Known2);
3199 }
3200 break;
3201 }
3202 case ISD::VSCALE: {
3204 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
3205 Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits();
3206 break;
3207 }
3208 case ISD::CONCAT_VECTORS: {
3209 if (Op.getValueType().isScalableVector())
3210 break;
3211 // Split DemandedElts and test each of the demanded subvectors.
3212 Known.Zero.setAllBits(); Known.One.setAllBits();
3213 EVT SubVectorVT = Op.getOperand(0).getValueType();
3214 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3215 unsigned NumSubVectors = Op.getNumOperands();
3216 for (unsigned i = 0; i != NumSubVectors; ++i) {
3217 APInt DemandedSub =
3218 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
3219 if (!!DemandedSub) {
3220 SDValue Sub = Op.getOperand(i);
3221 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
3222 Known = Known.intersectWith(Known2);
3223 }
3224 // If we don't know any bits, early out.
3225 if (Known.isUnknown())
3226 break;
3227 }
3228 break;
3229 }
3230 case ISD::INSERT_SUBVECTOR: {
3231 if (Op.getValueType().isScalableVector())
3232 break;
3233 // Demand any elements from the subvector and the remainder from the src its
3234 // inserted into.
3235 SDValue Src = Op.getOperand(0);
3236 SDValue Sub = Op.getOperand(1);
3237 uint64_t Idx = Op.getConstantOperandVal(2);
3238 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3239 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3240 APInt DemandedSrcElts = DemandedElts;
3241 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
3242
3243 Known.One.setAllBits();
3244 Known.Zero.setAllBits();
3245 if (!!DemandedSubElts) {
3246 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
3247 if (Known.isUnknown())
3248 break; // early-out.
3249 }
3250 if (!!DemandedSrcElts) {
3251 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3252 Known = Known.intersectWith(Known2);
3253 }
3254 break;
3255 }
3257 // Offset the demanded elts by the subvector index.
3258 SDValue Src = Op.getOperand(0);
3259 // Bail until we can represent demanded elements for scalable vectors.
3260 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3261 break;
3262 uint64_t Idx = Op.getConstantOperandVal(1);
3263 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3264 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3265 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3266 break;
3267 }
3268 case ISD::SCALAR_TO_VECTOR: {
3269 if (Op.getValueType().isScalableVector())
3270 break;
3271 // We know about scalar_to_vector as much as we know about it source,
3272 // which becomes the first element of otherwise unknown vector.
3273 if (DemandedElts != 1)
3274 break;
3275
3276 SDValue N0 = Op.getOperand(0);
3277 Known = computeKnownBits(N0, Depth + 1);
3278 if (N0.getValueSizeInBits() != BitWidth)
3279 Known = Known.trunc(BitWidth);
3280
3281 break;
3282 }
3283 case ISD::BITCAST: {
3284 if (Op.getValueType().isScalableVector())
3285 break;
3286
3287 SDValue N0 = Op.getOperand(0);
3288 EVT SubVT = N0.getValueType();
3289 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3290
3291 // Ignore bitcasts from unsupported types.
3292 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3293 break;
3294
3295 // Fast handling of 'identity' bitcasts.
3296 if (BitWidth == SubBitWidth) {
3297 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
3298 break;
3299 }
3300
3301 bool IsLE = getDataLayout().isLittleEndian();
3302
3303 // Bitcast 'small element' vector to 'large element' scalar/vector.
3304 if ((BitWidth % SubBitWidth) == 0) {
3305 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3306
3307 // Collect known bits for the (larger) output by collecting the known
3308 // bits from each set of sub elements and shift these into place.
3309 // We need to separately call computeKnownBits for each set of
3310 // sub elements as the knownbits for each is likely to be different.
3311 unsigned SubScale = BitWidth / SubBitWidth;
3312 APInt SubDemandedElts(NumElts * SubScale, 0);
3313 for (unsigned i = 0; i != NumElts; ++i)
3314 if (DemandedElts[i])
3315 SubDemandedElts.setBit(i * SubScale);
3316
3317 for (unsigned i = 0; i != SubScale; ++i) {
3318 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
3319 Depth + 1);
3320 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3321 Known.insertBits(Known2, SubBitWidth * Shifts);
3322 }
3323 }
3324
3325 // Bitcast 'large element' scalar/vector to 'small element' vector.
3326 if ((SubBitWidth % BitWidth) == 0) {
3327 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3328
3329 // Collect known bits for the (smaller) output by collecting the known
3330 // bits from the overlapping larger input elements and extracting the
3331 // sub sections we actually care about.
3332 unsigned SubScale = SubBitWidth / BitWidth;
3333 APInt SubDemandedElts =
3334 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
3335 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
3336
3337 Known.Zero.setAllBits(); Known.One.setAllBits();
3338 for (unsigned i = 0; i != NumElts; ++i)
3339 if (DemandedElts[i]) {
3340 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3341 unsigned Offset = (Shifts % SubScale) * BitWidth;
3342 Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));
3343 // If we don't know any bits, early out.
3344 if (Known.isUnknown())
3345 break;
3346 }
3347 }
3348 break;
3349 }
3350 case ISD::AND:
3351 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3352 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3353
3354 Known &= Known2;
3355 break;
3356 case ISD::OR:
3357 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3358 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3359
3360 Known |= Known2;
3361 break;
3362 case ISD::XOR:
3363 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3364 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3365
3366 Known ^= Known2;
3367 break;
3368 case ISD::MUL: {
3369 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3370 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3371 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3372 // TODO: SelfMultiply can be poison, but not undef.
3373 if (SelfMultiply)
3374 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3375 Op.getOperand(0), DemandedElts, false, Depth + 1);
3376 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3377
3378 // If the multiplication is known not to overflow, the product of a number
3379 // with itself is non-negative. Only do this if we didn't already computed
3380 // the opposite value for the sign bit.
3381 if (Op->getFlags().hasNoSignedWrap() &&
3382 Op.getOperand(0) == Op.getOperand(1) &&
3383 !Known.isNegative())
3384 Known.makeNonNegative();
3385 break;
3386 }
3387 case ISD::MULHU: {
3388 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3389 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3390 Known = KnownBits::mulhu(Known, Known2);
3391 break;
3392 }
3393 case ISD::MULHS: {
3394 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3395 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3396 Known = KnownBits::mulhs(Known, Known2);
3397 break;
3398 }
3399 case ISD::UMUL_LOHI: {
3400 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3401 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3402 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3403 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3404 if (Op.getResNo() == 0)
3405 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3406 else
3407 Known = KnownBits::mulhu(Known, Known2);
3408 break;
3409 }
3410 case ISD::SMUL_LOHI: {
3411 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3412 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3413 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3414 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3415 if (Op.getResNo() == 0)
3416 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3417 else
3418 Known = KnownBits::mulhs(Known, Known2);
3419 break;
3420 }
3421 case ISD::AVGCEILU: {
3422 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3423 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3424 Known = Known.zext(BitWidth + 1);
3425 Known2 = Known2.zext(BitWidth + 1);
3427 Known = KnownBits::computeForAddCarry(Known, Known2, One);
3428 Known = Known.extractBits(BitWidth, 1);
3429 break;
3430 }
3431 case ISD::SELECT:
3432 case ISD::VSELECT:
3433 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3434 // If we don't know any bits, early out.
3435 if (Known.isUnknown())
3436 break;
3437 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3438
3439 // Only known if known in both the LHS and RHS.
3440 Known = Known.intersectWith(Known2);
3441 break;
3442 case ISD::SELECT_CC:
3443 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3444 // If we don't know any bits, early out.
3445 if (Known.isUnknown())
3446 break;
3447 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3448
3449 // Only known if known in both the LHS and RHS.
3450 Known = Known.intersectWith(Known2);
3451 break;
3452 case ISD::SMULO:
3453 case ISD::UMULO:
3454 if (Op.getResNo() != 1)
3455 break;
3456 // The boolean result conforms to getBooleanContents.
3457 // If we know the result of a setcc has the top bits zero, use this info.
3458 // We know that we have an integer-based boolean since these operations
3459 // are only available for integer.
3460 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3462 BitWidth > 1)
3463 Known.Zero.setBitsFrom(1);
3464 break;
3465 case ISD::SETCC:
3466 case ISD::SETCCCARRY:
3467 case ISD::STRICT_FSETCC:
3468 case ISD::STRICT_FSETCCS: {
3469 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3470 // If we know the result of a setcc has the top bits zero, use this info.
3471 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3473 BitWidth > 1)
3474 Known.Zero.setBitsFrom(1);
3475 break;
3476 }
3477 case ISD::SHL:
3478 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3479 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3480 Known = KnownBits::shl(Known, Known2);
3481
3482 // Minimum shift low bits are known zero.
3483 if (const APInt *ShMinAmt =
3485 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3486 break;
3487 case ISD::SRL:
3488 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3489 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3490 Known = KnownBits::lshr(Known, Known2, /*ShAmtNonZero=*/false,
3491 Op->getFlags().hasExact());
3492
3493 // Minimum shift high bits are known zero.
3494 if (const APInt *ShMinAmt =
3496 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3497 break;
3498 case ISD::SRA:
3499 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3500 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3501 Known = KnownBits::ashr(Known, Known2, /*ShAmtNonZero=*/false,
3502 Op->getFlags().hasExact());
3503 break;
3504 case ISD::FSHL:
3505 case ISD::FSHR:
3506 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3507 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3508
3509 // For fshl, 0-shift returns the 1st arg.
3510 // For fshr, 0-shift returns the 2nd arg.
3511 if (Amt == 0) {
3512 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3513 DemandedElts, Depth + 1);
3514 break;
3515 }
3516
3517 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3518 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3519 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3520 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3521 if (Opcode == ISD::FSHL) {
3522 Known.One <<= Amt;
3523 Known.Zero <<= Amt;
3524 Known2.One.lshrInPlace(BitWidth - Amt);
3525 Known2.Zero.lshrInPlace(BitWidth - Amt);
3526 } else {
3527 Known.One <<= BitWidth - Amt;
3528 Known.Zero <<= BitWidth - Amt;
3529 Known2.One.lshrInPlace(Amt);
3530 Known2.Zero.lshrInPlace(Amt);
3531 }
3532 Known = Known.unionWith(Known2);
3533 }
3534 break;
3535 case ISD::SHL_PARTS:
3536 case ISD::SRA_PARTS:
3537 case ISD::SRL_PARTS: {
3538 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3539
3540 // Collect lo/hi source values and concatenate.
3541 unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();
3542 unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();
3543 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3544 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3545 Known = Known2.concat(Known);
3546
3547 // Collect shift amount.
3548 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3549
3550 if (Opcode == ISD::SHL_PARTS)
3551 Known = KnownBits::shl(Known, Known2);
3552 else if (Opcode == ISD::SRA_PARTS)
3553 Known = KnownBits::ashr(Known, Known2);
3554 else // if (Opcode == ISD::SRL_PARTS)
3555 Known = KnownBits::lshr(Known, Known2);
3556
3557 // TODO: Minimum shift low/high bits are known zero.
3558
3559 if (Op.getResNo() == 0)
3560 Known = Known.extractBits(LoBits, 0);
3561 else
3562 Known = Known.extractBits(HiBits, LoBits);
3563 break;
3564 }
3566 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3567 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3568 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3569 break;
3570 }
3571 case ISD::CTTZ:
3572 case ISD::CTTZ_ZERO_UNDEF: {
3573 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3574 // If we have a known 1, its position is our upper bound.
3575 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3576 unsigned LowBits = llvm::bit_width(PossibleTZ);
3577 Known.Zero.setBitsFrom(LowBits);
3578 break;
3579 }
3580 case ISD::CTLZ:
3581 case ISD::CTLZ_ZERO_UNDEF: {
3582 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3583 // If we have a known 1, its position is our upper bound.
3584 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3585 unsigned LowBits = llvm::bit_width(PossibleLZ);
3586 Known.Zero.setBitsFrom(LowBits);
3587 break;
3588 }
3589 case ISD::CTPOP: {
3590 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3591 // If we know some of the bits are zero, they can't be one.
3592 unsigned PossibleOnes = Known2.countMaxPopulation();
3593 Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));
3594 break;
3595 }
3596 case ISD::PARITY: {
3597 // Parity returns 0 everywhere but the LSB.
3598 Known.Zero.setBitsFrom(1);
3599 break;
3600 }
3601 case ISD::LOAD: {
3602 LoadSDNode *LD = cast<LoadSDNode>(Op);
3603 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3604 if (ISD::isNON_EXTLoad(LD) && Cst) {
3605 // Determine any common known bits from the loaded constant pool value.
3606 Type *CstTy = Cst->getType();
3607 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3608 !Op.getValueType().isScalableVector()) {
3609 // If its a vector splat, then we can (quickly) reuse the scalar path.
3610 // NOTE: We assume all elements match and none are UNDEF.
3611 if (CstTy->isVectorTy()) {
3612 if (const Constant *Splat = Cst->getSplatValue()) {
3613 Cst = Splat;
3614 CstTy = Cst->getType();
3615 }
3616 }
3617 // TODO - do we need to handle different bitwidths?
3618 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3619 // Iterate across all vector elements finding common known bits.
3620 Known.One.setAllBits();
3621 Known.Zero.setAllBits();
3622 for (unsigned i = 0; i != NumElts; ++i) {
3623 if (!DemandedElts[i])
3624 continue;
3625 if (Constant *Elt = Cst->getAggregateElement(i)) {
3626 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3627 const APInt &Value = CInt->getValue();
3628 Known.One &= Value;
3629 Known.Zero &= ~Value;
3630 continue;
3631 }
3632 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3633 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3634 Known.One &= Value;
3635 Known.Zero &= ~Value;
3636 continue;
3637 }
3638 }
3639 Known.One.clearAllBits();
3640 Known.Zero.clearAllBits();
3641 break;
3642 }
3643 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3644 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3645 Known = KnownBits::makeConstant(CInt->getValue());
3646 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3647 Known =
3648 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3649 }
3650 }
3651 }
3652 } else if (Op.getResNo() == 0) {
3653 KnownBits Known0(!LD->getMemoryVT().isScalableVT()
3654 ? LD->getMemoryVT().getFixedSizeInBits()
3655 : BitWidth);
3656 EVT VT = Op.getValueType();
3657 // Fill in any known bits from range information. There are 3 types being
3658 // used. The results VT (same vector elt size as BitWidth), the loaded
3659 // MemoryVT (which may or may not be vector) and the range VTs original
3660 // type. The range matadata needs the full range (i.e
3661 // MemoryVT().getSizeInBits()), which is truncated to the correct elt size
3662 // if it is know. These are then extended to the original VT sizes below.
3663 if (const MDNode *MD = LD->getRanges()) {
3665 if (VT.isVector()) {
3666 // Handle truncation to the first demanded element.
3667 // TODO: Figure out which demanded elements are covered
3668 if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
3669 break;
3670 Known0 = Known0.trunc(BitWidth);
3671 }
3672 }
3673
3674 if (LD->getMemoryVT().isVector())
3675 Known0 = Known0.trunc(LD->getMemoryVT().getScalarSizeInBits());
3676
3677 // Extend the Known bits from memory to the size of the result.
3678 if (ISD::isZEXTLoad(Op.getNode()))
3679 Known = Known0.zext(BitWidth);
3680 else if (ISD::isSEXTLoad(Op.getNode()))
3681 Known = Known0.sext(BitWidth);
3682 else if (ISD::isEXTLoad(Op.getNode()))
3683 Known = Known0.anyext(BitWidth);
3684 else
3685 Known = Known0;
3686 assert(Known.getBitWidth() == BitWidth);
3687 return Known;
3688 }
3689 break;
3690 }
3692 if (Op.getValueType().isScalableVector())
3693 break;
3694 EVT InVT = Op.getOperand(0).getValueType();
3695 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3696 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3697 Known = Known.zext(BitWidth);
3698 break;
3699 }
3700 case ISD::ZERO_EXTEND: {
3701 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3702 Known = Known.zext(BitWidth);
3703 break;
3704 }
3706 if (Op.getValueType().isScalableVector())
3707 break;
3708 EVT InVT = Op.getOperand(0).getValueType();
3709 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3710 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3711 // If the sign bit is known to be zero or one, then sext will extend
3712 // it to the top bits, else it will just zext.
3713 Known = Known.sext(BitWidth);
3714 break;
3715 }
3716 case ISD::SIGN_EXTEND: {
3717 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3718 // If the sign bit is known to be zero or one, then sext will extend
3719 // it to the top bits, else it will just zext.
3720 Known = Known.sext(BitWidth);
3721 break;
3722 }
3724 if (Op.getValueType().isScalableVector())
3725 break;
3726 EVT InVT = Op.getOperand(0).getValueType();
3727 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3728 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3729 Known = Known.anyext(BitWidth);
3730 break;
3731 }
3732 case ISD::ANY_EXTEND: {
3733 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3734 Known = Known.anyext(BitWidth);
3735 break;
3736 }
3737 case ISD::TRUNCATE: {
3738 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3739 Known = Known.trunc(BitWidth);
3740 break;
3741 }
3742 case ISD::AssertZext: {
3743 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3745 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3746 Known.Zero |= (~InMask);
3747 Known.One &= (~Known.Zero);
3748 break;
3749 }
3750 case ISD::AssertAlign: {
3751 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3752 assert(LogOfAlign != 0);
3753
3754 // TODO: Should use maximum with source
3755 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3756 // well as clearing one bits.
3757 Known.Zero.setLowBits(LogOfAlign);
3758 Known.One.clearLowBits(LogOfAlign);
3759 break;
3760 }
3761 case ISD::FGETSIGN:
3762 // All bits are zero except the low bit.
3763 Known.Zero.setBitsFrom(1);
3764 break;
3765 case ISD::ADD:
3766 case ISD::SUB: {
3767 SDNodeFlags Flags = Op.getNode()->getFlags();
3768 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3769 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3771 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(),
3772 Flags.hasNoUnsignedWrap(), Known, Known2);
3773 break;
3774 }
3775 case ISD::USUBO:
3776 case ISD::SSUBO:
3777 case ISD::USUBO_CARRY:
3778 case ISD::SSUBO_CARRY:
3779 if (Op.getResNo() == 1) {
3780 // If we know the result of a setcc has the top bits zero, use this info.
3781 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3783 BitWidth > 1)
3784 Known.Zero.setBitsFrom(1);
3785 break;
3786 }
3787 [[fallthrough]];
3788 case ISD::SUBC: {
3789 assert(Op.getResNo() == 0 &&
3790 "We only compute knownbits for the difference here.");
3791
3792 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
3793 KnownBits Borrow(1);
3794 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
3795 Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3796 // Borrow has bit width 1
3797 Borrow = Borrow.trunc(1);
3798 } else {
3799 Borrow.setAllZero();
3800 }
3801
3802 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3803 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3804 Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
3805 break;
3806 }
3807 case ISD::UADDO:
3808 case ISD::SADDO:
3809 case ISD::UADDO_CARRY:
3810 case ISD::SADDO_CARRY:
3811 if (Op.getResNo() == 1) {
3812 // If we know the result of a setcc has the top bits zero, use this info.
3813 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3815 BitWidth > 1)
3816 Known.Zero.setBitsFrom(1);
3817 break;
3818 }
3819 [[fallthrough]];
3820 case ISD::ADDC:
3821 case ISD::ADDE: {
3822 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3823
3824 // With ADDE and UADDO_CARRY, a carry bit may be added in.
3825 KnownBits Carry(1);
3826 if (Opcode == ISD::ADDE)
3827 // Can't track carry from glue, set carry to unknown.
3828 Carry.resetAll();
3829 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
3830 Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3831 // Carry has bit width 1
3832 Carry = Carry.trunc(1);
3833 } else {
3834 Carry.setAllZero();
3835 }
3836
3837 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3838 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3839 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3840 break;
3841 }
3842 case ISD::UDIV: {
3843 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3844 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3845 Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());
3846 break;
3847 }
3848 case ISD::SDIV: {
3849 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3850 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3851 Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());
3852 break;
3853 }
3854 case ISD::SREM: {
3855 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3856 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3857 Known = KnownBits::srem(Known, Known2);
3858 break;
3859 }
3860 case ISD::UREM: {
3861 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3862 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3863 Known = KnownBits::urem(Known, Known2);
3864 break;
3865 }
3866 case ISD::EXTRACT_ELEMENT: {
3867 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3868 const unsigned Index = Op.getConstantOperandVal(1);
3869 const unsigned EltBitWidth = Op.getValueSizeInBits();
3870
3871 // Remove low part of known bits mask
3872 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3873 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3874
3875 // Remove high part of known bit mask
3876 Known = Known.trunc(EltBitWidth);
3877 break;
3878 }
3880 SDValue InVec = Op.getOperand(0);
3881 SDValue EltNo = Op.getOperand(1);
3882 EVT VecVT = InVec.getValueType();
3883 // computeKnownBits not yet implemented for scalable vectors.
3884 if (VecVT.isScalableVector())
3885 break;
3886 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3887 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3888
3889 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3890 // anything about the extended bits.
3891 if (BitWidth > EltBitWidth)
3892 Known = Known.trunc(EltBitWidth);
3893
3894 // If we know the element index, just demand that vector element, else for
3895 // an unknown element index, ignore DemandedElts and demand them all.
3896 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
3897 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3898 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3899 DemandedSrcElts =
3900 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3901
3902 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3903 if (BitWidth > EltBitWidth)
3904 Known = Known.anyext(BitWidth);
3905 break;
3906 }
3908 if (Op.getValueType().isScalableVector())
3909 break;
3910
3911 // If we know the element index, split the demand between the
3912 // source vector and the inserted element, otherwise assume we need
3913 // the original demanded vector elements and the value.
3914 SDValue InVec = Op.getOperand(0);
3915 SDValue InVal = Op.getOperand(1);
3916 SDValue EltNo = Op.getOperand(2);
3917 bool DemandedVal = true;
3918 APInt DemandedVecElts = DemandedElts;
3919 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3920 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3921 unsigned EltIdx = CEltNo->getZExtValue();
3922 DemandedVal = !!DemandedElts[EltIdx];
3923 DemandedVecElts.clearBit(EltIdx);
3924 }
3925 Known.One.setAllBits();
3926 Known.Zero.setAllBits();
3927 if (DemandedVal) {
3928 Known2 = computeKnownBits(InVal, Depth + 1);
3929 Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));
3930 }
3931 if (!!DemandedVecElts) {
3932 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3933 Known = Known.intersectWith(Known2);
3934 }
3935 break;
3936 }
3937 case ISD::BITREVERSE: {
3938 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3939 Known = Known2.reverseBits();
3940 break;
3941 }
3942 case ISD::BSWAP: {
3943 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3944 Known = Known2.byteSwap();
3945 break;
3946 }
3947 case ISD::ABS: {
3948 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3949 Known = Known2.abs();
3950 break;
3951 }
3952 case ISD::USUBSAT: {
3953 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3954 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3955 Known = KnownBits::usub_sat(Known, Known2);
3956 break;
3957 }
3958 case ISD::UMIN: {
3959 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3960 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3961 Known = KnownBits::umin(Known, Known2);
3962 break;
3963 }
3964 case ISD::UMAX: {
3965 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3966 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3967 Known = KnownBits::umax(Known, Known2);
3968 break;
3969 }
3970 case ISD::SMIN:
3971 case ISD::SMAX: {
3972 // If we have a clamp pattern, we know that the number of sign bits will be
3973 // the minimum of the clamp min/max range.
3974 bool IsMax = (Opcode == ISD::SMAX);
3975 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3976 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3977 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3978 CstHigh =
3979 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3980 if (CstLow && CstHigh) {
3981 if (!IsMax)
3982 std::swap(CstLow, CstHigh);
3983
3984 const APInt &ValueLow = CstLow->getAPIntValue();
3985 const APInt &ValueHigh = CstHigh->getAPIntValue();
3986 if (ValueLow.sle(ValueHigh)) {
3987 unsigned LowSignBits = ValueLow.getNumSignBits();
3988 unsigned HighSignBits = ValueHigh.getNumSignBits();
3989 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3990 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3991 Known.One.setHighBits(MinSignBits);
3992 break;
3993 }
3994 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3995 Known.Zero.setHighBits(MinSignBits);
3996 break;
3997 }
3998 }
3999 }
4000
4001 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4002 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4003 if (IsMax)
4004 Known = KnownBits::smax(Known, Known2);
4005 else
4006 Known = KnownBits::smin(Known, Known2);
4007
4008 // For SMAX, if CstLow is non-negative we know the result will be
4009 // non-negative and thus all sign bits are 0.
4010 // TODO: There's an equivalent of this for smin with negative constant for
4011 // known ones.
4012 if (IsMax && CstLow) {
4013 const APInt &ValueLow = CstLow->getAPIntValue();
4014 if (ValueLow.isNonNegative()) {
4015 unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4016 Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));
4017 }
4018 }
4019
4020 break;
4021 }
4022 case ISD::UINT_TO_FP: {
4023 Known.makeNonNegative();
4024 break;
4025 }
4026 case ISD::SINT_TO_FP: {
4027 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4028 if (Known2.isNonNegative())
4029 Known.makeNonNegative();
4030 else if (Known2.isNegative())
4031 Known.makeNegative();
4032 break;
4033 }
4034 case ISD::FP_TO_UINT_SAT: {
4035 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4036 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4038 break;
4039 }
4041 if (Op.getResNo() == 1) {
4042 // The boolean result conforms to getBooleanContents.
4043 // If we know the result of a setcc has the top bits zero, use this info.
4044 // We know that we have an integer-based boolean since these operations
4045 // are only available for integer.
4046 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
4048 BitWidth > 1)
4049 Known.Zero.setBitsFrom(1);
4050 break;
4051 }
4052 [[fallthrough]];
4054 case ISD::ATOMIC_SWAP:
4066 case ISD::ATOMIC_LOAD: {
4067 unsigned MemBits =
4068 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4069 // If we are looking at the loaded value.
4070 if (Op.getResNo() == 0) {
4072 Known.Zero.setBitsFrom(MemBits);
4073 }
4074 break;
4075 }
4076 case ISD::FrameIndex:
4078 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
4079 Known, getMachineFunction());
4080 break;
4081
4082 default:
4083 if (Opcode < ISD::BUILTIN_OP_END)
4084 break;
4085 [[fallthrough]];
4089 // TODO: Probably okay to remove after audit; here to reduce change size
4090 // in initial enablement patch for scalable vectors
4091 if (Op.getValueType().isScalableVector())
4092 break;
4093
4094 // Allow the target to implement this method for its nodes.
4095 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
4096 break;
4097 }
4098
4099 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
4100 return Known;
4101}
4102
4103/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4105 switch (OR) {
4113 }
4114 llvm_unreachable("Unknown OverflowResult");
4115}
4116
4119 // X + 0 never overflow
4120 if (isNullConstant(N1))
4121 return OFK_Never;
4122
4123 // If both operands each have at least two sign bits, the addition
4124 // cannot overflow.
4125 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4126 return OFK_Never;
4127
4128 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4129 return OFK_Sometime;
4130}
4131
4134 // X + 0 never overflow
4135 if (isNullConstant(N1))
4136 return OFK_Never;
4137
4138 // mulhi + 1 never overflow
4139 KnownBits N1Known = computeKnownBits(N1);
4140 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4141 N1Known.getMaxValue().ult(2))
4142 return OFK_Never;
4143
4144 KnownBits N0Known = computeKnownBits(N0);
4145 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4146 N0Known.getMaxValue().ult(2))
4147 return OFK_Never;
4148
4149 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4150 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4151 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4152 return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));
4153}
4154
4157 // X - 0 never overflow
4158 if (isNullConstant(N1))
4159 return OFK_Never;
4160
4161 // If both operands each have at least two sign bits, the subtraction
4162 // cannot overflow.
4163 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4164 return OFK_Never;
4165
4166 KnownBits N0Known = computeKnownBits(N0);
4167 KnownBits N1Known = computeKnownBits(N1);
4168 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);
4169 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);
4170 return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));
4171}
4172
4175 // X - 0 never overflow
4176 if (isNullConstant(N1))
4177 return OFK_Never;
4178
4179 KnownBits N0Known = computeKnownBits(N0);
4180 KnownBits N1Known = computeKnownBits(N1);
4181 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4182 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4183 return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));
4184}
4185
4188 // X * 0 and X * 1 never overflow.
4189 if (isNullConstant(N1) || isOneConstant(N1))
4190 return OFK_Never;
4191
4192 KnownBits N0Known = computeKnownBits(N0);
4193 KnownBits N1Known = computeKnownBits(N1);
4194 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4195 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4196 return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));
4197}
4198
4201 // X * 0 and X * 1 never overflow.
4202 if (isNullConstant(N1) || isOneConstant(N1))
4203 return OFK_Never;
4204
4205 // Get the size of the result.
4206 unsigned BitWidth = N0.getScalarValueSizeInBits();
4207
4208 // Sum of the sign bits.
4209 unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);
4210
4211 // If we have enough sign bits, then there's no overflow.
4212 if (SignBits > BitWidth + 1)
4213 return OFK_Never;
4214
4215 if (SignBits == BitWidth + 1) {
4216 // The overflow occurs when the true multiplication of the
4217 // the operands is the minimum negative number.
4218 KnownBits N0Known = computeKnownBits(N0);
4219 KnownBits N1Known = computeKnownBits(N1);
4220 // If one of the operands is non-negative, then there's no
4221 // overflow.
4222 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4223 return OFK_Never;
4224 }
4225
4226 return OFK_Sometime;
4227}
4228
4230 if (Depth >= MaxRecursionDepth)
4231 return false; // Limit search depth.
4232
4233 EVT OpVT = Val.getValueType();
4234 unsigned BitWidth = OpVT.getScalarSizeInBits();
4235
4236 // Is the constant a known power of 2?
4238 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4239 }))
4240 return true;
4241
4242 // A left-shift of a constant one will have exactly one bit set because
4243 // shifting the bit off the end is undefined.
4244 if (Val.getOpcode() == ISD::SHL) {
4245 auto *C = isConstOrConstSplat(Val.getOperand(0));
4246 if (C && C->getAPIntValue() == 1)
4247 return true;
4248 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4249 isKnownNeverZero(Val, Depth);
4250 }
4251
4252 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4253 // one bit set.
4254 if (Val.getOpcode() == ISD::SRL) {
4255 auto *C = isConstOrConstSplat(Val.getOperand(0));
4256 if (C && C->getAPIntValue().isSignMask())
4257 return true;
4258 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4259 isKnownNeverZero(Val, Depth);
4260 }
4261
4262 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4263 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4264
4265 // Are all operands of a build vector constant powers of two?
4266 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4267 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
4268 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4269 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4270 return false;
4271 }))
4272 return true;
4273
4274 // Is the operand of a splat vector a constant power of two?
4275 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4276 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
4277 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
4278 return true;
4279
4280 // vscale(power-of-two) is a power-of-two for some targets
4281 if (Val.getOpcode() == ISD::VSCALE &&
4282 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4284 return true;
4285
4286 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4287 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4288 return isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1) &&
4290
4291 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4292 return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) &&
4294
4295 // Looking for `x & -x` pattern:
4296 // If x == 0:
4297 // x & -x -> 0
4298 // If x != 0:
4299 // x & -x -> non-zero pow2
4300 // so if we find the pattern return whether we know `x` is non-zero.
4301 SDValue X;
4302 if (sd_match(Val, m_And(m_Value(X), m_Sub(m_Zero(), m_Deferred(X)))))
4303 return isKnownNeverZero(X, Depth);
4304
4305 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4306 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4307
4308 // More could be done here, though the above checks are enough
4309 // to handle some common cases.
4310 return false;
4311}
4312
4314 EVT VT = Op.getValueType();
4315
4316 // Since the number of lanes in a scalable vector is unknown at compile time,
4317 // we track one bit which is implicitly broadcast to all lanes. This means
4318 // that all lanes in a scalable vector are considered demanded.
4319 APInt DemandedElts = VT.isFixedLengthVector()
4321 : APInt(1, 1);
4322 return ComputeNumSignBits(Op, DemandedElts, Depth);
4323}
4324
4325unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4326 unsigned Depth) const {
4327 EVT VT = Op.getValueType();
4328 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4329 unsigned VTBits = VT.getScalarSizeInBits();
4330 unsigned NumElts = DemandedElts.getBitWidth();
4331 unsigned Tmp, Tmp2;
4332 unsigned FirstAnswer = 1;
4333
4334 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4335 const APInt &Val = C->getAPIntValue();
4336 return Val.getNumSignBits();
4337 }
4338
4339 if (Depth >= MaxRecursionDepth)
4340 return 1; // Limit search depth.
4341
4342 if (!DemandedElts)
4343 return 1; // No demanded elts, better to assume we don't know anything.
4344
4345 unsigned Opcode = Op.getOpcode();
4346 switch (Opcode) {
4347 default: break;
4348 case ISD::AssertSext:
4349 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4350 return VTBits-Tmp+1;
4351 case ISD::AssertZext:
4352 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4353 return VTBits-Tmp;
4354 case ISD::MERGE_VALUES:
4355 return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,
4356 Depth + 1);
4357 case ISD::SPLAT_VECTOR: {
4358 // Check if the sign bits of source go down as far as the truncated value.
4359 unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();
4360 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4361 if (NumSrcSignBits > (NumSrcBits - VTBits))
4362 return NumSrcSignBits - (NumSrcBits - VTBits);
4363 break;
4364 }
4365 case ISD::BUILD_VECTOR:
4366 assert(!VT.isScalableVector());
4367 Tmp = VTBits;
4368 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4369 if (!DemandedElts[i])
4370 continue;
4371
4372 SDValue SrcOp = Op.getOperand(i);
4373 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4374 // for constant nodes to ensure we only look at the sign bits.
4375 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SrcOp)) {
4376 APInt T = C->getAPIntValue().trunc(VTBits);
4377 Tmp2 = T.getNumSignBits();
4378 } else {
4379 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
4380
4381 if (SrcOp.getValueSizeInBits() != VTBits) {
4382 assert(SrcOp.getValueSizeInBits() > VTBits &&
4383 "Expected BUILD_VECTOR implicit truncation");
4384 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4385 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4386 }
4387 }
4388 Tmp = std::min(Tmp, Tmp2);
4389 }
4390 return Tmp;
4391
4392 case ISD::VECTOR_SHUFFLE: {
4393 // Collect the minimum number of sign bits that are shared by every vector
4394 // element referenced by the shuffle.
4395 APInt DemandedLHS, DemandedRHS;
4396 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
4397 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4398 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
4399 DemandedLHS, DemandedRHS))
4400 return 1;
4401
4402 Tmp = std::numeric_limits<unsigned>::max();
4403 if (!!DemandedLHS)
4404 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
4405 if (!!DemandedRHS) {
4406 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
4407 Tmp = std::min(Tmp, Tmp2);
4408 }
4409 // If we don't know anything, early out and try computeKnownBits fall-back.
4410 if (Tmp == 1)
4411 break;
4412 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4413 return Tmp;
4414 }
4415
4416 case ISD::BITCAST: {
4417 if (VT.isScalableVector())
4418 break;
4419 SDValue N0 = Op.getOperand(0);
4420 EVT SrcVT = N0.getValueType();
4421 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4422
4423 // Ignore bitcasts from unsupported types..
4424 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4425 break;
4426
4427 // Fast handling of 'identity' bitcasts.
4428 if (VTBits == SrcBits)
4429 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
4430
4431 bool IsLE = getDataLayout().isLittleEndian();
4432
4433 // Bitcast 'large element' scalar/vector to 'small element' vector.
4434 if ((SrcBits % VTBits) == 0) {
4435 assert(VT.isVector() && "Expected bitcast to vector");
4436
4437 unsigned Scale = SrcBits / VTBits;
4438 APInt SrcDemandedElts =
4439 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
4440
4441 // Fast case - sign splat can be simply split across the small elements.
4442 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
4443 if (Tmp == SrcBits)
4444 return VTBits;
4445
4446 // Slow case - determine how far the sign extends into each sub-element.
4447 Tmp2 = VTBits;
4448 for (unsigned i = 0; i != NumElts; ++i)
4449 if (DemandedElts[i]) {
4450 unsigned SubOffset = i % Scale;
4451 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4452 SubOffset = SubOffset * VTBits;
4453 if (Tmp <= SubOffset)
4454 return 1;
4455 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4456 }
4457 return Tmp2;
4458 }
4459 break;
4460 }
4461
4463 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4464 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4465 return VTBits - Tmp + 1;
4466 case ISD::SIGN_EXTEND:
4467 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
4468 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
4470 // Max of the input and what this extends.
4471 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4472 Tmp = VTBits-Tmp+1;
4473 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4474 return std::max(Tmp, Tmp2);
4476 if (VT.isScalableVector())
4477 break;
4478 SDValue Src = Op.getOperand(0);
4479 EVT SrcVT = Src.getValueType();
4480 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
4481 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4482 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
4483 }
4484 case ISD::SRA:
4485 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4486 // SRA X, C -> adds C sign bits.
4487 if (const APInt *ShAmt =
4489 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
4490 return Tmp;
4491 case ISD::SHL:
4492 if (const APInt *ShAmt =
4493 getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
4494 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4495 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4496 if (ShAmt->ult(Tmp))
4497 return Tmp - ShAmt->getZExtValue();
4498 }
4499 break;
4500 case ISD::AND:
4501 case ISD::OR:
4502 case ISD::XOR: // NOT is handled here.
4503 // Logical binary ops preserve the number of sign bits at the worst.
4504 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4505 if (Tmp != 1) {
4506 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4507 FirstAnswer = std::min(Tmp, Tmp2);
4508 // We computed what we know about the sign bits as our first
4509 // answer. Now proceed to the generic code that uses
4510 // computeKnownBits, and pick whichever answer is better.
4511 }
4512 break;
4513
4514 case ISD::SELECT:
4515 case ISD::VSELECT:
4516 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4517 if (Tmp == 1) return 1; // Early out.
4518 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4519 return std::min(Tmp, Tmp2);
4520 case ISD::SELECT_CC:
4521 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4522 if (Tmp == 1) return 1; // Early out.
4523 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
4524 return std::min(Tmp, Tmp2);
4525
4526 case ISD::SMIN:
4527 case ISD::SMAX: {
4528 // If we have a clamp pattern, we know that the number of sign bits will be
4529 // the minimum of the clamp min/max range.
4530 bool IsMax = (Opcode == ISD::SMAX);
4531 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4532 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4533 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4534 CstHigh =
4535 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4536 if (CstLow && CstHigh) {
4537 if (!IsMax)
4538 std::swap(CstLow, CstHigh);
4539 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
4540 Tmp = CstLow->getAPIntValue().getNumSignBits();
4541 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4542 return std::min(Tmp, Tmp2);
4543 }
4544 }
4545
4546 // Fallback - just get the minimum number of sign bits of the operands.
4547 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4548 if (Tmp == 1)
4549 return 1; // Early out.
4550 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4551 return std::min(Tmp, Tmp2);
4552 }
4553 case ISD::UMIN:
4554 case ISD::UMAX:
4555 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4556 if (Tmp == 1)
4557 return 1; // Early out.
4558 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4559 return std::min(Tmp, Tmp2);
4560 case ISD::SADDO:
4561 case ISD::UADDO:
4562 case ISD::SADDO_CARRY:
4563 case ISD::UADDO_CARRY:
4564 case ISD::SSUBO:
4565 case ISD::USUBO:
4566 case ISD::SSUBO_CARRY:
4567 case ISD::USUBO_CARRY:
4568 case ISD::SMULO:
4569 case ISD::UMULO:
4570 if (Op.getResNo() != 1)
4571 break;
4572 // The boolean result conforms to getBooleanContents. Fall through.
4573 // If setcc returns 0/-1, all bits are sign bits.
4574 // We know that we have an integer-based boolean since these operations
4575 // are only available for integer.
4576 if (TLI->getBooleanContents(VT.isVector(), false) ==
4578 return VTBits;
4579 break;
4580 case ISD::SETCC:
4581 case ISD::SETCCCARRY:
4582 case ISD::STRICT_FSETCC:
4583 case ISD::STRICT_FSETCCS: {
4584 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
4585 // If setcc returns 0/-1, all bits are sign bits.
4586 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
4588 return VTBits;
4589 break;
4590 }
4591 case ISD::ROTL:
4592 case ISD::ROTR:
4593 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4594
4595 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
4596 if (Tmp == VTBits)
4597 return VTBits;
4598
4599 if (ConstantSDNode *C =
4600 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
4601 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
4602
4603 // Handle rotate right by N like a rotate left by 32-N.
4604 if (Opcode == ISD::ROTR)
4605 RotAmt = (VTBits - RotAmt) % VTBits;
4606
4607 // If we aren't rotating out all of the known-in sign bits, return the
4608 // number that are left. This handles rotl(sext(x), 1) for example.
4609 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
4610 }
4611 break;
4612 case ISD::ADD:
4613 case ISD::ADDC:
4614 // Add can have at most one carry bit. Thus we know that the output
4615 // is, at worst, one more bit than the inputs.
4616 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4617 if (Tmp == 1) return 1; // Early out.
4618
4619 // Special case decrementing a value (ADD X, -1):
4620 if (ConstantSDNode *CRHS =
4621 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
4622 if (CRHS->isAllOnes()) {
4623 KnownBits Known =
4624 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4625
4626 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4627 // sign bits set.
4628 if ((Known.Zero | 1).isAllOnes())
4629 return VTBits;
4630
4631 // If we are subtracting one from a positive number, there is no carry
4632 // out of the result.
4633 if (Known.isNonNegative())
4634 return Tmp;
4635 }
4636
4637 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4638 if (Tmp2 == 1) return 1; // Early out.
4639 return std::min(Tmp, Tmp2) - 1;
4640 case ISD::SUB:
4641 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4642 if (Tmp2 == 1) return 1; // Early out.
4643
4644 // Handle NEG.
4645 if (ConstantSDNode *CLHS =
4646 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
4647 if (CLHS->isZero()) {
4648 KnownBits Known =
4649 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4650 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4651 // sign bits set.
4652 if ((Known.Zero | 1).isAllOnes())
4653 return VTBits;
4654
4655 // If the input is known to be positive (the sign bit is known clear),
4656 // the output of the NEG has the same number of sign bits as the input.
4657 if (Known.isNonNegative())
4658 return Tmp2;
4659
4660 // Otherwise, we treat this like a SUB.
4661 }
4662
4663 // Sub can have at most one carry bit. Thus we know that the output
4664 // is, at worst, one more bit than the inputs.
4665 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4666 if (Tmp == 1) return 1; // Early out.
4667 return std::min(Tmp, Tmp2) - 1;
4668 case ISD::MUL: {
4669 // The output of the Mul can be at most twice the valid bits in the inputs.
4670 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4671 if (SignBitsOp0 == 1)
4672 break;
4673 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4674 if (SignBitsOp1 == 1)
4675 break;
4676 unsigned OutValidBits =
4677 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4678 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4679 }
4680 case ISD::SREM:
4681 // The sign bit is the LHS's sign bit, except when the result of the
4682 // remainder is zero. The magnitude of the result should be less than or
4683 // equal to the magnitude of the LHS. Therefore, the result should have
4684 // at least as many sign bits as the left hand side.
4685 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4686 case ISD::TRUNCATE: {
4687 // Check if the sign bits of source go down as far as the truncated value.
4688 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4689 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4690 if (NumSrcSignBits > (NumSrcBits - VTBits))
4691 return NumSrcSignBits - (NumSrcBits - VTBits);
4692 break;
4693 }
4694 case ISD::EXTRACT_ELEMENT: {
4695 if (VT.isScalableVector())
4696 break;
4697 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4698 const int BitWidth = Op.getValueSizeInBits();
4699 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4700
4701 // Get reverse index (starting from 1), Op1 value indexes elements from
4702 // little end. Sign starts at big end.
4703 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4704
4705 // If the sign portion ends in our element the subtraction gives correct
4706 // result. Otherwise it gives either negative or > bitwidth result
4707 return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth);
4708 }
4710 if (VT.isScalableVector())
4711 break;
4712 // If we know the element index, split the demand between the
4713 // source vector and the inserted element, otherwise assume we need
4714 // the original demanded vector elements and the value.
4715 SDValue InVec = Op.getOperand(0);
4716 SDValue InVal = Op.getOperand(1);
4717 SDValue EltNo = Op.getOperand(2);
4718 bool DemandedVal = true;
4719 APInt DemandedVecElts = DemandedElts;
4720 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4721 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4722 unsigned EltIdx = CEltNo->getZExtValue();
4723 DemandedVal = !!DemandedElts[EltIdx];
4724 DemandedVecElts.clearBit(EltIdx);
4725 }
4726 Tmp = std::numeric_limits<unsigned>::max();
4727 if (DemandedVal) {
4728 // TODO - handle implicit truncation of inserted elements.
4729 if (InVal.getScalarValueSizeInBits() != VTBits)
4730 break;
4731 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4732 Tmp = std::min(Tmp, Tmp2);
4733 }
4734 if (!!DemandedVecElts) {
4735 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4736 Tmp = std::min(Tmp, Tmp2);
4737 }
4738 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4739 return Tmp;
4740 }
4742 assert(!VT.isScalableVector());
4743 SDValue InVec = Op.getOperand(0);
4744 SDValue EltNo = Op.getOperand(1);
4745 EVT VecVT = InVec.getValueType();
4746 // ComputeNumSignBits not yet implemented for scalable vectors.
4747 if (VecVT.isScalableVector())
4748 break;
4749 const unsigned BitWidth = Op.getValueSizeInBits();
4750 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4751 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4752
4753 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4754 // anything about sign bits. But if the sizes match we can derive knowledge
4755 // about sign bits from the vector operand.
4756 if (BitWidth != EltBitWidth)
4757 break;
4758
4759 // If we know the element index, just demand that vector element, else for
4760 // an unknown element index, ignore DemandedElts and demand them all.
4761 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4762 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4763 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4764 DemandedSrcElts =
4765 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4766
4767 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4768 }
4770 // Offset the demanded elts by the subvector index.
4771 SDValue Src = Op.getOperand(0);
4772 // Bail until we can represent demanded elements for scalable vectors.
4773 if (Src.getValueType().isScalableVector())
4774 break;
4775 uint64_t Idx = Op.getConstantOperandVal(1);
4776 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4777 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
4778 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4779 }
4780 case ISD::CONCAT_VECTORS: {
4781 if (VT.isScalableVector())
4782 break;
4783 // Determine the minimum number of sign bits across all demanded
4784 // elts of the input vectors. Early out if the result is already 1.
4785 Tmp = std::numeric_limits<unsigned>::max();
4786 EVT SubVectorVT = Op.getOperand(0).getValueType();
4787 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4788 unsigned NumSubVectors = Op.getNumOperands();
4789 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4790 APInt DemandedSub =
4791 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4792 if (!DemandedSub)
4793 continue;
4794 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4795 Tmp = std::min(Tmp, Tmp2);
4796 }
4797 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4798 return Tmp;
4799 }
4800 case ISD::INSERT_SUBVECTOR: {
4801 if (VT.isScalableVector())
4802 break;
4803 // Demand any elements from the subvector and the remainder from the src its
4804 // inserted into.
4805 SDValue Src = Op.getOperand(0);
4806 SDValue Sub = Op.getOperand(1);
4807 uint64_t Idx = Op.getConstantOperandVal(2);
4808 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4809 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4810 APInt DemandedSrcElts = DemandedElts;
4811 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
4812
4813 Tmp = std::numeric_limits<unsigned>::max();
4814 if (!!DemandedSubElts) {
4815 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4816 if (Tmp == 1)
4817 return 1; // early-out
4818 }
4819 if (!!DemandedSrcElts) {
4820 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4821 Tmp = std::min(Tmp, Tmp2);
4822 }
4823 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4824 return Tmp;
4825 }
4826 case ISD::LOAD: {
4827 LoadSDNode *LD = cast<LoadSDNode>(Op);
4828 if (const MDNode *Ranges = LD->getRanges()) {
4829 if (DemandedElts != 1)
4830 break;
4831
4833 if (VTBits > CR.getBitWidth()) {
4834 switch (LD->getExtensionType()) {
4835 case ISD::SEXTLOAD:
4836 CR = CR.signExtend(VTBits);
4837 break;
4838 case ISD::ZEXTLOAD:
4839 CR = CR.zeroExtend(VTBits);
4840 break;
4841 default:
4842 break;
4843 }
4844 }
4845
4846 if (VTBits != CR.getBitWidth())
4847 break;
4848 return std::min(CR.getSignedMin().getNumSignBits(),
4850 }
4851
4852 break;
4853 }
4856 case ISD::ATOMIC_SWAP:
4868 case ISD::ATOMIC_LOAD: {
4869 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4870 // If we are looking at the loaded value.
4871 if (Op.getResNo() == 0) {
4872 if (Tmp == VTBits)
4873 return 1; // early-out
4875 return VTBits - Tmp + 1;
4877 return VTBits - Tmp;
4878 }
4879 break;
4880 }
4881 }
4882
4883 // If we are looking at the loaded value of the SDNode.
4884 if (Op.getResNo() == 0) {
4885 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4886 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4887 unsigned ExtType = LD->getExtensionType();
4888 switch (ExtType) {
4889 default: break;
4890 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4891 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4892 return VTBits - Tmp + 1;
4893 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4894 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4895 return VTBits - Tmp;
4896 case ISD::NON_EXTLOAD:
4897 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4898 // We only need to handle vectors - computeKnownBits should handle
4899 // scalar cases.
4900 Type *CstTy = Cst->getType();
4901 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
4902 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
4903 VTBits == CstTy->getScalarSizeInBits()) {
4904 Tmp = VTBits;
4905 for (unsigned i = 0; i != NumElts; ++i) {
4906 if (!DemandedElts[i])
4907 continue;
4908 if (Constant *Elt = Cst->getAggregateElement(i)) {
4909 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4910 const APInt &Value = CInt->getValue();
4911 Tmp = std::min(Tmp, Value.getNumSignBits());
4912 continue;
4913 }
4914 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4915 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4916 Tmp = std::min(Tmp, Value.getNumSignBits());
4917 continue;
4918 }
4919 }
4920 // Unknown type. Conservatively assume no bits match sign bit.
4921 return 1;
4922 }
4923 return Tmp;
4924 }
4925 }
4926 break;
4927 }
4928 }
4929 }
4930
4931 // Allow the target to implement this method for its nodes.
4932 if (Opcode >= ISD::BUILTIN_OP_END ||
4933 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4934 Opcode == ISD::INTRINSIC_W_CHAIN ||
4935 Opcode == ISD::INTRINSIC_VOID) {
4936 // TODO: This can probably be removed once target code is audited. This
4937 // is here purely to reduce patch size and review complexity.
4938 if (!VT.isScalableVector()) {
4939 unsigned NumBits =
4940 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4941 if (NumBits > 1)
4942 FirstAnswer = std::max(FirstAnswer, NumBits);
4943 }
4944 }
4945
4946 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4947 // use this information.
4948 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4949 return std::max(FirstAnswer, Known.countMinSignBits());
4950}
4951
4953 unsigned Depth) const {
4954 unsigned SignBits = ComputeNumSignBits(Op, Depth);
4955 return Op.getScalarValueSizeInBits() - SignBits + 1;
4956}
4957
4959 const APInt &DemandedElts,
4960 unsigned Depth) const {
4961 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
4962 return Op.getScalarValueSizeInBits() - SignBits + 1;
4963}
4964
4966 unsigned Depth) const {
4967 // Early out for FREEZE.
4968 if (Op.getOpcode() == ISD::FREEZE)
4969 return true;
4970
4971 // TODO: Assume we don't know anything for now.
4972 EVT VT = Op.getValueType();
4973 if (VT.isScalableVector())
4974 return false;
4975
4976 APInt DemandedElts = VT.isVector()
4978 : APInt(1, 1);
4979 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4980}
4981
4983 const APInt &DemandedElts,
4984 bool PoisonOnly,
4985 unsigned Depth) const {
4986 unsigned Opcode = Op.getOpcode();
4987
4988 // Early out for FREEZE.
4989 if (Opcode == ISD::FREEZE)
4990 return true;
4991
4992 if (Depth >= MaxRecursionDepth)
4993 return false; // Limit search depth.
4994
4995 if (isIntOrFPConstant(Op))
4996 return true;
4997
4998 switch (Opcode) {
4999 case ISD::CONDCODE:
5000 case ISD::VALUETYPE:
5001 case ISD::FrameIndex:
5003 return true;
5004
5005 case ISD::UNDEF:
5006 return PoisonOnly;
5007
5008 case ISD::BUILD_VECTOR:
5009 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5010 // this shouldn't affect the result.
5011 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5012 if (!DemandedElts[i])
5013 continue;
5015 Depth + 1))
5016 return false;
5017 }
5018 return true;
5019
5020 // TODO: Search for noundef attributes from library functions.
5021
5022 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5023
5024 default:
5025 // Allow the target to implement this method for its nodes.
5026 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5027 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5029 Op, DemandedElts, *this, PoisonOnly, Depth);
5030 break;
5031 }
5032
5033 // If Op can't create undef/poison and none of its operands are undef/poison
5034 // then Op is never undef/poison.
5035 // NOTE: TargetNodes should handle this in themselves in
5036 // isGuaranteedNotToBeUndefOrPoisonForTargetNode.
5037 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5038 Depth) &&
5039 all_of(Op->ops(), [&](SDValue V) {
5040 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
5041 });
5042}
5043
5045 bool ConsiderFlags,
5046 unsigned Depth) const {
5047 // TODO: Assume we don't know anything for now.
5048 EVT VT = Op.getValueType();
5049 if (VT.isScalableVector())
5050 return true;
5051
5052 APInt DemandedElts = VT.isVector()
5054 : APInt(1, 1);
5055 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5056 Depth);
5057}
5058
5060 bool PoisonOnly, bool ConsiderFlags,
5061 unsigned Depth) const {
5062 // TODO: Assume we don't know anything for now.
5063 EVT VT = Op.getValueType();
5064 if (VT.isScalableVector())
5065 return true;
5066
5067 unsigned Opcode = Op.getOpcode();
5068 switch (Opcode) {
5069 case ISD::FREEZE:
5072 case ISD::AND:
5073 case ISD::XOR:
5074 case ISD::ROTL:
5075 case ISD::ROTR:
5076 case ISD::FSHL:
5077 case ISD::FSHR:
5078 case ISD::BSWAP:
5079 case ISD::CTPOP:
5080 case ISD::BITREVERSE:
5081 case ISD::PARITY:
5082 case ISD::SIGN_EXTEND:
5083 case ISD::TRUNCATE:
5087 case ISD::BITCAST:
5088 case ISD::BUILD_VECTOR:
5089 case ISD::BUILD_PAIR:
5090 return false;
5091
5092 case ISD::SETCC: {
5093 // Integer setcc cannot create undef or poison.
5094 if (Op.getOperand(0).getValueType().isInteger())
5095 return false;
5096
5097 // FP compares are more complicated. They can create poison for nan/infinity
5098 // based on options and flags. The options and flags also cause special
5099 // nonan condition codes to be used. Those condition codes may be preserved
5100 // even if the nonan flag is dropped somewhere.
5101 ISD::CondCode CCCode = cast<CondCodeSDNode>(Op.getOperand(2))->get();
5102 if (((unsigned)CCCode & 0x10U))
5103 return true;
5104
5106 return Options.NoNaNsFPMath || Options.NoInfsFPMath ||
5107 (ConsiderFlags &&
5108 (Op->getFlags().hasNoNaNs() || Op->getFlags().hasNoInfs()));
5109 }
5110
5111 // Matches hasPoisonGeneratingFlags().
5112 case ISD::ZERO_EXTEND:
5113 return ConsiderFlags && Op->getFlags().hasNonNeg();
5114
5115 case ISD::ADD:
5116 case ISD::SUB:
5117 case ISD::MUL:
5118 // Matches hasPoisonGeneratingFlags().
5119 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5120 Op->getFlags().hasNoUnsignedWrap());
5121
5122 case ISD::SHL:
5123 // If the max shift amount isn't in range, then the shift can create poison.
5124 if (!getValidMaximumShiftAmountConstant(Op, DemandedElts))
5125 return true;
5126
5127 // Matches hasPoisonGeneratingFlags().
5128 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5129 Op->getFlags().hasNoUnsignedWrap());
5130
5131 // Matches hasPoisonGeneratingFlags().
5132 case ISD::OR:
5133 return ConsiderFlags && Op->getFlags().hasDisjoint();
5134
5136 // Ensure that the element index is in bounds.
5137 EVT VecVT = Op.getOperand(0).getValueType();
5138 KnownBits KnownIdx = computeKnownBits(Op.getOperand(2), Depth + 1);
5139 return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
5140 }
5141
5142 default:
5143 // Allow the target to implement this method for its nodes.
5144 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5145 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5147 Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth);
5148 break;
5149 }
5150
5151 // Be conservative and return true.
5152 return true;
5153}
5154
5156 unsigned Opcode = Op.getOpcode();
5157 if (Opcode == ISD::OR)
5158 return Op->getFlags().hasDisjoint() ||
5159 haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));
5160 if (Opcode == ISD::XOR)
5161 return isMinSignedConstant(Op.getOperand(1));
5162 return false;
5163}
5164
5166 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
5167 !isa<ConstantSDNode>(Op.getOperand(1)))
5168 return false;
5169
5170 if (Op.getOpcode() == ISD::OR &&
5171 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
5172 return false;
5173
5174 return true;
5175}
5176
5177bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5178 // If we're told that NaNs won't happen, assume they won't.
5179 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5180 return true;
5181
5182 if (Depth >= MaxRecursionDepth)
5183 return false; // Limit search depth.
5184
5185 // If the value is a constant, we can obviously see if it is a NaN or not.
5186 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
5187 return !C->getValueAPF().isNaN() ||
5188 (SNaN && !C->getValueAPF().isSignaling());
5189 }
5190
5191 unsigned Opcode = Op.getOpcode();
5192 switch (Opcode) {
5193 case ISD::FADD:
5194 case ISD::FSUB:
5195 case ISD::FMUL:
5196 case ISD::FDIV:
5197 case ISD::FREM:
5198 case ISD::FSIN:
5199 case ISD::FCOS:
5200 case ISD::FMA:
5201 case ISD::FMAD: {
5202 if (SNaN)
5203 return true;
5204 // TODO: Need isKnownNeverInfinity
5205 return false;
5206 }
5207 case ISD::FCANONICALIZE:
5208 case ISD::FEXP:
5209 case ISD::FEXP2:
5210 case ISD::FEXP10:
5211 case ISD::FTRUNC:
5212 case ISD::FFLOOR:
5213 case ISD::FCEIL:
5214 case ISD::FROUND:
5215 case ISD::FROUNDEVEN:
5216 case ISD::FRINT:
5217 case ISD::LRINT:
5218 case ISD::LLRINT:
5219 case ISD::FNEARBYINT:
5220 case ISD::FLDEXP: {
5221 if (SNaN)
5222 return true;
5223 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5224 }
5225 case ISD::FABS:
5226 case ISD::FNEG:
5227 case ISD::FCOPYSIGN: {
5228 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5229 }
5230 case ISD::SELECT:
5231 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5232 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5233 case ISD::FP_EXTEND:
5234 case ISD::FP_ROUND: {
5235 if (SNaN)
5236 return true;
5237 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5238 }
5239 case ISD::SINT_TO_FP:
5240 case ISD::UINT_TO_FP:
5241 return true;
5242 case ISD::FSQRT: // Need is known positive
5243 case ISD::FLOG:
5244 case ISD::FLOG2:
5245 case ISD::FLOG10:
5246 case ISD::FPOWI:
5247 case ISD::FPOW: {
5248 if (SNaN)
5249 return true;
5250 // TODO: Refine on operand
5251 return false;
5252 }
5253 case ISD::FMINNUM:
5254 case ISD::FMAXNUM: {
5255 // Only one needs to be known not-nan, since it will be returned if the
5256 // other ends up being one.
5257 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
5258 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5259 }
5260 case ISD::FMINNUM_IEEE:
5261 case ISD::FMAXNUM_IEEE: {
5262 if (SNaN)
5263 return true;
5264 // This can return a NaN if either operand is an sNaN, or if both operands
5265 // are NaN.
5266 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
5267 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
5268 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
5269 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
5270 }
5271 case ISD::FMINIMUM:
5272 case ISD::FMAXIMUM: {
5273 // TODO: Does this quiet or return the origina NaN as-is?
5274 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5275 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5276 }
5278 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5279 }
5280 case ISD::BUILD_VECTOR: {
5281 for (const SDValue &Opnd : Op->ops())
5282 if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
5283 return false;
5284 return true;
5285 }
5286 default:
5287 if (Opcode >= ISD::BUILTIN_OP_END ||
5288 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5289 Opcode == ISD::INTRINSIC_W_CHAIN ||
5290 Opcode == ISD::INTRINSIC_VOID) {
5291 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
5292 }
5293
5294 return false;
5295 }
5296}
5297
5299 assert(Op.getValueType().isFloatingPoint() &&
5300 "Floating point type expected");
5301
5302 // If the value is a constant, we can obviously see if it is a zero or not.
5304 Op, [](ConstantFPSDNode *C) { return !C->isZero(); });
5305}
5306
5308 if (Depth >= MaxRecursionDepth)
5309 return false; // Limit search depth.
5310
5311 assert(!Op.getValueType().isFloatingPoint() &&
5312 "Floating point types unsupported - use isKnownNeverZeroFloat");
5313
5314 // If the value is a constant, we can obviously see if it is a zero or not.
5316 [](ConstantSDNode *C) { return !C->isZero(); }))
5317 return true;
5318
5319 // TODO: Recognize more cases here. Most of the cases are also incomplete to
5320 // some degree.
5321 switch (Op.getOpcode()) {
5322 default:
5323 break;
5324
5325 case ISD::OR:
5326 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5327 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5328
5329 case ISD::VSELECT:
5330 case ISD::SELECT:
5331 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5332 isKnownNeverZero(Op.getOperand(2), Depth + 1);
5333
5334 case ISD::SHL: {
5335 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5336 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5337 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5338 // 1 << X is never zero.
5339 if (ValKnown.One[0])
5340 return true;
5341 // If max shift cnt of known ones is non-zero, result is non-zero.
5342 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5343 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5344 !ValKnown.One.shl(MaxCnt).isZero())
5345 return true;
5346 break;
5347 }
5348 case ISD::UADDSAT:
5349 case ISD::UMAX:
5350 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5351 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5352
5353 // TODO for smin/smax: If either operand is known negative/positive
5354 // respectively we don't need the other to be known at all.
5355 case ISD::SMAX:
5356 case ISD::SMIN:
5357 case ISD::UMIN:
5358 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5359 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5360
5361 case ISD::ROTL:
5362 case ISD::ROTR:
5363 case ISD::BITREVERSE:
5364 case ISD::BSWAP:
5365 case ISD::CTPOP:
5366 case ISD::ABS:
5367 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5368
5369 case ISD::SRA:
5370 case ISD::SRL: {
5371 if (Op->getFlags().hasExact())
5372 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5373 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5374 if (ValKnown.isNegative())
5375 return true;
5376 // If max shift cnt of known ones is non-zero, result is non-zero.
5377 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5378 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5379 !ValKnown.One.lshr(MaxCnt).isZero())
5380 return true;
5381 break;
5382 }
5383 case ISD::UDIV:
5384 case ISD::SDIV:
5385 // div exact can only produce a zero if the dividend is zero.
5386 // TODO: For udiv this is also true if Op1 u<= Op0
5387 if (Op->getFlags().hasExact())
5388 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5389 break;
5390
5391 case ISD::ADD:
5392 if (Op->getFlags().hasNoUnsignedWrap())
5393 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5394 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5395 return true;
5396 // TODO: There are a lot more cases we can prove for add.
5397 break;
5398
5399 case ISD::SUB: {
5400 if (isNullConstant(Op.getOperand(0)))
5401 return isKnownNeverZero(Op.getOperand(1), Depth + 1);
5402
5403 std::optional<bool> ne =
5404 KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1),
5405 computeKnownBits(Op.getOperand(1), Depth + 1));
5406 return ne && *ne;
5407 }
5408
5409 case ISD::MUL:
5410 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5411 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5412 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5413 return true;
5414 break;
5415
5416 case ISD::ZERO_EXTEND:
5417 case ISD::SIGN_EXTEND:
5418 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5419 }
5420
5422}
5423
5425 // Check the obvious case.
5426 if (A == B) return true;
5427
5428 // For negative and positive zero.
5429 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
5430 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
5431 if (CA->isZero() && CB->isZero()) return true;
5432
5433 // Otherwise they may not be equal.
5434 return false;
5435}
5436
5437// Only bits set in Mask must be negated, other bits may be arbitrary.
5439 if (isBitwiseNot(V, AllowUndefs))
5440 return V.getOperand(0);
5441
5442 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
5443 // bits in the non-extended part.
5444 ConstantSDNode *MaskC = isConstOrConstSplat(Mask);
5445 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5446 return SDValue();
5447 SDValue ExtArg = V.getOperand(0);
5448 if (ExtArg.getScalarValueSizeInBits() >=
5449 MaskC->getAPIntValue().getActiveBits() &&
5450 isBitwiseNot(ExtArg, AllowUndefs) &&
5451 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&
5452 ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())
5453 return ExtArg.getOperand(0).getOperand(0);
5454 return SDValue();
5455}
5456
5458 // Match masked merge pattern (X & ~M) op (Y & M)
5459 // Including degenerate case (X & ~M) op M
5460 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
5461 SDValue Other) {
5462 if (SDValue NotOperand =
5463 getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {
5464 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
5465 NotOperand->getOpcode() == ISD::TRUNCATE)
5466 NotOperand = NotOperand->getOperand(0);
5467
5468 if (Other == NotOperand)
5469 return true;
5470 if (Other->getOpcode() == ISD::AND)
5471 return NotOperand == Other->getOperand(0) ||
5472 NotOperand == Other->getOperand(1);
5473 }
5474 return false;
5475 };
5476
5477 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
5478 A = A->getOperand(0);
5479
5480 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
5481 B = B->getOperand(0);
5482
5483 if (A->getOpcode() == ISD::AND)
5484 return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||
5485 MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);
5486 return false;
5487}
5488
5489// FIXME: unify with llvm::haveNoCommonBitsSet.
5491 assert(A.getValueType() == B.getValueType() &&
5492 "Values must have the same type");
5495 return true;
5498}
5499
5500static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
5501 SelectionDAG &DAG) {
5502 if (cast<ConstantSDNode>(Step)->isZero())
5503 return DAG.getConstant(0, DL, VT);
5504
5505 return SDValue();
5506}
5507
5510 SelectionDAG &DAG) {
5511 int NumOps = Ops.size();
5512 assert(NumOps != 0 && "Can't build an empty vector!");
5513 assert(!VT.isScalableVector() &&
5514 "BUILD_VECTOR cannot be used with scalable types");
5515 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
5516 "Incorrect element count in BUILD_VECTOR!");
5517
5518 // BUILD_VECTOR of UNDEFs is UNDEF.
5519 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5520 return DAG.getUNDEF(VT);
5521
5522 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
5523 SDValue IdentitySrc;
5524 bool IsIdentity = true;
5525 for (int i = 0; i != NumOps; ++i) {
5526 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5527 Ops[i].getOperand(0).getValueType() != VT ||
5528 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
5529 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
5530 Ops[i].getConstantOperandAPInt(1) != i) {
5531 IsIdentity = false;
5532 break;
5533 }
5534 IdentitySrc = Ops[i].getOperand(0);
5535 }
5536 if (IsIdentity)
5537 return IdentitySrc;
5538
5539 return SDValue();
5540}
5541
5542/// Try to simplify vector concatenation to an input value, undef, or build
5543/// vector.
5546 SelectionDAG &DAG) {
5547 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
5548 assert(llvm::all_of(Ops,
5549 [Ops](SDValue Op) {
5550 return Ops[0].getValueType() == Op.getValueType();
5551 }) &&
5552 "Concatenation of vectors with inconsistent value types!");
5553 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
5554 VT.getVectorElementCount() &&
5555 "Incorrect element count in vector concatenation!");
5556
5557 if (Ops.size() == 1)
5558 return Ops[0];
5559
5560 // Concat of UNDEFs is UNDEF.
5561 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5562 return DAG.getUNDEF(VT);
5563
5564 // Scan the operands and look for extract operations from a single source
5565 // that correspond to insertion at the same location via this concatenation:
5566 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
5567 SDValue IdentitySrc;
5568 bool IsIdentity = true;
5569 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
5570 SDValue Op = Ops[i];
5571 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
5572 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5573 Op.getOperand(0).getValueType() != VT ||
5574 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
5575 Op.getConstantOperandVal(1) != IdentityIndex) {
5576 IsIdentity = false;
5577 break;
5578 }
5579 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
5580 "Unexpected identity source vector for concat of extracts");
5581 IdentitySrc = Op.getOperand(0);
5582 }
5583 if (IsIdentity) {
5584 assert(IdentitySrc && "Failed to set source vector of extracts");
5585 return IdentitySrc;
5586 }
5587
5588 // The code below this point is only designed to work for fixed width
5589 // vectors, so we bail out for now.
5590 if (VT.isScalableVector())
5591 return SDValue();
5592
5593 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
5594 // simplified to one big BUILD_VECTOR.
5595 // FIXME: Add support for SCALAR_TO_VECTOR as well.
5596 EVT SVT = VT.getScalarType();
5598 for (SDValue Op : Ops) {
5599 EVT OpVT = Op.getValueType();
5600 if (Op.isUndef())
5601 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
5602 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
5603 Elts.append(Op->op_begin(), Op->op_end());
5604 else
5605 return SDValue();
5606 }
5607
5608 // BUILD_VECTOR requires all inputs to be of the same type, find the
5609 // maximum type and extend them all.
5610 for (SDValue Op : Elts)
5611 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
5612
5613 if (SVT.bitsGT(VT.getScalarType())) {
5614 for (SDValue &Op : Elts) {
5615 if (Op.isUndef())
5616 Op = DAG.getUNDEF(SVT);
5617 else
5618 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
5619 ? DAG.getZExtOrTrunc(Op, DL, SVT)
5620 : DAG.getSExtOrTrunc(Op, DL, SVT);
5621 }
5622 }
5623
5624 SDValue V = DAG.getBuildVector(VT, DL, Elts);
5625 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
5626 return V;
5627}
5628
5629/// Gets or creates the specified node.
5630SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
5632 AddNodeIDNode(ID, Opcode, getVTList(VT), std::nullopt);
5633 void *IP = nullptr;
5634 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5635 return SDValue(E, 0);
5636
5637 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5638 getVTList(VT));
5639 CSEMap.InsertNode(N, IP);
5640
5641 InsertNode(N);
5642 SDValue V = SDValue(N, 0);
5643 NewSDValueDbgMsg(V, "Creating new node: ", this);
5644 return V;
5645}
5646
5647SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5648 SDValue N1) {
5649 SDNodeFlags Flags;
5650 if (Inserter)
5651 Flags = Inserter->getFlags();
5652 return getNode(Opcode, DL, VT, N1, Flags);
5653}
5654
5655SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5656 SDValue N1, const SDNodeFlags Flags) {
5657 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
5658
5659 // Constant fold unary operations with a vector integer or float operand.
5660 switch (Opcode) {
5661 default:
5662 // FIXME: Entirely reasonable to perform folding of other unary
5663 // operations here as the need arises.
5664 break;
5665 case ISD::FNEG:
5666 case ISD::FABS:
5667 case ISD::FCEIL:
5668 case ISD::FTRUNC:
5669 case ISD::FFLOOR:
5670 case ISD::FP_EXTEND:
5671 case ISD::FP_TO_SINT:
5672 case ISD::FP_TO_UINT:
5673 case ISD::FP_TO_FP16:
5674 case ISD::FP_TO_BF16:
5675 case ISD::TRUNCATE:
5676 case ISD::ANY_EXTEND:
5677 case ISD::ZERO_EXTEND:
5678 case ISD::SIGN_EXTEND:
5679 case ISD::UINT_TO_FP:
5680 case ISD::SINT_TO_FP:
5681 case ISD::FP16_TO_FP:
5682 case ISD::BF16_TO_FP:
5683 case ISD::BITCAST:
5684 case ISD::ABS:
5685 case ISD::BITREVERSE:
5686 case ISD::BSWAP:
5687 case ISD::CTLZ:
5689 case ISD::CTTZ:
5691 case ISD::CTPOP:
5692 case ISD::STEP_VECTOR: {
5693 SDValue Ops = {N1};
5694 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
5695 return Fold;
5696 }
5697 }
5698
5699 unsigned OpOpcode = N1.getNode()->getOpcode();
5700 switch (Opcode) {
5701 case ISD::STEP_VECTOR:
5702 assert(VT.isScalableVector() &&
5703 "STEP_VECTOR can only be used with scalable types");
5704 assert(OpOpcode == ISD::TargetConstant &&
5705 VT.getVectorElementType() == N1.getValueType() &&
5706 "Unexpected step operand");
5707 break;
5708 case ISD::FREEZE:
5709 assert(VT == N1.getValueType() && "Unexpected VT!");
5710 if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false,
5711 /*Depth*/ 1))
5712 return N1;
5713 break;
5714 case ISD::TokenFactor:
5715 case ISD::MERGE_VALUES:
5717 return N1; // Factor, merge or concat of one node? No need.
5718 case ISD::BUILD_VECTOR: {
5719 // Attempt to simplify BUILD_VECTOR.
5720 SDValue Ops[] = {N1};
5721 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5722 return V;
5723 break;
5724 }
5725 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
5726 case ISD::FP_EXTEND:
5728 "Invalid FP cast!");
5729 if (N1.getValueType() == VT) return N1; // noop conversion.
5730 assert((!VT.isVector() || VT.getVectorElementCount() ==
5732 "Vector element count mismatch!");
5733 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
5734 if (N1.isUndef())
5735 return getUNDEF(VT);
5736 break;
5737 case ISD::FP_TO_SINT:
5738 case ISD::FP_TO_UINT:
5739 if (N1.isUndef())
5740 return getUNDEF(VT);
5741 break;
5742 case ISD::SINT_TO_FP:
5743 case ISD::UINT_TO_FP:
5744 // [us]itofp(undef) = 0, because the result value is bounded.
5745 if (N1.isUndef())
5746 return getConstantFP(0.0, DL, VT);
5747 break;
5748 case ISD::SIGN_EXTEND:
5749 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5750 "Invalid SIGN_EXTEND!");
5751 assert(VT.isVector() == N1.getValueType().isVector() &&
5752 "SIGN_EXTEND result type type should be vector iff the operand "
5753 "type is vector!");
5754 if (N1.getValueType() == VT) return N1; // noop extension
5755 assert((!VT.isVector() || VT.getVectorElementCount() ==
5757 "Vector element count mismatch!");
5758 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
5759 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
5760 SDNodeFlags Flags;
5761 if (OpOpcode == ISD::ZERO_EXTEND)
5762 Flags.setNonNeg(N1->getFlags().hasNonNeg());
5763 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
5764 }
5765 if (OpOpcode == ISD::UNDEF)
5766 // sext(undef) = 0, because the top bits will all be the same.
5767 return getConstant(0, DL, VT);
5768 break;
5769 case ISD::ZERO_EXTEND:
5770 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5771 "Invalid ZERO_EXTEND!");
5772 assert(VT.isVector() == N1.getValueType().isVector() &&
5773 "ZERO_EXTEND result type type should be vector iff the operand "
5774 "type is vector!");
5775 if (N1.getValueType() == VT) return N1; // noop extension
5776 assert((!VT.isVector() || VT.getVectorElementCount() ==
5778 "Vector element count mismatch!");
5779 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
5780 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
5781 SDNodeFlags Flags;
5782 Flags.setNonNeg(N1->getFlags().hasNonNeg());
5783 return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);
5784 }
5785 if (OpOpcode == ISD::UNDEF)
5786 // zext(undef) = 0, because the top bits will be zero.
5787 return getConstant(0, DL, VT);
5788
5789 // Skip unnecessary zext_inreg pattern:
5790 // (zext (trunc x)) -> x iff the upper bits are known zero.
5791 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
5792 // use to recognise zext_inreg patterns.
5793 if (OpOpcode == ISD::TRUNCATE) {
5794 SDValue OpOp = N1.getOperand(0);
5795 if (OpOp.getValueType() == VT) {
5796 if (OpOp.getOpcode() != ISD::AND) {
5799 if (MaskedValueIsZero(OpOp, HiBits)) {
5800 transferDbgValues(N1, OpOp);
5801 return OpOp;
5802 }
5803 }
5804 }
5805 }
5806 break;
5807 case ISD::ANY_EXTEND:
5808 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5809 "Invalid ANY_EXTEND!");
5810 assert(VT.isVector() == N1.getValueType().isVector() &&
5811 "ANY_EXTEND result type type should be vector iff the operand "
5812 "type is vector!");
5813 if (N1.getValueType() == VT) return N1; // noop extension
5814 assert((!VT.isVector() || VT.getVectorElementCount() ==
5816 "Vector element count mismatch!");
5817 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
5818
5819 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5820 OpOpcode == ISD::ANY_EXTEND) {
5821 SDNodeFlags Flags;
5822 if (OpOpcode == ISD::ZERO_EXTEND)
5823 Flags.setNonNeg(N1->getFlags().hasNonNeg());
5824 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
5825 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
5826 }
5827 if (OpOpcode == ISD::UNDEF)
5828 return getUNDEF(VT);
5829
5830 // (ext (trunc x)) -> x
5831 if (OpOpcode == ISD::TRUNCATE) {
5832 SDValue OpOp = N1.getOperand(0);
5833 if (OpOp.getValueType() == VT) {
5834 transferDbgValues(N1, OpOp);
5835 return OpOp;
5836 }
5837 }
5838 break;
5839 case ISD::TRUNCATE:
5840 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5841 "Invalid TRUNCATE!");
5842 assert(VT.isVector() == N1.getValueType().isVector() &&
5843 "TRUNCATE result type type should be vector iff the operand "
5844 "type is vector!");
5845 if (N1.getValueType() == VT) return N1; // noop truncate
5846 assert((!VT.isVector() || VT.getVectorElementCount() ==
5848 "Vector element count mismatch!");
5849 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
5850 if (OpOpcode == ISD::TRUNCATE)
5851 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5852 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5853 OpOpcode == ISD::ANY_EXTEND) {
5854 // If the source is smaller than the dest, we still need an extend.
5856 VT.getScalarType()))
5857 return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5858 if (N1.getOperand(0).getValueType().bitsGT(VT))
5859 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5860 return N1.getOperand(0);
5861 }
5862 if (OpOpcode == ISD::UNDEF)
5863 return getUNDEF(VT);
5864 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
5865 return getVScale(DL, VT,
5867 break;
5871 assert(VT.isVector() && "This DAG node is restricted to vector types.");
5872 assert(N1.getValueType().bitsLE(VT) &&
5873 "The input must be the same size or smaller than the result.");
5876 "The destination vector type must have fewer lanes than the input.");
5877 break;
5878 case ISD::ABS:
5879 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
5880 if (OpOpcode == ISD::UNDEF)
5881 return getConstant(0, DL, VT);
5882 break;
5883 case ISD::BSWAP:
5884 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
5885 assert((VT.getScalarSizeInBits() % 16 == 0) &&
5886 "BSWAP types must be a multiple of 16 bits!");
5887 if (OpOpcode == ISD::UNDEF)
5888 return getUNDEF(VT);
5889 // bswap(bswap(X)) -> X.
5890 if (OpOpcode == ISD::BSWAP)
5891 return N1.getOperand(0);
5892 break;
5893 case ISD::BITREVERSE:
5894 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
5895 if (OpOpcode == ISD::UNDEF)
5896 return getUNDEF(VT);
5897 break;
5898 case ISD::BITCAST:
5900 "Cannot BITCAST between types of different sizes!");
5901 if (VT == N1.getValueType()) return N1; // noop conversion.
5902 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5903 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));
5904 if (OpOpcode == ISD::UNDEF)
5905 return getUNDEF(VT);
5906 break;
5908 assert(VT.isVector() && !N1.getValueType().isVector() &&
5909 (VT.getVectorElementType() == N1.getValueType() ||
5911 N1.getValueType().isInteger() &&
5913 "Illegal SCALAR_TO_VECTOR node!");
5914 if (OpOpcode == ISD::UNDEF)
5915 return getUNDEF(VT);
5916 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5917 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5918 isa<ConstantSDNode>(N1.getOperand(1)) &&
5919 N1.getConstantOperandVal(1) == 0 &&
5920 N1.getOperand(0).getValueType() == VT)
5921 return N1.getOperand(0);
5922 break;
5923 case ISD::FNEG:
5924 // Negation of an unknown bag of bits is still completely undefined.
5925 if (OpOpcode == ISD::UNDEF)
5926 return getUNDEF(VT);
5927
5928 if (OpOpcode == ISD::FNEG) // --X -> X
5929 return N1.getOperand(0);
5930 break;
5931 case ISD::FABS:
5932 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5933 return getNode(ISD::FABS, DL, VT, N1.getOperand(0));
5934 break;
5935 case ISD::VSCALE:
5936 assert(VT == N1.getValueType() && "Unexpected VT!");
5937 break;
5938 case ISD::CTPOP:
5939 if (N1.getValueType().getScalarType() == MVT::i1)
5940 return N1;
5941 break;
5942 case ISD::CTLZ:
5943 case ISD::CTTZ:
5944 if (N1.getValueType().getScalarType() == MVT::i1)
5945 return getNOT(DL, N1, N1.getValueType());
5946 break;
5947 case ISD::VECREDUCE_ADD:
5948 if (N1.getValueType().getScalarType() == MVT::i1)
5949 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);
5950 break;
5953 if (N1.getValueType().getScalarType() == MVT::i1)
5954 return getNode(ISD::VECREDUCE_OR, DL, VT, N1);
5955 break;
5958 if (N1.getValueType().getScalarType() == MVT::i1)
5959 return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
5960 break;
5961 }
5962
5963 SDNode *N;
5964 SDVTList VTs = getVTList(VT);
5965 SDValue Ops[] = {N1};
5966 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
5968 AddNodeIDNode(ID, Opcode, VTs, Ops);
5969 void *IP = nullptr;
5970 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
5971 E->intersectFlagsWith(Flags);
5972 return SDValue(E, 0);
5973 }
5974
5975 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5976 N->setFlags(Flags);
5977 createOperands(N, Ops);
5978 CSEMap.InsertNode(N, IP);
5979 } else {
5980 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
5981 createOperands(N, Ops);
5982 }
5983
5984 InsertNode(N);
5985 SDValue V = SDValue(N, 0);
5986 NewSDValueDbgMsg(V, "Creating new node: ", this);
5987 return V;
5988}
5989
5990static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
5991 const APInt &C2) {
5992 switch (Opcode) {
5993 case ISD::ADD: return C1 + C2;
5994 case ISD::SUB: return C1 - C2;
5995 case ISD::MUL: return C1 * C2;
5996 case ISD::AND: return C1 & C2;
5997 case ISD::OR: return C1 | C2;
5998 case ISD::XOR: return C1 ^ C2;
5999 case ISD::SHL: return C1 << C2;
6000 case ISD::SRL: return C1.lshr(C2);
6001 case ISD::SRA: return C1.ashr(C2);
6002 case ISD::ROTL: return C1.rotl(C2);
6003 case ISD::ROTR: return C1.rotr(C2);
6004 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
6005 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
6006 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
6007 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
6008 case ISD::SADDSAT: return C1.sadd_sat(C2);
6009 case ISD::UADDSAT: return C1.uadd_sat(C2);
6010 case ISD::SSUBSAT: return C1.ssub_sat(C2);
6011 case ISD::USUBSAT: return C1.usub_sat(C2);
6012 case ISD::SSHLSAT: return C1.sshl_sat(C2);
6013 case ISD::USHLSAT: return C1.ushl_sat(C2);
6014 case ISD::UDIV:
6015 if (!C2.getBoolValue())
6016 break;
6017 return C1.udiv(C2);
6018 case ISD::UREM:
6019 if (!C2.getBoolValue())
6020 break;
6021 return C1.urem(C2);
6022 case ISD::SDIV:
6023 if (!C2.getBoolValue())
6024 break;
6025 return C1.sdiv(C2);
6026 case ISD::SREM:
6027 if (!C2.getBoolValue())
6028 break;
6029 return C1.srem(C2);
6030 case ISD::MULHS: {
6031 unsigned FullWidth = C1.getBitWidth() * 2;
6032 APInt C1Ext = C1.sext(FullWidth);
6033 APInt C2Ext = C2.sext(FullWidth);
6034 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
6035 }
6036 case ISD::MULHU: {
6037 unsigned FullWidth = C1.getBitWidth() * 2;
6038 APInt C1Ext = C1.zext(FullWidth);
6039 APInt C2Ext = C2.zext(FullWidth);
6040 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
6041 }
6042 case ISD::AVGFLOORS:
6043 return APIntOps::avgFloorS(C1, C2);
6044 case ISD::AVGFLOORU:
6045 return APIntOps::avgFloorU(C1, C2);
6046 case ISD::AVGCEILS:
6047 return APIntOps::avgCeilS(C1, C2);
6048 case ISD::AVGCEILU:
6049 return APIntOps::avgCeilU(C1, C2);
6050 case ISD::ABDS:
6051 return APIntOps::abds(C1, C2);
6052 case ISD::ABDU:
6053 return APIntOps::abdu(C1, C2);
6054 }
6055 return std::nullopt;
6056}
6057
6058// Handle constant folding with UNDEF.
6059// TODO: Handle more cases.
6060static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6061 bool IsUndef1, const APInt &C2,
6062 bool IsUndef2) {
6063 if (!(IsUndef1 || IsUndef2))
6064 return FoldValue(Opcode, C1, C2);
6065
6066 // Fold and(x, undef) -> 0
6067 // Fold mul(x, undef) -> 0
6068 if (Opcode == ISD::AND || Opcode == ISD::MUL)
6069 return APInt::getZero(C1.getBitWidth());
6070
6071 return std::nullopt;
6072}
6073
6075 const GlobalAddressSDNode *GA,
6076 const SDNode *N2) {
6077 if (GA->getOpcode() != ISD::GlobalAddress)
6078 return SDValue();
6079 if (!TLI->isOffsetFoldingLegal(GA))
6080 return SDValue();
6081 auto *C2 = dyn_cast<ConstantSDNode>(N2);
6082 if (!C2)
6083 return SDValue();
6084 int64_t Offset = C2->getSExtValue();
6085 switch (Opcode) {
6086 case ISD::ADD: break;
6087 case ISD::SUB: Offset = -uint64_t(Offset); break;
6088 default: return SDValue();
6089 }
6090 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
6091 GA->getOffset() + uint64_t(Offset));
6092}
6093
6094bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6095 switch (Opcode) {
6096 case ISD::SDIV:
6097 case ISD::UDIV:
6098 case ISD::SREM:
6099 case ISD::UREM: {
6100 // If a divisor is zero/undef or any element of a divisor vector is
6101 // zero/undef, the whole op is undef.
6102 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6103 SDValue Divisor = Ops[1];
6104 if (Divisor.isUndef() || isNullConstant(Divisor))
6105 return true;
6106
6107 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
6108 llvm::any_of(Divisor->op_values(),
6109 [](SDValue V) { return V.isUndef() ||
6110 isNullConstant(V); });
6111 // TODO: Handle signed overflow.
6112 }
6113 // TODO: Handle oversized shifts.
6114 default:
6115 return false;
6116 }
6117}
6118
6120 EVT VT, ArrayRef<SDValue> Ops) {
6121 // If the opcode is a target-specific ISD node, there's nothing we can
6122 // do here and the operand rules may not line up with the below, so
6123 // bail early.
6124 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6125 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6126 // foldCONCAT_VECTORS in getNode before this is called.
6127 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6128 return SDValue();
6129
6130 unsigned NumOps = Ops.size();
6131 if (NumOps == 0)
6132 return SDValue();
6133
6134 if (isUndef(Opcode, Ops))
6135 return getUNDEF(VT);
6136
6137 // Handle unary special cases.
6138 if (NumOps == 1) {
6139 SDValue N1 = Ops[0];
6140
6141 // Constant fold unary operations with an integer constant operand. Even
6142 // opaque constant will be folded, because the folding of unary operations
6143 // doesn't create new constants with different values. Nevertheless, the
6144 // opaque flag is preserved during folding to prevent future folding with
6145 // other constants.
6146 if (auto *C = dyn_cast<ConstantSDNode>(N1)) {
6147 const APInt &Val = C->getAPIntValue();
6148 switch (Opcode) {
6149 case ISD::SIGN_EXTEND:
6150 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6151 C->isTargetOpcode(), C->isOpaque());
6152 case ISD::TRUNCATE:
6153 if (C->isOpaque())
6154 break;
6155 [[fallthrough]];
6156 case ISD::ZERO_EXTEND:
6157 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6158 C->isTargetOpcode(), C->isOpaque());
6159 case ISD::ANY_EXTEND:
6160 // Some targets like RISCV prefer to sign extend some types.
6161 if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))
6162 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6163 C->isTargetOpcode(), C->isOpaque());
6164 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6165 C->isTargetOpcode(), C->isOpaque());
6166 case ISD::ABS:
6167 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
6168 C->isOpaque());
6169 case ISD::BITREVERSE:
6170 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
6171 C->isOpaque());
6172 case ISD::BSWAP:
6173 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
6174 C->isOpaque());
6175 case ISD::CTPOP:
6176 return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),
6177 C->isOpaque());
6178 case ISD::CTLZ:
6180 return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),
6181 C->isOpaque());
6182 case ISD::CTTZ:
6184 return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
6185 C->isOpaque());
6186 case ISD::UINT_TO_FP:
6187 case ISD::SINT_TO_FP: {
6190 (void)apf.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,
6192 return getConstantFP(apf, DL, VT);
6193 }
6194 case ISD::FP16_TO_FP:
6195 case ISD::BF16_TO_FP: {
6196 bool Ignored;
6197 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6198 : APFloat::BFloat(),
6199 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
6200
6201 // This can return overflow, underflow, or inexact; we don't care.
6202 // FIXME need to be more flexible about rounding mode.
6203 (void)FPV.convert(EVTToAPFloatSemantics(VT),
6205 return getConstantFP(FPV, DL, VT);
6206 }
6207 case ISD::STEP_VECTOR:
6208 if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))
6209 return V;
6210 break;
6211 case ISD::BITCAST:
6212 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
6213 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6214 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
6215 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6216 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
6217 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6218 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
6219 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
6220 break;
6221 }
6222 }
6223
6224 // Constant fold unary operations with a floating point constant operand.
6225 if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {
6226 APFloat V = C->getValueAPF(); // make copy
6227 switch (Opcode) {
6228 case ISD::FNEG:
6229 V.changeSign();
6230 return getConstantFP(V, DL, VT);
6231 case ISD::FABS:
6232 V.clearSign();
6233 return getConstantFP(V, DL, VT);
6234 case ISD::FCEIL: {
6235 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
6236 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6237 return getConstantFP(V, DL, VT);
6238 return SDValue();
6239 }
6240 case ISD::FTRUNC: {
6241 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
6242 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6243 return getConstantFP(V, DL, VT);
6244 return SDValue();
6245 }
6246 case ISD::FFLOOR: {
6247 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
6248 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6249 return getConstantFP(V, DL, VT);
6250 return SDValue();
6251 }
6252 case ISD::FP_EXTEND: {
6253 bool ignored;
6254 // This can return overflow, underflow, or inexact; we don't care.
6255 // FIXME need to be more flexible about rounding mode.
6257 &ignored);
6258 return getConstantFP(V, DL, VT);
6259 }
6260 case ISD::FP_TO_SINT:
6261 case ISD::FP_TO_UINT: {
6262 bool ignored;
6263 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6264 // FIXME need to be more flexible about rounding mode.
6266 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
6267 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6268 break;
6269 return getConstant(IntVal, DL, VT);
6270 }
6271 case ISD::FP_TO_FP16:
6272 case ISD::FP_TO_BF16: {
6273 bool Ignored;
6274 // This can return overflow, underflow, or inexact; we don't care.
6275 // FIXME need to be more flexible about rounding mode.
6276 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6277 : APFloat::BFloat(),
6279 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6280 }
6281 case ISD::BITCAST:
6282 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
6283 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6284 VT);
6285 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
6286 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6287 VT);
6288 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
6289 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6290 VT);
6291 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
6292 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6293 break;
6294 }
6295 }
6296
6297 // Early-out if we failed to constant fold a bitcast.
6298 if (Opcode == ISD::BITCAST)
6299 return SDValue();
6300 }
6301
6302 // Handle binops special cases.
6303 if (NumOps == 2) {
6304 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6305 return CFP;
6306
6307 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6308 if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6309 if (C1->isOpaque() || C2->isOpaque())
6310 return SDValue();
6311
6312 std::optional<APInt> FoldAttempt =
6313 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6314 if (!FoldAttempt)
6315 return SDValue();
6316
6317 SDValue Folded = getConstant(*FoldAttempt, DL, VT);
6318 assert((!Folded || !VT.isVector()) &&
6319 "Can't fold vectors ops with scalar operands");
6320 return Folded;
6321 }
6322 }
6323
6324 // fold (add Sym, c) -> Sym+c
6325 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[0]))
6326 return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());
6327 if (TLI->isCommutativeBinOp(Opcode))
6328 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[1]))
6329 return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());
6330 }
6331
6332 // This is for vector folding only from here on.
6333 if (!VT.isVector())
6334 return SDValue();
6335
6336 ElementCount NumElts = VT.getVectorElementCount();
6337
6338 // See if we can fold through bitcasted integer ops.
6339 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6340 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6341 Ops[0].getOpcode() == ISD::BITCAST &&
6342 Ops[1].getOpcode() == ISD::BITCAST) {
6343 SDValue N1 = peekThroughBitcasts(Ops[0]);
6344 SDValue N2 = peekThroughBitcasts(Ops[1]);
6345 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6346 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6347 EVT BVVT = N1.getValueType();
6348 if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
6349 bool IsLE = getDataLayout().isLittleEndian();
6350 unsigned EltBits = VT.getScalarSizeInBits();
6351 SmallVector<APInt> RawBits1, RawBits2;
6352 BitVector UndefElts1, UndefElts2;
6353 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6354 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6355 SmallVector<APInt> RawBits;
6356 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6357 std::optional<APInt> Fold = FoldValueWithUndef(
6358 Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);
6359 if (!Fold)
6360 break;
6361 RawBits.push_back(*Fold);
6362 }
6363 if (RawBits.size() == NumElts.getFixedValue()) {
6364 // We have constant folded, but we need to cast this again back to
6365 // the original (possibly legalized) type.
6366 SmallVector<APInt> DstBits;
6367 BitVector DstUndefs;
6369 DstBits, RawBits, DstUndefs,
6370 BitVector(RawBits.size(), false));
6371 EVT BVEltVT = BV1->getOperand(0).getValueType();
6372 unsigned BVEltBits = BVEltVT.getSizeInBits();
6373 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
6374 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6375 if (DstUndefs[I])
6376 continue;
6377 Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);
6378 }
6379 return getBitcast(VT, getBuildVector(BVVT, DL, Ops));
6380 }
6381 }
6382 }
6383 }
6384
6385 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6386 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6387 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6388 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6389 APInt RHSVal;
6390 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) {
6391 APInt NewStep = Opcode == ISD::MUL
6392 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6393 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6394 return getStepVector(DL, VT, NewStep);
6395 }
6396 }
6397
6398 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6399 return !Op.getValueType().isVector() ||
6400 Op.getValueType().getVectorElementCount() == NumElts;
6401 };
6402
6403 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6404 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6405 Op.getOpcode() == ISD::BUILD_VECTOR ||
6406 Op.getOpcode() == ISD::SPLAT_VECTOR;
6407 };
6408
6409 // All operands must be vector types with the same number of elements as
6410 // the result type and must be either UNDEF or a build/splat vector
6411 // or UNDEF scalars.
6412 if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
6413 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
6414 return SDValue();
6415
6416 // If we are comparing vectors, then the result needs to be a i1 boolean that
6417 // is then extended back to the legal result type depending on how booleans
6418 // are represented.
6419 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
6420 ISD::NodeType ExtendCode =
6421 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
6424
6425 // Find legal integer scalar type for constant promotion and
6426 // ensure that its scalar size is at least as large as source.
6427 EVT LegalSVT = VT.getScalarType();
6428 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
6429 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
6430 if (LegalSVT.bitsLT(VT.getScalarType()))
6431 return SDValue();
6432 }
6433
6434 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
6435 // only have one operand to check. For fixed-length vector types we may have
6436 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
6437 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
6438
6439 // Constant fold each scalar lane separately.
6440 SmallVector<SDValue, 4> ScalarResults;
6441 for (unsigned I = 0; I != NumVectorElts; I++) {
6442 SmallVector<SDValue, 4> ScalarOps;
6443 for (SDValue Op : Ops) {
6444 EVT InSVT = Op.getValueType().getScalarType();
6445 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
6446 Op.getOpcode() != ISD::SPLAT_VECTOR) {
6447 if (Op.isUndef())
6448 ScalarOps.push_back(getUNDEF(InSVT));
6449 else
6450 ScalarOps.push_back(Op);
6451 continue;
6452 }
6453
6454 SDValue ScalarOp =
6455 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
6456 EVT ScalarVT = ScalarOp.getValueType();
6457
6458 // Build vector (integer) scalar operands may need implicit
6459 // truncation - do this before constant folding.
6460 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
6461 // Don't create illegally-typed nodes unless they're constants or undef
6462 // - if we fail to constant fold we can't guarantee the (dead) nodes
6463 // we're creating will be cleaned up before being visited for
6464 // legalization.
6465 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
6466 !isa<ConstantSDNode>(ScalarOp) &&
6467 TLI->getTypeAction(*getContext(), InSVT) !=
6469 return SDValue();
6470 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
6471 }
6472
6473 ScalarOps.push_back(ScalarOp);
6474 }
6475
6476 // Constant fold the scalar operands.
6477 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps);
6478
6479 // Legalize the (integer) scalar constant if necessary.
6480 if (LegalSVT != SVT)
6481 ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);
6482
6483 // Scalar folding only succeeded if the result is a constant or UNDEF.
6484 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
6485 ScalarResult.getOpcode() != ISD::ConstantFP)
6486 return SDValue();
6487 ScalarResults.push_back(ScalarResult);
6488 }
6489
6490 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
6491 : getBuildVector(VT, DL, ScalarResults);
6492 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
6493 return V;
6494}
6495
6497 EVT VT, ArrayRef<SDValue> Ops) {
6498 // TODO: Add support for unary/ternary fp opcodes.
6499 if (Ops.size() != 2)
6500 return SDValue();
6501
6502 // TODO: We don't do any constant folding for strict FP opcodes here, but we
6503 // should. That will require dealing with a potentially non-default
6504 // rounding mode, checking the "opStatus" return value from the APFloat
6505 // math calculations, and possibly other variations.
6506 SDValue N1 = Ops[0];
6507 SDValue N2 = Ops[1];
6508 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
6509 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
6510 if (N1CFP && N2CFP) {
6511 APFloat C1 = N1CFP->getValueAPF(); // make copy
6512 const APFloat &C2 = N2CFP->getValueAPF();
6513 switch (Opcode) {
6514 case ISD::FADD:
6516 return getConstantFP(C1, DL, VT);
6517 case ISD::FSUB:
6519 return getConstantFP(C1, DL, VT);
6520 case ISD::FMUL:
6522 return getConstantFP(C1, DL, VT);
6523 case ISD::FDIV:
6525 return getConstantFP(C1, DL, VT);
6526 case ISD::FREM:
6527 C1.mod(C2);
6528 return getConstantFP(C1, DL, VT);
6529 case ISD::FCOPYSIGN:
6530 C1.copySign(C2);
6531 return getConstantFP(C1, DL, VT);
6532 case ISD::FMINNUM:
6533 return getConstantFP(minnum(C1, C2), DL, VT);
6534 case ISD::FMAXNUM:
6535 return getConstantFP(maxnum(C1, C2), DL, VT);
6536 case ISD::FMINIMUM:
6537 return getConstantFP(minimum(C1, C2), DL, VT);
6538 case ISD::FMAXIMUM:
6539 return getConstantFP(maximum(C1, C2), DL, VT);
6540 default: break;
6541 }
6542 }
6543 if (N1CFP && Opcode == ISD::FP_ROUND) {
6544 APFloat C1 = N1CFP->getValueAPF(); // make copy
6545 bool Unused;
6546 // This can return overflow, underflow, or inexact; we don't care.
6547 // FIXME need to be more flexible about rounding mode.
6549 &Unused);
6550 return getConstantFP(C1, DL, VT);
6551 }
6552
6553 switch (Opcode) {
6554 case ISD::FSUB:
6555 // -0.0 - undef --> undef (consistent with "fneg undef")
6556 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
6557 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
6558 return getUNDEF(VT);
6559 [[fallthrough]];
6560
6561 case ISD::FADD:
6562 case ISD::FMUL:
6563 case ISD::FDIV:
6564 case ISD::FREM:
6565 // If both operands are undef, the result is undef. If 1 operand is undef,
6566 // the result is NaN. This should match the behavior of the IR optimizer.
6567 if (N1.isUndef() && N2.isUndef())
6568 return getUNDEF(VT);
6569 if (N1.isUndef() || N2.isUndef())
6571 }
6572 return SDValue();
6573}
6574
6576 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
6577
6578 // There's no need to assert on a byte-aligned pointer. All pointers are at
6579 // least byte aligned.
6580 if (A == Align(1))
6581 return Val;
6582
6585 ID.AddInteger(A.value());
6586
6587 void *IP = nullptr;
6588 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6589 return SDValue(E, 0);
6590
6591 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
6592 Val.getValueType(), A);
6593 createOperands(N, {Val});
6594
6595 CSEMap.InsertNode(N, IP);
6596 InsertNode(N);
6597
6598 SDValue V(N, 0);
6599 NewSDValueDbgMsg(V, "Creating new node: ", this);
6600 return V;
6601}
6602
6603SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6604 SDValue N1, SDValue N2) {
6605 SDNodeFlags Flags;
6606 if (Inserter)
6607 Flags = Inserter->getFlags();
6608 return getNode(Opcode, DL, VT, N1, N2, Flags);
6609}
6610
6612 SDValue &N2) const {
6613 if (!TLI->isCommutativeBinOp(Opcode))
6614 return;
6615
6616 // Canonicalize:
6617 // binop(const, nonconst) -> binop(nonconst, const)
6622 if ((N1C && !N2C) || (N1CFP && !N2CFP))
6623 std::swap(N1, N2);
6624
6625 // Canonicalize:
6626 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
6627 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6629 std::swap(N1, N2);
6630}
6631
6632SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6633 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
6635 N2.getOpcode() != ISD::DELETED_NODE &&
6636 "Operand is DELETED_NODE!");
6637
6638 canonicalizeCommutativeBinop(Opcode, N1, N2);
6639
6640 auto *N1C = dyn_cast<ConstantSDNode>(N1);
6641 auto *N2C = dyn_cast<ConstantSDNode>(N2);
6642
6643 // Don't allow undefs in vector splats - we might be returning N2 when folding
6644 // to zero etc.
6645 ConstantSDNode *N2CV =
6646 isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
6647
6648 switch (Opcode) {
6649 default: break;
6650 case ISD::TokenFactor:
6651 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
6652 N2.getValueType() == MVT::Other && "Invalid token factor!");
6653 // Fold trivial token factors.
6654 if (N1.getOpcode() == ISD::EntryToken) return N2;
6655 if (N2.getOpcode() == ISD::EntryToken) return N1;
6656 if (N1 == N2) return N1;
6657 break;
6658 case ISD::BUILD_VECTOR: {
6659 // Attempt to simplify BUILD_VECTOR.
6660 SDValue Ops[] = {N1, N2};
6661 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6662 return V;
6663 break;
6664 }
6665 case ISD::CONCAT_VECTORS: {
6666 SDValue Ops[] = {N1, N2};
6667 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
6668 return V;
6669 break;
6670 }
6671 case ISD::AND:
6672 assert(VT.isInteger() && "This operator does not apply to FP types!");
6673 assert(N1.getValueType() == N2.getValueType() &&
6674 N1.getValueType() == VT && "Binary operator types must match!");
6675 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
6676 // worth handling here.
6677 if (N2CV && N2CV->isZero())
6678 return N2;
6679 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
6680 return N1;
6681 break;
6682 case ISD::OR:
6683 case ISD::XOR:
6684 case ISD::ADD:
6685 case ISD::SUB:
6686 assert(VT.isInteger() && "This operator does not apply to FP types!");
6687 assert(N1.getValueType() == N2.getValueType() &&
6688 N1.getValueType() == VT && "Binary operator types must match!");
6689 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
6690 // it's worth handling here.
6691 if (N2CV && N2CV->isZero())
6692 return N1;
6693 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
6694 VT.getVectorElementType() == MVT::i1)
6695 return getNode(ISD::XOR, DL, VT, N1, N2);
6696 break;
6697 case ISD::MUL:
6698 assert(VT.isInteger() && "This operator does not apply to FP types!");
6699 assert(N1.getValueType() == N2.getValueType() &&
6700 N1.getValueType() == VT && "Binary operator types must match!");
6701 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6702 return getNode(ISD::AND, DL, VT, N1, N2);
6703 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6704 const APInt &MulImm = N1->getConstantOperandAPInt(0);
6705 const APInt &N2CImm = N2C->getAPIntValue();
6706 return getVScale(DL, VT, MulImm * N2CImm);
6707 }
6708 break;
6709 case ISD::UDIV:
6710 case ISD::UREM:
6711 case ISD::MULHU:
6712 case ISD::MULHS:
6713 case ISD::SDIV:
6714 case ISD::SREM:
6715 case ISD::SADDSAT:
6716 case ISD::SSUBSAT:
6717 case ISD::UADDSAT:
6718 case ISD::USUBSAT:
6719 assert(VT.isInteger() && "This operator does not apply to FP types!");
6720 assert(N1.getValueType() == N2.getValueType() &&
6721 N1.getValueType() == VT && "Binary operator types must match!");
6722 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
6723 // fold (add_sat x, y) -> (or x, y) for bool types.
6724 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
6725 return getNode(ISD::OR, DL, VT, N1, N2);
6726 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
6727 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
6728 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
6729 }
6730 break;
6731 case ISD::ABDS:
6732 case ISD::ABDU:
6733 assert(VT.isInteger() && "This operator does not apply to FP types!");
6734 assert(N1.getValueType() == N2.getValueType() &&
6735 N1.getValueType() == VT && "Binary operator types must match!");
6736 break;
6737 case ISD::SMIN:
6738 case ISD::UMAX:
6739 assert(VT.isInteger() && "This operator does not apply to FP types!");
6740 assert(N1.getValueType() == N2.getValueType() &&
6741 N1.getValueType() == VT && "Binary operator types must match!");
6742 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6743 return getNode(ISD::OR, DL, VT, N1, N2);
6744 break;
6745 case ISD::SMAX:
6746 case ISD::UMIN:
6747 assert(VT.isInteger() && "This operator does not apply to FP types!");
6748 assert(N1.getValueType() == N2.getValueType() &&
6749 N1.getValueType() == VT && "Binary operator types must match!");
6750 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6751 return getNode(ISD::AND, DL, VT, N1, N2);
6752 break;
6753 case ISD::FADD:
6754 case ISD::FSUB:
6755 case ISD::FMUL:
6756 case ISD::FDIV:
6757 case ISD::FREM:
6758 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
6759 assert(N1.getValueType() == N2.getValueType() &&
6760 N1.getValueType() == VT && "Binary operator types must match!");
6761 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
6762 return V;
6763 break;
6764 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
6765 assert(N1.getValueType() == VT &&
6768 "Invalid FCOPYSIGN!");
6769 break;
6770 case ISD::SHL:
6771 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6772 const APInt &MulImm = N1->getConstantOperandAPInt(0);
6773 const APInt &ShiftImm = N2C->getAPIntValue();
6774 return getVScale(DL, VT, MulImm << ShiftImm);
6775 }
6776 [[fallthrough]];
6777 case ISD::SRA:
6778 case ISD::SRL:
6779 if (SDValue V = simplifyShift(N1, N2))
6780 return V;
6781 [[fallthrough]];
6782 case ISD::ROTL:
6783 case ISD::ROTR:
6784 assert(VT == N1.getValueType() &&
6785 "Shift operators return type must be the same as their first arg");
6786 assert(VT.isInteger() && N2.getValueType().isInteger() &&
6787 "Shifts only work on integers");
6788 assert((!VT.isVector() || VT == N2.getValueType()) &&
6789 "Vector shift amounts must be in the same as their first arg");
6790 // Verify that the shift amount VT is big enough to hold valid shift
6791 // amounts. This catches things like trying to shift an i1024 value by an
6792 // i8, which is easy to fall into in generic code that uses
6793 // TLI.getShiftAmount().
6796 "Invalid use of small shift amount with oversized value!");
6797
6798 // Always fold shifts of i1 values so the code generator doesn't need to
6799 // handle them. Since we know the size of the shift has to be less than the
6800 // size of the value, the shift/rotate count is guaranteed to be zero.
6801 if (VT == MVT::i1)
6802 return N1;
6803 if (N2CV && N2CV->isZero())
6804 return N1;
6805 break;
6806 case ISD::FP_ROUND:
6807 assert(VT.isFloatingPoint() &&
6809 VT.bitsLE(N1.getValueType()) &&
6810 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
6811 "Invalid FP_ROUND!");
6812 if (N1.getValueType() == VT) return N1; // noop conversion.
6813 break;
6814 case ISD::AssertSext:
6815 case ISD::AssertZext: {
6816 EVT EVT = cast<VTSDNode>(N2)->getVT();
6817 assert(VT == N1.getValueType() && "Not an inreg extend!");
6818 assert(VT.isInteger() && EVT.isInteger() &&
6819 "Cannot *_EXTEND_INREG FP types");
6820 assert(!EVT.isVector() &&
6821 "AssertSExt/AssertZExt type should be the vector element type "
6822 "rather than the vector type!");
6823 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
6824 if (VT.getScalarType() == EVT) return N1; // noop assertion.
6825 break;
6826 }
6828 EVT EVT = cast<VTSDNode>(N2)->getVT();
6829 assert(VT == N1.getValueType() && "Not an inreg extend!");
6830 assert(VT.isInteger() && EVT.isInteger() &&
6831 "Cannot *_EXTEND_INREG FP types");
6832 assert(EVT.isVector() == VT.isVector() &&
6833 "SIGN_EXTEND_INREG type should be vector iff the operand "
6834 "type is vector!");
6835 assert((!EVT.isVector() ||
6837 "Vector element counts must match in SIGN_EXTEND_INREG");
6838 assert(EVT.bitsLE(VT) && "Not extending!");
6839 if (EVT == VT) return N1; // Not actually extending
6840
6841 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6842 unsigned FromBits = EVT.getScalarSizeInBits();
6843 Val <<= Val.getBitWidth() - FromBits;
6844 Val.ashrInPlace(Val.getBitWidth() - FromBits);
6845 return getConstant(Val, DL, ConstantVT);
6846 };
6847
6848 if (N1C) {
6849 const APInt &Val = N1C->getAPIntValue();
6850 return SignExtendInReg(Val, VT);
6851 }
6852
6855 llvm::EVT OpVT = N1.getOperand(0).getValueType();
6856 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
6857 SDValue Op = N1.getOperand(i);
6858 if (Op.isUndef()) {
6859 Ops.push_back(getUNDEF(OpVT));
6860 continue;
6861 }
6862 ConstantSDNode *C = cast<ConstantSDNode>(Op);
6863 APInt Val = C->getAPIntValue();
6864 Ops.push_back(SignExtendInReg(Val, OpVT));
6865 }
6866 return getBuildVector(VT, DL, Ops);
6867 }
6868
6869 if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6870 isa<ConstantSDNode>(N1.getOperand(0)))
6871 return getNode(
6872 ISD::SPLAT_VECTOR, DL, VT,
6873 SignExtendInReg(N1.getConstantOperandAPInt(0),
6874 N1.getOperand(0).getValueType()));
6875 break;
6876 }
6878 case ISD::FP_TO_UINT_SAT: {
6879 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
6880 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
6881 assert(N1.getValueType().isVector() == VT.isVector() &&
6882 "FP_TO_*INT_SAT type should be vector iff the operand type is "
6883 "vector!");
6884 assert((!VT.isVector() || VT.getVectorElementCount() ==
6886 "Vector element counts must match in FP_TO_*INT_SAT");
6887 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
6888 "Type to saturate to must be a scalar.");
6889 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
6890 "Not extending!");
6891 break;
6892 }
6895 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
6896 element type of the vector.");
6897
6898 // Extract from an undefined value or using an undefined index is undefined.
6899 if (N1.isUndef() || N2.isUndef())
6900 return getUNDEF(VT);
6901
6902 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
6903 // vectors. For scalable vectors we will provide appropriate support for
6904 // dealing with arbitrary indices.
6905 if (N2C && N1.getValueType().isFixedLengthVector() &&
6906 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
6907 return getUNDEF(VT);
6908
6909 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
6910 // expanding copies of large vectors from registers. This only works for
6911 // fixed length vectors, since we need to know the exact number of
6912 // elements.
6913 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
6915 unsigned Factor =
6918 N1.getOperand(N2C->getZExtValue() / Factor),
6919 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
6920 }
6921
6922 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
6923 // lowering is expanding large vector constants.
6924 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
6925 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
6928 "BUILD_VECTOR used for scalable vectors");
6929 unsigned Index =
6930 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
6931 SDValue Elt = N1.getOperand(Index);
6932
6933 if (VT != Elt.getValueType())
6934 // If the vector element type is not legal, the BUILD_VECTOR operands
6935 // are promoted and implicitly truncated, and the result implicitly
6936 // extended. Make that explicit here.
6937 Elt = getAnyExtOrTrunc(Elt, DL, VT);
6938
6939 return Elt;
6940 }
6941
6942 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
6943 // operations are lowered to scalars.
6944 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
6945 // If the indices are the same, return the inserted element else
6946 // if the indices are known different, extract the element from
6947 // the original vector.
6948 SDValue N1Op2 = N1.getOperand(2);
6949 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
6950
6951 if (N1Op2C && N2C) {
6952 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
6953 if (VT == N1.getOperand(1).getValueType())
6954 return N1.getOperand(1);
6955 if (VT.isFloatingPoint()) {
6957 return getFPExtendOrRound(N1.getOperand(1), DL, VT);
6958 }
6959 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
6960 }
6961 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
6962 }
6963 }
6964
6965 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
6966 // when vector types are scalarized and v1iX is legal.
6967 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
6968 // Here we are completely ignoring the extract element index (N2),
6969 // which is fine for fixed width vectors, since any index other than 0
6970 // is undefined anyway. However, this cannot be ignored for scalable
6971 // vectors - in theory we could support this, but we don't want to do this
6972 // without a profitability check.
6973 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6975 N1.getValueType().getVectorNumElements() == 1) {
6976 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
6977 N1.getOperand(1));
6978 }
6979 break;
6981 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
6982 assert(!N1.getValueType().isVector() && !VT.isVector() &&
6983 (N1.getValueType().isInteger() == VT.isInteger()) &&
6984 N1.getValueType() != VT &&
6985 "Wrong types for EXTRACT_ELEMENT!");
6986
6987 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
6988 // 64-bit integers into 32-bit parts. Instead of building the extract of
6989 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
6990 if (N1.getOpcode() == ISD::BUILD_PAIR)
6991 return N1.getOperand(N2C->getZExtValue());
6992
6993 // EXTRACT_ELEMENT of a constant int is also very common.
6994 if (N1C) {
6995 unsigned ElementSize = VT.getSizeInBits();
6996 unsigned Shift = ElementSize * N2C->getZExtValue();
6997 const APInt &Val = N1C->getAPIntValue();
6998 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
6999 }
7000 break;
7002 EVT N1VT = N1.getValueType();
7003 assert(VT.isVector() && N1VT.isVector() &&
7004 "Extract subvector VTs must be vectors!");
7006 "Extract subvector VTs must have the same element type!");
7007 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
7008 "Cannot extract a scalable vector from a fixed length vector!");
7009 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7011 "Extract subvector must be from larger vector to smaller vector!");
7012 assert(N2C && "Extract subvector index must be a constant");
7013 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7014 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
7015 N1VT.getVectorMinNumElements()) &&
7016 "Extract subvector overflow!");
7017 assert(N2C->getAPIntValue().getBitWidth() ==
7018 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7019 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
7020
7021 // Trivial extraction.
7022 if (VT == N1VT)
7023 return N1;
7024
7025 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
7026 if (N1.isUndef())
7027 return getUNDEF(VT);
7028
7029 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
7030 // the concat have the same type as the extract.
7031 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
7032 VT == N1.getOperand(0).getValueType()) {
7033 unsigned Factor = VT.getVectorMinNumElements();
7034 return N1.getOperand(N2C->getZExtValue() / Factor);
7035 }
7036
7037 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
7038 // during shuffle legalization.
7039 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
7040 VT == N1.getOperand(1).getValueType())
7041 return N1.getOperand(1);
7042 break;
7043 }
7044 }
7045
7046 // Perform trivial constant folding.
7047 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
7048 return SV;
7049
7050 // Canonicalize an UNDEF to the RHS, even over a constant.
7051 if (N1.isUndef()) {
7052 if (TLI->isCommutativeBinOp(Opcode)) {
7053 std::swap(N1, N2);
7054 } else {
7055 switch (Opcode) {
7056 case ISD::SUB:
7057 return getUNDEF(VT); // fold op(undef, arg2) -> undef
7059 case ISD::UDIV:
7060 case ISD::SDIV:
7061 case ISD::UREM:
7062 case ISD::SREM:
7063 case ISD::SSUBSAT:
7064 case ISD::USUBSAT:
7065 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
7066 }
7067 }
7068 }
7069
7070 // Fold a bunch of operators when the RHS is undef.
7071 if (N2.isUndef()) {
7072 switch (Opcode) {
7073 case ISD::XOR:
7074 if (N1.isUndef())
7075 // Handle undef ^ undef -> 0 special case. This is a common
7076 // idiom (misuse).
7077 return getConstant(0, DL, VT);
7078 [[fallthrough]];
7079 case ISD::ADD:
7080 case ISD::SUB:
7081 case ISD::UDIV:
7082 case ISD::SDIV:
7083 case ISD::UREM:
7084 case ISD::SREM:
7085 return getUNDEF(VT); // fold op(arg1, undef) -> undef
7086 case ISD::MUL:
7087 case ISD::AND:
7088 case ISD::SSUBSAT:
7089 case ISD::USUBSAT:
7090 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
7091 case ISD::OR:
7092 case ISD::SADDSAT:
7093 case ISD::UADDSAT:
7094 return getAllOnesConstant(DL, VT);
7095 }
7096 }
7097
7098 // Memoize this node if possible.
7099 SDNode *N;
7100 SDVTList VTs = getVTList(VT);
7101 SDValue Ops[] = {N1, N2};
7102 if (VT != MVT::Glue) {
7104 AddNodeIDNode(ID, Opcode, VTs, Ops);
7105 void *IP = nullptr;
7106 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7107 E->intersectFlagsWith(Flags);
7108 return SDValue(E, 0);
7109 }
7110
7111 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7112 N->setFlags(Flags);
7113 createOperands(N, Ops);
7114 CSEMap.InsertNode(N, IP);
7115 } else {
7116 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7117 createOperands(N, Ops);
7118 }
7119
7120 InsertNode(N);
7121 SDValue V = SDValue(N, 0);
7122 NewSDValueDbgMsg(V, "Creating new node: ", this);
7123 return V;
7124}
7125
7126SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7127 SDValue N1, SDValue N2, SDValue N3) {
7128 SDNodeFlags Flags;
7129 if (Inserter)
7130 Flags = Inserter->getFlags();
7131 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7132}
7133
7134SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7135 SDValue N1, SDValue N2, SDValue N3,
7136 const SDNodeFlags Flags) {
7138 N2.getOpcode() != ISD::DELETED_NODE &&
7139 N3.getOpcode() != ISD::DELETED_NODE &&
7140 "Operand is DELETED_NODE!");
7141 // Perform various simplifications.
7142 switch (Opcode) {
7143 case ISD::FMA:
7144 case ISD::FMAD: {
7145 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7146 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7147 N3.getValueType() == VT && "FMA types must match!");
7148 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
7149 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
7150 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
7151 if (N1CFP && N2CFP && N3CFP) {
7152 APFloat V1 = N1CFP->getValueAPF();
7153 const APFloat &V2 = N2CFP->getValueAPF();
7154 const APFloat &V3 = N3CFP->getValueAPF();
7155 if (Opcode == ISD::FMAD) {
7158 } else
7160 return getConstantFP(V1, DL, VT);
7161 }
7162 break;
7163 }
7164 case ISD::BUILD_VECTOR: {
7165 // Attempt to simplify BUILD_VECTOR.
7166 SDValue Ops[] = {N1, N2, N3};
7167 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7168 return V;
7169 break;
7170 }
7171 case ISD::CONCAT_VECTORS: {
7172 SDValue Ops[] = {N1, N2, N3};
7173 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7174 return V;
7175 break;
7176 }
7177 case ISD::SETCC: {
7178 assert(VT.isInteger() && "SETCC result type must be an integer!");
7179 assert(N1.getValueType() == N2.getValueType() &&
7180 "SETCC operands must have the same type!");
7181 assert(VT.isVector() == N1.getValueType().isVector() &&
7182 "SETCC type should be vector iff the operand type is vector!");
7183 assert((!VT.isVector() || VT.getVectorElementCount() ==
7185 "SETCC vector element counts must match!");
7186 // Use FoldSetCC to simplify SETCC's.
7187 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
7188 return V;
7189 // Vector constant folding.
7190 SDValue Ops[] = {N1, N2, N3};
7191 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7192 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
7193 return V;
7194 }
7195 break;
7196 }
7197 case ISD::SELECT:
7198 case ISD::VSELECT:
7199 if (SDValue V = simplifySelect(N1, N2, N3))
7200 return V;
7201 break;
7203 llvm_unreachable("should use getVectorShuffle constructor!");
7204 case ISD::VECTOR_SPLICE: {
7205 if (cast<ConstantSDNode>(N3)->isZero())
7206 return N1;
7207 break;
7208 }
7210 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
7211 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7212 // for scalable vectors where we will generate appropriate code to
7213 // deal with out-of-bounds cases correctly.
7214 if (N3C && N1.getValueType().isFixedLengthVector() &&
7216 return getUNDEF(VT);
7217
7218 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7219 if (N3.isUndef())
7220 return getUNDEF(VT);
7221
7222 // If the inserted element is an UNDEF, just use the input vector.
7223 if (N2.isUndef())
7224 return N1;
7225
7226 break;
7227 }
7228 case ISD::INSERT_SUBVECTOR: {
7229 // Inserting undef into undef is still undef.
7230 if (N1.isUndef() && N2.isUndef())
7231 return getUNDEF(VT);
7232
7233 EVT N2VT = N2.getValueType();
7234 assert(VT == N1.getValueType() &&
7235 "Dest and insert subvector source types must match!");
7236 assert(VT.isVector() && N2VT.isVector() &&
7237 "Insert subvector VTs must be vectors!");
7239 "Insert subvector VTs must have the same element type!");
7240 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7241 "Cannot insert a scalable vector into a fixed length vector!");
7242 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7244 "Insert subvector must be from smaller vector to larger vector!");
7245 assert(isa<ConstantSDNode>(N3) &&
7246 "Insert subvector index must be constant");
7247 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7248 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7250 "Insert subvector overflow!");
7252 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7253 "Constant index for INSERT_SUBVECTOR has an invalid size");
7254
7255 // Trivial insertion.
7256 if (VT == N2VT)
7257 return N2;
7258
7259 // If this is an insert of an extracted vector into an undef vector, we
7260 // can just use the input to the extract.
7261 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7262 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
7263 return N2.getOperand(0);
7264 break;
7265 }
7266 case ISD::BITCAST:
7267 // Fold bit_convert nodes from a type to themselves.
7268 if (N1.getValueType() == VT)
7269 return N1;
7270 break;
7271 case ISD::VP_TRUNCATE:
7272 case ISD::VP_SIGN_EXTEND:
7273 case ISD::VP_ZERO_EXTEND:
7274 // Don't create noop casts.
7275 if (N1.getValueType() == VT)
7276 return N1;
7277 break;
7278 }
7279
7280 // Memoize node if it doesn't produce a glue result.
7281 SDNode *N;
7282 SDVTList VTs = getVTList(VT);
7283 SDValue Ops[] = {N1, N2, N3};
7284 if (VT != MVT::Glue) {
7286 AddNodeIDNode(ID, Opcode, VTs, Ops);
7287 void *IP = nullptr;
7288 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7289 E->intersectFlagsWith(Flags);
7290 return SDValue(E, 0);
7291 }
7292
7293 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7294 N->setFlags(Flags);
7295 createOperands(N, Ops);
7296 CSEMap.InsertNode(N, IP);
7297 } else {
7298 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7299 createOperands(N, Ops);
7300 }
7301
7302 InsertNode(N);
7303 SDValue V = SDValue(N, 0);
7304 NewSDValueDbgMsg(V, "Creating new node: ", this);
7305 return V;
7306}
7307
7308SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7309 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7310 SDValue Ops[] = { N1, N2, N3, N4 };
7311 return getNode(Opcode, DL, VT, Ops);
7312}
7313
7314SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7315 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7316 SDValue N5) {
7317 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7318 return getNode(Opcode, DL, VT, Ops);
7319}
7320
7321/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7322/// the incoming stack arguments to be loaded from the stack.
7324 SmallVector<SDValue, 8> ArgChains;
7325
7326 // Include the original chain at the beginning of the list. When this is
7327 // used by target LowerCall hooks, this helps legalize find the
7328 // CALLSEQ_BEGIN node.
7329 ArgChains.push_back(Chain);
7330
7331 // Add a chain value for each stack argument.
7332 for (SDNode *U : getEntryNode().getNode()->uses())
7333 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))
7334 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
7335 if (FI->getIndex() < 0)
7336 ArgChains.push_back(SDValue(L, 1));
7337
7338 // Build a tokenfactor for all the chains.
7339 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7340}
7341
7342/// getMemsetValue - Vectorized representation of the memset value
7343/// operand.
7345 const SDLoc &dl) {
7346 assert(!Value.isUndef());
7347
7348 unsigned NumBits = VT.getScalarSizeInBits();
7349 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
7350 assert(C->getAPIntValue().getBitWidth() == 8);
7351 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
7352 if (VT.isInteger()) {
7353 bool IsOpaque = VT.getSizeInBits() > 64 ||
7354 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
7355 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
7356 }
7357 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
7358 VT);
7359 }
7360
7361 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7362 EVT IntVT = VT.getScalarType();
7363 if (!IntVT.isInteger())
7364 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
7365
7366 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
7367 if (NumBits > 8) {
7368 // Use a multiplication with 0x010101... to extend the input to the
7369 // required length.
7370 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
7371 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
7372 DAG.getConstant(Magic, dl, IntVT));
7373 }
7374
7375 if (VT != Value.getValueType() && !VT.isInteger())
7376 Value = DAG.getBitcast(VT.getScalarType(), Value);
7377 if (VT != Value.getValueType())
7378 Value = DAG.getSplatBuildVector(VT, dl, Value);
7379
7380 return Value;
7381}
7382
7383/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
7384/// used when a memcpy is turned into a memset when the source is a constant
7385/// string ptr.
7387 const TargetLowering &TLI,
7388 const ConstantDataArraySlice &Slice) {
7389 // Handle vector with all elements zero.
7390 if (Slice.Array == nullptr) {
7391 if (VT.isInteger())
7392 return DAG.getConstant(0, dl, VT);
7393 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7394 return DAG.getConstantFP(0.0, dl, VT);
7395 if (VT.isVector()) {
7396 unsigned NumElts = VT.getVectorNumElements();
7397 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
7398 return DAG.getNode(ISD::BITCAST, dl, VT,
7399 DAG.getConstant(0, dl,
7401 EltVT, NumElts)));
7402 }
7403 llvm_unreachable("Expected type!");
7404 }
7405
7406 assert(!VT.isVector() && "Can't handle vector type here!");
7407 unsigned NumVTBits = VT.getSizeInBits();
7408 unsigned NumVTBytes = NumVTBits / 8;
7409 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
7410
7411 APInt Val(NumVTBits, 0);
7412 if (DAG.getDataLayout().isLittleEndian()) {
7413 for (unsigned i = 0; i != NumBytes; ++i)
7414 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
7415 } else {
7416 for (unsigned i = 0; i != NumBytes; ++i)
7417 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7418 }
7419
7420 // If the "cost" of materializing the integer immediate is less than the cost
7421 // of a load, then it is cost effective to turn the load into the immediate.
7422 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
7423 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
7424 return DAG.getConstant(Val, dl, VT);
7425 return SDValue();
7426}
7427
7429 const SDLoc &DL,
7430 const SDNodeFlags Flags) {
7431 EVT VT = Base.getValueType();
7432 SDValue Index;
7433
7434 if (Offset.isScalable())
7435 Index = getVScale(DL, Base.getValueType(),
7436 APInt(Base.getValueSizeInBits().getFixedValue(),
7437 Offset.getKnownMinValue()));
7438 else
7439 Index = getConstant(Offset.getFixedValue(), DL, VT);
7440
7441 return getMemBasePlusOffset(Base, Index, DL, Flags);
7442}
7443
7445 const SDLoc &DL,
7446 const SDNodeFlags Flags) {
7447 assert(Offset.getValueType().isInteger());
7448 EVT BasePtrVT = Ptr.getValueType();
7449 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
7450}
7451
7452/// Returns true if memcpy source is constant data.
7454 uint64_t SrcDelta = 0;
7455 GlobalAddressSDNode *G = nullptr;
7456 if (Src.getOpcode() == ISD::GlobalAddress)
7457 G = cast<GlobalAddressSDNode>(Src);
7458 else if (Src.getOpcode() == ISD::ADD &&
7459 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
7460 Src.getOperand(1).getOpcode() == ISD::Constant) {
7461 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
7462 SrcDelta = Src.getConstantOperandVal(1);
7463 }
7464 if (!G)
7465 return false;
7466
7467 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
7468 SrcDelta + G->getOffset());
7469}
7470
7472 SelectionDAG &DAG) {
7473 // On Darwin, -Os means optimize for size without hurting performance, so
7474 // only really optimize for size when -Oz (MinSize) is used.
7476 return MF.getFunction().hasMinSize();
7477 return DAG.shouldOptForSize();
7478}
7479
7481 SmallVector<SDValue, 32> &OutChains, unsigned From,
7482 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
7483 SmallVector<SDValue, 16> &OutStoreChains) {
7484 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
7485 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
7486 SmallVector<SDValue, 16> GluedLoadChains;
7487 for (unsigned i = From; i < To; ++i) {
7488 OutChains.push_back(OutLoadChains[i]);
7489 GluedLoadChains.push_back(OutLoadChains[i]);
7490 }
7491
7492 // Chain for all loads.
7493 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7494 GluedLoadChains);
7495
7496 for (unsigned i = From; i < To; ++i) {
7497 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
7498 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
7499 ST->getBasePtr(), ST->getMemoryVT(),
7500 ST->getMemOperand());
7501 OutChains.push_back(NewStore);
7502 }
7503}
7504
7506 SDValue Chain, SDValue Dst, SDValue Src,
7507 uint64_t Size, Align Alignment,
7508 bool isVol, bool AlwaysInline,
7509 MachinePointerInfo DstPtrInfo,
7510 MachinePointerInfo SrcPtrInfo,
7511 const AAMDNodes &AAInfo, AAResults *AA) {
7512 // Turn a memcpy of undef to nop.
7513 // FIXME: We need to honor volatile even is Src is undef.
7514 if (Src.isUndef())
7515 return Chain;
7516
7517 // Expand memcpy to a series of load and store ops if the size operand falls
7518 // below a certain threshold.
7519 // TODO: In the AlwaysInline case, if the size is big then generate a loop
7520 // rather than maybe a humongous number of loads and stores.
7521 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7522 const DataLayout &DL = DAG.getDataLayout();
7523 LLVMContext &C = *DAG.getContext();
7524 std::vector<EVT> MemOps;
7525 bool DstAlignCanChange = false;
7527 MachineFrameInfo &MFI = MF.getFrameInfo();
7528 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7529 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7530 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7531 DstAlignCanChange = true;
7532 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7533 if (!SrcAlign || Alignment > *SrcAlign)
7534 SrcAlign = Alignment;
7535 assert(SrcAlign && "SrcAlign must be set");
7537 // If marked as volatile, perform a copy even when marked as constant.
7538 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
7539 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
7540 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
7541 const MemOp Op = isZeroConstant
7542 ? MemOp::Set(Size, DstAlignCanChange, Alignment,
7543 /*IsZeroMemset*/ true, isVol)
7544 : MemOp::Copy(Size, DstAlignCanChange, Alignment,
7545 *SrcAlign, isVol, CopyFromConstant);
7546 if (!TLI.findOptimalMemOpLowering(
7547 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
7548 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
7549 return SDValue();
7550
7551 if (DstAlignCanChange) {
7552 Type *Ty = MemOps[0].getTypeForEVT(C);
7553 Align NewAlign = DL.getABITypeAlign(Ty);
7554
7555 // Don't promote to an alignment that would require dynamic stack
7556 // realignment which may conflict with optimizations such as tail call
7557 // optimization.
7559 if (!TRI->hasStackRealignment(MF))
7560 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7561 NewAlign = NewAlign.previous();
7562
7563 if (NewAlign > Alignment) {
7564 // Give the stack frame object a larger alignment if needed.
7565 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7566 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7567 Alignment = NewAlign;
7568 }
7569 }
7570
7571 // Prepare AAInfo for loads/stores after lowering this memcpy.
7572 AAMDNodes NewAAInfo = AAInfo;
7573 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7574
7575 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
7576 bool isConstant =
7577 AA && SrcVal &&
7578 AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
7579
7580 MachineMemOperand::Flags MMOFlags =
7582 SmallVector<SDValue, 16> OutLoadChains;
7583 SmallVector<SDValue, 16> OutStoreChains;
7584 SmallVector<SDValue, 32> OutChains;
7585 unsigned NumMemOps = MemOps.size();
7586 uint64_t SrcOff = 0, DstOff = 0;
7587 for (unsigned i = 0; i != NumMemOps; ++i) {
7588 EVT VT = MemOps[i];
7589 unsigned VTSize = VT.getSizeInBits() / 8;
7590 SDValue Value, Store;
7591
7592 if (VTSize > Size) {
7593 // Issuing an unaligned load / store pair that overlaps with the previous
7594 // pair. Adjust the offset accordingly.
7595 assert(i == NumMemOps-1 && i != 0);
7596 SrcOff -= VTSize - Size;
7597 DstOff -= VTSize - Size;
7598 }
7599
7600 if (CopyFromConstant &&
7601 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
7602 // It's unlikely a store of a vector immediate can be done in a single
7603 // instruction. It would require a load from a constantpool first.
7604 // We only handle zero vectors here.
7605 // FIXME: Handle other cases where store of vector immediate is done in
7606 // a single instruction.
7607 ConstantDataArraySlice SubSlice;
7608 if (SrcOff < Slice.Length) {
7609 SubSlice = Slice;
7610 SubSlice.move(SrcOff);
7611 } else {
7612 // This is an out-of-bounds access and hence UB. Pretend we read zero.
7613 SubSlice.Array = nullptr;
7614 SubSlice.Offset = 0;
7615 SubSlice.Length = VTSize;
7616 }
7617 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
7618 if (Value.getNode()) {
7619 Store = DAG.getStore(
7620 Chain, dl, Value,
7621 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7622 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7623 OutChains.push_back(Store);
7624 }
7625 }
7626
7627 if (!Store.getNode()) {
7628 // The type might not be legal for the target. This should only happen
7629 // if the type is smaller than a legal type, as on PPC, so the right
7630 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
7631 // to Load/Store if NVT==VT.
7632 // FIXME does the case above also need this?
7633 EVT NVT = TLI.getTypeToTransformTo(C, VT);
7634 assert(NVT.bitsGE(VT));
7635
7636 bool isDereferenceable =
7637 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7638 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7639 if (isDereferenceable)
7641 if (isConstant)
7642 SrcMMOFlags |= MachineMemOperand::MOInvariant;
7643
7644 Value = DAG.getExtLoad(
7645 ISD::EXTLOAD, dl, NVT, Chain,
7646 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7647 SrcPtrInfo.getWithOffset(SrcOff), VT,
7648 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
7649 OutLoadChains.push_back(Value.getValue(1));
7650
7651 Store = DAG.getTruncStore(
7652 Chain, dl, Value,
7653 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7654 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
7655 OutStoreChains.push_back(Store);
7656 }
7657 SrcOff += VTSize;
7658 DstOff += VTSize;
7659 Size -= VTSize;
7660 }
7661
7662 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
7664 unsigned NumLdStInMemcpy = OutStoreChains.size();
7665
7666 if (NumLdStInMemcpy) {
7667 // It may be that memcpy might be converted to memset if it's memcpy
7668 // of constants. In such a case, we won't have loads and stores, but
7669 // just stores. In the absence of loads, there is nothing to gang up.
7670 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
7671 // If target does not care, just leave as it.
7672 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7673 OutChains.push_back(OutLoadChains[i]);
7674 OutChains.push_back(OutStoreChains[i]);
7675 }
7676 } else {
7677 // Ld/St less than/equal limit set by target.
7678 if (NumLdStInMemcpy <= GluedLdStLimit) {
7679 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7680 NumLdStInMemcpy, OutLoadChains,
7681 OutStoreChains);
7682 } else {
7683 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
7684 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7685 unsigned GlueIter = 0;
7686
7687 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7688 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7689 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
7690
7691 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
7692 OutLoadChains, OutStoreChains);
7693 GlueIter += GluedLdStLimit;
7694 }
7695
7696 // Residual ld/st.
7697 if (RemainingLdStInMemcpy) {
7698 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7699 RemainingLdStInMemcpy, OutLoadChains,
7700 OutStoreChains);
7701 }
7702 }
7703 }
7704 }
7705 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7706}
7707
7709 SDValue Chain, SDValue Dst, SDValue Src,
7710 uint64_t Size, Align Alignment,
7711 bool isVol, bool AlwaysInline,
7712 MachinePointerInfo DstPtrInfo,
7713 MachinePointerInfo SrcPtrInfo,
7714 const AAMDNodes &AAInfo) {
7715 // Turn a memmove of undef to nop.
7716 // FIXME: We need to honor volatile even is Src is undef.
7717 if (Src.isUndef())
7718 return Chain;
7719
7720 // Expand memmove to a series of load and store ops if the size operand falls
7721 // below a certain threshold.
7722 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7723 const DataLayout &DL = DAG.getDataLayout();
7724 LLVMContext &C = *DAG.getContext();
7725 std::vector<EVT> MemOps;
7726 bool DstAlignCanChange = false;
7728 MachineFrameInfo &MFI = MF.getFrameInfo();
7729 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7730 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7731 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7732 DstAlignCanChange = true;
7733 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7734 if (!SrcAlign || Alignment > *SrcAlign)
7735 SrcAlign = Alignment;
7736 assert(SrcAlign && "SrcAlign must be set");
7737 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
7738 if (!TLI.findOptimalMemOpLowering(
7739 MemOps, Limit,
7740 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
7741 /*IsVolatile*/ true),
7742 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
7743 MF.getFunction().getAttributes()))
7744 return SDValue();
7745
7746 if (DstAlignCanChange) {
7747 Type *Ty = MemOps[0].getTypeForEVT(C);
7748 Align NewAlign = DL.getABITypeAlign(Ty);
7749
7750 // Don't promote to an alignment that would require dynamic stack
7751 // realignment which may conflict with optimizations such as tail call
7752 // optimization.
7754 if (!TRI->hasStackRealignment(MF))
7755 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7756 NewAlign = NewAlign.previous();
7757
7758 if (NewAlign > Alignment) {
7759 // Give the stack frame object a larger alignment if needed.
7760 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7761 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7762 Alignment = NewAlign;
7763 }
7764 }
7765
7766 // Prepare AAInfo for loads/stores after lowering this memmove.
7767 AAMDNodes NewAAInfo = AAInfo;
7768 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7769
7770 MachineMemOperand::Flags MMOFlags =
7772 uint64_t SrcOff = 0, DstOff = 0;
7773 SmallVector<SDValue, 8> LoadValues;
7774 SmallVector<SDValue, 8> LoadChains;
7775 SmallVector<SDValue, 8> OutChains;
7776 unsigned NumMemOps = MemOps.size();
7777 for (unsigned i = 0; i < NumMemOps; i++) {
7778 EVT VT = MemOps[i];
7779 unsigned VTSize = VT.getSizeInBits() / 8;
7780 SDValue Value;
7781
7782 bool isDereferenceable =
7783 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7784 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7785 if (isDereferenceable)
7787
7788 Value = DAG.getLoad(
7789 VT, dl, Chain,
7790 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7791 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
7792 LoadValues.push_back(Value);
7793 LoadChains.push_back(Value.getValue(1));
7794 SrcOff += VTSize;
7795 }
7796 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7797 OutChains.clear();
7798 for (unsigned i = 0; i < NumMemOps; i++) {
7799 EVT VT = MemOps[i];
7800 unsigned VTSize = VT.getSizeInBits() / 8;
7801 SDValue Store;
7802
7803 Store = DAG.getStore(
7804 Chain, dl, LoadValues[i],
7805 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7806 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7807 OutChains.push_back(Store);
7808 DstOff += VTSize;
7809 }
7810
7811 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7812}
7813
7814/// Lower the call to 'memset' intrinsic function into a series of store
7815/// operations.
7816///
7817/// \param DAG Selection DAG where lowered code is placed.
7818/// \param dl Link to corresponding IR location.
7819/// \param Chain Control flow dependency.
7820/// \param Dst Pointer to destination memory location.
7821/// \param Src Value of byte to write into the memory.
7822/// \param Size Number of bytes to write.
7823/// \param Alignment Alignment of the destination in bytes.
7824/// \param isVol True if destination is volatile.
7825/// \param AlwaysInline Makes sure no function call is generated.
7826/// \param DstPtrInfo IR information on the memory pointer.
7827/// \returns New head in the control flow, if lowering was successful, empty
7828/// SDValue otherwise.
7829///
7830/// The function tries to replace 'llvm.memset' intrinsic with several store
7831/// operations and value calculation code. This is usually profitable for small
7832/// memory size or when the semantic requires inlining.
7834 SDValue Chain, SDValue Dst, SDValue Src,
7835 uint64_t Size, Align Alignment, bool isVol,
7836 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
7837 const AAMDNodes &AAInfo) {
7838 // Turn a memset of undef to nop.
7839 // FIXME: We need to honor volatile even is Src is undef.
7840 if (Src.isUndef())
7841 return Chain;
7842
7843 // Expand memset to a series of load/store ops if the size operand
7844 // falls below a certain threshold.
7845 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7846 std::vector<EVT> MemOps;
7847 bool DstAlignCanChange = false;
7849 MachineFrameInfo &MFI = MF.getFrameInfo();
7850 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7851 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7852 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7853 DstAlignCanChange = true;
7854 bool IsZeroVal = isNullConstant(Src);
7855 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
7856
7857 if (!TLI.findOptimalMemOpLowering(
7858 MemOps, Limit,
7859 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
7860 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
7861 return SDValue();
7862
7863 if (DstAlignCanChange) {
7864 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
7865 const DataLayout &DL = DAG.getDataLayout();
7866 Align NewAlign = DL.getABITypeAlign(Ty);
7867
7868 // Don't promote to an alignment that would require dynamic stack
7869 // realignment which may conflict with optimizations such as tail call
7870 // optimization.
7872 if (!TRI->hasStackRealignment(MF))
7873 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7874 NewAlign = NewAlign.previous();
7875
7876 if (NewAlign > Alignment) {
7877 // Give the stack frame object a larger alignment if needed.
7878 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7879 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7880 Alignment = NewAlign;
7881 }
7882 }
7883
7884 SmallVector<SDValue, 8> OutChains;
7885 uint64_t DstOff = 0;
7886 unsigned NumMemOps = MemOps.size();
7887
7888 // Find the largest store and generate the bit pattern for it.
7889 EVT LargestVT = MemOps[0];
7890 for (unsigned i = 1; i < NumMemOps; i++)
7891 if (MemOps[i].bitsGT(LargestVT))
7892 LargestVT = MemOps[i];
7893 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
7894
7895 // Prepare AAInfo for loads/stores after lowering this memset.
7896 AAMDNodes NewAAInfo = AAInfo;
7897 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7898
7899 for (unsigned i = 0; i < NumMemOps; i++) {
7900 EVT VT = MemOps[i];
7901 unsigned VTSize = VT.getSizeInBits() / 8;
7902 if (VTSize > Size) {
7903 // Issuing an unaligned load / store pair that overlaps with the previous
7904 // pair. Adjust the offset accordingly.
7905 assert(i == NumMemOps-1 && i != 0);
7906 DstOff -= VTSize - Size;
7907 }
7908
7909 // If this store is smaller than the largest store see whether we can get
7910 // the smaller value for free with a truncate or extract vector element and
7911 // then store.
7912 SDValue Value = MemSetValue;
7913 if (VT.bitsLT(LargestVT)) {
7914 unsigned Index;
7915 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
7916 EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);
7917 if (!LargestVT.isVector() && !VT.isVector() &&
7918 TLI.isTruncateFree(LargestVT, VT))
7919 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
7920 else if (LargestVT.isVector() && !VT.isVector() &&
7922 LargestVT.getTypeForEVT(*DAG.getContext()),
7923 VT.getSizeInBits(), Index) &&
7924 TLI.isTypeLegal(SVT) &&
7925 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
7926 // Target which can combine store(extractelement VectorTy, Idx) can get
7927 // the smaller value for free.
7928 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);
7929 Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, TailValue,
7930 DAG.getVectorIdxConstant(Index, dl));
7931 } else
7932 Value = getMemsetValue(Src, VT, DAG, dl);
7933 }
7934 assert(Value.getValueType() == VT && "Value with wrong type.");
7935 SDValue Store = DAG.getStore(
7936 Chain, dl, Value,
7937 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7938 DstPtrInfo.getWithOffset(DstOff), Alignment,
7940 NewAAInfo);
7941 OutChains.push_back(Store);
7942 DstOff += VT.getSizeInBits() / 8;
7943 Size -= VTSize;
7944 }
7945
7946 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7947}
7948
7950 unsigned AS) {
7951 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
7952 // pointer operands can be losslessly bitcasted to pointers of address space 0
7953 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
7954 report_fatal_error("cannot lower memory intrinsic in address space " +
7955 Twine(AS));
7956 }
7957}
7958
7960 SDValue Src, SDValue Size, Align Alignment,
7961 bool isVol, bool AlwaysInline, bool isTailCall,
7962 MachinePointerInfo DstPtrInfo,
7963 MachinePointerInfo SrcPtrInfo,
7964 const AAMDNodes &AAInfo, AAResults *AA) {
7965 // Check to see if we should lower the memcpy to loads and stores first.
7966 // For cases within the target-specified limits, this is the best choice.
7967 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
7968 if (ConstantSize) {
7969 // Memcpy with size zero? Just return the original chain.
7970 if (ConstantSize->isZero())
7971 return Chain;
7972
7974 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
7975 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7976 if (Result.getNode())
7977 return Result;
7978 }
7979
7980 // Then check to see if we should lower the memcpy with target-specific
7981 // code. If the target chooses to do this, this is the next best.
7982 if (TSI) {
7983 SDValue Result = TSI->EmitTargetCodeForMemcpy(
7984 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
7985 DstPtrInfo, SrcPtrInfo);
7986 if (Result.getNode())
7987 return Result;
7988 }
7989
7990 // If we really need inline code and the target declined to provide it,
7991 // use a (potentially long) sequence of loads and stores.
7992 if (AlwaysInline) {
7993 assert(ConstantSize && "AlwaysInline requires a constant size!");
7995 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
7996 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
7997 }
7998
8001
8002 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
8003 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
8004 // respect volatile, so they may do things like read or write memory
8005 // beyond the given memory regions. But fixing this isn't easy, and most
8006 // people don't care.
8007
8008 // Emit a library call.
8011 Entry.Ty = PointerType::getUnqual(*getContext());
8012 Entry.Node = Dst; Args.push_back(Entry);
8013 Entry.Node = Src; Args.push_back(Entry);
8014
8015 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8016 Entry.Node = Size; Args.push_back(Entry);
8017 // FIXME: pass in SDLoc
8019 CLI.setDebugLoc(dl)
8020 .setChain(Chain)
8021 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
8022 Dst.getValueType().getTypeForEVT(*getContext()),
8023 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
8024 TLI->getPointerTy(getDataLayout())),
8025 std::move(Args))
8027 .setTailCall(isTailCall);
8028
8029 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8030 return CallResult.second;
8031}
8032
8034 SDValue Dst, SDValue Src, SDValue Size,
8035 Type *SizeTy, unsigned ElemSz,
8036 bool isTailCall,
8037 MachinePointerInfo DstPtrInfo,
8038 MachinePointerInfo SrcPtrInfo) {
8039 // Emit a library call.
8042 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8043 Entry.Node = Dst;
8044 Args.push_back(Entry);
8045
8046 Entry.Node = Src;
8047 Args.push_back(Entry);
8048
8049 Entry.Ty = SizeTy;
8050 Entry.Node = Size;
8051 Args.push_back(Entry);
8052
8053 RTLIB::Libcall LibraryCall =
8055 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8056 report_fatal_error("Unsupported element size");
8057
8059 CLI.setDebugLoc(dl)
8060 .setChain(Chain)
8061 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8063 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8064 TLI->getPointerTy(getDataLayout())),
8065 std::move(Args))
8067 .setTailCall(isTailCall);
8068
8069 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8070 return CallResult.second;
8071}
8072
8074 SDValue Src, SDValue Size, Align Alignment,
8075 bool isVol, bool isTailCall,
8076 MachinePointerInfo DstPtrInfo,
8077 MachinePointerInfo SrcPtrInfo,
8078 const AAMDNodes &AAInfo, AAResults *AA) {
8079 // Check to see if we should lower the memmove to loads and stores first.
8080 // For cases within the target-specified limits, this is the best choice.
8081 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8082 if (ConstantSize) {
8083 // Memmove with size zero? Just return the original chain.
8084 if (ConstantSize->isZero())
8085 return Chain;
8086
8088 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8089 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
8090 if (Result.getNode())
8091 return Result;
8092 }
8093
8094 // Then check to see if we should lower the memmove with target-specific
8095 // code. If the target chooses to do this, this is the next best.
8096 if (TSI) {
8097 SDValue Result =
8098 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
8099 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8100 if (Result.getNode())
8101 return Result;
8102 }
8103
8106
8107 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8108 // not be safe. See memcpy above for more details.
8109
8110 // Emit a library call.
8113 Entry.Ty = PointerType::getUnqual(*getContext());
8114 Entry.Node = Dst; Args.push_back(Entry);
8115 Entry.Node = Src; Args.push_back(Entry);
8116
8117 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8118 Entry.Node = Size; Args.push_back(Entry);
8119 // FIXME: pass in SDLoc
8121 CLI.setDebugLoc(dl)
8122 .setChain(Chain)
8123 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
8124 Dst.getValueType().getTypeForEVT(*getContext()),
8125 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
8126 TLI->getPointerTy(getDataLayout())),
8127 std::move(Args))
8129 .setTailCall(isTailCall);
8130
8131 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8132 return CallResult.second;
8133}
8134
8136 SDValue Dst, SDValue Src, SDValue Size,
8137 Type *SizeTy, unsigned ElemSz,
8138 bool isTailCall,
8139 MachinePointerInfo DstPtrInfo,
8140 MachinePointerInfo SrcPtrInfo) {
8141 // Emit a library call.
8144 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8145 Entry.Node = Dst;
8146 Args.push_back(Entry);
8147
8148 Entry.Node = Src;
8149 Args.push_back(Entry);
8150
8151 Entry.Ty = SizeTy;
8152 Entry.Node = Size;
8153 Args.push_back(Entry);
8154
8155 RTLIB::Libcall LibraryCall =
8157 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8158 report_fatal_error("Unsupported element size");
8159
8161 CLI.setDebugLoc(dl)
8162 .setChain(Chain)
8163 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8165 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8166 TLI->getPointerTy(getDataLayout())),
8167 std::move(Args))
8169 .setTailCall(isTailCall);
8170
8171 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8172 return CallResult.second;
8173}
8174
8176 SDValue Src, SDValue Size, Align Alignment,
8177 bool isVol, bool AlwaysInline, bool isTailCall,
8178 MachinePointerInfo DstPtrInfo,
8179 const AAMDNodes &AAInfo) {
8180 // Check to see if we should lower the memset to stores first.
8181 // For cases within the target-specified limits, this is the best choice.
8182 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8183 if (ConstantSize) {
8184 // Memset with size zero? Just return the original chain.
8185 if (ConstantSize->isZero())
8186 return Chain;
8187
8188 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8189 ConstantSize->getZExtValue(), Alignment,
8190 isVol, false, DstPtrInfo, AAInfo);
8191
8192 if (Result.getNode())
8193 return Result;
8194 }
8195
8196 // Then check to see if we should lower the memset with target-specific
8197 // code. If the target chooses to do this, this is the next best.
8198 if (TSI) {
8199 SDValue Result = TSI->EmitTargetCodeForMemset(
8200 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8201 if (Result.getNode())
8202 return Result;
8203 }
8204
8205 // If we really need inline code and the target declined to provide it,
8206 // use a (potentially long) sequence of loads and stores.
8207 if (AlwaysInline) {
8208 assert(ConstantSize && "AlwaysInline requires a constant size!");
8209 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8210 ConstantSize->getZExtValue(), Alignment,
8211 isVol, true, DstPtrInfo, AAInfo);
8212 assert(Result &&
8213 "getMemsetStores must return a valid sequence when AlwaysInline");
8214 return Result;
8215 }
8216
8218
8219 // Emit a library call.
8220 auto &Ctx = *getContext();
8221 const auto& DL = getDataLayout();
8222
8224 // FIXME: pass in SDLoc
8225 CLI.setDebugLoc(dl).setChain(Chain);
8226
8227 const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);
8228
8229 // Helper function to create an Entry from Node and Type.
8230 const auto CreateEntry = [](SDValue Node, Type *Ty) {
8232 Entry.Node = Node;
8233 Entry.Ty = Ty;
8234 return Entry;
8235 };
8236
8237 // If zeroing out and bzero is present, use it.
8238 if (isNullConstant(Src) && BzeroName) {
8240 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8241 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8242 CLI.setLibCallee(
8243 TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),
8244 getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));
8245 } else {
8247 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8248 Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8249 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8250 CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
8251 Dst.getValueType().getTypeForEVT(Ctx),
8252 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
8253 TLI->getPointerTy(DL)),
8254 std::move(Args));
8255 }
8256
8257 CLI.setDiscardResult().setTailCall(isTailCall);
8258
8259 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8260 return CallResult.second;
8261}
8262
8265 Type *SizeTy, unsigned ElemSz,
8266 bool isTailCall,
8267 MachinePointerInfo DstPtrInfo) {
8268 // Emit a library call.
8271 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8272 Entry.Node = Dst;
8273 Args.push_back(Entry);
8274
8275 Entry.Ty = Type::getInt8Ty(*getContext());
8276 Entry.Node = Value;
8277 Args.push_back(Entry);
8278
8279 Entry.Ty = SizeTy;
8280 Entry.Node = Size;
8281 Args.push_back(Entry);
8282
8283 RTLIB::Libcall LibraryCall =
8285 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8286 report_fatal_error("Unsupported element size");
8287
8289 CLI.setDebugLoc(dl)
8290 .setChain(Chain)
8291 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8293 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8294 TLI->getPointerTy(getDataLayout())),
8295 std::move(Args))
8297 .setTailCall(isTailCall);
8298
8299 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8300 return CallResult.second;
8301}
8302
8303SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8304 SDVTList VTList, ArrayRef<SDValue> Ops,
8305 MachineMemOperand *MMO) {
8307 ID.AddInteger(MemVT.getRawBits());
8308 AddNodeIDNode(ID, Opcode, VTList, Ops);
8309 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8310 ID.AddInteger(MMO->getFlags());
8311 void* IP = nullptr;
8312 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8313 cast<AtomicSDNode>(E)->refineAlignment(MMO);
8314 return SDValue(E, 0);
8315 }
8316
8317 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8318 VTList, MemVT, MMO);
8319 createOperands(N, Ops);
8320
8321 CSEMap.InsertNode(N, IP);
8322 InsertNode(N);
8323 return SDValue(N, 0);
8324}
8325
8327 EVT MemVT, SDVTList VTs, SDValue Chain,
8328 SDValue Ptr, SDValue Cmp, SDValue Swp,
8329 MachineMemOperand *MMO) {
8330 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8332 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8333
8334 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8335 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8336}
8337
8338SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8339 SDValue Chain, SDValue Ptr, SDValue Val,
8340 MachineMemOperand *MMO) {
8341 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
8342 Opcode == ISD::ATOMIC_LOAD_SUB ||
8343 Opcode == ISD::ATOMIC_LOAD_AND ||
8344 Opcode == ISD::ATOMIC_LOAD_CLR ||
8345 Opcode == ISD::ATOMIC_LOAD_OR ||
8346 Opcode == ISD::ATOMIC_LOAD_XOR ||
8347 Opcode == ISD::ATOMIC_LOAD_NAND ||
8348 Opcode == ISD::ATOMIC_LOAD_MIN ||
8349 Opcode == ISD::ATOMIC_LOAD_MAX ||
8350 Opcode == ISD::ATOMIC_LOAD_UMIN ||
8351 Opcode == ISD::ATOMIC_LOAD_UMAX ||
8352 Opcode == ISD::ATOMIC_LOAD_FADD ||
8353 Opcode == ISD::ATOMIC_LOAD_FSUB ||
8354 Opcode == ISD::ATOMIC_LOAD_FMAX ||
8355 Opcode == ISD::ATOMIC_LOAD_FMIN ||
8356 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
8357 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
8358 Opcode == ISD::ATOMIC_SWAP ||
8359 Opcode == ISD::ATOMIC_STORE) &&
8360 "Invalid Atomic Op");
8361
8362 EVT VT = Val.getValueType();
8363
8364 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
8365 getVTList(VT, MVT::Other);
8366 SDValue Ops[] = {Chain, Ptr, Val};
8367 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8368}
8369
8370SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8371 EVT VT, SDValue Chain, SDValue Ptr,
8372 MachineMemOperand *MMO) {
8373 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
8374
8375 SDVTList VTs = getVTList(VT, MVT::Other);
8376 SDValue Ops[] = {Chain, Ptr};
8377 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8378}
8379
8380/// getMergeValues - Create a MERGE_VALUES node from the given operands.
8382 if (Ops.size() == 1)
8383 return Ops[0];
8384
8386 VTs.reserve(Ops.size());
8387 for (const SDValue &Op : Ops)
8388 VTs.push_back(Op.getValueType());
8389 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
8390}
8391
8393 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
8394 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
8395 MachineMemOperand::Flags Flags, uint64_t Size, const AAMDNodes &AAInfo) {
8396 if (!Size && MemVT.isScalableVector())
8398 else if (!Size)
8399 Size = MemVT.getStoreSize();
8400
8402 MachineMemOperand *MMO =
8403 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
8404
8405 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
8406}
8407
8409 SDVTList VTList,
8410 ArrayRef<SDValue> Ops, EVT MemVT,
8411 MachineMemOperand *MMO) {
8412 assert((Opcode == ISD::INTRINSIC_VOID ||
8413 Opcode == ISD::INTRINSIC_W_CHAIN ||
8414 Opcode == ISD::PREFETCH ||
8415 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
8416 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
8417 "Opcode is not a memory-accessing opcode!");
8418
8419 // Memoize the node unless it returns a glue result.
8421 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8423 AddNodeIDNode(ID, Opcode, VTList, Ops);
8424 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8425 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
8426 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8427 ID.AddInteger(MMO->getFlags());
8428 ID.AddInteger(MemVT.getRawBits());
8429 void *IP = nullptr;
8430 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8431 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
8432 return SDValue(E, 0);
8433 }
8434
8435 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8436 VTList, MemVT, MMO);
8437 createOperands(N, Ops);
8438
8439 CSEMap.InsertNode(N, IP);
8440 } else {
8441 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8442 VTList, MemVT, MMO);
8443 createOperands(N, Ops);
8444 }
8445 InsertNode(N);
8446 SDValue V(N, 0);
8447 NewSDValueDbgMsg(V, "Creating new node: ", this);
8448 return V;
8449}
8450
8452 SDValue Chain, int FrameIndex,
8453 int64_t Size, int64_t Offset) {
8454 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
8455 const auto VTs = getVTList(MVT::Other);
8456 SDValue Ops[2] = {
8457 Chain,
8458 getFrameIndex(FrameIndex,
8459 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
8460 true)};
8461
8463 AddNodeIDNode(ID, Opcode, VTs, Ops);
8464 ID.AddInteger(FrameIndex);
8465 ID.AddInteger(Size);
8466 ID.AddInteger(Offset);
8467 void *IP = nullptr;
8468 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8469 return SDValue(E, 0);
8470
8471 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
8472 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
8473 createOperands(N, Ops);
8474 CSEMap.InsertNode(N, IP);
8475 InsertNode(N);
8476 SDValue V(N, 0);
8477 NewSDValueDbgMsg(V, "Creating new node: ", this);
8478 return V;
8479}
8480
8482 uint64_t Guid, uint64_t Index,
8483 uint32_t Attr) {
8484 const unsigned Opcode = ISD::PSEUDO_PROBE;
8485 const auto VTs = getVTList(MVT::Other);
8486 SDValue Ops[] = {Chain};
8488 AddNodeIDNode(ID, Opcode, VTs, Ops);
8489 ID.AddInteger(Guid);
8490 ID.AddInteger(Index);
8491 void *IP = nullptr;
8492 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
8493 return SDValue(E, 0);
8494
8495 auto *N = newSDNode<PseudoProbeSDNode>(
8496 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
8497 createOperands(N, Ops);
8498 CSEMap.InsertNode(N, IP);
8499 InsertNode(N);
8500 SDValue V(N, 0);
8501 NewSDValueDbgMsg(V, "Creating new node: ", this);
8502 return V;
8503}
8504
8505/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8506/// MachinePointerInfo record from it. This is particularly useful because the
8507/// code generator has many cases where it doesn't bother passing in a
8508/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8510 SelectionDAG &DAG, SDValue Ptr,
8511 int64_t Offset = 0) {
8512 // If this is FI+Offset, we can model it.
8513 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
8515 FI->getIndex(), Offset);
8516
8517 // If this is (FI+Offset1)+Offset2, we can model it.
8518 if (Ptr.getOpcode() != ISD::ADD ||
8519 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
8520 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
8521 return Info;
8522
8523 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
8525 DAG.getMachineFunction(), FI,
8526 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
8527}
8528
8529/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8530/// MachinePointerInfo record from it. This is particularly useful because the
8531/// code generator has many cases where it doesn't bother passing in a
8532/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8534 SelectionDAG &DAG, SDValue Ptr,
8535 SDValue OffsetOp) {
8536 // If the 'Offset' value isn't a constant, we can't handle this.
8537 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
8538 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
8539 if (OffsetOp.isUndef())
8540 return InferPointerInfo(Info, DAG, Ptr);
8541 return Info;
8542}
8543
8545 EVT VT, const SDLoc &dl, SDValue Chain,
8547 MachinePointerInfo PtrInfo, EVT MemVT,
8548 Align Alignment,
8549 MachineMemOperand::Flags MMOFlags,
8550 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8551 assert(Chain.getValueType() == MVT::Other &&
8552 "Invalid chain type");
8553
8554 MMOFlags |= MachineMemOperand::MOLoad;
8555 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8556 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8557 // clients.
8558 if (PtrInfo.V.isNull())
8559 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8560
8563 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8564 Alignment, AAInfo, Ranges);
8565 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
8566}
8567
8569 EVT VT, const SDLoc &dl, SDValue Chain,
8570 SDValue Ptr, SDValue Offset, EVT MemVT,
8571 MachineMemOperand *MMO) {
8572 if (VT == MemVT) {
8573 ExtType = ISD::NON_EXTLOAD;
8574 } else if (ExtType == ISD::NON_EXTLOAD) {
8575 assert(VT == MemVT && "Non-extending load from different memory type!");
8576 } else {
8577 // Extending load.
8578 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
8579 "Should only be an extending load, not truncating!");
8580 assert(VT.isInteger() == MemVT.isInteger() &&
8581 "Cannot convert from FP to Int or Int -> FP!");
8582 assert(VT.isVector() == MemVT.isVector() &&
8583 "Cannot use an ext load to convert to or from a vector!");
8584 assert((!VT.isVector() ||
8586 "Cannot use an ext load to change the number of vector elements!");
8587 }
8588
8589 bool Indexed = AM != ISD::UNINDEXED;
8590 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8591
8592 SDVTList VTs = Indexed ?
8593 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
8594 SDValue Ops[] = { Chain, Ptr, Offset };
8596 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
8597 ID.AddInteger(MemVT.getRawBits());
8598 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
8599 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
8600 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8601 ID.AddInteger(MMO->getFlags());
8602 void *IP = nullptr;
8603 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8604 cast<LoadSDNode>(E)->refineAlignment(MMO);
8605 return SDValue(E, 0);
8606 }
8607 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8608 ExtType, MemVT, MMO);
8609 createOperands(N, Ops);
8610
8611 CSEMap.InsertNode(N, IP);
8612 InsertNode(N);
8613 SDValue V(N, 0);
8614 NewSDValueDbgMsg(V, "Creating new node: ", this);
8615 return V;
8616}
8617
8620 MaybeAlign Alignment,
8621 MachineMemOperand::Flags MMOFlags,
8622 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8623 SDValue Undef = getUNDEF(Ptr.getValueType());
8624 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8625 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
8626}
8627
8630 SDValue Undef = getUNDEF(Ptr.getValueType());
8631 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8632 VT, MMO);
8633}
8634
8636 EVT VT, SDValue Chain, SDValue Ptr,
8637 MachinePointerInfo PtrInfo, EVT MemVT,
8638 MaybeAlign Alignment,
8639 MachineMemOperand::Flags MMOFlags,
8640 const AAMDNodes &AAInfo) {
8641 SDValue Undef = getUNDEF(Ptr.getValueType());
8642 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
8643 MemVT, Alignment, MMOFlags, AAInfo);
8644}
8645
8647 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
8648 MachineMemOperand *MMO) {
8649 SDValue Undef = getUNDEF(Ptr.getValueType());
8650 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
8651 MemVT, MMO);
8652}
8653
8657 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
8658 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8659 // Don't propagate the invariant or dereferenceable flags.
8660 auto MMOFlags =
8661 LD->getMemOperand()->getFlags() &
8663 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8664 LD->getChain(), Base, Offset, LD->getPointerInfo(),
8665 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
8666}
8667
8670 Align Alignment,
8671 MachineMemOperand::Flags MMOFlags,
8672 const AAMDNodes &AAInfo) {
8673 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8674
8675 MMOFlags |= MachineMemOperand::MOStore;
8676 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8677
8678 if (PtrInfo.V.isNull())
8679 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8680
8682 uint64_t Size =
8684 MachineMemOperand *MMO =
8685 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
8686 return getStore(Chain, dl, Val, Ptr, MMO);
8687}
8688
8691 assert(Chain.getValueType() == MVT::Other &&
8692 "Invalid chain type");
8693 EVT VT = Val.getValueType();
8694 SDVTList VTs = getVTList(MVT::Other);
8695 SDValue Undef = getUNDEF(Ptr.getValueType());
8696 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8698 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8699 ID.AddInteger(VT.getRawBits());
8700 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8701 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
8702 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8703 ID.AddInteger(MMO->getFlags());
8704 void *IP = nullptr;
8705 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8706 cast<StoreSDNode>(E)->refineAlignment(MMO);
8707 return SDValue(E, 0);
8708 }
8709 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8710 ISD::UNINDEXED, false, VT, MMO);
8711 createOperands(N, Ops);
8712
8713 CSEMap.InsertNode(N, IP);
8714 InsertNode(N);
8715 SDValue V(N, 0);
8716 NewSDValueDbgMsg(V, "Creating new node: ", this);
8717 return V;
8718}
8719
8722 EVT SVT, Align Alignment,
8723 MachineMemOperand::Flags MMOFlags,
8724 const AAMDNodes &AAInfo) {
8725 assert(Chain.getValueType() == MVT::Other &&
8726 "Invalid chain type");
8727
8728 MMOFlags |= MachineMemOperand::MOStore;
8729 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8730
8731 if (PtrInfo.V.isNull())
8732 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8733
8736 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
8737 Alignment, AAInfo);
8738 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
8739}
8740
8742 SDValue Ptr, EVT SVT,
8743 MachineMemOperand *MMO) {
8744 EVT VT = Val.getValueType();
8745
8746 assert(Chain.getValueType() == MVT::Other &&
8747 "Invalid chain type");
8748 if (VT == SVT)
8749 return getStore(Chain, dl, Val, Ptr, MMO);
8750
8752 "Should only be a truncating store, not extending!");
8753 assert(VT.isInteger() == SVT.isInteger() &&
8754 "Can't do FP-INT conversion!");
8755 assert(VT.isVector() == SVT.isVector() &&
8756 "Cannot use trunc store to convert to or from a vector!");
8757 assert((!VT.isVector() ||
8759 "Cannot use trunc store to change the number of vector elements!");
8760
8761 SDVTList VTs = getVTList(MVT::Other);
8762 SDValue Undef = getUNDEF(Ptr.getValueType());
8763 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8765 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8766 ID.AddInteger(SVT.getRawBits());
8767 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8768 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
8769 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8770 ID.AddInteger(MMO->getFlags());
8771 void *IP = nullptr;
8772 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8773 cast<StoreSDNode>(E)->refineAlignment(MMO);
8774 return SDValue(E, 0);
8775 }
8776 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8777 ISD::UNINDEXED, true, SVT, MMO);
8778 createOperands(N, Ops);
8779
8780 CSEMap.InsertNode(N, IP);
8781 InsertNode(N);
8782 SDValue V(N, 0);
8783 NewSDValueDbgMsg(V, "Creating new node: ", this);
8784 return V;
8785}
8786
8790 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
8791 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
8792 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8793 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
8795 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8796 ID.AddInteger(ST->getMemoryVT().getRawBits());
8797 ID.AddInteger(ST->getRawSubclassData());
8798 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
8799 ID.AddInteger(ST->getMemOperand()->getFlags());
8800 void *IP = nullptr;
8801 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8802 return SDValue(E, 0);
8803
8804 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8805 ST->isTruncatingStore(), ST->getMemoryVT(),
8806 ST->getMemOperand());
8807 createOperands(N, Ops);
8808
8809 CSEMap.InsertNode(N, IP);
8810 InsertNode(N);
8811 SDValue V(N, 0);
8812 NewSDValueDbgMsg(V, "Creating new node: ", this);
8813 return V;
8814}
8815
8817 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
8818 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
8819 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
8820 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
8821 const MDNode *Ranges, bool IsExpanding) {
8822 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8823
8824 MMOFlags |= MachineMemOperand::MOLoad;
8825 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8826 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8827 // clients.
8828 if (PtrInfo.V.isNull())
8829 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8830
8833 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8834 Alignment, AAInfo, Ranges);
8835 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
8836 MMO, IsExpanding);
8837}
8838
8840 ISD::LoadExtType ExtType, EVT VT,
8841 const SDLoc &dl, SDValue Chain, SDValue Ptr,
8842 SDValue Offset, SDValue Mask, SDValue EVL,
8843 EVT MemVT, MachineMemOperand *MMO,
8844 bool IsExpanding) {
8845 bool Indexed = AM != ISD::UNINDEXED;
8846 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8847
8848 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
8849 : getVTList(VT, MVT::Other);
8850 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
8852 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops);
8853 ID.AddInteger(MemVT.getRawBits());
8854 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
8855 dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
8856 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8857 ID.AddInteger(MMO->getFlags());
8858 void *IP = nullptr;
8859 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8860 cast<VPLoadSDNode>(E)->refineAlignment(MMO);
8861 return SDValue(E, 0);
8862 }
8863 auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8864 ExtType, IsExpanding, MemVT, MMO);
8865 createOperands(N, Ops);
8866
8867 CSEMap.InsertNode(N, IP);
8868 InsertNode(N);
8869 SDValue V(N, 0);
8870 NewSDValueDbgMsg(V, "Creating new node: ", this);
8871 return V;
8872}
8873
8875 SDValue Ptr, SDValue Mask, SDValue EVL,
8876 MachinePointerInfo PtrInfo,
8877 MaybeAlign Alignment,
8878 MachineMemOperand::Flags MMOFlags,
8879 const AAMDNodes &AAInfo, const MDNode *Ranges,
8880 bool IsExpanding) {
8881 SDValue Undef = getUNDEF(Ptr.getValueType());
8882 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8883 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
8884 IsExpanding);
8885}
8886
8888 SDValue Ptr, SDValue Mask, SDValue EVL,
8889 MachineMemOperand *MMO, bool IsExpanding) {
8890 SDValue Undef = getUNDEF(Ptr.getValueType());
8891 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8892 Mask, EVL, VT, MMO, IsExpanding);
8893}
8894
8896 EVT VT, SDValue Chain, SDValue Ptr,
8897 SDValue Mask, SDValue EVL,
8898 MachinePointerInfo PtrInfo, EVT MemVT,
8899 MaybeAlign Alignment,
8900 MachineMemOperand::Flags MMOFlags,
8901 const AAMDNodes &AAInfo, bool IsExpanding) {
8902 SDValue Undef = getUNDEF(Ptr.getValueType());
8903 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8904 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,
8905 IsExpanding);
8906}
8907
8909 EVT VT, SDValue Chain, SDValue Ptr,
8910 SDValue Mask, SDValue EVL, EVT MemVT,
8911 MachineMemOperand *MMO, bool IsExpanding) {
8912 SDValue Undef = getUNDEF(Ptr.getValueType());
8913 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8914 EVL, MemVT, MMO, IsExpanding);
8915}
8916
8920 auto *LD = cast<VPLoadSDNode>(OrigLoad);
8921 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8922 // Don't propagate the invariant or dereferenceable flags.
8923 auto MMOFlags =
8924 LD->getMemOperand()->getFlags() &
8926 return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8927 LD->getChain(), Base, Offset, LD->getMask(),
8928 LD->getVectorLength(), LD->getPointerInfo(),
8929 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
8930 nullptr, LD->isExpandingLoad());
8931}
8932
8935 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
8936 ISD::MemIndexedMode AM, bool IsTruncating,
8937 bool IsCompressing) {
8938 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8939 bool Indexed = AM != ISD::UNINDEXED;
8940 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
8941 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
8942 : getVTList(MVT::Other);
8943 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
8945 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8946 ID.AddInteger(MemVT.getRawBits());
8947 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
8948 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
8949 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8950 ID.AddInteger(MMO->getFlags());
8951 void *IP = nullptr;
8952 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8953 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
8954 return SDValue(E, 0);
8955 }
8956 auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8957 IsTruncating, IsCompressing, MemVT, MMO);
8958 createOperands(N, Ops);
8959
8960 CSEMap.InsertNode(N, IP);
8961 InsertNode(N);
8962 SDValue V(N, 0);
8963 NewSDValueDbgMsg(V, "Creating new node: ", this);
8964 return V;
8965}
8966
8968 SDValue Val, SDValue Ptr, SDValue Mask,
8969 SDValue EVL, MachinePointerInfo PtrInfo,
8970 EVT SVT, Align Alignment,
8971 MachineMemOperand::Flags MMOFlags,
8972 const AAMDNodes &AAInfo,
8973 bool IsCompressing) {
8974 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8975
8976 MMOFlags |= MachineMemOperand::MOStore;
8977 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8978
8979 if (PtrInfo.V.isNull())
8980 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8981
8984 PtrInfo, MMOFlags, MemoryLocation::getSizeOrUnknown(SVT.getStoreSize()),
8985 Alignment, AAInfo);
8986 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
8987 IsCompressing);
8988}
8989
8991 SDValue Val, SDValue Ptr, SDValue Mask,
8992 SDValue EVL, EVT SVT,
8993 MachineMemOperand *MMO,
8994 bool IsCompressing) {
8995 EVT VT = Val.getValueType();
8996
8997 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8998 if (VT == SVT)
8999 return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask,
9000 EVL, VT, MMO, ISD::UNINDEXED,
9001 /*IsTruncating*/ false, IsCompressing);
9002
9004 "Should only be a truncating store, not extending!");
9005 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9006 assert(VT.isVector() == SVT.isVector() &&
9007 "Cannot use trunc store to convert to or from a vector!");
9008 assert((!VT.isVector() ||
9010 "Cannot use trunc store to change the number of vector elements!");
9011
9012 SDVTList VTs = getVTList(MVT::Other);
9013 SDValue Undef = getUNDEF(Ptr.getValueType());
9014 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
9016 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9017 ID.AddInteger(SVT.getRawBits());
9018 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9019 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9020 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9021 ID.AddInteger(MMO->getFlags());
9022 void *IP = nullptr;
9023 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9024 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9025 return SDValue(E, 0);
9026 }
9027 auto *N =
9028 newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9029 ISD::UNINDEXED, true, IsCompressing, SVT, MMO);
9030 createOperands(N, Ops);
9031
9032 CSEMap.InsertNode(N, IP);
9033 InsertNode(N);
9034 SDValue V(N, 0);
9035 NewSDValueDbgMsg(V, "Creating new node: ", this);
9036 return V;
9037}
9038
9042 auto *ST = cast<VPStoreSDNode>(OrigStore);
9043 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
9044 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9045 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
9046 Offset, ST->getMask(), ST->getVectorLength()};
9048 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9049 ID.AddInteger(ST->getMemoryVT().getRawBits());
9050 ID.AddInteger(ST->getRawSubclassData());
9051 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9052 ID.AddInteger(ST->getMemOperand()->getFlags());
9053 void *IP = nullptr;
9054 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9055 return SDValue(E, 0);
9056
9057 auto *N = newSDNode<VPStoreSDNode>(
9058 dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(),
9059 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
9060 createOperands(N, Ops);
9061
9062 CSEMap.InsertNode(N, IP);
9063 InsertNode(N);
9064 SDValue V(N, 0);
9065 NewSDValueDbgMsg(V, "Creating new node: ", this);
9066 return V;
9067}
9068
9070 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9071 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9072 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9073 bool Indexed = AM != ISD::UNINDEXED;
9074 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9075
9076 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9077 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9078 : getVTList(VT, MVT::Other);
9080 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
9081 ID.AddInteger(VT.getRawBits());
9082 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9083 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9084 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9085
9086 void *IP = nullptr;
9087 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9088 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9089 return SDValue(E, 0);
9090 }
9091
9092 auto *N =
9093 newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,
9094 ExtType, IsExpanding, MemVT, MMO);
9095 createOperands(N, Ops);
9096 CSEMap.InsertNode(N, IP);
9097 InsertNode(N);
9098 SDValue V(N, 0);
9099 NewSDValueDbgMsg(V, "Creating new node: ", this);
9100 return V;
9101}
9102
9104 SDValue Ptr, SDValue Stride,
9105 SDValue Mask, SDValue EVL,
9106 MachineMemOperand *MMO,
9107 bool IsExpanding) {
9108 SDValue Undef = getUNDEF(Ptr.getValueType());
9110 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9111}
9112
9114 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9115 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9116 MachineMemOperand *MMO, bool IsExpanding) {
9117 SDValue Undef = getUNDEF(Ptr.getValueType());
9118 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9119 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9120}
9121
9123 SDValue Val, SDValue Ptr,
9124 SDValue Offset, SDValue Stride,
9125 SDValue Mask, SDValue EVL, EVT MemVT,
9126 MachineMemOperand *MMO,
9128 bool IsTruncating, bool IsCompressing) {
9129 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9130 bool Indexed = AM != ISD::UNINDEXED;
9131 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9132 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9133 : getVTList(MVT::Other);
9134 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9136 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9137 ID.AddInteger(MemVT.getRawBits());
9138 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9139 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9140 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9141 void *IP = nullptr;
9142 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9143 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9144 return SDValue(E, 0);
9145 }
9146 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9147 VTs, AM, IsTruncating,
9148 IsCompressing, MemVT, MMO);
9149 createOperands(N, Ops);
9150
9151 CSEMap.InsertNode(N, IP);
9152 InsertNode(N);
9153 SDValue V(N, 0);
9154 NewSDValueDbgMsg(V, "Creating new node: ", this);
9155 return V;
9156}
9157
9159 SDValue Val, SDValue Ptr,
9160 SDValue Stride, SDValue Mask,
9161 SDValue EVL, EVT SVT,
9162 MachineMemOperand *MMO,
9163 bool IsCompressing) {
9164 EVT VT = Val.getValueType();
9165
9166 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9167 if (VT == SVT)
9168 return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()),
9169 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED,
9170 /*IsTruncating*/ false, IsCompressing);
9171
9173 "Should only be a truncating store, not extending!");
9174 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9175 assert(VT.isVector() == SVT.isVector() &&
9176 "Cannot use trunc store to convert to or from a vector!");
9177 assert((!VT.isVector() ||
9179 "Cannot use trunc store to change the number of vector elements!");
9180
9181 SDVTList VTs = getVTList(MVT::Other);
9182 SDValue Undef = getUNDEF(Ptr.getValueType());
9183 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9185 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9186 ID.AddInteger(SVT.getRawBits());
9187 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9188 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9189 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9190 void *IP = nullptr;
9191 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9192 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9193 return SDValue(E, 0);
9194 }
9195 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9196 VTs, ISD::UNINDEXED, true,
9197 IsCompressing, SVT, MMO);
9198 createOperands(N, Ops);
9199
9200 CSEMap.InsertNode(N, IP);
9201 InsertNode(N);
9202 SDValue V(N, 0);
9203 NewSDValueDbgMsg(V, "Creating new node: ", this);
9204 return V;
9205}
9206
9209 ISD::MemIndexType IndexType) {
9210 assert(Ops.size() == 6 && "Incompatible number of operands");
9211
9213 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops);
9214 ID.AddInteger(VT.getRawBits());
9215 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9216 dl.getIROrder(), VTs, VT, MMO, IndexType));
9217 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9218 ID.AddInteger(MMO->getFlags());
9219 void *IP = nullptr;
9220 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9221 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9222 return SDValue(E, 0);
9223 }
9224
9225 auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9226 VT, MMO, IndexType);
9227 createOperands(N, Ops);
9228
9229 assert(N->getMask().getValueType().getVectorElementCount() ==
9230 N->getValueType(0).getVectorElementCount() &&
9231 "Vector width mismatch between mask and data");
9232 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9233 N->getValueType(0).getVectorElementCount().isScalable() &&
9234 "Scalable flags of index and data do not match");
9236 N->getIndex().getValueType().getVectorElementCount(),
9237 N->getValueType(0).getVectorElementCount()) &&
9238 "Vector width mismatch between index and data");
9239 assert(isa<ConstantSDNode>(N->getScale()) &&
9240 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9241 "Scale should be a constant power of 2");
9242
9243 CSEMap.InsertNode(N, IP);
9244 InsertNode(N);
9245 SDValue V(N, 0);
9246 NewSDValueDbgMsg(V, "Creating new node: ", this);
9247 return V;
9248}
9249
9252 MachineMemOperand *MMO,
9253 ISD::MemIndexType IndexType) {
9254 assert(Ops.size() == 7 && "Incompatible number of operands");
9255
9257 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops);
9258 ID.AddInteger(VT.getRawBits());
9259 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9260 dl.getIROrder(), VTs, VT, MMO, IndexType));
9261 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9262 ID.AddInteger(MMO->getFlags());
9263 void *IP = nullptr;
9264 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9265 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9266 return SDValue(E, 0);
9267 }
9268 auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9269 VT, MMO, IndexType);
9270 createOperands(N, Ops);
9271
9272 assert(N->getMask().getValueType().getVectorElementCount() ==
9273 N->getValue().getValueType().getVectorElementCount() &&
9274 "Vector width mismatch between mask and data");
9275 assert(
9276 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9277 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9278 "Scalable flags of index and data do not match");
9280 N->getIndex().getValueType().getVectorElementCount(),
9281 N->getValue().getValueType().getVectorElementCount()) &&
9282 "Vector width mismatch between index and data");
9283 assert(isa<ConstantSDNode>(N->getScale()) &&
9284 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9285 "Scale should be a constant power of 2");
9286
9287 CSEMap.InsertNode(N, IP);
9288 InsertNode(N);
9289 SDValue V(N, 0);
9290 NewSDValueDbgMsg(V, "Creating new node: ", this);
9291 return V;
9292}
9293
9296 SDValue PassThru, EVT MemVT,
9297 MachineMemOperand *MMO,
9299 ISD::LoadExtType ExtTy, bool isExpanding) {
9300 bool Indexed = AM != ISD::UNINDEXED;
9301 assert((Indexed || Offset.isUndef()) &&
9302 "Unindexed masked load with an offset!");
9303 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9304 : getVTList(VT, MVT::Other);
9305 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9307 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
9308 ID.AddInteger(MemVT.getRawBits());
9309 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9310 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9311 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9312 ID.AddInteger(MMO->getFlags());
9313 void *IP = nullptr;
9314 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9315 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9316 return SDValue(E, 0);
9317 }
9318 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9319 AM, ExtTy, isExpanding, MemVT, MMO);
9320 createOperands(N, Ops);
9321
9322 CSEMap.InsertNode(N, IP);
9323 InsertNode(N);
9324 SDValue V(N, 0);
9325 NewSDValueDbgMsg(V, "Creating new node: ", this);
9326 return V;
9327}
9328
9332 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
9333 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9334 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
9335 Offset, LD->getMask(), LD->getPassThru(),
9336 LD->getMemoryVT(), LD->getMemOperand(), AM,
9337 LD->getExtensionType(), LD->isExpandingLoad());
9338}
9339
9342 SDValue Mask, EVT MemVT,
9343 MachineMemOperand *MMO,
9344 ISD::MemIndexedMode AM, bool IsTruncating,
9345 bool IsCompressing) {
9346 assert(Chain.getValueType() == MVT::Other &&
9347 "Invalid chain type");
9348 bool Indexed = AM != ISD::UNINDEXED;
9349 assert((Indexed || Offset.isUndef()) &&
9350 "Unindexed masked store with an offset!");
9351 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9352 : getVTList(MVT::Other);
9353 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
9355 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
9356 ID.AddInteger(MemVT.getRawBits());
9357 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9358 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9359 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9360 ID.AddInteger(MMO->getFlags());
9361 void *IP = nullptr;
9362 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9363 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
9364 return SDValue(E, 0);
9365 }
9366 auto *N =
9367 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9368 IsTruncating, IsCompressing, MemVT, MMO);
9369 createOperands(N, Ops);
9370
9371 CSEMap.InsertNode(N, IP);
9372 InsertNode(N);
9373 SDValue V(N, 0);
9374 NewSDValueDbgMsg(V, "Creating new node: ", this);
9375 return V;
9376}
9377
9381 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
9382 assert(ST->getOffset().isUndef() &&
9383 "Masked store is already a indexed store!");
9384 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
9385 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
9386 AM, ST->isTruncatingStore(), ST->isCompressingStore());
9387}
9388
9391 MachineMemOperand *MMO,
9392 ISD::MemIndexType IndexType,
9393 ISD::LoadExtType ExtTy) {
9394 assert(Ops.size() == 6 && "Incompatible number of operands");
9395
9397 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
9398 ID.AddInteger(MemVT.getRawBits());
9399 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9400 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
9401 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9402 ID.AddInteger(MMO->getFlags());
9403 void *IP = nullptr;
9404 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9405 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
9406 return SDValue(E, 0);
9407 }
9408
9409 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9410 VTs, MemVT, MMO, IndexType, ExtTy);
9411 createOperands(N, Ops);
9412
9413 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
9414 "Incompatible type of the PassThru value in MaskedGatherSDNode");
9415 assert(N->getMask().getValueType().getVectorElementCount() ==
9416 N->getValueType(0).getVectorElementCount() &&
9417 "Vector width mismatch between mask and data");
9418 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9419 N->getValueType(0).getVectorElementCount().isScalable() &&
9420 "Scalable flags of index and data do not match");
9422 N->getIndex().getValueType().getVectorElementCount(),
9423 N->getValueType(0).getVectorElementCount()) &&
9424 "Vector width mismatch between index and data");
9425 assert(isa<ConstantSDNode>(N->getScale()) &&
9426 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9427 "Scale should be a constant power of 2");
9428
9429 CSEMap.InsertNode(N, IP);
9430 InsertNode(N);
9431 SDValue V(N, 0);
9432 NewSDValueDbgMsg(V, "Creating new node: ", this);
9433 return V;
9434}
9435
9438 MachineMemOperand *MMO,
9439 ISD::MemIndexType IndexType,
9440 bool IsTrunc) {
9441 assert(Ops.size() == 6 && "Incompatible number of operands");
9442
9444 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
9445 ID.AddInteger(MemVT.getRawBits());
9446 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9447 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
9448 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9449 ID.AddInteger(MMO->getFlags());
9450 void *IP = nullptr;
9451 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9452 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
9453 return SDValue(E, 0);
9454 }
9455
9456 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9457 VTs, MemVT, MMO, IndexType, IsTrunc);
9458 createOperands(N, Ops);
9459
9460 assert(N->getMask().getValueType().getVectorElementCount() ==
9461 N->getValue().getValueType().getVectorElementCount() &&
9462 "Vector width mismatch between mask and data");
9463 assert(
9464 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9465 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9466 "Scalable flags of index and data do not match");
9468 N->getIndex().getValueType().getVectorElementCount(),
9469 N->getValue().getValueType().getVectorElementCount()) &&
9470 "Vector width mismatch between index and data");
9471 assert(isa<ConstantSDNode>(N->getScale()) &&
9472 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9473 "Scale should be a constant power of 2");
9474
9475 CSEMap.InsertNode(N, IP);
9476 InsertNode(N);
9477 SDValue V(N, 0);
9478 NewSDValueDbgMsg(V, "Creating new node: ", this);
9479 return V;
9480}
9481
9483 EVT MemVT, MachineMemOperand *MMO) {
9484 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9485 SDVTList VTs = getVTList(MVT::Other);
9486 SDValue Ops[] = {Chain, Ptr};
9489 ID.AddInteger(MemVT.getRawBits());
9490 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9491 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9492 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9493 ID.AddInteger(MMO->getFlags());
9494 void *IP = nullptr;
9495 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9496 return SDValue(E, 0);
9497
9498 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),
9499 dl.getDebugLoc(), VTs, MemVT, MMO);
9500 createOperands(N, Ops);
9501
9502 CSEMap.InsertNode(N, IP);
9503 InsertNode(N);
9504 SDValue V(N, 0);
9505 NewSDValueDbgMsg(V, "Creating new node: ", this);
9506 return V;
9507}
9508
9510 EVT MemVT, MachineMemOperand *MMO) {
9511 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9512 SDVTList VTs = getVTList(MVT::Other);
9513 SDValue Ops[] = {Chain, Ptr};
9516 ID.AddInteger(MemVT.getRawBits());
9517 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9518 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9519 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9520 ID.AddInteger(MMO->getFlags());
9521 void *IP = nullptr;
9522 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9523 return SDValue(E, 0);
9524
9525 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),
9526 dl.getDebugLoc(), VTs, MemVT, MMO);
9527 createOperands(N, Ops);
9528
9529 CSEMap.InsertNode(N, IP);
9530 InsertNode(N);
9531 SDValue V(N, 0);
9532 NewSDValueDbgMsg(V, "Creating new node: ", this);
9533 return V;
9534}
9535
9537 // select undef, T, F --> T (if T is a constant), otherwise F
9538 // select, ?, undef, F --> F
9539 // select, ?, T, undef --> T
9540 if (Cond.isUndef())
9541 return isConstantValueOfAnyType(T) ? T : F;
9542 if (T.isUndef())
9543 return F;
9544 if (F.isUndef())
9545 return T;
9546
9547 // select true, T, F --> T
9548 // select false, T, F --> F
9549 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
9550 return CondC->isZero() ? F : T;
9551
9552 // TODO: This should simplify VSELECT with non-zero constant condition using
9553 // something like this (but check boolean contents to be complete?):
9554 if (ConstantSDNode *CondC = isConstOrConstSplat(Cond, /*AllowUndefs*/ false,
9555 /*AllowTruncation*/ true))
9556 if (CondC->isZero())
9557 return F;
9558
9559 // select ?, T, T --> T
9560 if (T == F)
9561 return T;
9562
9563 return SDValue();
9564}
9565
9567 // shift undef, Y --> 0 (can always assume that the undef value is 0)
9568 if (X.isUndef())
9569 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
9570 // shift X, undef --> undef (because it may shift by the bitwidth)
9571 if (Y.isUndef())
9572 return getUNDEF(X.getValueType());
9573
9574 // shift 0, Y --> 0
9575 // shift X, 0 --> X
9577 return X;
9578
9579 // shift X, C >= bitwidth(X) --> undef
9580 // All vector elements must be too big (or undef) to avoid partial undefs.
9581 auto isShiftTooBig = [X](ConstantSDNode *Val) {
9582 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
9583 };
9584 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
9585 return getUNDEF(X.getValueType());
9586
9587 return SDValue();
9588}
9589
9591 SDNodeFlags Flags) {
9592 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
9593 // (an undef operand can be chosen to be Nan/Inf), then the result of this
9594 // operation is poison. That result can be relaxed to undef.
9595 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
9596 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
9597 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9598 (YC && YC->getValueAPF().isNaN());
9599 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9600 (YC && YC->getValueAPF().isInfinity());
9601
9602 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
9603 return getUNDEF(X.getValueType());
9604
9605 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
9606 return getUNDEF(X.getValueType());
9607
9608 if (!YC)
9609 return SDValue();
9610
9611 // X + -0.0 --> X
9612 if (Opcode == ISD::FADD)
9613 if (YC->getValueAPF().isNegZero())
9614 return X;
9615
9616 // X - +0.0 --> X
9617 if (Opcode == ISD::FSUB)
9618 if (YC->getValueAPF().isPosZero())
9619 return X;
9620
9621 // X * 1.0 --> X
9622 // X / 1.0 --> X
9623 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
9624 if (YC->getValueAPF().isExactlyValue(1.0))
9625 return X;
9626
9627 // X * 0.0 --> 0.0
9628 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
9629 if (YC->getValueAPF().isZero())
9630 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
9631
9632 return SDValue();
9633}
9634
9636 SDValue Ptr, SDValue SV, unsigned Align) {
9637 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
9638 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
9639}
9640
9641SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9642 ArrayRef<SDUse> Ops) {
9643 switch (Ops.size()) {
9644 case 0: return getNode(Opcode, DL, VT);
9645 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
9646 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
9647 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
9648 default: break;
9649 }
9650
9651 // Copy from an SDUse array into an SDValue array for use with
9652 // the regular getNode logic.
9653 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
9654 return getNode(Opcode, DL, VT, NewOps);
9655}
9656
9657SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9658 ArrayRef<SDValue> Ops) {
9659 SDNodeFlags Flags;
9660 if (Inserter)
9661 Flags = Inserter->getFlags();
9662 return getNode(Opcode, DL, VT, Ops, Flags);
9663}
9664
9665SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9666 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9667 unsigned NumOps = Ops.size();
9668 switch (NumOps) {
9669 case 0: return getNode(Opcode, DL, VT);
9670 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
9671 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
9672 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
9673 default: break;
9674 }
9675
9676#ifndef NDEBUG
9677 for (const auto &Op : Ops)
9678 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9679 "Operand is DELETED_NODE!");
9680#endif
9681
9682 switch (Opcode) {
9683 default: break;
9684 case ISD::BUILD_VECTOR:
9685 // Attempt to simplify BUILD_VECTOR.
9686 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
9687 return V;
9688 break;
9690 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
9691 return V;
9692 break;
9693 case ISD::SELECT_CC:
9694 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
9695 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
9696 "LHS and RHS of condition must have same type!");
9697 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9698 "True and False arms of SelectCC must have same type!");
9699 assert(Ops[2].getValueType() == VT &&
9700 "select_cc node must be of same type as true and false value!");
9701 assert((!Ops[0].getValueType().isVector() ||
9702 Ops[0].getValueType().getVectorElementCount() ==
9703 VT.getVectorElementCount()) &&
9704 "Expected select_cc with vector result to have the same sized "
9705 "comparison type!");
9706 break;
9707 case ISD::BR_CC:
9708 assert(NumOps == 5 && "BR_CC takes 5 operands!");
9709 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9710 "LHS/RHS of comparison should match types!");
9711 break;
9712 case ISD::VP_ADD:
9713 case ISD::VP_SUB:
9714 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
9715 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9716 Opcode = ISD::VP_XOR;
9717 break;
9718 case ISD::VP_MUL:
9719 // If it is VP_MUL mask operation then turn it to VP_AND
9720 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9721 Opcode = ISD::VP_AND;
9722 break;
9723 case ISD::VP_REDUCE_MUL:
9724 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
9725 if (VT == MVT::i1)
9726 Opcode = ISD::VP_REDUCE_AND;
9727 break;
9728 case ISD::VP_REDUCE_ADD:
9729 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
9730 if (VT == MVT::i1)
9731 Opcode = ISD::VP_REDUCE_XOR;
9732 break;
9733 case ISD::VP_REDUCE_SMAX:
9734 case ISD::VP_REDUCE_UMIN:
9735 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
9736 // VP_REDUCE_AND.
9737 if (VT == MVT::i1)
9738 Opcode = ISD::VP_REDUCE_AND;
9739 break;
9740 case ISD::VP_REDUCE_SMIN:
9741 case ISD::VP_REDUCE_UMAX:
9742 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
9743 // VP_REDUCE_OR.
9744 if (VT == MVT::i1)
9745 Opcode = ISD::VP_REDUCE_OR;
9746 break;
9747 }
9748
9749 // Memoize nodes.
9750 SDNode *N;
9751 SDVTList VTs = getVTList(VT);
9752
9753 if (VT != MVT::Glue) {
9755 AddNodeIDNode(ID, Opcode, VTs, Ops);
9756 void *IP = nullptr;
9757
9758 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9759 return SDValue(E, 0);
9760
9761 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9762 createOperands(N, Ops);
9763
9764 CSEMap.InsertNode(N, IP);
9765 } else {
9766 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9767 createOperands(N, Ops);
9768 }
9769
9770 N->setFlags(Flags);
9771 InsertNode(N);
9772 SDValue V(N, 0);
9773 NewSDValueDbgMsg(V, "Creating new node: ", this);
9774 return V;
9775}
9776
9777SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9778 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
9779 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
9780}
9781
9782SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9783 ArrayRef<SDValue> Ops) {
9784 SDNodeFlags Flags;
9785 if (Inserter)
9786 Flags = Inserter->getFlags();
9787 return getNode(Opcode, DL, VTList, Ops, Flags);
9788}
9789
9790SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9791 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9792 if (VTList.NumVTs == 1)
9793 return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
9794
9795#ifndef NDEBUG
9796 for (const auto &Op : Ops)
9797 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9798 "Operand is DELETED_NODE!");
9799#endif
9800
9801 switch (Opcode) {
9802 case ISD::SADDO:
9803 case ISD::UADDO:
9804 case ISD::SSUBO:
9805 case ISD::USUBO: {
9806 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9807 "Invalid add/sub overflow op!");
9808 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9809 Ops[0].getValueType() == Ops[1].getValueType() &&
9810 Ops[0].getValueType() == VTList.VTs[0] &&
9811 "Binary operator types must match!");
9812 SDValue N1 = Ops[0], N2 = Ops[1];
9813 canonicalizeCommutativeBinop(Opcode, N1, N2);
9814
9815 // (X +- 0) -> X with zero-overflow.
9816 ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,
9817 /*AllowTruncation*/ true);
9818 if (N2CV && N2CV->isZero()) {
9819 SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
9820 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
9821 }
9822
9823 if (VTList.VTs[0].isVector() &&
9824 VTList.VTs[0].getVectorElementType() == MVT::i1 &&
9825 VTList.VTs[1].getVectorElementType() == MVT::i1) {
9826 SDValue F1 = getFreeze(N1);
9827 SDValue F2 = getFreeze(N2);
9828 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
9829 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
9830 return getNode(ISD::MERGE_VALUES, DL, VTList,
9831 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9832 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
9833 Flags);
9834 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
9835 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
9836 SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
9837 return getNode(ISD::MERGE_VALUES, DL, VTList,
9838 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9839 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
9840 Flags);
9841 }
9842 }
9843 break;
9844 }
9845 case ISD::SMUL_LOHI:
9846 case ISD::UMUL_LOHI: {
9847 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
9848 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
9849 VTList.VTs[0] == Ops[0].getValueType() &&
9850 VTList.VTs[0] == Ops[1].getValueType() &&
9851 "Binary operator types must match!");
9852 // Constant fold.
9853 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Ops[0]);
9854 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ops[1]);
9855 if (LHS && RHS) {
9856 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
9857 unsigned OutWidth = Width * 2;
9858 APInt Val = LHS->getAPIntValue();
9859 APInt Mul = RHS->getAPIntValue();
9860 if (Opcode == ISD::SMUL_LOHI) {
9861 Val = Val.sext(OutWidth);
9862 Mul = Mul.sext(OutWidth);
9863 } else {
9864 Val = Val.zext(OutWidth);
9865 Mul = Mul.zext(OutWidth);
9866 }
9867 Val *= Mul;
9868
9869 SDValue Hi =
9870 getConstant(Val.extractBits(Width, Width), DL, VTList.VTs[0]);
9871 SDValue Lo = getConstant(Val.trunc(Width), DL, VTList.VTs[0]);
9872 return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags);
9873 }
9874 break;
9875 }
9876 case ISD::FFREXP: {
9877 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
9878 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
9879 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
9880
9881 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Ops[0])) {
9882 int FrexpExp;
9883 APFloat FrexpMant =
9884 frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);
9885 SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);
9886 SDValue Result1 =
9887 getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]);
9888 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);
9889 }
9890
9891 break;
9892 }
9894 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9895 "Invalid STRICT_FP_EXTEND!");
9896 assert(VTList.VTs[0].isFloatingPoint() &&
9897 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
9898 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9899 "STRICT_FP_EXTEND result type should be vector iff the operand "
9900 "type is vector!");
9901 assert((!VTList.VTs[0].isVector() ||
9902 VTList.VTs[0].getVectorElementCount() ==
9903 Ops[1].getValueType().getVectorElementCount()) &&
9904 "Vector element count mismatch!");
9905 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
9906 "Invalid fpext node, dst <= src!");
9907 break;
9909 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
9910 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9911 "STRICT_FP_ROUND result type should be vector iff the operand "
9912 "type is vector!");
9913 assert((!VTList.VTs[0].isVector() ||
9914 VTList.VTs[0].getVectorElementCount() ==
9915 Ops[1].getValueType().getVectorElementCount()) &&
9916 "Vector element count mismatch!");
9917 assert(VTList.VTs[0].isFloatingPoint() &&
9918 Ops[1].getValueType().isFloatingPoint() &&
9919 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
9920 isa<ConstantSDNode>(Ops[2]) &&
9921 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
9922 "Invalid STRICT_FP_ROUND!");
9923 break;
9924#if 0
9925 // FIXME: figure out how to safely handle things like
9926 // int foo(int x) { return 1 << (x & 255); }
9927 // int bar() { return foo(256); }
9928 case ISD::SRA_PARTS:
9929 case ISD::SRL_PARTS:
9930 case ISD::SHL_PARTS:
9931 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
9932 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
9933 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9934 else if (N3.getOpcode() == ISD::AND)
9935 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
9936 // If the and is only masking out bits that cannot effect the shift,
9937 // eliminate the and.
9938 unsigned NumBits = VT.getScalarSizeInBits()*2;
9939 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
9940 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9941 }
9942 break;
9943#endif
9944 }
9945
9946 // Memoize the node unless it returns a glue result.
9947 SDNode *N;
9948 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9950 AddNodeIDNode(ID, Opcode, VTList, Ops);
9951 void *IP = nullptr;
9952 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9953 return SDValue(E, 0);
9954
9955 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
9956 createOperands(N, Ops);
9957 CSEMap.InsertNode(N, IP);
9958 } else {
9959 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
9960 createOperands(N, Ops);
9961 }
9962
9963 N->setFlags(Flags);
9964 InsertNode(N);
9965 SDValue V(N, 0);
9966 NewSDValueDbgMsg(V, "Creating new node: ", this);
9967 return V;
9968}
9969
9970SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9971 SDVTList VTList) {
9972 return getNode(Opcode, DL, VTList, std::nullopt);
9973}
9974
9975SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9976 SDValue N1) {
9977 SDValue Ops[] = { N1 };
9978 return getNode(Opcode, DL, VTList, Ops);
9979}
9980
9981SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9982 SDValue N1, SDValue N2) {
9983 SDValue Ops[] = { N1, N2 };
9984 return getNode(Opcode, DL, VTList, Ops);
9985}
9986
9987SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9988 SDValue N1, SDValue N2, SDValue N3) {
9989 SDValue Ops[] = { N1, N2, N3 };
9990 return getNode(Opcode, DL, VTList, Ops);
9991}
9992
9993SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9994 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
9995 SDValue Ops[] = { N1, N2, N3, N4 };
9996 return getNode(Opcode, DL, VTList, Ops);
9997}
9998
9999SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10000 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10001 SDValue N5) {
10002 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10003 return getNode(Opcode, DL, VTList, Ops);
10004}
10005
10007 return makeVTList(SDNode::getValueTypeList(VT), 1);
10008}
10009
10012 ID.AddInteger(2U);
10013 ID.AddInteger(VT1.getRawBits());
10014 ID.AddInteger(VT2.getRawBits());
10015
10016 void *IP = nullptr;
10017 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10018 if (!Result) {
10019 EVT *Array = Allocator.Allocate<EVT>(2);
10020 Array[0] = VT1;
10021 Array[1] = VT2;
10022 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10023 VTListMap.InsertNode(Result, IP);
10024 }
10025 return Result->getSDVTList();
10026}
10027
10030 ID.AddInteger(3U);
10031 ID.AddInteger(VT1.getRawBits());
10032 ID.AddInteger(VT2.getRawBits());
10033 ID.AddInteger(VT3.getRawBits());
10034
10035 void *IP = nullptr;
10036 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10037 if (!Result) {
10038 EVT *Array = Allocator.Allocate<EVT>(3);
10039 Array[0] = VT1;
10040 Array[1] = VT2;
10041 Array[2] = VT3;
10042 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10043 VTListMap.InsertNode(Result, IP);
10044 }
10045 return Result->getSDVTList();
10046}
10047
10050 ID.AddInteger(4U);
10051 ID.AddInteger(VT1.getRawBits());
10052 ID.AddInteger(VT2.getRawBits());
10053 ID.AddInteger(VT3.getRawBits());
10054 ID.AddInteger(VT4.getRawBits());
10055
10056 void *IP = nullptr;
10057 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10058 if (!Result) {
10059 EVT *Array = Allocator.Allocate<EVT>(4);
10060 Array[0] = VT1;
10061 Array[1] = VT2;
10062 Array[2] = VT3;
10063 Array[3] = VT4;
10064 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10065 VTListMap.InsertNode(Result, IP);
10066 }
10067 return Result->getSDVTList();
10068}
10069
10071 unsigned NumVTs = VTs.size();
10073 ID.AddInteger(NumVTs);
10074 for (unsigned index = 0; index < NumVTs; index++) {
10075 ID.AddInteger(VTs[index].getRawBits());
10076 }
10077
10078 void *IP = nullptr;
10079 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10080 if (!Result) {
10081 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
10082 llvm::copy(VTs, Array);
10083 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10084 VTListMap.InsertNode(Result, IP);
10085 }
10086 return Result->getSDVTList();
10087}
10088
10089
10090/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10091/// specified operands. If the resultant node already exists in the DAG,
10092/// this does not modify the specified node, instead it returns the node that
10093/// already exists. If the resultant node does not exist in the DAG, the
10094/// input node is returned. As a degenerate case, if you specify the same
10095/// input operands as the node already has, the input node is returned.
10097 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10098
10099 // Check to see if there is no change.
10100 if (Op == N->getOperand(0)) return N;
10101
10102 // See if the modified node already exists.
10103 void *InsertPos = nullptr;
10104 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10105 return Existing;
10106
10107 // Nope it doesn't. Remove the node from its current place in the maps.
10108 if (InsertPos)
10109 if (!RemoveNodeFromCSEMaps(N))
10110 InsertPos = nullptr;
10111
10112 // Now we update the operands.
10113 N->OperandList[0].set(Op);
10114
10116 // If this gets put into a CSE map, add it.
10117 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10118 return N;
10119}
10120
10122 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10123
10124 // Check to see if there is no change.
10125 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
10126 return N; // No operands changed, just return the input node.
10127
10128 // See if the modified node already exists.
10129 void *InsertPos = nullptr;
10130 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10131 return Existing;
10132
10133 // Nope it doesn't. Remove the node from its current place in the maps.
10134 if (InsertPos)
10135 if (!RemoveNodeFromCSEMaps(N))
10136 InsertPos = nullptr;
10137
10138 // Now we update the operands.
10139 if (N->OperandList[0] != Op1)
10140 N->OperandList[0].set(Op1);
10141 if (N->OperandList[1] != Op2)
10142 N->OperandList[1].set(Op2);
10143
10145 // If this gets put into a CSE map, add it.
10146 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10147 return N;
10148}
10149
10152 SDValue Ops[] = { Op1, Op2, Op3 };
10153 return UpdateNodeOperands(N, Ops);
10154}
10155
10158 SDValue Op3, SDValue Op4) {
10159 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10160 return UpdateNodeOperands(N, Ops);
10161}
10162
10165 SDValue Op3, SDValue Op4, SDValue Op5) {
10166 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10167 return UpdateNodeOperands(N, Ops);
10168}
10169
10172 unsigned NumOps = Ops.size();
10173 assert(N->getNumOperands() == NumOps &&
10174 "Update with wrong number of operands");
10175
10176 // If no operands changed just return the input node.
10177 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
10178 return N;
10179
10180 // See if the modified node already exists.
10181 void *InsertPos = nullptr;
10182 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10183 return Existing;
10184
10185 // Nope it doesn't. Remove the node from its current place in the maps.
10186 if (InsertPos)
10187 if (!RemoveNodeFromCSEMaps(N))
10188 InsertPos = nullptr;
10189
10190 // Now we update the operands.
10191 for (unsigned i = 0; i != NumOps; ++i)
10192 if (N->OperandList[i] != Ops[i])
10193 N->OperandList[i].set(Ops[i]);
10194
10196 // If this gets put into a CSE map, add it.
10197 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10198 return N;
10199}
10200
10201/// DropOperands - Release the operands and set this node to have
10202/// zero operands.
10204 // Unlike the code in MorphNodeTo that does this, we don't need to
10205 // watch for dead nodes here.
10206 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10207 SDUse &Use = *I++;
10208 Use.set(SDValue());
10209 }
10210}
10211
10213 ArrayRef<MachineMemOperand *> NewMemRefs) {
10214 if (NewMemRefs.empty()) {
10215 N->clearMemRefs();
10216 return;
10217 }
10218
10219 // Check if we can avoid allocating by storing a single reference directly.
10220 if (NewMemRefs.size() == 1) {
10221 N->MemRefs = NewMemRefs[0];
10222 N->NumMemRefs = 1;
10223 return;
10224 }
10225
10226 MachineMemOperand **MemRefsBuffer =
10227 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
10228 llvm::copy(NewMemRefs, MemRefsBuffer);
10229 N->MemRefs = MemRefsBuffer;
10230 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10231}
10232
10233/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10234/// machine opcode.
10235///
10237 EVT VT) {
10238 SDVTList VTs = getVTList(VT);
10239 return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10240}
10241
10243 EVT VT, SDValue Op1) {
10244 SDVTList VTs = getVTList(VT);
10245 SDValue Ops[] = { Op1 };
10246 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10247}
10248
10250 EVT VT, SDValue Op1,
10251 SDValue Op2) {
10252 SDVTList VTs = getVTList(VT);
10253 SDValue Ops[] = { Op1, Op2 };
10254 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10255}
10256
10258 EVT VT, SDValue Op1,
10259 SDValue Op2, SDValue Op3) {
10260 SDVTList VTs = getVTList(VT);
10261 SDValue Ops[] = { Op1, Op2, Op3 };
10262 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10263}
10264
10266 EVT VT, ArrayRef<SDValue> Ops) {
10267 SDVTList VTs = getVTList(VT);
10268 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10269}
10270
10272 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10273 SDVTList VTs = getVTList(VT1, VT2);
10274 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10275}
10276
10278 EVT VT1, EVT VT2) {
10279 SDVTList VTs = getVTList(VT1, VT2);
10280 return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10281}
10282
10284 EVT VT1, EVT VT2, EVT VT3,
10285 ArrayRef<SDValue> Ops) {
10286 SDVTList VTs = getVTList(VT1, VT2, VT3);
10287 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10288}
10289
10291 EVT VT1, EVT VT2,
10292 SDValue Op1, SDValue Op2) {
10293 SDVTList VTs = getVTList(VT1, VT2);
10294 SDValue Ops[] = { Op1, Op2 };
10295 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10296}
10297
10299 SDVTList VTs,ArrayRef<SDValue> Ops) {
10300 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
10301 // Reset the NodeID to -1.
10302 New->setNodeId(-1);
10303 if (New != N) {
10304 ReplaceAllUsesWith(N, New);
10306 }
10307 return New;
10308}
10309
10310/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
10311/// the line number information on the merged node since it is not possible to
10312/// preserve the information that operation is associated with multiple lines.
10313/// This will make the debugger working better at -O0, were there is a higher
10314/// probability having other instructions associated with that line.
10315///
10316/// For IROrder, we keep the smaller of the two
10317SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
10318 DebugLoc NLoc = N->getDebugLoc();
10319 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
10320 N->setDebugLoc(DebugLoc());
10321 }
10322 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
10323 N->setIROrder(Order);
10324 return N;
10325}
10326
10327/// MorphNodeTo - This *mutates* the specified node to have the specified
10328/// return type, opcode, and operands.
10329///
10330/// Note that MorphNodeTo returns the resultant node. If there is already a
10331/// node of the specified opcode and operands, it returns that node instead of
10332/// the current one. Note that the SDLoc need not be the same.
10333///
10334/// Using MorphNodeTo is faster than creating a new node and swapping it in
10335/// with ReplaceAllUsesWith both because it often avoids allocating a new
10336/// node, and because it doesn't require CSE recalculation for any of
10337/// the node's users.
10338///
10339/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
10340/// As a consequence it isn't appropriate to use from within the DAG combiner or
10341/// the legalizer which maintain worklists that would need to be updated when
10342/// deleting things.
10344 SDVTList VTs, ArrayRef<SDValue> Ops) {
10345 // If an identical node already exists, use it.
10346 void *IP = nullptr;
10347 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
10349 AddNodeIDNode(ID, Opc, VTs, Ops);
10350 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
10351 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
10352 }
10353
10354 if (!RemoveNodeFromCSEMaps(N))
10355 IP = nullptr;
10356
10357 // Start the morphing.
10358 N->NodeType = Opc;
10359 N->ValueList = VTs.VTs;
10360 N->NumValues = VTs.NumVTs;
10361
10362 // Clear the operands list, updating used nodes to remove this from their
10363 // use list. Keep track of any operands that become dead as a result.
10364 SmallPtrSet<SDNode*, 16> DeadNodeSet;
10365 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
10366 SDUse &Use = *I++;
10367 SDNode *Used = Use.getNode();
10368 Use.set(SDValue());
10369 if (Used->use_empty())
10370 DeadNodeSet.insert(Used);
10371 }
10372
10373 // For MachineNode, initialize the memory references information.
10374 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
10375 MN->clearMemRefs();
10376
10377 // Swap for an appropriately sized array from the recycler.
10378 removeOperands(N);
10379 createOperands(N, Ops);
10380
10381 // Delete any nodes that are still dead after adding the uses for the
10382 // new operands.
10383 if (!DeadNodeSet.empty()) {
10384 SmallVector<SDNode *, 16> DeadNodes;
10385 for (SDNode *N : DeadNodeSet)
10386 if (N->use_empty())
10387 DeadNodes.push_back(N);
10388 RemoveDeadNodes(DeadNodes);
10389 }
10390
10391 if (IP)
10392 CSEMap.InsertNode(N, IP); // Memoize the new node.
10393 return N;
10394}
10395
10397 unsigned OrigOpc = Node->getOpcode();
10398 unsigned NewOpc;
10399 switch (OrigOpc) {
10400 default:
10401 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
10402#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10403 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10404#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10405 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10406#include "llvm/IR/ConstrainedOps.def"
10407 }
10408
10409 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
10410
10411 // We're taking this node out of the chain, so we need to re-link things.
10412 SDValue InputChain = Node->getOperand(0);
10413 SDValue OutputChain = SDValue(Node, 1);
10414 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
10415
10417 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10418 Ops.push_back(Node->getOperand(i));
10419
10420 SDVTList VTs = getVTList(Node->getValueType(0));
10421 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
10422
10423 // MorphNodeTo can operate in two ways: if an existing node with the
10424 // specified operands exists, it can just return it. Otherwise, it
10425 // updates the node in place to have the requested operands.
10426 if (Res == Node) {
10427 // If we updated the node in place, reset the node ID. To the isel,
10428 // this should be just like a newly allocated machine node.
10429 Res->setNodeId(-1);
10430 } else {
10431 ReplaceAllUsesWith(Node, Res);
10432 RemoveDeadNode(Node);
10433 }
10434
10435 return Res;
10436}
10437
10438/// getMachineNode - These are used for target selectors to create a new node
10439/// with specified return type(s), MachineInstr opcode, and operands.
10440///
10441/// Note that getMachineNode returns the resultant node. If there is already a
10442/// node of the specified opcode and operands, it returns that node instead of
10443/// the current one.
10445 EVT VT) {
10446 SDVTList VTs = getVTList(VT);
10447 return getMachineNode(Opcode, dl, VTs, std::nullopt);
10448}
10449
10451 EVT VT, SDValue Op1) {
10452 SDVTList VTs = getVTList(VT);
10453 SDValue Ops[] = { Op1 };
10454 return getMachineNode(Opcode, dl, VTs, Ops);
10455}
10456
10458 EVT VT, SDValue Op1, SDValue Op2) {
10459 SDVTList VTs = getVTList(VT);
10460 SDValue Ops[] = { Op1, Op2 };
10461 return getMachineNode(Opcode, dl, VTs, Ops);
10462}
10463
10465 EVT VT, SDValue Op1, SDValue Op2,
10466 SDValue Op3) {
10467 SDVTList VTs = getVTList(VT);
10468 SDValue Ops[] = { Op1, Op2, Op3 };
10469 return getMachineNode(Opcode, dl, VTs, Ops);
10470}
10471
10473 EVT VT, ArrayRef<SDValue> Ops) {
10474 SDVTList VTs = getVTList(VT);
10475 return getMachineNode(Opcode, dl, VTs, Ops);
10476}
10477
10479 EVT VT1, EVT VT2, SDValue Op1,
10480 SDValue Op2) {
10481 SDVTList VTs = getVTList(VT1, VT2);
10482 SDValue Ops[] = { Op1, Op2 };
10483 return getMachineNode(Opcode, dl, VTs, Ops);
10484}
10485
10487 EVT VT1, EVT VT2, SDValue Op1,
10488 SDValue Op2, SDValue Op3) {
10489 SDVTList VTs = getVTList(VT1, VT2);
10490 SDValue Ops[] = { Op1, Op2, Op3 };
10491 return getMachineNode(Opcode, dl, VTs, Ops);
10492}
10493
10495 EVT VT1, EVT VT2,
10496 ArrayRef<SDValue> Ops) {
10497 SDVTList VTs = getVTList(VT1, VT2);
10498 return getMachineNode(Opcode, dl, VTs, Ops);
10499}
10500
10502 EVT VT1, EVT VT2, EVT VT3,
10503 SDValue Op1, SDValue Op2) {
10504 SDVTList VTs = getVTList(VT1, VT2, VT3);
10505 SDValue Ops[] = { Op1, Op2 };
10506 return getMachineNode(Opcode, dl, VTs, Ops);
10507}
10508
10510 EVT VT1, EVT VT2, EVT VT3,
10511 SDValue Op1, SDValue Op2,
10512 SDValue Op3) {
10513 SDVTList VTs = getVTList(VT1, VT2, VT3);
10514 SDValue Ops[] = { Op1, Op2, Op3 };
10515 return getMachineNode(Opcode, dl, VTs, Ops);
10516}
10517
10519 EVT VT1, EVT VT2, EVT VT3,
10520 ArrayRef<SDValue> Ops) {
10521 SDVTList VTs = getVTList(VT1, VT2, VT3);
10522 return getMachineNode(Opcode, dl, VTs, Ops);
10523}
10524
10526 ArrayRef<EVT> ResultTys,
10527 ArrayRef<SDValue> Ops) {
10528 SDVTList VTs = getVTList(ResultTys);
10529 return getMachineNode(Opcode, dl, VTs, Ops);
10530}
10531
10533 SDVTList VTs,
10534 ArrayRef<SDValue> Ops) {
10535 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
10537 void *IP = nullptr;
10538
10539 if (DoCSE) {
10541 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
10542 IP = nullptr;
10543 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
10544 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
10545 }
10546 }
10547
10548 // Allocate a new MachineSDNode.
10549 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
10550 createOperands(N, Ops);
10551
10552 if (DoCSE)
10553 CSEMap.InsertNode(N, IP);
10554
10555 InsertNode(N);
10556 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
10557 return N;
10558}
10559
10560/// getTargetExtractSubreg - A convenience function for creating
10561/// TargetOpcode::EXTRACT_SUBREG nodes.
10563 SDValue Operand) {
10564 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10565 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
10566 VT, Operand, SRIdxVal);
10567 return SDValue(Subreg, 0);
10568}
10569
10570/// getTargetInsertSubreg - A convenience function for creating
10571/// TargetOpcode::INSERT_SUBREG nodes.
10573 SDValue Operand, SDValue Subreg) {
10574 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10575 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
10576 VT, Operand, Subreg, SRIdxVal);
10577 return SDValue(Result, 0);
10578}
10579
10580/// getNodeIfExists - Get the specified node if it's already available, or
10581/// else return NULL.
10583 ArrayRef<SDValue> Ops) {
10584 SDNodeFlags Flags;
10585 if (Inserter)
10586 Flags = Inserter->getFlags();
10587 return getNodeIfExists(Opcode, VTList, Ops, Flags);
10588}
10589
10592 const SDNodeFlags Flags) {
10593 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10595 AddNodeIDNode(ID, Opcode, VTList, Ops);
10596 void *IP = nullptr;
10597 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
10598 E->intersectFlagsWith(Flags);
10599 return E;
10600 }
10601 }
10602 return nullptr;
10603}
10604
10605/// doesNodeExist - Check if a node exists without modifying its flags.
10606bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
10607 ArrayRef<SDValue> Ops) {
10608 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10610 AddNodeIDNode(ID, Opcode, VTList, Ops);
10611 void *IP = nullptr;
10612 if (FindNodeOrInsertPos(ID, SDLoc(), IP))
10613 return true;
10614 }
10615 return false;
10616}
10617
10618/// getDbgValue - Creates a SDDbgValue node.
10619///
10620/// SDNode
10622 SDNode *N, unsigned R, bool IsIndirect,
10623 const DebugLoc &DL, unsigned O) {
10624 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10625 "Expected inlined-at fields to agree");
10626 return new (DbgInfo->getAlloc())
10627 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
10628 {}, IsIndirect, DL, O,
10629 /*IsVariadic=*/false);
10630}
10631
10632/// Constant
10634 DIExpression *Expr,
10635 const Value *C,
10636 const DebugLoc &DL, unsigned O) {
10637 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10638 "Expected inlined-at fields to agree");
10639 return new (DbgInfo->getAlloc())
10640 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
10641 /*IsIndirect=*/false, DL, O,
10642 /*IsVariadic=*/false);
10643}
10644
10645/// FrameIndex
10647 DIExpression *Expr, unsigned FI,
10648 bool IsIndirect,
10649 const DebugLoc &DL,
10650 unsigned O) {
10651 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10652 "Expected inlined-at fields to agree");
10653 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
10654}
10655
10656/// FrameIndex with dependencies
10658 DIExpression *Expr, unsigned FI,
10659 ArrayRef<SDNode *> Dependencies,
10660 bool IsIndirect,
10661 const DebugLoc &DL,
10662 unsigned O) {
10663 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10664 "Expected inlined-at fields to agree");
10665 return new (DbgInfo->getAlloc())
10666 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
10667 Dependencies, IsIndirect, DL, O,
10668 /*IsVariadic=*/false);
10669}
10670
10671/// VReg
10673 unsigned VReg, bool IsIndirect,
10674 const DebugLoc &DL, unsigned O) {
10675 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10676 "Expected inlined-at fields to agree");
10677 return new (DbgInfo->getAlloc())
10678 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
10679 {}, IsIndirect, DL, O,
10680 /*IsVariadic=*/false);
10681}
10682
10685 ArrayRef<SDNode *> Dependencies,
10686 bool IsIndirect, const DebugLoc &DL,
10687 unsigned O, bool IsVariadic) {
10688 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10689 "Expected inlined-at fields to agree");
10690 return new (DbgInfo->getAlloc())
10691 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
10692 DL, O, IsVariadic);
10693}
10694
10696 unsigned OffsetInBits, unsigned SizeInBits,
10697 bool InvalidateDbg) {
10698 SDNode *FromNode = From.getNode();
10699 SDNode *ToNode = To.getNode();
10700 assert(FromNode && ToNode && "Can't modify dbg values");
10701
10702 // PR35338
10703 // TODO: assert(From != To && "Redundant dbg value transfer");
10704 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
10705 if (From == To || FromNode == ToNode)
10706 return;
10707
10708 if (!FromNode->getHasDebugValue())
10709 return;
10710
10711 SDDbgOperand FromLocOp =
10712 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
10714
10716 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
10717 if (Dbg->isInvalidated())
10718 continue;
10719
10720 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
10721
10722 // Create a new location ops vector that is equal to the old vector, but
10723 // with each instance of FromLocOp replaced with ToLocOp.
10724 bool Changed = false;
10725 auto NewLocOps = Dbg->copyLocationOps();
10726 std::replace_if(
10727 NewLocOps.begin(), NewLocOps.end(),
10728 [&Changed, FromLocOp](const SDDbgOperand &Op) {
10729 bool Match = Op == FromLocOp;
10730 Changed |= Match;
10731 return Match;
10732 },
10733 ToLocOp);
10734 // Ignore this SDDbgValue if we didn't find a matching location.
10735 if (!Changed)
10736 continue;
10737
10738 DIVariable *Var = Dbg->getVariable();
10739 auto *Expr = Dbg->getExpression();
10740 // If a fragment is requested, update the expression.
10741 if (SizeInBits) {
10742 // When splitting a larger (e.g., sign-extended) value whose
10743 // lower bits are described with an SDDbgValue, do not attempt
10744 // to transfer the SDDbgValue to the upper bits.
10745 if (auto FI = Expr->getFragmentInfo())
10746 if (OffsetInBits + SizeInBits > FI->SizeInBits)
10747 continue;
10748 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
10749 SizeInBits);
10750 if (!Fragment)
10751 continue;
10752 Expr = *Fragment;
10753 }
10754
10755 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
10756 // Clone the SDDbgValue and move it to To.
10757 SDDbgValue *Clone = getDbgValueList(
10758 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
10759 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
10760 Dbg->isVariadic());
10761 ClonedDVs.push_back(Clone);
10762
10763 if (InvalidateDbg) {
10764 // Invalidate value and indicate the SDDbgValue should not be emitted.
10765 Dbg->setIsInvalidated();
10766 Dbg->setIsEmitted();
10767 }
10768 }
10769
10770 for (SDDbgValue *Dbg : ClonedDVs) {
10771 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
10772 "Transferred DbgValues should depend on the new SDNode");
10773 AddDbgValue(Dbg, false);
10774 }
10775}
10776
10778 if (!N.getHasDebugValue())
10779 return;
10780
10782 for (auto *DV : GetDbgValues(&N)) {
10783 if (DV->isInvalidated())
10784 continue;
10785 switch (N.getOpcode()) {
10786 default:
10787 break;
10788 case ISD::ADD: {
10789 SDValue N0 = N.getOperand(0);
10790 SDValue N1 = N.getOperand(1);
10791 if (!isa<ConstantSDNode>(N0)) {
10792 bool RHSConstant = isa<ConstantSDNode>(N1);
10794 if (RHSConstant)
10795 Offset = N.getConstantOperandVal(1);
10796 // We are not allowed to turn indirect debug values variadic, so
10797 // don't salvage those.
10798 if (!RHSConstant && DV->isIndirect())
10799 continue;
10800
10801 // Rewrite an ADD constant node into a DIExpression. Since we are
10802 // performing arithmetic to compute the variable's *value* in the
10803 // DIExpression, we need to mark the expression with a
10804 // DW_OP_stack_value.
10805 auto *DIExpr = DV->getExpression();
10806 auto NewLocOps = DV->copyLocationOps();
10807 bool Changed = false;
10808 size_t OrigLocOpsSize = NewLocOps.size();
10809 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
10810 // We're not given a ResNo to compare against because the whole
10811 // node is going away. We know that any ISD::ADD only has one
10812 // result, so we can assume any node match is using the result.
10813 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10814 NewLocOps[i].getSDNode() != &N)
10815 continue;
10816 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10817 if (RHSConstant) {
10820 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
10821 } else {
10822 // Convert to a variadic expression (if not already).
10823 // convertToVariadicExpression() returns a const pointer, so we use
10824 // a temporary const variable here.
10825 const auto *TmpDIExpr =
10829 ExprOps.push_back(NewLocOps.size());
10830 ExprOps.push_back(dwarf::DW_OP_plus);
10833 NewLocOps.push_back(RHS);
10834 DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);
10835 }
10836 Changed = true;
10837 }
10838 (void)Changed;
10839 assert(Changed && "Salvage target doesn't use N");
10840
10841 bool IsVariadic =
10842 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
10843
10844 auto AdditionalDependencies = DV->getAdditionalDependencies();
10845 SDDbgValue *Clone = getDbgValueList(
10846 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
10847 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
10848 ClonedDVs.push_back(Clone);
10849 DV->setIsInvalidated();
10850 DV->setIsEmitted();
10851 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
10852 N0.getNode()->dumprFull(this);
10853 dbgs() << " into " << *DIExpr << '\n');
10854 }
10855 break;
10856 }
10857 case ISD::TRUNCATE: {
10858 SDValue N0 = N.getOperand(0);
10859 TypeSize FromSize = N0.getValueSizeInBits();
10860 TypeSize ToSize = N.getValueSizeInBits(0);
10861
10862 DIExpression *DbgExpression = DV->getExpression();
10863 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);
10864 auto NewLocOps = DV->copyLocationOps();
10865 bool Changed = false;
10866 for (size_t i = 0; i < NewLocOps.size(); ++i) {
10867 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10868 NewLocOps[i].getSDNode() != &N)
10869 continue;
10870
10871 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10872 DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);
10873 Changed = true;
10874 }
10875 assert(Changed && "Salvage target doesn't use N");
10876 (void)Changed;
10877
10878 SDDbgValue *Clone =
10879 getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,
10880 DV->getAdditionalDependencies(), DV->isIndirect(),
10881 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
10882
10883 ClonedDVs.push_back(Clone);
10884 DV->setIsInvalidated();
10885 DV->setIsEmitted();
10886 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
10887 dbgs() << " into " << *DbgExpression << '\n');
10888 break;
10889 }
10890 }
10891 }
10892
10893 for (SDDbgValue *Dbg : ClonedDVs) {
10894 assert(!Dbg->getSDNodes().empty() &&
10895 "Salvaged DbgValue should depend on a new SDNode");
10896 AddDbgValue(Dbg, false);
10897 }
10898}
10899
10900/// Creates a SDDbgLabel node.
10902 const DebugLoc &DL, unsigned O) {
10903 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
10904 "Expected inlined-at fields to agree");
10905 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
10906}
10907
10908namespace {
10909
10910/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
10911/// pointed to by a use iterator is deleted, increment the use iterator
10912/// so that it doesn't dangle.
10913///
10914class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
10917
10918 void NodeDeleted(SDNode *N, SDNode *E) override {
10919 // Increment the iterator as needed.
10920 while (UI != UE && N == *UI)
10921 ++UI;
10922 }
10923
10924public:
10925 RAUWUpdateListener(SelectionDAG &d,
10928 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
10929};
10930
10931} // end anonymous namespace
10932
10933/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
10934/// This can cause recursive merging of nodes in the DAG.
10935///
10936/// This version assumes From has a single result value.
10937///
10939 SDNode *From = FromN.getNode();
10940 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
10941 "Cannot replace with this method!");
10942 assert(From != To.getNode() && "Cannot replace uses of with self");
10943
10944 // Preserve Debug Values
10945 transferDbgValues(FromN, To);
10946 // Preserve extra info.
10947 copyExtraInfo(From, To.getNode());
10948
10949 // Iterate over all the existing uses of From. New uses will be added
10950 // to the beginning of the use list, which we avoid visiting.
10951 // This specifically avoids visiting uses of From that arise while the
10952 // replacement is happening, because any such uses would be the result
10953 // of CSE: If an existing node looks like From after one of its operands
10954 // is replaced by To, we don't want to replace of all its users with To
10955 // too. See PR3018 for more info.
10956 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
10957 RAUWUpdateListener Listener(*this, UI, UE);
10958 while (UI != UE) {
10959 SDNode *User = *UI;
10960
10961 // This node is about to morph, remove its old self from the CSE maps.
10962 RemoveNodeFromCSEMaps(User);
10963
10964 // A user can appear in a use list multiple times, and when this
10965 // happens the uses are usually next to each other in the list.
10966 // To help reduce the number of CSE recomputations, process all
10967 // the uses of this user that we can find this way.
10968 do {
10969 SDUse &Use = UI.getUse();
10970 ++UI;
10971 Use.set(To);
10972 if (To->isDivergent() != From->isDivergent())
10974 } while (UI != UE && *UI == User);
10975 // Now that we have modified User, add it back to the CSE maps. If it
10976 // already exists there, recursively merge the results together.
10977 AddModifiedNodeToCSEMaps(User);
10978 }
10979
10980 // If we just RAUW'd the root, take note.
10981 if (FromN == getRoot())
10982 setRoot(To);
10983}
10984
10985/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
10986/// This can cause recursive merging of nodes in the DAG.
10987///
10988/// This version assumes that for each value of From, there is a
10989/// corresponding value in To in the same position with the same type.
10990///
10992#ifndef NDEBUG
10993 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
10994 assert((!From->hasAnyUseOfValue(i) ||
10995 From->getValueType(i) == To->getValueType(i)) &&
10996 "Cannot use this version of ReplaceAllUsesWith!");
10997#endif
10998
10999 // Handle the trivial case.
11000 if (From == To)
11001 return;
11002
11003 // Preserve Debug Info. Only do this if there's a use.
11004 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11005 if (From->hasAnyUseOfValue(i)) {
11006 assert((i < To->getNumValues()) && "Invalid To location");
11008 }
11009 // Preserve extra info.
11010 copyExtraInfo(From, To);
11011
11012 // Iterate over just the existing users of From. See the comments in
11013 // the ReplaceAllUsesWith above.
11014 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11015 RAUWUpdateListener Listener(*this, UI, UE);
11016 while (UI != UE) {
11017 SDNode *User = *UI;
11018
11019 // This node is about to morph, remove its old self from the CSE maps.
11020 RemoveNodeFromCSEMaps(User);
11021
11022 // A user can appear in a use list multiple times, and when this
11023 // happens the uses are usually next to each other in the list.
11024 // To help reduce the number of CSE recomputations, process all
11025 // the uses of this user that we can find this way.
11026 do {
11027 SDUse &Use = UI.getUse();
11028 ++UI;
11029 Use.setNode(To);
11030 if (To->isDivergent() != From->isDivergent())
11032 } while (UI != UE && *UI == User);
11033
11034 // Now that we have modified User, add it back to the CSE maps. If it
11035 // already exists there, recursively merge the results together.
11036 AddModifiedNodeToCSEMaps(User);
11037 }
11038
11039 // If we just RAUW'd the root, take note.
11040 if (From == getRoot().getNode())
11041 setRoot(SDValue(To, getRoot().getResNo()));
11042}
11043
11044/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11045/// This can cause recursive merging of nodes in the DAG.
11046///
11047/// This version can replace From with any result values. To must match the
11048/// number and types of values returned by From.
11050 if (From->getNumValues() == 1) // Handle the simple case efficiently.
11051 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
11052
11053 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11054 // Preserve Debug Info.
11055 transferDbgValues(SDValue(From, i), To[i]);
11056 // Preserve extra info.
11057 copyExtraInfo(From, To[i].getNode());
11058 }
11059
11060 // Iterate over just the existing users of From. See the comments in
11061 // the ReplaceAllUsesWith above.
11062 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11063 RAUWUpdateListener Listener(*this, UI, UE);
11064 while (UI != UE) {
11065 SDNode *User = *UI;
11066
11067 // This node is about to morph, remove its old self from the CSE maps.
11068 RemoveNodeFromCSEMaps(User);
11069
11070 // A user can appear in a use list multiple times, and when this happens the
11071 // uses are usually next to each other in the list. To help reduce the
11072 // number of CSE and divergence recomputations, process all the uses of this
11073 // user that we can find this way.
11074 bool To_IsDivergent = false;
11075 do {
11076 SDUse &Use = UI.getUse();
11077 const SDValue &ToOp = To[Use.getResNo()];
11078 ++UI;
11079 Use.set(ToOp);
11080 To_IsDivergent |= ToOp->isDivergent();
11081 } while (UI != UE && *UI == User);
11082
11083 if (To_IsDivergent != From->isDivergent())
11085
11086 // Now that we have modified User, add it back to the CSE maps. If it
11087 // already exists there, recursively merge the results together.
11088 AddModifiedNodeToCSEMaps(User);
11089 }
11090
11091 // If we just RAUW'd the root, take note.
11092 if (From == getRoot().getNode())
11093 setRoot(SDValue(To[getRoot().getResNo()]));
11094}
11095
11096/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11097/// uses of other values produced by From.getNode() alone. The Deleted
11098/// vector is handled the same way as for ReplaceAllUsesWith.
11100 // Handle the really simple, really trivial case efficiently.
11101 if (From == To) return;
11102
11103 // Handle the simple, trivial, case efficiently.
11104 if (From.getNode()->getNumValues() == 1) {
11106 return;
11107 }
11108
11109 // Preserve Debug Info.
11111 copyExtraInfo(From.getNode(), To.getNode());
11112
11113 // Iterate over just the existing users of From. See the comments in
11114 // the ReplaceAllUsesWith above.
11115 SDNode::use_iterator UI = From.getNode()->use_begin(),
11116 UE = From.getNode()->use_end();
11117 RAUWUpdateListener Listener(*this, UI, UE);
11118 while (UI != UE) {
11119 SDNode *User = *UI;
11120 bool UserRemovedFromCSEMaps = false;
11121
11122 // A user can appear in a use list multiple times, and when this
11123 // happens the uses are usually next to each other in the list.
11124 // To help reduce the number of CSE recomputations, process all
11125 // the uses of this user that we can find this way.
11126 do {
11127 SDUse &Use = UI.getUse();
11128
11129 // Skip uses of different values from the same node.
11130 if (Use.getResNo() != From.getResNo()) {
11131 ++UI;
11132 continue;
11133 }
11134
11135 // If this node hasn't been modified yet, it's still in the CSE maps,
11136 // so remove its old self from the CSE maps.
11137 if (!UserRemovedFromCSEMaps) {
11138 RemoveNodeFromCSEMaps(User);
11139 UserRemovedFromCSEMaps = true;
11140 }
11141
11142 ++UI;
11143 Use.set(To);
11144 if (To->isDivergent() != From->isDivergent())
11146 } while (UI != UE && *UI == User);
11147 // We are iterating over all uses of the From node, so if a use
11148 // doesn't use the specific value, no changes are made.
11149 if (!UserRemovedFromCSEMaps)
11150 continue;
11151
11152 // Now that we have modified User, add it back to the CSE maps. If it
11153 // already exists there, recursively merge the results together.
11154 AddModifiedNodeToCSEMaps(User);
11155 }
11156
11157 // If we just RAUW'd the root, take note.
11158 if (From == getRoot())
11159 setRoot(To);
11160}
11161
11162namespace {
11163
11164/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11165/// to record information about a use.
11166struct UseMemo {
11167 SDNode *User;
11168 unsigned Index;
11169 SDUse *Use;
11170};
11171
11172/// operator< - Sort Memos by User.
11173bool operator<(const UseMemo &L, const UseMemo &R) {
11174 return (intptr_t)L.User < (intptr_t)R.User;
11175}
11176
11177/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11178/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11179/// the node already has been taken care of recursively.
11180class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11182
11183 void NodeDeleted(SDNode *N, SDNode *E) override {
11184 for (UseMemo &Memo : Uses)
11185 if (Memo.User == N)
11186 Memo.User = nullptr;
11187 }
11188
11189public:
11190 RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses)
11191 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11192};
11193
11194} // end anonymous namespace
11195
11197 if (TLI->isSDNodeAlwaysUniform(N)) {
11198 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11199 "Conflicting divergence information!");
11200 return false;
11201 }
11202 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11203 return true;
11204 for (const auto &Op : N->ops()) {
11205 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
11206 return true;
11207 }
11208 return false;
11209}
11210
11212 SmallVector<SDNode *, 16> Worklist(1, N);
11213 do {
11214 N = Worklist.pop_back_val();
11215 bool IsDivergent = calculateDivergence(N);
11216 if (N->SDNodeBits.IsDivergent != IsDivergent) {
11217 N->SDNodeBits.IsDivergent = IsDivergent;
11218 llvm::append_range(Worklist, N->uses());
11219 }
11220 } while (!Worklist.empty());
11221}
11222
11223void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11225 Order.reserve(AllNodes.size());
11226 for (auto &N : allnodes()) {
11227 unsigned NOps = N.getNumOperands();
11228 Degree[&N] = NOps;
11229 if (0 == NOps)
11230 Order.push_back(&N);
11231 }
11232 for (size_t I = 0; I != Order.size(); ++I) {
11233 SDNode *N = Order[I];
11234 for (auto *U : N->uses()) {
11235 unsigned &UnsortedOps = Degree[U];
11236 if (0 == --UnsortedOps)
11237 Order.push_back(U);
11238 }
11239 }
11240}
11241
11242#ifndef NDEBUG
11244 std::vector<SDNode *> TopoOrder;
11245 CreateTopologicalOrder(TopoOrder);
11246 for (auto *N : TopoOrder) {
11247 assert(calculateDivergence(N) == N->isDivergent() &&
11248 "Divergence bit inconsistency detected");
11249 }
11250}
11251#endif
11252
11253/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11254/// uses of other values produced by From.getNode() alone. The same value
11255/// may appear in both the From and To list. The Deleted vector is
11256/// handled the same way as for ReplaceAllUsesWith.
11258 const SDValue *To,
11259 unsigned Num){
11260 // Handle the simple, trivial case efficiently.
11261 if (Num == 1)
11262 return ReplaceAllUsesOfValueWith(*From, *To);
11263
11264 transferDbgValues(*From, *To);
11265 copyExtraInfo(From->getNode(), To->getNode());
11266
11267 // Read up all the uses and make records of them. This helps
11268 // processing new uses that are introduced during the
11269 // replacement process.
11271 for (unsigned i = 0; i != Num; ++i) {
11272 unsigned FromResNo = From[i].getResNo();
11273 SDNode *FromNode = From[i].getNode();
11274 for (SDNode::use_iterator UI = FromNode->use_begin(),
11275 E = FromNode->use_end(); UI != E; ++UI) {
11276 SDUse &Use = UI.getUse();
11277 if (Use.getResNo() == FromResNo) {
11278 UseMemo Memo = { *UI, i, &Use };
11279 Uses.push_back(Memo);
11280 }
11281 }
11282 }
11283
11284 // Sort the uses, so that all the uses from a given User are together.
11286 RAUOVWUpdateListener Listener(*this, Uses);
11287
11288 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
11289 UseIndex != UseIndexEnd; ) {
11290 // We know that this user uses some value of From. If it is the right
11291 // value, update it.
11292 SDNode *User = Uses[UseIndex].User;
11293 // If the node has been deleted by recursive CSE updates when updating
11294 // another node, then just skip this entry.
11295 if (User == nullptr) {
11296 ++UseIndex;
11297 continue;
11298 }
11299
11300 // This node is about to morph, remove its old self from the CSE maps.
11301 RemoveNodeFromCSEMaps(User);
11302
11303 // The Uses array is sorted, so all the uses for a given User
11304 // are next to each other in the list.
11305 // To help reduce the number of CSE recomputations, process all
11306 // the uses of this user that we can find this way.
11307 do {
11308 unsigned i = Uses[UseIndex].Index;
11309 SDUse &Use = *Uses[UseIndex].Use;
11310 ++UseIndex;
11311
11312 Use.set(To[i]);
11313 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
11314
11315 // Now that we have modified User, add it back to the CSE maps. If it
11316 // already exists there, recursively merge the results together.
11317 AddModifiedNodeToCSEMaps(User);
11318 }
11319}
11320
11321/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
11322/// based on their topological order. It returns the maximum id and a vector
11323/// of the SDNodes* in assigned order by reference.
11325 unsigned DAGSize = 0;
11326
11327 // SortedPos tracks the progress of the algorithm. Nodes before it are
11328 // sorted, nodes after it are unsorted. When the algorithm completes
11329 // it is at the end of the list.
11330 allnodes_iterator SortedPos = allnodes_begin();
11331
11332 // Visit all the nodes. Move nodes with no operands to the front of
11333 // the list immediately. Annotate nodes that do have operands with their
11334 // operand count. Before we do this, the Node Id fields of the nodes
11335 // may contain arbitrary values. After, the Node Id fields for nodes
11336 // before SortedPos will contain the topological sort index, and the
11337 // Node Id fields for nodes At SortedPos and after will contain the
11338 // count of outstanding operands.
11340 checkForCycles(&N, this);
11341 unsigned Degree = N.getNumOperands();
11342 if (Degree == 0) {
11343 // A node with no uses, add it to the result array immediately.
11344 N.setNodeId(DAGSize++);
11345 allnodes_iterator Q(&N);
11346 if (Q != SortedPos)
11347 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
11348 assert(SortedPos != AllNodes.end() && "Overran node list");
11349 ++SortedPos;
11350 } else {
11351 // Temporarily use the Node Id as scratch space for the degree count.
11352 N.setNodeId(Degree);
11353 }
11354 }
11355
11356 // Visit all the nodes. As we iterate, move nodes into sorted order,
11357 // such that by the time the end is reached all nodes will be sorted.
11358 for (SDNode &Node : allnodes()) {
11359 SDNode *N = &Node;
11360 checkForCycles(N, this);
11361 // N is in sorted position, so all its uses have one less operand
11362 // that needs to be sorted.
11363 for (SDNode *P : N->uses()) {
11364 unsigned Degree = P->getNodeId();
11365 assert(Degree != 0 && "Invalid node degree");
11366 --Degree;
11367 if (Degree == 0) {
11368 // All of P's operands are sorted, so P may sorted now.
11369 P->setNodeId(DAGSize++);
11370 if (P->getIterator() != SortedPos)
11371 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
11372 assert(SortedPos != AllNodes.end() && "Overran node list");
11373 ++SortedPos;
11374 } else {
11375 // Update P's outstanding operand count.
11376 P->setNodeId(Degree);
11377 }
11378 }
11379 if (Node.getIterator() == SortedPos) {
11380#ifndef NDEBUG
11382 SDNode *S = &*++I;
11383 dbgs() << "Overran sorted position:\n";
11384 S->dumprFull(this); dbgs() << "\n";
11385 dbgs() << "Checking if this is due to cycles\n";
11386 checkForCycles(this, true);
11387#endif
11388 llvm_unreachable(nullptr);
11389 }
11390 }
11391
11392 assert(SortedPos == AllNodes.end() &&
11393 "Topological sort incomplete!");
11394 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
11395 "First node in topological sort is not the entry token!");
11396 assert(AllNodes.front().getNodeId() == 0 &&
11397 "First node in topological sort has non-zero id!");
11398 assert(AllNodes.front().getNumOperands() == 0 &&
11399 "First node in topological sort has operands!");
11400 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
11401 "Last node in topologic sort has unexpected id!");
11402 assert(AllNodes.back().use_empty() &&
11403 "Last node in topologic sort has users!");
11404 assert(DAGSize == allnodes_size() && "Node count mismatch!");
11405 return DAGSize;
11406}
11407
11408/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
11409/// value is produced by SD.
11410void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
11411 for (SDNode *SD : DB->getSDNodes()) {
11412 if (!SD)
11413 continue;
11414 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
11415 SD->setHasDebugValue(true);
11416 }
11417 DbgInfo->add(DB, isParameter);
11418}
11419
11420void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
11421
11423 SDValue NewMemOpChain) {
11424 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
11425 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
11426 // The new memory operation must have the same position as the old load in
11427 // terms of memory dependency. Create a TokenFactor for the old load and new
11428 // memory operation and update uses of the old load's output chain to use that
11429 // TokenFactor.
11430 if (OldChain == NewMemOpChain || OldChain.use_empty())
11431 return NewMemOpChain;
11432
11433 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
11434 OldChain, NewMemOpChain);
11435 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
11436 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
11437 return TokenFactor;
11438}
11439
11441 SDValue NewMemOp) {
11442 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
11443 SDValue OldChain = SDValue(OldLoad, 1);
11444 SDValue NewMemOpChain = NewMemOp.getValue(1);
11445 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
11446}
11447
11449 Function **OutFunction) {
11450 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
11451
11452 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
11453 auto *Module = MF->getFunction().getParent();
11454 auto *Function = Module->getFunction(Symbol);
11455
11456 if (OutFunction != nullptr)
11457 *OutFunction = Function;
11458
11459 if (Function != nullptr) {
11460 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
11461 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
11462 }
11463
11464 std::string ErrorStr;
11465 raw_string_ostream ErrorFormatter(ErrorStr);
11466 ErrorFormatter << "Undefined external symbol ";
11467 ErrorFormatter << '"' << Symbol << '"';
11468 report_fatal_error(Twine(ErrorFormatter.str()));
11469}
11470
11471//===----------------------------------------------------------------------===//
11472// SDNode Class
11473//===----------------------------------------------------------------------===//
11474
11476 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11477 return Const != nullptr && Const->isZero();
11478}
11479
11481 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
11482 return Const != nullptr && Const->isZero() && !Const->isNegative();
11483}
11484
11486 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11487 return Const != nullptr && Const->isAllOnes();
11488}
11489
11491 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11492 return Const != nullptr && Const->isOne();
11493}
11494
11496 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11497 return Const != nullptr && Const->isMinSignedValue();
11498}
11499
11500bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
11501 unsigned OperandNo) {
11502 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
11503 // TODO: Target-specific opcodes could be added.
11504 if (auto *Const = isConstOrConstSplat(V)) {
11505 switch (Opcode) {
11506 case ISD::ADD:
11507 case ISD::OR:
11508 case ISD::XOR:
11509 case ISD::UMAX:
11510 return Const->isZero();
11511 case ISD::MUL:
11512 return Const->isOne();
11513 case ISD::AND:
11514 case ISD::UMIN:
11515 return Const->isAllOnes();
11516 case ISD::SMAX:
11517 return Const->isMinSignedValue();
11518 case ISD::SMIN:
11519 return Const->isMaxSignedValue();
11520 case ISD::SUB:
11521 case ISD::SHL:
11522 case ISD::SRA:
11523 case ISD::SRL:
11524 return OperandNo == 1 && Const->isZero();
11525 case ISD::UDIV:
11526 case ISD::SDIV:
11527 return OperandNo == 1 && Const->isOne();
11528 }
11529 } else if (auto *ConstFP = isConstOrConstSplatFP(V)) {
11530 switch (Opcode) {
11531 case ISD::FADD:
11532 return ConstFP->isZero() &&
11533 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
11534 case ISD::FSUB:
11535 return OperandNo == 1 && ConstFP->isZero() &&
11536 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11537 case ISD::FMUL:
11538 return ConstFP->isExactlyValue(1.0);
11539 case ISD::FDIV:
11540 return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
11541 case ISD::FMINNUM:
11542 case ISD::FMAXNUM: {
11543 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
11544 EVT VT = V.getValueType();
11546 APFloat NeutralAF = !Flags.hasNoNaNs()
11547 ? APFloat::getQNaN(Semantics)
11548 : !Flags.hasNoInfs()
11549 ? APFloat::getInf(Semantics)
11550 : APFloat::getLargest(Semantics);
11551 if (Opcode == ISD::FMAXNUM)
11552 NeutralAF.changeSign();
11553
11554 return ConstFP->isExactlyValue(NeutralAF);
11555 }
11556 }
11557 }
11558 return false;
11559}
11560
11562 while (V.getOpcode() == ISD::BITCAST)
11563 V = V.getOperand(0);
11564 return V;
11565}
11566
11568 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
11569 V = V.getOperand(0);
11570 return V;
11571}
11572
11574 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
11575 V = V.getOperand(0);
11576 return V;
11577}
11578
11580 while (V.getOpcode() == ISD::TRUNCATE)
11581 V = V.getOperand(0);
11582 return V;
11583}
11584
11585bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
11586 if (V.getOpcode() != ISD::XOR)
11587 return false;
11588 V = peekThroughBitcasts(V.getOperand(1));
11589 unsigned NumBits = V.getScalarValueSizeInBits();
11590 ConstantSDNode *C =
11591 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
11592 return C && (C->getAPIntValue().countr_one() >= NumBits);
11593}
11594
11596 bool AllowTruncation) {
11597 EVT VT = N.getValueType();
11598 APInt DemandedElts = VT.isFixedLengthVector()
11600 : APInt(1, 1);
11601 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
11602}
11603
11605 bool AllowUndefs,
11606 bool AllowTruncation) {
11607 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
11608 return CN;
11609
11610 // SplatVectors can truncate their operands. Ignore that case here unless
11611 // AllowTruncation is set.
11612 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
11613 EVT VecEltVT = N->getValueType(0).getVectorElementType();
11614 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
11615 EVT CVT = CN->getValueType(0);
11616 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
11617 if (AllowTruncation || CVT == VecEltVT)
11618 return CN;
11619 }
11620 }
11621
11622 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11623 BitVector UndefElements;
11624 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
11625
11626 // BuildVectors can truncate their operands. Ignore that case here unless
11627 // AllowTruncation is set.
11628 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11629 if (CN && (UndefElements.none() || AllowUndefs)) {
11630 EVT CVT = CN->getValueType(0);
11631 EVT NSVT = N.getValueType().getScalarType();
11632 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
11633 if (AllowTruncation || (CVT == NSVT))
11634 return CN;
11635 }
11636 }
11637
11638 return nullptr;
11639}
11640
11642 EVT VT = N.getValueType();
11643 APInt DemandedElts = VT.isFixedLengthVector()
11645 : APInt(1, 1);
11646 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
11647}
11648
11650 const APInt &DemandedElts,
11651 bool AllowUndefs) {
11652 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
11653 return CN;
11654
11655 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11656 BitVector UndefElements;
11657 ConstantFPSDNode *CN =
11658 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
11659 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11660 if (CN && (UndefElements.none() || AllowUndefs))
11661 return CN;
11662 }
11663
11664 if (N.getOpcode() == ISD::SPLAT_VECTOR)
11665 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
11666 return CN;
11667
11668 return nullptr;
11669}
11670
11671bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
11672 // TODO: may want to use peekThroughBitcast() here.
11673 ConstantSDNode *C =
11674 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
11675 return C && C->isZero();
11676}
11677
11678bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
11679 ConstantSDNode *C =
11680 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
11681 return C && C->isOne();
11682}
11683
11684bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
11686 unsigned BitWidth = N.getScalarValueSizeInBits();
11687 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
11688 return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
11689}
11690
11692 DropOperands();
11693}
11694
11695GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
11696 const DebugLoc &DL,
11697 const GlobalValue *GA, EVT VT,
11698 int64_t o, unsigned TF)
11699 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
11700 TheGlobal = GA;
11701}
11702
11704 EVT VT, unsigned SrcAS,
11705 unsigned DestAS)
11706 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
11707 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
11708
11709MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
11710 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
11711 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
11712 MemSDNodeBits.IsVolatile = MMO->isVolatile();
11713 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
11714 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
11715 MemSDNodeBits.IsInvariant = MMO->isInvariant();
11716
11717 // We check here that the size of the memory operand fits within the size of
11718 // the MMO. This is because the MMO might indicate only a possible address
11719 // range instead of specifying the affected memory addresses precisely.
11720 // TODO: Make MachineMemOperands aware of scalable vectors.
11722 "Size mismatch!");
11723}
11724
11725/// Profile - Gather unique data for the node.
11726///
11728 AddNodeIDNode(ID, this);
11729}
11730
11731namespace {
11732
11733 struct EVTArray {
11734 std::vector<EVT> VTs;
11735
11736 EVTArray() {
11737 VTs.reserve(MVT::VALUETYPE_SIZE);
11738 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
11739 VTs.push_back(MVT((MVT::SimpleValueType)i));
11740 }
11741 };
11742
11743} // end anonymous namespace
11744
11745/// getValueTypeList - Return a pointer to the specified value type.
11746///
11747const EVT *SDNode::getValueTypeList(EVT VT) {
11748 static std::set<EVT, EVT::compareRawBits> EVTs;
11749 static EVTArray SimpleVTArray;
11750 static sys::SmartMutex<true> VTMutex;
11751
11752 if (VT.isExtended()) {
11753 sys::SmartScopedLock<true> Lock(VTMutex);
11754 return &(*EVTs.insert(VT).first);
11755 }
11756 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
11757 return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
11758}
11759
11760/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
11761/// indicated value. This method ignores uses of other values defined by this
11762/// operation.
11763bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
11764 assert(Value < getNumValues() && "Bad value!");
11765
11766 // TODO: Only iterate over uses of a given value of the node
11767 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
11768 if (UI.getUse().getResNo() == Value) {
11769 if (NUses == 0)
11770 return false;
11771 --NUses;
11772 }
11773 }
11774
11775 // Found exactly the right number of uses?
11776 return NUses == 0;
11777}
11778
11779/// hasAnyUseOfValue - Return true if there are any use of the indicated
11780/// value. This method ignores uses of other values defined by this operation.
11781bool SDNode::hasAnyUseOfValue(unsigned Value) const {
11782 assert(Value < getNumValues() && "Bad value!");
11783
11784 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
11785 if (UI.getUse().getResNo() == Value)
11786 return true;
11787
11788 return false;
11789}
11790
11791/// isOnlyUserOf - Return true if this node is the only use of N.
11792bool SDNode::isOnlyUserOf(const SDNode *N) const {
11793 bool Seen = false;
11794 for (const SDNode *User : N->uses()) {
11795 if (User == this)
11796 Seen = true;
11797 else
11798 return false;
11799 }
11800
11801 return Seen;
11802}
11803
11804/// Return true if the only users of N are contained in Nodes.
11806 bool Seen = false;
11807 for (const SDNode *User : N->uses()) {
11808 if (llvm::is_contained(Nodes, User))
11809 Seen = true;
11810 else
11811 return false;
11812 }
11813
11814 return Seen;
11815}
11816
11817/// isOperand - Return true if this node is an operand of N.
11818bool SDValue::isOperandOf(const SDNode *N) const {
11819 return is_contained(N->op_values(), *this);
11820}
11821
11822bool SDNode::isOperandOf(const SDNode *N) const {
11823 return any_of(N->op_values(),
11824 [this](SDValue Op) { return this == Op.getNode(); });
11825}
11826
11827/// reachesChainWithoutSideEffects - Return true if this operand (which must
11828/// be a chain) reaches the specified operand without crossing any
11829/// side-effecting instructions on any chain path. In practice, this looks
11830/// through token factors and non-volatile loads. In order to remain efficient,
11831/// this only looks a couple of nodes in, it does not do an exhaustive search.
11832///
11833/// Note that we only need to examine chains when we're searching for
11834/// side-effects; SelectionDAG requires that all side-effects are represented
11835/// by chains, even if another operand would force a specific ordering. This
11836/// constraint is necessary to allow transformations like splitting loads.
11838 unsigned Depth) const {
11839 if (*this == Dest) return true;
11840
11841 // Don't search too deeply, we just want to be able to see through
11842 // TokenFactor's etc.
11843 if (Depth == 0) return false;
11844
11845 // If this is a token factor, all inputs to the TF happen in parallel.
11846 if (getOpcode() == ISD::TokenFactor) {
11847 // First, try a shallow search.
11848 if (is_contained((*this)->ops(), Dest)) {
11849 // We found the chain we want as an operand of this TokenFactor.
11850 // Essentially, we reach the chain without side-effects if we could
11851 // serialize the TokenFactor into a simple chain of operations with
11852 // Dest as the last operation. This is automatically true if the
11853 // chain has one use: there are no other ordering constraints.
11854 // If the chain has more than one use, we give up: some other
11855 // use of Dest might force a side-effect between Dest and the current
11856 // node.
11857 if (Dest.hasOneUse())
11858 return true;
11859 }
11860 // Next, try a deep search: check whether every operand of the TokenFactor
11861 // reaches Dest.
11862 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
11863 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
11864 });
11865 }
11866
11867 // Loads don't have side effects, look through them.
11868 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
11869 if (Ld->isUnordered())
11870 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
11871 }
11872 return false;
11873}
11874
11875bool SDNode::hasPredecessor(const SDNode *N) const {
11878 Worklist.push_back(this);
11879 return hasPredecessorHelper(N, Visited, Worklist);
11880}
11881
11883 this->Flags.intersectWith(Flags);
11884}
11885
11886SDValue
11888 ArrayRef<ISD::NodeType> CandidateBinOps,
11889 bool AllowPartials) {
11890 // The pattern must end in an extract from index 0.
11891 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
11892 !isNullConstant(Extract->getOperand(1)))
11893 return SDValue();
11894
11895 // Match against one of the candidate binary ops.
11896 SDValue Op = Extract->getOperand(0);
11897 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
11898 return Op.getOpcode() == unsigned(BinOp);
11899 }))
11900 return SDValue();
11901
11902 // Floating-point reductions may require relaxed constraints on the final step
11903 // of the reduction because they may reorder intermediate operations.
11904 unsigned CandidateBinOp = Op.getOpcode();
11905 if (Op.getValueType().isFloatingPoint()) {
11906 SDNodeFlags Flags = Op->getFlags();
11907 switch (CandidateBinOp) {
11908 case ISD::FADD:
11909 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
11910 return SDValue();
11911 break;
11912 default:
11913 llvm_unreachable("Unhandled FP opcode for binop reduction");
11914 }
11915 }
11916
11917 // Matching failed - attempt to see if we did enough stages that a partial
11918 // reduction from a subvector is possible.
11919 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
11920 if (!AllowPartials || !Op)
11921 return SDValue();
11922 EVT OpVT = Op.getValueType();
11923 EVT OpSVT = OpVT.getScalarType();
11924 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
11925 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
11926 return SDValue();
11927 BinOp = (ISD::NodeType)CandidateBinOp;
11928 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
11930 };
11931
11932 // At each stage, we're looking for something that looks like:
11933 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
11934 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
11935 // i32 undef, i32 undef, i32 undef, i32 undef>
11936 // %a = binop <8 x i32> %op, %s
11937 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
11938 // we expect something like:
11939 // <4,5,6,7,u,u,u,u>
11940 // <2,3,u,u,u,u,u,u>
11941 // <1,u,u,u,u,u,u,u>
11942 // While a partial reduction match would be:
11943 // <2,3,u,u,u,u,u,u>
11944 // <1,u,u,u,u,u,u,u>
11945 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
11946 SDValue PrevOp;
11947 for (unsigned i = 0; i < Stages; ++i) {
11948 unsigned MaskEnd = (1 << i);
11949
11950 if (Op.getOpcode() != CandidateBinOp)
11951 return PartialReduction(PrevOp, MaskEnd);
11952
11953 SDValue Op0 = Op.getOperand(0);
11954 SDValue Op1 = Op.getOperand(1);
11955
11956 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
11957 if (Shuffle) {
11958 Op = Op1;
11959 } else {
11960 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
11961 Op = Op0;
11962 }
11963
11964 // The first operand of the shuffle should be the same as the other operand
11965 // of the binop.
11966 if (!Shuffle || Shuffle->getOperand(0) != Op)
11967 return PartialReduction(PrevOp, MaskEnd);
11968
11969 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
11970 for (int Index = 0; Index < (int)MaskEnd; ++Index)
11971 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
11972 return PartialReduction(PrevOp, MaskEnd);
11973
11974 PrevOp = Op;
11975 }
11976
11977 // Handle subvector reductions, which tend to appear after the shuffle
11978 // reduction stages.
11979 while (Op.getOpcode() == CandidateBinOp) {
11980 unsigned NumElts = Op.getValueType().getVectorNumElements();
11981 SDValue Op0 = Op.getOperand(0);
11982 SDValue Op1 = Op.getOperand(1);
11983 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
11985 Op0.getOperand(0) != Op1.getOperand(0))
11986 break;
11987 SDValue Src = Op0.getOperand(0);
11988 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
11989 if (NumSrcElts != (2 * NumElts))
11990 break;
11991 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
11992 Op1.getConstantOperandAPInt(1) == NumElts) &&
11993 !(Op1.getConstantOperandAPInt(1) == 0 &&
11994 Op0.getConstantOperandAPInt(1) == NumElts))
11995 break;
11996 Op = Src;
11997 }
11998
11999 BinOp = (ISD::NodeType)CandidateBinOp;
12000 return Op;
12001}
12002
12004 EVT VT = N->getValueType(0);
12005 EVT EltVT = VT.getVectorElementType();
12006 unsigned NE = VT.getVectorNumElements();
12007
12008 SDLoc dl(N);
12009
12010 // If ResNE is 0, fully unroll the vector op.
12011 if (ResNE == 0)
12012 ResNE = NE;
12013 else if (NE > ResNE)
12014 NE = ResNE;
12015
12016 if (N->getNumValues() == 2) {
12017 SmallVector<SDValue, 8> Scalars0, Scalars1;
12018 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12019 EVT VT1 = N->getValueType(1);
12020 EVT EltVT1 = VT1.getVectorElementType();
12021
12022 unsigned i;
12023 for (i = 0; i != NE; ++i) {
12024 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12025 SDValue Operand = N->getOperand(j);
12026 EVT OperandVT = Operand.getValueType();
12027
12028 // A vector operand; extract a single element.
12029 EVT OperandEltVT = OperandVT.getVectorElementType();
12030 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12031 Operand, getVectorIdxConstant(i, dl));
12032 }
12033
12034 SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);
12035 Scalars0.push_back(EltOp);
12036 Scalars1.push_back(EltOp.getValue(1));
12037 }
12038
12039 SDValue Vec0 = getBuildVector(VT, dl, Scalars0);
12040 SDValue Vec1 = getBuildVector(VT1, dl, Scalars1);
12041 return getMergeValues({Vec0, Vec1}, dl);
12042 }
12043
12044 assert(N->getNumValues() == 1 &&
12045 "Can't unroll a vector with multiple results!");
12046
12048 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12049
12050 unsigned i;
12051 for (i= 0; i != NE; ++i) {
12052 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12053 SDValue Operand = N->getOperand(j);
12054 EVT OperandVT = Operand.getValueType();
12055 if (OperandVT.isVector()) {
12056 // A vector operand; extract a single element.
12057 EVT OperandEltVT = OperandVT.getVectorElementType();
12058 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12059 Operand, getVectorIdxConstant(i, dl));
12060 } else {
12061 // A scalar operand; just use it as is.
12062 Operands[j] = Operand;
12063 }
12064 }
12065
12066 switch (N->getOpcode()) {
12067 default: {
12068 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
12069 N->getFlags()));
12070 break;
12071 }
12072 case ISD::VSELECT:
12073 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
12074 break;
12075 case ISD::SHL:
12076 case ISD::SRA:
12077 case ISD::SRL:
12078 case ISD::ROTL:
12079 case ISD::ROTR:
12080 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
12082 Operands[1])));
12083 break;
12085 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
12086 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
12087 Operands[0],
12088 getValueType(ExtVT)));
12089 }
12090 }
12091 }
12092
12093 for (; i < ResNE; ++i)
12094 Scalars.push_back(getUNDEF(EltVT));
12095
12096 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
12097 return getBuildVector(VecVT, dl, Scalars);
12098}
12099
12100std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12101 SDNode *N, unsigned ResNE) {
12102 unsigned Opcode = N->getOpcode();
12103 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12104 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12105 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12106 "Expected an overflow opcode");
12107
12108 EVT ResVT = N->getValueType(0);
12109 EVT OvVT = N->getValueType(1);
12110 EVT ResEltVT = ResVT.getVectorElementType();
12111 EVT OvEltVT = OvVT.getVectorElementType();
12112 SDLoc dl(N);
12113
12114 // If ResNE is 0, fully unroll the vector op.
12115 unsigned NE = ResVT.getVectorNumElements();
12116 if (ResNE == 0)
12117 ResNE = NE;
12118 else if (NE > ResNE)
12119 NE = ResNE;
12120
12121 SmallVector<SDValue, 8> LHSScalars;
12122 SmallVector<SDValue, 8> RHSScalars;
12123 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
12124 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
12125
12126 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
12127 SDVTList VTs = getVTList(ResEltVT, SVT);
12128 SmallVector<SDValue, 8> ResScalars;
12129 SmallVector<SDValue, 8> OvScalars;
12130 for (unsigned i = 0; i < NE; ++i) {
12131 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12132 SDValue Ov =
12133 getSelect(dl, OvEltVT, Res.getValue(1),
12134 getBoolConstant(true, dl, OvEltVT, ResVT),
12135 getConstant(0, dl, OvEltVT));
12136
12137 ResScalars.push_back(Res);
12138 OvScalars.push_back(Ov);
12139 }
12140
12141 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
12142 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
12143
12144 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
12145 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
12146 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
12147 getBuildVector(NewOvVT, dl, OvScalars));
12148}
12149
12152 unsigned Bytes,
12153 int Dist) const {
12154 if (LD->isVolatile() || Base->isVolatile())
12155 return false;
12156 // TODO: probably too restrictive for atomics, revisit
12157 if (!LD->isSimple())
12158 return false;
12159 if (LD->isIndexed() || Base->isIndexed())
12160 return false;
12161 if (LD->getChain() != Base->getChain())
12162 return false;
12163 EVT VT = LD->getMemoryVT();
12164 if (VT.getSizeInBits() / 8 != Bytes)
12165 return false;
12166
12167 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
12168 auto LocDecomp = BaseIndexOffset::match(LD, *this);
12169
12170 int64_t Offset = 0;
12171 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
12172 return (Dist * (int64_t)Bytes == Offset);
12173 return false;
12174}
12175
12176/// InferPtrAlignment - Infer alignment of a load / store address. Return
12177/// std::nullopt if it cannot be inferred.
12179 // If this is a GlobalAddress + cst, return the alignment.
12180 const GlobalValue *GV = nullptr;
12181 int64_t GVOffset = 0;
12182 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
12183 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12184 KnownBits Known(PtrWidth);
12186 unsigned AlignBits = Known.countMinTrailingZeros();
12187 if (AlignBits)
12188 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
12189 }
12190
12191 // If this is a direct reference to a stack slot, use information about the
12192 // stack slot's alignment.
12193 int FrameIdx = INT_MIN;
12194 int64_t FrameOffset = 0;
12195 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
12196 FrameIdx = FI->getIndex();
12197 } else if (isBaseWithConstantOffset(Ptr) &&
12198 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
12199 // Handle FI+Cst
12200 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
12201 FrameOffset = Ptr.getConstantOperandVal(1);
12202 }
12203
12204 if (FrameIdx != INT_MIN) {
12206 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
12207 }
12208
12209 return std::nullopt;
12210}
12211
12212/// Split the scalar node with EXTRACT_ELEMENT using the provided
12213/// VTs and return the low/high part.
12214std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12215 const SDLoc &DL,
12216 const EVT &LoVT,
12217 const EVT &HiVT) {
12218 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12219 "Split node must be a scalar type");
12220 SDValue Lo =
12222 SDValue Hi =
12224 return std::make_pair(Lo, Hi);
12225}
12226
12227/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12228/// which is split (or expanded) into two not necessarily identical pieces.
12229std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12230 // Currently all types are split in half.
12231 EVT LoVT, HiVT;
12232 if (!VT.isVector())
12233 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
12234 else
12235 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
12236
12237 return std::make_pair(LoVT, HiVT);
12238}
12239
12240/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12241/// type, dependent on an enveloping VT that has been split into two identical
12242/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12243std::pair<EVT, EVT>
12245 bool *HiIsEmpty) const {
12246 EVT EltTp = VT.getVectorElementType();
12247 // Examples:
12248 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
12249 // custom VL=9 with enveloping VL=8/8 yields 8/1
12250 // custom VL=10 with enveloping VL=8/8 yields 8/2
12251 // etc.
12252 ElementCount VTNumElts = VT.getVectorElementCount();
12253 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12254 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12255 "Mixing fixed width and scalable vectors when enveloping a type");
12256 EVT LoVT, HiVT;
12257 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12258 LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12259 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
12260 *HiIsEmpty = false;
12261 } else {
12262 // Flag that hi type has zero storage size, but return split envelop type
12263 // (this would be easier if vector types with zero elements were allowed).
12264 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
12265 HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12266 *HiIsEmpty = true;
12267 }
12268 return std::make_pair(LoVT, HiVT);
12269}
12270
12271/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12272/// low/high part.
12273std::pair<SDValue, SDValue>
12274SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
12275 const EVT &HiVT) {
12276 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
12277 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
12278 "Splitting vector with an invalid mixture of fixed and scalable "
12279 "vector types");
12281 N.getValueType().getVectorMinNumElements() &&
12282 "More vector elements requested than available!");
12283 SDValue Lo, Hi;
12284 Lo =
12286 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
12287 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
12288 // IDX with the runtime scaling factor of the result vector type. For
12289 // fixed-width result vectors, that runtime scaling factor is 1.
12292 return std::make_pair(Lo, Hi);
12293}
12294
12295std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
12296 const SDLoc &DL) {
12297 // Split the vector length parameter.
12298 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
12299 EVT VT = N.getValueType();
12301 "Expecting the mask to be an evenly-sized vector");
12302 unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
12303 SDValue HalfNumElts =
12304 VecVT.isFixedLengthVector()
12305 ? getConstant(HalfMinNumElts, DL, VT)
12306 : getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
12307 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);
12308 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);
12309 return std::make_pair(Lo, Hi);
12310}
12311
12312/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
12314 EVT VT = N.getValueType();
12317 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
12319}
12320
12323 unsigned Start, unsigned Count,
12324 EVT EltVT) {
12325 EVT VT = Op.getValueType();
12326 if (Count == 0)
12327 Count = VT.getVectorNumElements();
12328 if (EltVT == EVT())
12329 EltVT = VT.getVectorElementType();
12330 SDLoc SL(Op);
12331 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
12332 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
12333 getVectorIdxConstant(i, SL)));
12334 }
12335}
12336
12337// getAddressSpace - Return the address space this GlobalAddress belongs to.
12339 return getGlobal()->getType()->getAddressSpace();
12340}
12341
12344 return Val.MachineCPVal->getType();
12345 return Val.ConstVal->getType();
12346}
12347
12348bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
12349 unsigned &SplatBitSize,
12350 bool &HasAnyUndefs,
12351 unsigned MinSplatBits,
12352 bool IsBigEndian) const {
12353 EVT VT = getValueType(0);
12354 assert(VT.isVector() && "Expected a vector type");
12355 unsigned VecWidth = VT.getSizeInBits();
12356 if (MinSplatBits > VecWidth)
12357 return false;
12358
12359 // FIXME: The widths are based on this node's type, but build vectors can
12360 // truncate their operands.
12361 SplatValue = APInt(VecWidth, 0);
12362 SplatUndef = APInt(VecWidth, 0);
12363
12364 // Get the bits. Bits with undefined values (when the corresponding element
12365 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
12366 // in SplatValue. If any of the values are not constant, give up and return
12367 // false.
12368 unsigned int NumOps = getNumOperands();
12369 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
12370 unsigned EltWidth = VT.getScalarSizeInBits();
12371
12372 for (unsigned j = 0; j < NumOps; ++j) {
12373 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12374 SDValue OpVal = getOperand(i);
12375 unsigned BitPos = j * EltWidth;
12376
12377 if (OpVal.isUndef())
12378 SplatUndef.setBits(BitPos, BitPos + EltWidth);
12379 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
12380 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
12381 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
12382 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
12383 else
12384 return false;
12385 }
12386
12387 // The build_vector is all constants or undefs. Find the smallest element
12388 // size that splats the vector.
12389 HasAnyUndefs = (SplatUndef != 0);
12390
12391 // FIXME: This does not work for vectors with elements less than 8 bits.
12392 while (VecWidth > 8) {
12393 // If we can't split in half, stop here.
12394 if (VecWidth & 1)
12395 break;
12396
12397 unsigned HalfSize = VecWidth / 2;
12398 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
12399 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
12400 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
12401 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
12402
12403 // If the two halves do not match (ignoring undef bits), stop here.
12404 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12405 MinSplatBits > HalfSize)
12406 break;
12407
12408 SplatValue = HighValue | LowValue;
12409 SplatUndef = HighUndef & LowUndef;
12410
12411 VecWidth = HalfSize;
12412 }
12413
12414 // FIXME: The loop above only tries to split in halves. But if the input
12415 // vector for example is <3 x i16> it wouldn't be able to detect a
12416 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
12417 // optimizations. I guess that back in the days when this helper was created
12418 // vectors normally was power-of-2 sized.
12419
12420 SplatBitSize = VecWidth;
12421 return true;
12422}
12423
12425 BitVector *UndefElements) const {
12426 unsigned NumOps = getNumOperands();
12427 if (UndefElements) {
12428 UndefElements->clear();
12429 UndefElements->resize(NumOps);
12430 }
12431 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12432 if (!DemandedElts)
12433 return SDValue();
12434 SDValue Splatted;
12435 for (unsigned i = 0; i != NumOps; ++i) {
12436 if (!DemandedElts[i])
12437 continue;
12438 SDValue Op = getOperand(i);
12439 if (Op.isUndef()) {
12440 if (UndefElements)
12441 (*UndefElements)[i] = true;
12442 } else if (!Splatted) {
12443 Splatted = Op;
12444 } else if (Splatted != Op) {
12445 return SDValue();
12446 }
12447 }
12448
12449 if (!Splatted) {
12450 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
12451 assert(getOperand(FirstDemandedIdx).isUndef() &&
12452 "Can only have a splat without a constant for all undefs.");
12453 return getOperand(FirstDemandedIdx);
12454 }
12455
12456 return Splatted;
12457}
12458
12460 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12461 return getSplatValue(DemandedElts, UndefElements);
12462}
12463
12465 SmallVectorImpl<SDValue> &Sequence,
12466 BitVector *UndefElements) const {
12467 unsigned NumOps = getNumOperands();
12468 Sequence.clear();
12469 if (UndefElements) {
12470 UndefElements->clear();
12471 UndefElements->resize(NumOps);
12472 }
12473 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12474 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
12475 return false;
12476
12477 // Set the undefs even if we don't find a sequence (like getSplatValue).
12478 if (UndefElements)
12479 for (unsigned I = 0; I != NumOps; ++I)
12480 if (DemandedElts[I] && getOperand(I).isUndef())
12481 (*UndefElements)[I] = true;
12482
12483 // Iteratively widen the sequence length looking for repetitions.
12484 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12485 Sequence.append(SeqLen, SDValue());
12486 for (unsigned I = 0; I != NumOps; ++I) {
12487 if (!DemandedElts[I])
12488 continue;
12489 SDValue &SeqOp = Sequence[I % SeqLen];
12491 if (Op.isUndef()) {
12492 if (!SeqOp)
12493 SeqOp = Op;
12494 continue;
12495 }
12496 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
12497 Sequence.clear();
12498 break;
12499 }
12500 SeqOp = Op;
12501 }
12502 if (!Sequence.empty())
12503 return true;
12504 }
12505
12506 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
12507 return false;
12508}
12509
12511 BitVector *UndefElements) const {
12512 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12513 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
12514}
12515
12518 BitVector *UndefElements) const {
12519 return dyn_cast_or_null<ConstantSDNode>(
12520 getSplatValue(DemandedElts, UndefElements));
12521}
12522
12525 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
12526}
12527
12530 BitVector *UndefElements) const {
12531 return dyn_cast_or_null<ConstantFPSDNode>(
12532 getSplatValue(DemandedElts, UndefElements));
12533}
12534
12537 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
12538}
12539
12540int32_t
12542 uint32_t BitWidth) const {
12543 if (ConstantFPSDNode *CN =
12544 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
12545 bool IsExact;
12546 APSInt IntVal(BitWidth);
12547 const APFloat &APF = CN->getValueAPF();
12548 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
12549 APFloat::opOK ||
12550 !IsExact)
12551 return -1;
12552
12553 return IntVal.exactLogBase2();
12554 }
12555 return -1;
12556}
12557
12559 bool IsLittleEndian, unsigned DstEltSizeInBits,
12560 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
12561 // Early-out if this contains anything but Undef/Constant/ConstantFP.
12562 if (!isConstant())
12563 return false;
12564
12565 unsigned NumSrcOps = getNumOperands();
12566 unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();
12567 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12568 "Invalid bitcast scale");
12569
12570 // Extract raw src bits.
12571 SmallVector<APInt> SrcBitElements(NumSrcOps,
12572 APInt::getZero(SrcEltSizeInBits));
12573 BitVector SrcUndeElements(NumSrcOps, false);
12574
12575 for (unsigned I = 0; I != NumSrcOps; ++I) {
12577 if (Op.isUndef()) {
12578 SrcUndeElements.set(I);
12579 continue;
12580 }
12581 auto *CInt = dyn_cast<ConstantSDNode>(Op);
12582 auto *CFP = dyn_cast<ConstantFPSDNode>(Op);
12583 assert((CInt || CFP) && "Unknown constant");
12584 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
12585 : CFP->getValueAPF().bitcastToAPInt();
12586 }
12587
12588 // Recast to dst width.
12589 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
12590 SrcBitElements, UndefElements, SrcUndeElements);
12591 return true;
12592}
12593
12594void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
12595 unsigned DstEltSizeInBits,
12596 SmallVectorImpl<APInt> &DstBitElements,
12597 ArrayRef<APInt> SrcBitElements,
12598 BitVector &DstUndefElements,
12599 const BitVector &SrcUndefElements) {
12600 unsigned NumSrcOps = SrcBitElements.size();
12601 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12602 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12603 "Invalid bitcast scale");
12604 assert(NumSrcOps == SrcUndefElements.size() &&
12605 "Vector size mismatch");
12606
12607 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
12608 DstUndefElements.clear();
12609 DstUndefElements.resize(NumDstOps, false);
12610 DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));
12611
12612 // Concatenate src elements constant bits together into dst element.
12613 if (SrcEltSizeInBits <= DstEltSizeInBits) {
12614 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
12615 for (unsigned I = 0; I != NumDstOps; ++I) {
12616 DstUndefElements.set(I);
12617 APInt &DstBits = DstBitElements[I];
12618 for (unsigned J = 0; J != Scale; ++J) {
12619 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12620 if (SrcUndefElements[Idx])
12621 continue;
12622 DstUndefElements.reset(I);
12623 const APInt &SrcBits = SrcBitElements[Idx];
12624 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
12625 "Illegal constant bitwidths");
12626 DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);
12627 }
12628 }
12629 return;
12630 }
12631
12632 // Split src element constant bits into dst elements.
12633 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
12634 for (unsigned I = 0; I != NumSrcOps; ++I) {
12635 if (SrcUndefElements[I]) {
12636 DstUndefElements.set(I * Scale, (I + 1) * Scale);
12637 continue;
12638 }
12639 const APInt &SrcBits = SrcBitElements[I];
12640 for (unsigned J = 0; J != Scale; ++J) {
12641 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12642 APInt &DstBits = DstBitElements[Idx];
12643 DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
12644 }
12645 }
12646}
12647
12649 for (const SDValue &Op : op_values()) {
12650 unsigned Opc = Op.getOpcode();
12651 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
12652 return false;
12653 }
12654 return true;
12655}
12656
12657std::optional<std::pair<APInt, APInt>>
12659 unsigned NumOps = getNumOperands();
12660 if (NumOps < 2)
12661 return std::nullopt;
12662
12663 if (!isa<ConstantSDNode>(getOperand(0)) ||
12664 !isa<ConstantSDNode>(getOperand(1)))
12665 return std::nullopt;
12666
12667 unsigned EltSize = getValueType(0).getScalarSizeInBits();
12668 APInt Start = getConstantOperandAPInt(0).trunc(EltSize);
12669 APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start;
12670
12671 if (Stride.isZero())
12672 return std::nullopt;
12673
12674 for (unsigned i = 2; i < NumOps; ++i) {
12675 if (!isa<ConstantSDNode>(getOperand(i)))
12676 return std::nullopt;
12677
12678 APInt Val = getConstantOperandAPInt(i).trunc(EltSize);
12679 if (Val != (Start + (Stride * i)))
12680 return std::nullopt;
12681 }
12682
12683 return std::make_pair(Start, Stride);
12684}
12685
12686bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
12687 // Find the first non-undef value in the shuffle mask.
12688 unsigned i, e;
12689 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
12690 /* search */;
12691
12692 // If all elements are undefined, this shuffle can be considered a splat
12693 // (although it should eventually get simplified away completely).
12694 if (i == e)
12695 return true;
12696
12697 // Make sure all remaining elements are either undef or the same as the first
12698 // non-undef value.
12699 for (int Idx = Mask[i]; i != e; ++i)
12700 if (Mask[i] >= 0 && Mask[i] != Idx)
12701 return false;
12702 return true;
12703}
12704
12705// Returns the SDNode if it is a constant integer BuildVector
12706// or constant integer.
12708 if (isa<ConstantSDNode>(N))
12709 return N.getNode();
12711 return N.getNode();
12712 // Treat a GlobalAddress supporting constant offset folding as a
12713 // constant integer.
12714 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
12715 if (GA->getOpcode() == ISD::GlobalAddress &&
12716 TLI->isOffsetFoldingLegal(GA))
12717 return GA;
12718 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12719 isa<ConstantSDNode>(N.getOperand(0)))
12720 return N.getNode();
12721 return nullptr;
12722}
12723
12724// Returns the SDNode if it is a constant float BuildVector
12725// or constant float.
12727 if (isa<ConstantFPSDNode>(N))
12728 return N.getNode();
12729
12731 return N.getNode();
12732
12733 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12734 isa<ConstantFPSDNode>(N.getOperand(0)))
12735 return N.getNode();
12736
12737 return nullptr;
12738}
12739
12740void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
12741 assert(!Node->OperandList && "Node already has operands");
12743 "too many operands to fit into SDNode");
12744 SDUse *Ops = OperandRecycler.allocate(
12745 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
12746
12747 bool IsDivergent = false;
12748 for (unsigned I = 0; I != Vals.size(); ++I) {
12749 Ops[I].setUser(Node);
12750 Ops[I].setInitial(Vals[I]);
12751 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
12752 IsDivergent |= Ops[I].getNode()->isDivergent();
12753 }
12754 Node->NumOperands = Vals.size();
12755 Node->OperandList = Ops;
12756 if (!TLI->isSDNodeAlwaysUniform(Node)) {
12757 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
12758 Node->SDNodeBits.IsDivergent = IsDivergent;
12759 }
12761}
12762
12765 size_t Limit = SDNode::getMaxNumOperands();
12766 while (Vals.size() > Limit) {
12767 unsigned SliceIdx = Vals.size() - Limit;
12768 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
12769 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
12770 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
12771 Vals.emplace_back(NewTF);
12772 }
12773 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
12774}
12775
12777 EVT VT, SDNodeFlags Flags) {
12778 switch (Opcode) {
12779 default:
12780 return SDValue();
12781 case ISD::ADD:
12782 case ISD::OR:
12783 case ISD::XOR:
12784 case ISD::UMAX:
12785 return getConstant(0, DL, VT);
12786 case ISD::MUL:
12787 return getConstant(1, DL, VT);
12788 case ISD::AND:
12789 case ISD::UMIN:
12790 return getAllOnesConstant(DL, VT);
12791 case ISD::SMAX:
12793 case ISD::SMIN:
12795 case ISD::FADD:
12796 return getConstantFP(-0.0, DL, VT);
12797 case ISD::FMUL:
12798 return getConstantFP(1.0, DL, VT);
12799 case ISD::FMINNUM:
12800 case ISD::FMAXNUM: {
12801 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12802 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12803 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
12804 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
12805 APFloat::getLargest(Semantics);
12806 if (Opcode == ISD::FMAXNUM)
12807 NeutralAF.changeSign();
12808
12809 return getConstantFP(NeutralAF, DL, VT);
12810 }
12811 case ISD::FMINIMUM:
12812 case ISD::FMAXIMUM: {
12813 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
12814 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12815 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
12816 : APFloat::getLargest(Semantics);
12817 if (Opcode == ISD::FMAXIMUM)
12818 NeutralAF.changeSign();
12819
12820 return getConstantFP(NeutralAF, DL, VT);
12821 }
12822
12823 }
12824}
12825
12826/// Helper used to make a call to a library function that has one argument of
12827/// pointer type.
12828///
12829/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
12830/// used to get or set floating-point state. They have one argument of pointer
12831/// type, which points to the memory region containing bits of the
12832/// floating-point state. The value returned by such function is ignored in the
12833/// created call.
12834///
12835/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
12836/// \param Ptr Pointer used to save/load state.
12837/// \param InChain Ingoing token chain.
12838/// \returns Outgoing chain token.
12840 SDValue InChain,
12841 const SDLoc &DLoc) {
12842 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
12845 Entry.Node = Ptr;
12846 Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext());
12847 Args.push_back(Entry);
12848 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
12849 SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),
12850 TLI->getPointerTy(getDataLayout()));
12852 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
12853 TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee,
12854 std::move(Args));
12855 return TLI->LowerCallTo(CLI).second;
12856}
12857
12859 assert(From && To && "Invalid SDNode; empty source SDValue?");
12860 auto I = SDEI.find(From);
12861 if (I == SDEI.end())
12862 return;
12863
12864 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
12865 // the iterator, hence the need to make a copy to prevent a use-after-free.
12866 NodeExtraInfo NEI = I->second;
12867 if (LLVM_LIKELY(!NEI.PCSections)) {
12868 // No deep copy required for the types of extra info set.
12869 //
12870 // FIXME: Investigate if other types of extra info also need deep copy. This
12871 // depends on the types of nodes they can be attached to: if some extra info
12872 // is only ever attached to nodes where a replacement To node is always the
12873 // node where later use and propagation of the extra info has the intended
12874 // semantics, no deep copy is required.
12875 SDEI[To] = std::move(NEI);
12876 return;
12877 }
12878
12879 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
12880 // through the replacement of From with To. Otherwise, replacements of a node
12881 // (From) with more complex nodes (To and its operands) may result in lost
12882 // extra info where the root node (To) is insignificant in further propagating
12883 // and using extra info when further lowering to MIR.
12884 //
12885 // In the first step pre-populate the visited set with the nodes reachable
12886 // from the old From node. This avoids copying NodeExtraInfo to parts of the
12887 // DAG that is not new and should be left untouched.
12888 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
12889 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
12890 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
12891 if (MaxDepth == 0) {
12892 // Remember this node in case we need to increase MaxDepth and continue
12893 // populating FromReach from this node.
12894 Leafs.emplace_back(N);
12895 return;
12896 }
12897 if (!FromReach.insert(N).second)
12898 return;
12899 for (const SDValue &Op : N->op_values())
12900 Self(Self, Op.getNode(), MaxDepth - 1);
12901 };
12902
12903 // Copy extra info to To and all its transitive operands (that are new).
12905 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
12906 if (FromReach.contains(N))
12907 return true;
12908 if (!Visited.insert(N).second)
12909 return true;
12910 if (getEntryNode().getNode() == N)
12911 return false;
12912 for (const SDValue &Op : N->op_values()) {
12913 if (!Self(Self, Op.getNode()))
12914 return false;
12915 }
12916 // Copy only if entry node was not reached.
12917 SDEI[N] = NEI;
12918 return true;
12919 };
12920
12921 // We first try with a lower MaxDepth, assuming that the path to common
12922 // operands between From and To is relatively short. This significantly
12923 // improves performance in the common case. The initial MaxDepth is big
12924 // enough to avoid retry in the common case; the last MaxDepth is large
12925 // enough to avoid having to use the fallback below (and protects from
12926 // potential stack exhaustion from recursion).
12927 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
12928 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
12929 // StartFrom is the previous (or initial) set of leafs reachable at the
12930 // previous maximum depth.
12932 std::swap(StartFrom, Leafs);
12933 for (const SDNode *N : StartFrom)
12934 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
12935 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
12936 return;
12937 // This should happen very rarely (reached the entry node).
12938 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
12939 assert(!Leafs.empty());
12940 }
12941
12942 // This should not happen - but if it did, that means the subgraph reachable
12943 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
12944 // could not visit all reachable common operands. Consequently, we were able
12945 // to reach the entry node.
12946 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
12947 assert(false && "From subgraph too complex - increase max. MaxDepth?");
12948 // Best-effort fallback if assertions disabled.
12949 SDEI[To] = std::move(NEI);
12950}
12951
12952#ifndef NDEBUG
12953static void checkForCyclesHelper(const SDNode *N,
12956 const llvm::SelectionDAG *DAG) {
12957 // If this node has already been checked, don't check it again.
12958 if (Checked.count(N))
12959 return;
12960
12961 // If a node has already been visited on this depth-first walk, reject it as
12962 // a cycle.
12963 if (!Visited.insert(N).second) {
12964 errs() << "Detected cycle in SelectionDAG\n";
12965 dbgs() << "Offending node:\n";
12966 N->dumprFull(DAG); dbgs() << "\n";
12967 abort();
12968 }
12969
12970 for (const SDValue &Op : N->op_values())
12971 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
12972
12973 Checked.insert(N);
12974 Visited.erase(N);
12975}
12976#endif
12977
12979 const llvm::SelectionDAG *DAG,
12980 bool force) {
12981#ifndef NDEBUG
12982 bool check = force;
12983#ifdef EXPENSIVE_CHECKS
12984 check = true;
12985#endif // EXPENSIVE_CHECKS
12986 if (check) {
12987 assert(N && "Checking nonexistent SDNode");
12990 checkForCyclesHelper(N, visited, checked, DAG);
12991 }
12992#endif // !NDEBUG
12993}
12994
12995void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
12996 checkForCycles(DAG->getRoot().getNode(), DAG, force);
12997}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
Definition: Compiler.h:460
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:240
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Given that RA is a live value
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines a hash set that can be used to remove duplication of nodes in a graph.
Rewrite Partial Register Uses
#define check(cond)
static const unsigned MaxDepth
static LVOptions Options
Definition: LVOptions.cpp:25
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:531
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
mir Rename Register Operands
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, SDVTList VTList, ArrayRef< SDValue > OpList)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static cl::opt< bool > EnableMemCpyDAGOpt("enable-memcpy-dag-opt", cl::Hidden, cl::init(true), cl::desc("Gang up loads and stores generated by inlining of memcpy"))
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList)
AddNodeIDValueTypes - Value type lists are intern'd so we can represent them solely with their pointe...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA)
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)
AddNodeIDOpcode - Add the node opcode to the NodeID data.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static cl::opt< int > MaxLdStGlue("ldstmemcpy-glue-max", cl::desc("Number limit for gluing ld/st of memcpy."), cl::Hidden, cl::init(0))
static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef< SDValue > Ops)
AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static void VerifySDNode(SDNode *N)
VerifySDNode - Check the given SDNode. Aborts if it is invalid.
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N)
If this is an SDNode with special info, add this info to the NodeID data.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS)
static Constant * ConstantFold(Instruction *I, const DataLayout &DL, const SmallDenseMap< Value *, Constant * > &ConstantPool)
Try to fold instruction I into a constant.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition: APFloat.h:988
opStatus divide(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1069
void copySign(const APFloat &RHS)
Definition: APFloat.h:1163
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5196
opStatus subtract(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1051
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: APFloat.h:1278
opStatus add(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1042
bool isFinite() const
Definition: APFloat.h:1300
bool isNaN() const
Definition: APFloat.h:1293
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition: APFloat.h:1193
opStatus multiply(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1060
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
Definition: APFloat.h:1096
bool isZero() const
Definition: APFloat.h:1291
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1006
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1185
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
opStatus mod(const APFloat &RHS)
Definition: APFloat.h:1087
bool isPosZero() const
Definition: APFloat.h:1306
bool isNegZero() const
Definition: APFloat.h:1307
void changeSign()
Definition: APFloat.h:1158
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Definition: APFloat.h:977
bool isInfinity() const
Definition: APFloat.h:1292
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt umul_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1977
APInt usub_sat(const APInt &RHS) const
Definition: APInt.cpp:2061
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1579
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1491
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
Definition: APInt.h:1370
unsigned popcount() const
Count the number of bits set.
Definition: APInt.h:1620
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
Definition: APInt.h:1364
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
Definition: APInt.cpp:608
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1002
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1463
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1308
APInt abs() const
Get the absolute value.
Definition: APInt.h:1737
APInt sadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2032
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:349
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:358
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition: APInt.cpp:1672
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1439
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1089
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:307
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1650
void clearAllBits()
Set every bit to 0.
Definition: APInt.h:1375
APInt rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
Definition: APInt.cpp:1124
APInt reverseBits() const
Definition: APInt.cpp:737
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:812
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition: APInt.h:1144
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1589
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
Definition: APInt.h:1578
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1548
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:620
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
APInt sshl_sat(const APInt &RHS) const
Definition: APInt.cpp:2092
APInt ushl_sat(const APInt &RHS) const
Definition: APInt.cpp:2106
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
APInt rotl(unsigned rotateAmt) const
Rotate left by rotateAmt.
Definition: APInt.cpp:1111
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:368
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
Definition: APInt.h:1395
unsigned logBase2() const
Definition: APInt.h:1703
APInt uadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2042
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:805
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1297
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
Definition: APInt.cpp:1742
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1128
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:954
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1345
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:851
APInt byteSwap() const
Definition: APInt.cpp:715
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1235
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:178
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
Definition: APInt.h:1367
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:453
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition: APInt.h:1215
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:836
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:829
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1199
APInt ssub_sat(const APInt &RHS) const
Definition: APInt.cpp:2051
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, unsigned SrcAS, unsigned DestAS)
Recycle small arrays allocated from a BumpPtrAllocator.
Definition: ArrayRecycler.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:195
This is an SDNode representing atomic operations.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
BitVector & reset()
Definition: BitVector.h:392
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
void clear()
clear - Removes all bits from the bitvector.
Definition: BitVector.h:335
BitVector & set()
Definition: BitVector.h:351
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:188
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
The address of a basic block.
Definition: Constants.h:888
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
std::optional< std::pair< APInt, APInt > > isConstantSequence() const
If this BuildVector is constant and represents the numerical series "<a, a+n, a+2n,...
SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
Definition: Allocator.h:148
void Reset()
Deallocate all but the current slab and reset the current pointer to the beginning of it,...
Definition: Allocator.h:123
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:267
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:147
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:144
bool isMachineConstantPoolEntry() const
This class represents a range of values.
Definition: ConstantRange.h:47
ConstantRange multiply(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
KnownBits toKnownBits() const
Return known bits for values in this range.
ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition: Constant.h:41
Constant * getSplatValue(bool AllowUndefs=false) const
If all elements of the vector constant have the same value, return that value.
Definition: Constants.cpp:1699
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:432
DWARF expression.
static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:238
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:878
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:865
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:763
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
A debug info location.
Definition: DebugLoc.h:33
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
const char * getSymbol() const
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Definition: FoldingSet.h:320
MachineBasicBlock * MBB
MBB - The current block.
Data structure describing the variable locations in a function.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:680
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:338
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:342
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
Definition: GlobalValue.h:263
unsigned getAddressSpace() const
Definition: GlobalValue.h:205
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
This class is used to form a handle around another node that is persistent and is updated across invo...
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate the offet and size that ar...
This class is used to represent ISD::LOAD nodes.
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
Machine Value Type.
SimpleValueType SimpleTy
static MVT getIntegerVT(unsigned BitWidth)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Abstract base class for all machine specific constantpool value subclasses.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
uint64_t getSize() const
Return the size in bytes of the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *MMO)
MachineMemOperand * MMO
Memory reference information.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
static uint64_t getSizeOrUnknown(const TypeSize &T)
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:191
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
The optimization diagnostic interface.
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Definition: DerivedTypes.h:679
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Definition: PointerUnion.h:142
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Keeps track of dbg_value information through SDISel.
Definition: SelectionDAG.h:159
BumpPtrAllocator & getAlloc()
Definition: SelectionDAG.h:188
void add(SDDbgValue *V, bool isParameter)
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node) const
Definition: SelectionDAG.h:194
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isDivergent() const
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
MemSDNodeBitfields MemSDNodeBits
void Profile(FoldingSetNodeID &ID) const
Gather unique data for the node.
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
void DropOperands()
Release the operands and set this node to have zero operands.
Represents a use of a SDNode.
SDNode * getNode() const
Convenience function for get().getNode().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const
Emit target-specific code that performs a memset.
virtual SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memmove.
virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memcpy.
SDNodeFlags getFlags() const
Definition: SelectionDAG.h:379
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:474
const APInt * getValidMaximumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
void updateDivergence(SDNode *N)
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
SDValue getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
bool isADDLike(SDValue Op) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
bool calculateDivergence(SDNode *N)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:387
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:531
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
Definition: SelectionDAG.h:659
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
const APInt * getValidShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has a constant or splat constant shift amount that is less than the element b...
SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, const SDLoc &DLoc)
Helper used to make a call to a library function that has one argument of pointer type.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, APInt StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
Definition: SelectionDAG.h:543
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, uint64_t Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
Definition: SelectionDAG.h:539
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:845
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing=false)
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
const APInt * getValidMinimumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:878
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:534
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void assign(size_type NumElts, ValueParamT Elt)
Definition: SmallVector.h:717
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void reserve(size_type N)
Definition: SmallVector.h:676
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
BooleanContent
Enum that describes how the target represents true/false values.
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
TargetOptions Options
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:542
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void set(Value *Val)
Definition: Value.h:882
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
Definition: TypeSize.h:176
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:224
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
SmartMutex - A mutex with a compile time constant parameter that indicates whether this mutex should ...
Definition: Mutex.h:28
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
Definition: APInt.h:2203
APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
Definition: APInt.cpp:3120
APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
Definition: APInt.cpp:3110
const APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
Definition: APInt.h:2198
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition: APInt.cpp:3011
APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
Definition: APInt.cpp:3105
APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
Definition: APInt.cpp:3115
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ TargetConstantPool
Definition: ISDOpcodes.h:168
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1171
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1276
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:497
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
Definition: ISDOpcodes.h:1060
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ TargetBlockAddress
Definition: ISDOpcodes.h:170
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:979
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:820
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1400
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1274
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
Definition: ISDOpcodes.h:1167
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1102
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:662
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ TargetExternalSymbol
Definition: ISDOpcodes.h:169
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ TargetJumpTable
Definition: ISDOpcodes.h:167
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
Definition: ISDOpcodes.h:177
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:758
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1275
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:646
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ RegisterMask
Definition: ISDOpcodes.h:75
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1277
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1266
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:978
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ TargetConstantFP
Definition: ISDOpcodes.h:159
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1329
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1248
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
@ ConstantPool
Definition: ISDOpcodes.h:82
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:809
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1304
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1191
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1279
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:158
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1007
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1324
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:657
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition: ISDOpcodes.h:636
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:831
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:907
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ BlockAddress
Definition: ISDOpcodes.h:84
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1278
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1012
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:313
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:165
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
unsigned getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1412
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed)
Returns true if the specified node is a vector where all elements can be truncated to the specified e...
bool matchUnaryPredicateImpl(SDValue Op, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1581
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
Definition: ISDOpcodes.h:1586
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1485
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1472
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:839
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:561
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
std::lock_guard< SmartMutex< mt_only > > SmartScopedLock
Definition: Mutex.h:69
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:361
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition: Utils.cpp:1456
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2415
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getAlign(const Function &F, unsigned index, unsigned &align)
bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition: bit.h:317
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition: APFloat.h:1436
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition: Utils.cpp:1438
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition: APFloat.h:1373
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 maximumNumber semantics.
Definition: APFloat.h:1410
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1745
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Other
Any other memory.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 minimumNumber semantics.
Definition: APFloat.h:1396
@ Mul
Product of integers.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1833
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1888
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:452
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:208
void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition: APFloat.h:1423
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:349
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
Definition: Metadata.h:780
MDNode * TBAA
The tag for type-based alias analysis.
Definition: Metadata.h:777
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:249
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition: APFloat.h:220
static constexpr roundingMode rmTowardNegative
Definition: APFloat.h:233
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:230
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:234
static const fltSemantics & IEEEquad() LLVM_READNONE
Definition: APFloat.cpp:251
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:250
static const fltSemantics & IEEEhalf() LLVM_READNONE
Definition: APFloat.cpp:247
static constexpr roundingMode rmTowardPositive
Definition: APFloat.h:232
static const fltSemantics & BFloat() LLVM_READNONE
Definition: APFloat.cpp:248
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:246
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Align previous() const
Definition: Alignment.h:88
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
intptr_t getRawBits() const
Definition: ValueTypes.h:495
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:246
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition: ValueTypes.h:141
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition: KnownBits.h:297
KnownBits sextInReg(unsigned SrcBitWidth) const
Return known bits for a in-register sign extension of the value we're tracking.
Definition: KnownBits.cpp:155
static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
Definition: KnownBits.cpp:872
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
Definition: KnownBits.h:251
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
Definition: KnownBits.cpp:208
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:104
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:77
void makeNonNegative()
Make this value non-negative.
Definition: KnownBits.h:120
static KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
Definition: KnownBits.cpp:761
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:238
static KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
Definition: KnownBits.cpp:422
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
Definition: KnownBits.cpp:1018
bool isUnknown() const
Returns true if we don't know any bits.
Definition: KnownBits.h:63
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:270
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
Definition: KnownBits.cpp:490
void makeNegative()
Make this value negative.
Definition: KnownBits.h:115
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition: KnownBits.h:157
KnownBits byteSwap() const
Definition: KnownBits.h:457
bool hasConflict() const
Returns true if there is conflicting information.
Definition: KnownBits.h:47
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:285
void setAllZero()
Make all bits known to be zero and discard any previous information.
Definition: KnownBits.h:89
KnownBits reverseBits() const
Definition: KnownBits.h:461
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
Definition: KnownBits.h:229
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
Definition: KnownBits.cpp:184
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition: KnownBits.h:168
void resetAll()
Resets the known state of all bits.
Definition: KnownBits.h:71
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
Definition: KnownBits.h:317
static KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
Definition: KnownBits.cpp:364
bool isNonZero() const
Returns true if this value is known to be non-zero.
Definition: KnownBits.h:107
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
Definition: KnownBits.h:221
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition: KnownBits.h:307
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition: KnownBits.h:176
static KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
Definition: KnownBits.cpp:143
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
Definition: KnownBits.h:192
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:141
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
Definition: KnownBits.cpp:221
static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
Definition: KnownBits.cpp:863
static KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
Definition: KnownBits.cpp:1037
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
Definition: KnownBits.cpp:976
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
Definition: KnownBits.cpp:57
static KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
Definition: KnownBits.cpp:917
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
Definition: KnownBits.h:328
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:101
static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
Definition: KnownBits.cpp:50
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition: KnownBits.h:276
void insertBits(const KnownBits &SubBits, unsigned BitPosition)
Insert the bits from a smaller known bits starting at bitPosition.
Definition: KnownBits.h:215
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
Definition: KnownBits.cpp:765
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
Definition: KnownBits.h:163
KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
Definition: KnownBits.cpp:544
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
Definition: KnownBits.cpp:279
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
Definition: KnownBits.cpp:202
This class contains a discriminated union of information about pointers in memory operands,...
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
These are IR-level optimization flags that may be propagated to SDNodes.
void intersectWith(const SDNodeFlags Flags)
Clear any flags in this flag set that aren't also set in Flags.
bool hasNonNeg() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned int NumVTs
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
Definition: SelectionDAG.h:307
DAGUpdateListener *const Next
Definition: SelectionDAG.h:308
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)