LLVM 19.0.0git
SelectionDAG.cpp
Go to the documentation of this file.
1//===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This implements the SelectionDAG class.
10//
11//===----------------------------------------------------------------------===//
12
14#include "SDNodeDbgValue.h"
15#include "llvm/ADT/APFloat.h"
16#include "llvm/ADT/APInt.h"
17#include "llvm/ADT/APSInt.h"
18#include "llvm/ADT/ArrayRef.h"
19#include "llvm/ADT/BitVector.h"
20#include "llvm/ADT/DenseSet.h"
21#include "llvm/ADT/FoldingSet.h"
22#include "llvm/ADT/STLExtras.h"
25#include "llvm/ADT/Twine.h"
50#include "llvm/IR/Constant.h"
52#include "llvm/IR/Constants.h"
53#include "llvm/IR/DataLayout.h"
55#include "llvm/IR/DebugLoc.h"
57#include "llvm/IR/Function.h"
58#include "llvm/IR/GlobalValue.h"
59#include "llvm/IR/Metadata.h"
60#include "llvm/IR/Type.h"
64#include "llvm/Support/Debug.h"
68#include "llvm/Support/Mutex.h"
74#include <algorithm>
75#include <cassert>
76#include <cstdint>
77#include <cstdlib>
78#include <limits>
79#include <set>
80#include <string>
81#include <utility>
82#include <vector>
83
84using namespace llvm;
85using namespace llvm::SDPatternMatch;
86
87/// makeVTList - Return an instance of the SDVTList struct initialized with the
88/// specified members.
89static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) {
90 SDVTList Res = {VTs, NumVTs};
91 return Res;
92}
93
94// Default null implementations of the callbacks.
98
99void SelectionDAG::DAGNodeDeletedListener::anchor() {}
100void SelectionDAG::DAGNodeInsertedListener::anchor() {}
101
102#define DEBUG_TYPE "selectiondag"
103
104static cl::opt<bool> EnableMemCpyDAGOpt("enable-memcpy-dag-opt",
105 cl::Hidden, cl::init(true),
106 cl::desc("Gang up loads and stores generated by inlining of memcpy"));
107
108static cl::opt<int> MaxLdStGlue("ldstmemcpy-glue-max",
109 cl::desc("Number limit for gluing ld/st of memcpy."),
110 cl::Hidden, cl::init(0));
111
113 LLVM_DEBUG(dbgs() << Msg; V.getNode()->dump(G););
114}
115
116//===----------------------------------------------------------------------===//
117// ConstantFPSDNode Class
118//===----------------------------------------------------------------------===//
119
120/// isExactlyValue - We don't rely on operator== working on double values, as
121/// it returns true for things that are clearly not equal, like -0.0 and 0.0.
122/// As such, this method can be used to do an exact bit-for-bit comparison of
123/// two floating point values.
125 return getValueAPF().bitwiseIsEqual(V);
126}
127
129 const APFloat& Val) {
130 assert(VT.isFloatingPoint() && "Can only convert between FP types");
131
132 // convert modifies in place, so make a copy.
133 APFloat Val2 = APFloat(Val);
134 bool losesInfo;
137 &losesInfo);
138 return !losesInfo;
139}
140
141//===----------------------------------------------------------------------===//
142// ISD Namespace
143//===----------------------------------------------------------------------===//
144
145bool ISD::isConstantSplatVector(const SDNode *N, APInt &SplatVal) {
146 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
147 unsigned EltSize =
148 N->getValueType(0).getVectorElementType().getSizeInBits();
149 if (auto *Op0 = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
150 SplatVal = Op0->getAPIntValue().trunc(EltSize);
151 return true;
152 }
153 if (auto *Op0 = dyn_cast<ConstantFPSDNode>(N->getOperand(0))) {
154 SplatVal = Op0->getValueAPF().bitcastToAPInt().trunc(EltSize);
155 return true;
156 }
157 }
158
159 auto *BV = dyn_cast<BuildVectorSDNode>(N);
160 if (!BV)
161 return false;
162
163 APInt SplatUndef;
164 unsigned SplatBitSize;
165 bool HasUndefs;
166 unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
167 // Endianness does not matter here. We are checking for a splat given the
168 // element size of the vector, and if we find such a splat for little endian
169 // layout, then that should be valid also for big endian (as the full vector
170 // size is known to be a multiple of the element size).
171 const bool IsBigEndian = false;
172 return BV->isConstantSplat(SplatVal, SplatUndef, SplatBitSize, HasUndefs,
173 EltSize, IsBigEndian) &&
174 EltSize == SplatBitSize;
175}
176
177// FIXME: AllOnes and AllZeros duplicate a lot of code. Could these be
178// specializations of the more general isConstantSplatVector()?
179
180bool ISD::isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly) {
181 // Look through a bit convert.
182 while (N->getOpcode() == ISD::BITCAST)
183 N = N->getOperand(0).getNode();
184
185 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
186 APInt SplatVal;
187 return isConstantSplatVector(N, SplatVal) && SplatVal.isAllOnes();
188 }
189
190 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
191
192 unsigned i = 0, e = N->getNumOperands();
193
194 // Skip over all of the undef values.
195 while (i != e && N->getOperand(i).isUndef())
196 ++i;
197
198 // Do not accept an all-undef vector.
199 if (i == e) return false;
200
201 // Do not accept build_vectors that aren't all constants or which have non-~0
202 // elements. We have to be a bit careful here, as the type of the constant
203 // may not be the same as the type of the vector elements due to type
204 // legalization (the elements are promoted to a legal type for the target and
205 // a vector of a type may be legal when the base element type is not).
206 // We only want to check enough bits to cover the vector elements, because
207 // we care if the resultant vector is all ones, not whether the individual
208 // constants are.
209 SDValue NotZero = N->getOperand(i);
210 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
211 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
212 if (CN->getAPIntValue().countr_one() < EltSize)
213 return false;
214 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
215 if (CFPN->getValueAPF().bitcastToAPInt().countr_one() < EltSize)
216 return false;
217 } else
218 return false;
219
220 // Okay, we have at least one ~0 value, check to see if the rest match or are
221 // undefs. Even with the above element type twiddling, this should be OK, as
222 // the same type legalization should have applied to all the elements.
223 for (++i; i != e; ++i)
224 if (N->getOperand(i) != NotZero && !N->getOperand(i).isUndef())
225 return false;
226 return true;
227}
228
229bool ISD::isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly) {
230 // Look through a bit convert.
231 while (N->getOpcode() == ISD::BITCAST)
232 N = N->getOperand(0).getNode();
233
234 if (!BuildVectorOnly && N->getOpcode() == ISD::SPLAT_VECTOR) {
235 APInt SplatVal;
236 return isConstantSplatVector(N, SplatVal) && SplatVal.isZero();
237 }
238
239 if (N->getOpcode() != ISD::BUILD_VECTOR) return false;
240
241 bool IsAllUndef = true;
242 for (const SDValue &Op : N->op_values()) {
243 if (Op.isUndef())
244 continue;
245 IsAllUndef = false;
246 // Do not accept build_vectors that aren't all constants or which have non-0
247 // elements. We have to be a bit careful here, as the type of the constant
248 // may not be the same as the type of the vector elements due to type
249 // legalization (the elements are promoted to a legal type for the target
250 // and a vector of a type may be legal when the base element type is not).
251 // We only want to check enough bits to cover the vector elements, because
252 // we care if the resultant vector is all zeros, not whether the individual
253 // constants are.
254 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
255 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Op)) {
256 if (CN->getAPIntValue().countr_zero() < EltSize)
257 return false;
258 } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Op)) {
259 if (CFPN->getValueAPF().bitcastToAPInt().countr_zero() < EltSize)
260 return false;
261 } else
262 return false;
263 }
264
265 // Do not accept an all-undef vector.
266 if (IsAllUndef)
267 return false;
268 return true;
269}
270
272 return isConstantSplatVectorAllOnes(N, /*BuildVectorOnly*/ true);
273}
274
276 return isConstantSplatVectorAllZeros(N, /*BuildVectorOnly*/ true);
277}
278
280 if (N->getOpcode() != ISD::BUILD_VECTOR)
281 return false;
282
283 for (const SDValue &Op : N->op_values()) {
284 if (Op.isUndef())
285 continue;
286 if (!isa<ConstantSDNode>(Op))
287 return false;
288 }
289 return true;
290}
291
293 if (N->getOpcode() != ISD::BUILD_VECTOR)
294 return false;
295
296 for (const SDValue &Op : N->op_values()) {
297 if (Op.isUndef())
298 continue;
299 if (!isa<ConstantFPSDNode>(Op))
300 return false;
301 }
302 return true;
303}
304
305bool ISD::isVectorShrinkable(const SDNode *N, unsigned NewEltSize,
306 bool Signed) {
307 assert(N->getValueType(0).isVector() && "Expected a vector!");
308
309 unsigned EltSize = N->getValueType(0).getScalarSizeInBits();
310 if (EltSize <= NewEltSize)
311 return false;
312
313 if (N->getOpcode() == ISD::ZERO_EXTEND) {
314 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
315 NewEltSize) &&
316 !Signed;
317 }
318 if (N->getOpcode() == ISD::SIGN_EXTEND) {
319 return (N->getOperand(0).getValueType().getScalarSizeInBits() <=
320 NewEltSize) &&
321 Signed;
322 }
323 if (N->getOpcode() != ISD::BUILD_VECTOR)
324 return false;
325
326 for (const SDValue &Op : N->op_values()) {
327 if (Op.isUndef())
328 continue;
329 if (!isa<ConstantSDNode>(Op))
330 return false;
331
332 APInt C = Op->getAsAPIntVal().trunc(EltSize);
333 if (Signed && C.trunc(NewEltSize).sext(EltSize) != C)
334 return false;
335 if (!Signed && C.trunc(NewEltSize).zext(EltSize) != C)
336 return false;
337 }
338
339 return true;
340}
341
343 // Return false if the node has no operands.
344 // This is "logically inconsistent" with the definition of "all" but
345 // is probably the desired behavior.
346 if (N->getNumOperands() == 0)
347 return false;
348 return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); });
349}
350
352 return N->getOpcode() == ISD::FREEZE && N->getOperand(0).isUndef();
353}
354
355template <typename ConstNodeType>
357 std::function<bool(ConstNodeType *)> Match,
358 bool AllowUndefs) {
359 // FIXME: Add support for scalar UNDEF cases?
360 if (auto *C = dyn_cast<ConstNodeType>(Op))
361 return Match(C);
362
363 // FIXME: Add support for vector UNDEF cases?
364 if (ISD::BUILD_VECTOR != Op.getOpcode() &&
365 ISD::SPLAT_VECTOR != Op.getOpcode())
366 return false;
367
368 EVT SVT = Op.getValueType().getScalarType();
369 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
370 if (AllowUndefs && Op.getOperand(i).isUndef()) {
371 if (!Match(nullptr))
372 return false;
373 continue;
374 }
375
376 auto *Cst = dyn_cast<ConstNodeType>(Op.getOperand(i));
377 if (!Cst || Cst->getValueType(0) != SVT || !Match(Cst))
378 return false;
379 }
380 return true;
381}
382// Build used template types.
383template bool ISD::matchUnaryPredicateImpl<ConstantSDNode>(
384 SDValue, std::function<bool(ConstantSDNode *)>, bool);
385template bool ISD::matchUnaryPredicateImpl<ConstantFPSDNode>(
386 SDValue, std::function<bool(ConstantFPSDNode *)>, bool);
387
389 SDValue LHS, SDValue RHS,
390 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
391 bool AllowUndefs, bool AllowTypeMismatch) {
392 if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType())
393 return false;
394
395 // TODO: Add support for scalar UNDEF cases?
396 if (auto *LHSCst = dyn_cast<ConstantSDNode>(LHS))
397 if (auto *RHSCst = dyn_cast<ConstantSDNode>(RHS))
398 return Match(LHSCst, RHSCst);
399
400 // TODO: Add support for vector UNDEF cases?
401 if (LHS.getOpcode() != RHS.getOpcode() ||
402 (LHS.getOpcode() != ISD::BUILD_VECTOR &&
403 LHS.getOpcode() != ISD::SPLAT_VECTOR))
404 return false;
405
406 EVT SVT = LHS.getValueType().getScalarType();
407 for (unsigned i = 0, e = LHS.getNumOperands(); i != e; ++i) {
408 SDValue LHSOp = LHS.getOperand(i);
409 SDValue RHSOp = RHS.getOperand(i);
410 bool LHSUndef = AllowUndefs && LHSOp.isUndef();
411 bool RHSUndef = AllowUndefs && RHSOp.isUndef();
412 auto *LHSCst = dyn_cast<ConstantSDNode>(LHSOp);
413 auto *RHSCst = dyn_cast<ConstantSDNode>(RHSOp);
414 if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef))
415 return false;
416 if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT ||
417 LHSOp.getValueType() != RHSOp.getValueType()))
418 return false;
419 if (!Match(LHSCst, RHSCst))
420 return false;
421 }
422 return true;
423}
424
426 switch (VecReduceOpcode) {
427 default:
428 llvm_unreachable("Expected VECREDUCE opcode");
431 case ISD::VP_REDUCE_FADD:
432 case ISD::VP_REDUCE_SEQ_FADD:
433 return ISD::FADD;
436 case ISD::VP_REDUCE_FMUL:
437 case ISD::VP_REDUCE_SEQ_FMUL:
438 return ISD::FMUL;
440 case ISD::VP_REDUCE_ADD:
441 return ISD::ADD;
443 case ISD::VP_REDUCE_MUL:
444 return ISD::MUL;
446 case ISD::VP_REDUCE_AND:
447 return ISD::AND;
449 case ISD::VP_REDUCE_OR:
450 return ISD::OR;
452 case ISD::VP_REDUCE_XOR:
453 return ISD::XOR;
455 case ISD::VP_REDUCE_SMAX:
456 return ISD::SMAX;
458 case ISD::VP_REDUCE_SMIN:
459 return ISD::SMIN;
461 case ISD::VP_REDUCE_UMAX:
462 return ISD::UMAX;
464 case ISD::VP_REDUCE_UMIN:
465 return ISD::UMIN;
467 case ISD::VP_REDUCE_FMAX:
468 return ISD::FMAXNUM;
470 case ISD::VP_REDUCE_FMIN:
471 return ISD::FMINNUM;
473 return ISD::FMAXIMUM;
475 return ISD::FMINIMUM;
476 }
477}
478
479bool ISD::isVPOpcode(unsigned Opcode) {
480 switch (Opcode) {
481 default:
482 return false;
483#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) \
484 case ISD::VPSD: \
485 return true;
486#include "llvm/IR/VPIntrinsics.def"
487 }
488}
489
490bool ISD::isVPBinaryOp(unsigned Opcode) {
491 switch (Opcode) {
492 default:
493 break;
494#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
495#define VP_PROPERTY_BINARYOP return true;
496#define END_REGISTER_VP_SDNODE(VPSD) break;
497#include "llvm/IR/VPIntrinsics.def"
498 }
499 return false;
500}
501
502bool ISD::isVPReduction(unsigned Opcode) {
503 switch (Opcode) {
504 default:
505 break;
506#define BEGIN_REGISTER_VP_SDNODE(VPSD, ...) case ISD::VPSD:
507#define VP_PROPERTY_REDUCTION(STARTPOS, ...) return true;
508#define END_REGISTER_VP_SDNODE(VPSD) break;
509#include "llvm/IR/VPIntrinsics.def"
510 }
511 return false;
512}
513
514/// The operand position of the vector mask.
515std::optional<unsigned> ISD::getVPMaskIdx(unsigned Opcode) {
516 switch (Opcode) {
517 default:
518 return std::nullopt;
519#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, ...) \
520 case ISD::VPSD: \
521 return MASKPOS;
522#include "llvm/IR/VPIntrinsics.def"
523 }
524}
525
526/// The operand position of the explicit vector length parameter.
527std::optional<unsigned> ISD::getVPExplicitVectorLengthIdx(unsigned Opcode) {
528 switch (Opcode) {
529 default:
530 return std::nullopt;
531#define BEGIN_REGISTER_VP_SDNODE(VPSD, LEGALPOS, TDNAME, MASKPOS, EVLPOS) \
532 case ISD::VPSD: \
533 return EVLPOS;
534#include "llvm/IR/VPIntrinsics.def"
535 }
536}
537
538std::optional<unsigned> ISD::getBaseOpcodeForVP(unsigned VPOpcode,
539 bool hasFPExcept) {
540 // FIXME: Return strict opcodes in case of fp exceptions.
541 switch (VPOpcode) {
542 default:
543 return std::nullopt;
544#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) case ISD::VPOPC:
545#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) return ISD::SDOPC;
546#define END_REGISTER_VP_SDNODE(VPOPC) break;
547#include "llvm/IR/VPIntrinsics.def"
548 }
549 return std::nullopt;
550}
551
552unsigned ISD::getVPForBaseOpcode(unsigned Opcode) {
553 switch (Opcode) {
554 default:
555 llvm_unreachable("can not translate this Opcode to VP.");
556#define BEGIN_REGISTER_VP_SDNODE(VPOPC, ...) break;
557#define VP_PROPERTY_FUNCTIONAL_SDOPC(SDOPC) case ISD::SDOPC:
558#define END_REGISTER_VP_SDNODE(VPOPC) return ISD::VPOPC;
559#include "llvm/IR/VPIntrinsics.def"
560 }
561}
562
564 switch (ExtType) {
565 case ISD::EXTLOAD:
566 return IsFP ? ISD::FP_EXTEND : ISD::ANY_EXTEND;
567 case ISD::SEXTLOAD:
568 return ISD::SIGN_EXTEND;
569 case ISD::ZEXTLOAD:
570 return ISD::ZERO_EXTEND;
571 default:
572 break;
573 }
574
575 llvm_unreachable("Invalid LoadExtType");
576}
577
579 // To perform this operation, we just need to swap the L and G bits of the
580 // operation.
581 unsigned OldL = (Operation >> 2) & 1;
582 unsigned OldG = (Operation >> 1) & 1;
583 return ISD::CondCode((Operation & ~6) | // Keep the N, U, E bits
584 (OldL << 1) | // New G bit
585 (OldG << 2)); // New L bit.
586}
587
589 unsigned Operation = Op;
590 if (isIntegerLike)
591 Operation ^= 7; // Flip L, G, E bits, but not U.
592 else
593 Operation ^= 15; // Flip all of the condition bits.
594
596 Operation &= ~8; // Don't let N and U bits get set.
597
598 return ISD::CondCode(Operation);
599}
600
602 return getSetCCInverseImpl(Op, Type.isInteger());
603}
604
606 bool isIntegerLike) {
607 return getSetCCInverseImpl(Op, isIntegerLike);
608}
609
610/// For an integer comparison, return 1 if the comparison is a signed operation
611/// and 2 if the result is an unsigned comparison. Return zero if the operation
612/// does not depend on the sign of the input (setne and seteq).
613static int isSignedOp(ISD::CondCode Opcode) {
614 switch (Opcode) {
615 default: llvm_unreachable("Illegal integer setcc operation!");
616 case ISD::SETEQ:
617 case ISD::SETNE: return 0;
618 case ISD::SETLT:
619 case ISD::SETLE:
620 case ISD::SETGT:
621 case ISD::SETGE: return 1;
622 case ISD::SETULT:
623 case ISD::SETULE:
624 case ISD::SETUGT:
625 case ISD::SETUGE: return 2;
626 }
627}
628
630 EVT Type) {
631 bool IsInteger = Type.isInteger();
632 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
633 // Cannot fold a signed integer setcc with an unsigned integer setcc.
634 return ISD::SETCC_INVALID;
635
636 unsigned Op = Op1 | Op2; // Combine all of the condition bits.
637
638 // If the N and U bits get set, then the resultant comparison DOES suddenly
639 // care about orderedness, and it is true when ordered.
640 if (Op > ISD::SETTRUE2)
641 Op &= ~16; // Clear the U bit if the N bit is set.
642
643 // Canonicalize illegal integer setcc's.
644 if (IsInteger && Op == ISD::SETUNE) // e.g. SETUGT | SETULT
645 Op = ISD::SETNE;
646
647 return ISD::CondCode(Op);
648}
649
651 EVT Type) {
652 bool IsInteger = Type.isInteger();
653 if (IsInteger && (isSignedOp(Op1) | isSignedOp(Op2)) == 3)
654 // Cannot fold a signed setcc with an unsigned setcc.
655 return ISD::SETCC_INVALID;
656
657 // Combine all of the condition bits.
658 ISD::CondCode Result = ISD::CondCode(Op1 & Op2);
659
660 // Canonicalize illegal integer setcc's.
661 if (IsInteger) {
662 switch (Result) {
663 default: break;
664 case ISD::SETUO : Result = ISD::SETFALSE; break; // SETUGT & SETULT
665 case ISD::SETOEQ: // SETEQ & SETU[LG]E
666 case ISD::SETUEQ: Result = ISD::SETEQ ; break; // SETUGE & SETULE
667 case ISD::SETOLT: Result = ISD::SETULT ; break; // SETULT & SETNE
668 case ISD::SETOGT: Result = ISD::SETUGT ; break; // SETUGT & SETNE
669 }
670 }
671
672 return Result;
673}
674
675//===----------------------------------------------------------------------===//
676// SDNode Profile Support
677//===----------------------------------------------------------------------===//
678
679/// AddNodeIDOpcode - Add the node opcode to the NodeID data.
680static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC) {
681 ID.AddInteger(OpC);
682}
683
684/// AddNodeIDValueTypes - Value type lists are intern'd so we can represent them
685/// solely with their pointer.
687 ID.AddPointer(VTList.VTs);
688}
689
690/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
692 ArrayRef<SDValue> Ops) {
693 for (const auto &Op : Ops) {
694 ID.AddPointer(Op.getNode());
695 ID.AddInteger(Op.getResNo());
696 }
697}
698
699/// AddNodeIDOperands - Various routines for adding operands to the NodeID data.
701 ArrayRef<SDUse> Ops) {
702 for (const auto &Op : Ops) {
703 ID.AddPointer(Op.getNode());
704 ID.AddInteger(Op.getResNo());
705 }
706}
707
708static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC,
709 SDVTList VTList, ArrayRef<SDValue> OpList) {
710 AddNodeIDOpcode(ID, OpC);
711 AddNodeIDValueTypes(ID, VTList);
712 AddNodeIDOperands(ID, OpList);
713}
714
715/// If this is an SDNode with special info, add this info to the NodeID data.
717 switch (N->getOpcode()) {
720 case ISD::MCSymbol:
721 llvm_unreachable("Should only be used on nodes with operands");
722 default: break; // Normal nodes don't need extra info.
724 case ISD::Constant: {
725 const ConstantSDNode *C = cast<ConstantSDNode>(N);
726 ID.AddPointer(C->getConstantIntValue());
727 ID.AddBoolean(C->isOpaque());
728 break;
729 }
731 case ISD::ConstantFP:
732 ID.AddPointer(cast<ConstantFPSDNode>(N)->getConstantFPValue());
733 break;
738 const GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(N);
739 ID.AddPointer(GA->getGlobal());
740 ID.AddInteger(GA->getOffset());
741 ID.AddInteger(GA->getTargetFlags());
742 break;
743 }
744 case ISD::BasicBlock:
745 ID.AddPointer(cast<BasicBlockSDNode>(N)->getBasicBlock());
746 break;
747 case ISD::Register:
748 ID.AddInteger(cast<RegisterSDNode>(N)->getReg());
749 break;
751 ID.AddPointer(cast<RegisterMaskSDNode>(N)->getRegMask());
752 break;
753 case ISD::SRCVALUE:
754 ID.AddPointer(cast<SrcValueSDNode>(N)->getValue());
755 break;
756 case ISD::FrameIndex:
758 ID.AddInteger(cast<FrameIndexSDNode>(N)->getIndex());
759 break;
762 if (cast<LifetimeSDNode>(N)->hasOffset()) {
763 ID.AddInteger(cast<LifetimeSDNode>(N)->getSize());
764 ID.AddInteger(cast<LifetimeSDNode>(N)->getOffset());
765 }
766 break;
768 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getGuid());
769 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getIndex());
770 ID.AddInteger(cast<PseudoProbeSDNode>(N)->getAttributes());
771 break;
772 case ISD::JumpTable:
774 ID.AddInteger(cast<JumpTableSDNode>(N)->getIndex());
775 ID.AddInteger(cast<JumpTableSDNode>(N)->getTargetFlags());
776 break;
779 const ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(N);
780 ID.AddInteger(CP->getAlign().value());
781 ID.AddInteger(CP->getOffset());
782 if (CP->isMachineConstantPoolEntry())
783 CP->getMachineCPVal()->addSelectionDAGCSEId(ID);
784 else
785 ID.AddPointer(CP->getConstVal());
786 ID.AddInteger(CP->getTargetFlags());
787 break;
788 }
789 case ISD::TargetIndex: {
790 const TargetIndexSDNode *TI = cast<TargetIndexSDNode>(N);
791 ID.AddInteger(TI->getIndex());
792 ID.AddInteger(TI->getOffset());
793 ID.AddInteger(TI->getTargetFlags());
794 break;
795 }
796 case ISD::LOAD: {
797 const LoadSDNode *LD = cast<LoadSDNode>(N);
798 ID.AddInteger(LD->getMemoryVT().getRawBits());
799 ID.AddInteger(LD->getRawSubclassData());
800 ID.AddInteger(LD->getPointerInfo().getAddrSpace());
801 ID.AddInteger(LD->getMemOperand()->getFlags());
802 break;
803 }
804 case ISD::STORE: {
805 const StoreSDNode *ST = cast<StoreSDNode>(N);
806 ID.AddInteger(ST->getMemoryVT().getRawBits());
807 ID.AddInteger(ST->getRawSubclassData());
808 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
809 ID.AddInteger(ST->getMemOperand()->getFlags());
810 break;
811 }
812 case ISD::VP_LOAD: {
813 const VPLoadSDNode *ELD = cast<VPLoadSDNode>(N);
814 ID.AddInteger(ELD->getMemoryVT().getRawBits());
815 ID.AddInteger(ELD->getRawSubclassData());
816 ID.AddInteger(ELD->getPointerInfo().getAddrSpace());
817 ID.AddInteger(ELD->getMemOperand()->getFlags());
818 break;
819 }
820 case ISD::VP_STORE: {
821 const VPStoreSDNode *EST = cast<VPStoreSDNode>(N);
822 ID.AddInteger(EST->getMemoryVT().getRawBits());
823 ID.AddInteger(EST->getRawSubclassData());
824 ID.AddInteger(EST->getPointerInfo().getAddrSpace());
825 ID.AddInteger(EST->getMemOperand()->getFlags());
826 break;
827 }
828 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD: {
829 const VPStridedLoadSDNode *SLD = cast<VPStridedLoadSDNode>(N);
830 ID.AddInteger(SLD->getMemoryVT().getRawBits());
831 ID.AddInteger(SLD->getRawSubclassData());
832 ID.AddInteger(SLD->getPointerInfo().getAddrSpace());
833 break;
834 }
835 case ISD::EXPERIMENTAL_VP_STRIDED_STORE: {
836 const VPStridedStoreSDNode *SST = cast<VPStridedStoreSDNode>(N);
837 ID.AddInteger(SST->getMemoryVT().getRawBits());
838 ID.AddInteger(SST->getRawSubclassData());
839 ID.AddInteger(SST->getPointerInfo().getAddrSpace());
840 break;
841 }
842 case ISD::VP_GATHER: {
843 const VPGatherSDNode *EG = cast<VPGatherSDNode>(N);
844 ID.AddInteger(EG->getMemoryVT().getRawBits());
845 ID.AddInteger(EG->getRawSubclassData());
846 ID.AddInteger(EG->getPointerInfo().getAddrSpace());
847 ID.AddInteger(EG->getMemOperand()->getFlags());
848 break;
849 }
850 case ISD::VP_SCATTER: {
851 const VPScatterSDNode *ES = cast<VPScatterSDNode>(N);
852 ID.AddInteger(ES->getMemoryVT().getRawBits());
853 ID.AddInteger(ES->getRawSubclassData());
854 ID.AddInteger(ES->getPointerInfo().getAddrSpace());
855 ID.AddInteger(ES->getMemOperand()->getFlags());
856 break;
857 }
858 case ISD::MLOAD: {
859 const MaskedLoadSDNode *MLD = cast<MaskedLoadSDNode>(N);
860 ID.AddInteger(MLD->getMemoryVT().getRawBits());
861 ID.AddInteger(MLD->getRawSubclassData());
862 ID.AddInteger(MLD->getPointerInfo().getAddrSpace());
863 ID.AddInteger(MLD->getMemOperand()->getFlags());
864 break;
865 }
866 case ISD::MSTORE: {
867 const MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N);
868 ID.AddInteger(MST->getMemoryVT().getRawBits());
869 ID.AddInteger(MST->getRawSubclassData());
870 ID.AddInteger(MST->getPointerInfo().getAddrSpace());
871 ID.AddInteger(MST->getMemOperand()->getFlags());
872 break;
873 }
874 case ISD::MGATHER: {
875 const MaskedGatherSDNode *MG = cast<MaskedGatherSDNode>(N);
876 ID.AddInteger(MG->getMemoryVT().getRawBits());
877 ID.AddInteger(MG->getRawSubclassData());
878 ID.AddInteger(MG->getPointerInfo().getAddrSpace());
879 ID.AddInteger(MG->getMemOperand()->getFlags());
880 break;
881 }
882 case ISD::MSCATTER: {
883 const MaskedScatterSDNode *MS = cast<MaskedScatterSDNode>(N);
884 ID.AddInteger(MS->getMemoryVT().getRawBits());
885 ID.AddInteger(MS->getRawSubclassData());
886 ID.AddInteger(MS->getPointerInfo().getAddrSpace());
887 ID.AddInteger(MS->getMemOperand()->getFlags());
888 break;
889 }
892 case ISD::ATOMIC_SWAP:
904 case ISD::ATOMIC_LOAD:
905 case ISD::ATOMIC_STORE: {
906 const AtomicSDNode *AT = cast<AtomicSDNode>(N);
907 ID.AddInteger(AT->getMemoryVT().getRawBits());
908 ID.AddInteger(AT->getRawSubclassData());
909 ID.AddInteger(AT->getPointerInfo().getAddrSpace());
910 ID.AddInteger(AT->getMemOperand()->getFlags());
911 break;
912 }
913 case ISD::VECTOR_SHUFFLE: {
914 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
915 for (int M : Mask)
916 ID.AddInteger(M);
917 break;
918 }
920 case ISD::BlockAddress: {
921 const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
922 ID.AddPointer(BA->getBlockAddress());
923 ID.AddInteger(BA->getOffset());
924 ID.AddInteger(BA->getTargetFlags());
925 break;
926 }
927 case ISD::AssertAlign:
928 ID.AddInteger(cast<AssertAlignSDNode>(N)->getAlign().value());
929 break;
930 case ISD::PREFETCH:
933 // Handled by MemIntrinsicSDNode check after the switch.
934 break;
935 } // end switch (N->getOpcode())
936
937 // MemIntrinsic nodes could also have subclass data, address spaces, and flags
938 // to check.
939 if (auto *MN = dyn_cast<MemIntrinsicSDNode>(N)) {
940 ID.AddInteger(MN->getRawSubclassData());
941 ID.AddInteger(MN->getPointerInfo().getAddrSpace());
942 ID.AddInteger(MN->getMemOperand()->getFlags());
943 ID.AddInteger(MN->getMemoryVT().getRawBits());
944 }
945}
946
947/// AddNodeIDNode - Generic routine for adding a nodes info to the NodeID
948/// data.
949static void AddNodeIDNode(FoldingSetNodeID &ID, const SDNode *N) {
950 AddNodeIDOpcode(ID, N->getOpcode());
951 // Add the return value info.
952 AddNodeIDValueTypes(ID, N->getVTList());
953 // Add the operand info.
954 AddNodeIDOperands(ID, N->ops());
955
956 // Handle SDNode leafs with special info.
958}
959
960//===----------------------------------------------------------------------===//
961// SelectionDAG Class
962//===----------------------------------------------------------------------===//
963
964/// doNotCSE - Return true if CSE should not be performed for this node.
965static bool doNotCSE(SDNode *N) {
966 if (N->getValueType(0) == MVT::Glue)
967 return true; // Never CSE anything that produces a glue result.
968
969 switch (N->getOpcode()) {
970 default: break;
971 case ISD::HANDLENODE:
972 case ISD::EH_LABEL:
973 return true; // Never CSE these nodes.
974 }
975
976 // Check that remaining values produced are not flags.
977 for (unsigned i = 1, e = N->getNumValues(); i != e; ++i)
978 if (N->getValueType(i) == MVT::Glue)
979 return true; // Never CSE anything that produces a glue result.
980
981 return false;
982}
983
984/// RemoveDeadNodes - This method deletes all unreachable nodes in the
985/// SelectionDAG.
987 // Create a dummy node (which is not added to allnodes), that adds a reference
988 // to the root node, preventing it from being deleted.
989 HandleSDNode Dummy(getRoot());
990
992
993 // Add all obviously-dead nodes to the DeadNodes worklist.
994 for (SDNode &Node : allnodes())
995 if (Node.use_empty())
996 DeadNodes.push_back(&Node);
997
998 RemoveDeadNodes(DeadNodes);
999
1000 // If the root changed (e.g. it was a dead load, update the root).
1001 setRoot(Dummy.getValue());
1002}
1003
1004/// RemoveDeadNodes - This method deletes the unreachable nodes in the
1005/// given list, and any nodes that become unreachable as a result.
1007
1008 // Process the worklist, deleting the nodes and adding their uses to the
1009 // worklist.
1010 while (!DeadNodes.empty()) {
1011 SDNode *N = DeadNodes.pop_back_val();
1012 // Skip to next node if we've already managed to delete the node. This could
1013 // happen if replacing a node causes a node previously added to the node to
1014 // be deleted.
1015 if (N->getOpcode() == ISD::DELETED_NODE)
1016 continue;
1017
1018 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1019 DUL->NodeDeleted(N, nullptr);
1020
1021 // Take the node out of the appropriate CSE map.
1022 RemoveNodeFromCSEMaps(N);
1023
1024 // Next, brutally remove the operand list. This is safe to do, as there are
1025 // no cycles in the graph.
1026 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
1027 SDUse &Use = *I++;
1028 SDNode *Operand = Use.getNode();
1029 Use.set(SDValue());
1030
1031 // Now that we removed this operand, see if there are no uses of it left.
1032 if (Operand->use_empty())
1033 DeadNodes.push_back(Operand);
1034 }
1035
1036 DeallocateNode(N);
1037 }
1038}
1039
1041 SmallVector<SDNode*, 16> DeadNodes(1, N);
1042
1043 // Create a dummy node that adds a reference to the root node, preventing
1044 // it from being deleted. (This matters if the root is an operand of the
1045 // dead node.)
1046 HandleSDNode Dummy(getRoot());
1047
1048 RemoveDeadNodes(DeadNodes);
1049}
1050
1052 // First take this out of the appropriate CSE map.
1053 RemoveNodeFromCSEMaps(N);
1054
1055 // Finally, remove uses due to operands of this node, remove from the
1056 // AllNodes list, and delete the node.
1057 DeleteNodeNotInCSEMaps(N);
1058}
1059
1060void SelectionDAG::DeleteNodeNotInCSEMaps(SDNode *N) {
1061 assert(N->getIterator() != AllNodes.begin() &&
1062 "Cannot delete the entry node!");
1063 assert(N->use_empty() && "Cannot delete a node that is not dead!");
1064
1065 // Drop all of the operands and decrement used node's use counts.
1066 N->DropOperands();
1067
1068 DeallocateNode(N);
1069}
1070
1071void SDDbgInfo::add(SDDbgValue *V, bool isParameter) {
1072 assert(!(V->isVariadic() && isParameter));
1073 if (isParameter)
1074 ByvalParmDbgValues.push_back(V);
1075 else
1076 DbgValues.push_back(V);
1077 for (const SDNode *Node : V->getSDNodes())
1078 if (Node)
1079 DbgValMap[Node].push_back(V);
1080}
1081
1082void SDDbgInfo::erase(const SDNode *Node) {
1083 DbgValMapType::iterator I = DbgValMap.find(Node);
1084 if (I == DbgValMap.end())
1085 return;
1086 for (auto &Val: I->second)
1087 Val->setIsInvalidated();
1088 DbgValMap.erase(I);
1089}
1090
1091void SelectionDAG::DeallocateNode(SDNode *N) {
1092 // If we have operands, deallocate them.
1093 removeOperands(N);
1094
1095 NodeAllocator.Deallocate(AllNodes.remove(N));
1096
1097 // Set the opcode to DELETED_NODE to help catch bugs when node
1098 // memory is reallocated.
1099 // FIXME: There are places in SDag that have grown a dependency on the opcode
1100 // value in the released node.
1101 __asan_unpoison_memory_region(&N->NodeType, sizeof(N->NodeType));
1102 N->NodeType = ISD::DELETED_NODE;
1103
1104 // If any of the SDDbgValue nodes refer to this SDNode, invalidate
1105 // them and forget about that node.
1106 DbgInfo->erase(N);
1107
1108 // Invalidate extra info.
1109 SDEI.erase(N);
1110}
1111
1112#ifndef NDEBUG
1113/// VerifySDNode - Check the given SDNode. Aborts if it is invalid.
1114static void VerifySDNode(SDNode *N) {
1115 switch (N->getOpcode()) {
1116 default:
1117 break;
1118 case ISD::BUILD_PAIR: {
1119 EVT VT = N->getValueType(0);
1120 assert(N->getNumValues() == 1 && "Too many results!");
1121 assert(!VT.isVector() && (VT.isInteger() || VT.isFloatingPoint()) &&
1122 "Wrong return type!");
1123 assert(N->getNumOperands() == 2 && "Wrong number of operands!");
1124 assert(N->getOperand(0).getValueType() == N->getOperand(1).getValueType() &&
1125 "Mismatched operand types!");
1126 assert(N->getOperand(0).getValueType().isInteger() == VT.isInteger() &&
1127 "Wrong operand type!");
1128 assert(VT.getSizeInBits() == 2 * N->getOperand(0).getValueSizeInBits() &&
1129 "Wrong return type size");
1130 break;
1131 }
1132 case ISD::BUILD_VECTOR: {
1133 assert(N->getNumValues() == 1 && "Too many results!");
1134 assert(N->getValueType(0).isVector() && "Wrong return type!");
1135 assert(N->getNumOperands() == N->getValueType(0).getVectorNumElements() &&
1136 "Wrong number of operands!");
1137 EVT EltVT = N->getValueType(0).getVectorElementType();
1138 for (const SDUse &Op : N->ops()) {
1139 assert((Op.getValueType() == EltVT ||
1140 (EltVT.isInteger() && Op.getValueType().isInteger() &&
1141 EltVT.bitsLE(Op.getValueType()))) &&
1142 "Wrong operand type!");
1143 assert(Op.getValueType() == N->getOperand(0).getValueType() &&
1144 "Operands must all have the same type");
1145 }
1146 break;
1147 }
1148 }
1149}
1150#endif // NDEBUG
1151
1152/// Insert a newly allocated node into the DAG.
1153///
1154/// Handles insertion into the all nodes list and CSE map, as well as
1155/// verification and other common operations when a new node is allocated.
1156void SelectionDAG::InsertNode(SDNode *N) {
1157 AllNodes.push_back(N);
1158#ifndef NDEBUG
1159 N->PersistentId = NextPersistentId++;
1160 VerifySDNode(N);
1161#endif
1162 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1163 DUL->NodeInserted(N);
1164}
1165
1166/// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that
1167/// correspond to it. This is useful when we're about to delete or repurpose
1168/// the node. We don't want future request for structurally identical nodes
1169/// to return N anymore.
1170bool SelectionDAG::RemoveNodeFromCSEMaps(SDNode *N) {
1171 bool Erased = false;
1172 switch (N->getOpcode()) {
1173 case ISD::HANDLENODE: return false; // noop.
1174 case ISD::CONDCODE:
1175 assert(CondCodeNodes[cast<CondCodeSDNode>(N)->get()] &&
1176 "Cond code doesn't exist!");
1177 Erased = CondCodeNodes[cast<CondCodeSDNode>(N)->get()] != nullptr;
1178 CondCodeNodes[cast<CondCodeSDNode>(N)->get()] = nullptr;
1179 break;
1181 Erased = ExternalSymbols.erase(cast<ExternalSymbolSDNode>(N)->getSymbol());
1182 break;
1184 ExternalSymbolSDNode *ESN = cast<ExternalSymbolSDNode>(N);
1185 Erased = TargetExternalSymbols.erase(std::pair<std::string, unsigned>(
1186 ESN->getSymbol(), ESN->getTargetFlags()));
1187 break;
1188 }
1189 case ISD::MCSymbol: {
1190 auto *MCSN = cast<MCSymbolSDNode>(N);
1191 Erased = MCSymbols.erase(MCSN->getMCSymbol());
1192 break;
1193 }
1194 case ISD::VALUETYPE: {
1195 EVT VT = cast<VTSDNode>(N)->getVT();
1196 if (VT.isExtended()) {
1197 Erased = ExtendedValueTypeNodes.erase(VT);
1198 } else {
1199 Erased = ValueTypeNodes[VT.getSimpleVT().SimpleTy] != nullptr;
1200 ValueTypeNodes[VT.getSimpleVT().SimpleTy] = nullptr;
1201 }
1202 break;
1203 }
1204 default:
1205 // Remove it from the CSE Map.
1206 assert(N->getOpcode() != ISD::DELETED_NODE && "DELETED_NODE in CSEMap!");
1207 assert(N->getOpcode() != ISD::EntryToken && "EntryToken in CSEMap!");
1208 Erased = CSEMap.RemoveNode(N);
1209 break;
1210 }
1211#ifndef NDEBUG
1212 // Verify that the node was actually in one of the CSE maps, unless it has a
1213 // glue result (which cannot be CSE'd) or is one of the special cases that are
1214 // not subject to CSE.
1215 if (!Erased && N->getValueType(N->getNumValues()-1) != MVT::Glue &&
1216 !N->isMachineOpcode() && !doNotCSE(N)) {
1217 N->dump(this);
1218 dbgs() << "\n";
1219 llvm_unreachable("Node is not in map!");
1220 }
1221#endif
1222 return Erased;
1223}
1224
1225/// AddModifiedNodeToCSEMaps - The specified node has been removed from the CSE
1226/// maps and modified in place. Add it back to the CSE maps, unless an identical
1227/// node already exists, in which case transfer all its users to the existing
1228/// node. This transfer can potentially trigger recursive merging.
1229void
1230SelectionDAG::AddModifiedNodeToCSEMaps(SDNode *N) {
1231 // For node types that aren't CSE'd, just act as if no identical node
1232 // already exists.
1233 if (!doNotCSE(N)) {
1234 SDNode *Existing = CSEMap.GetOrInsertNode(N);
1235 if (Existing != N) {
1236 // If there was already an existing matching node, use ReplaceAllUsesWith
1237 // to replace the dead one with the existing one. This can cause
1238 // recursive merging of other unrelated nodes down the line.
1239 ReplaceAllUsesWith(N, Existing);
1240
1241 // N is now dead. Inform the listeners and delete it.
1242 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1243 DUL->NodeDeleted(N, Existing);
1244 DeleteNodeNotInCSEMaps(N);
1245 return;
1246 }
1247 }
1248
1249 // If the node doesn't already exist, we updated it. Inform listeners.
1250 for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
1251 DUL->NodeUpdated(N);
1252}
1253
1254/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1255/// were replaced with those specified. If this node is never memoized,
1256/// return null, otherwise return a pointer to the slot it would take. If a
1257/// node already exists with these operands, the slot will be non-null.
1258SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, SDValue Op,
1259 void *&InsertPos) {
1260 if (doNotCSE(N))
1261 return nullptr;
1262
1263 SDValue Ops[] = { Op };
1265 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1267 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1268 if (Node)
1269 Node->intersectFlagsWith(N->getFlags());
1270 return Node;
1271}
1272
1273/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1274/// were replaced with those specified. If this node is never memoized,
1275/// return null, otherwise return a pointer to the slot it would take. If a
1276/// node already exists with these operands, the slot will be non-null.
1277SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N,
1278 SDValue Op1, SDValue Op2,
1279 void *&InsertPos) {
1280 if (doNotCSE(N))
1281 return nullptr;
1282
1283 SDValue Ops[] = { Op1, Op2 };
1285 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1287 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1288 if (Node)
1289 Node->intersectFlagsWith(N->getFlags());
1290 return Node;
1291}
1292
1293/// FindModifiedNodeSlot - Find a slot for the specified node if its operands
1294/// were replaced with those specified. If this node is never memoized,
1295/// return null, otherwise return a pointer to the slot it would take. If a
1296/// node already exists with these operands, the slot will be non-null.
1297SDNode *SelectionDAG::FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1298 void *&InsertPos) {
1299 if (doNotCSE(N))
1300 return nullptr;
1301
1303 AddNodeIDNode(ID, N->getOpcode(), N->getVTList(), Ops);
1305 SDNode *Node = FindNodeOrInsertPos(ID, SDLoc(N), InsertPos);
1306 if (Node)
1307 Node->intersectFlagsWith(N->getFlags());
1308 return Node;
1309}
1310
1312 Type *Ty = VT == MVT::iPTR ? PointerType::get(*getContext(), 0)
1313 : VT.getTypeForEVT(*getContext());
1314
1315 return getDataLayout().getABITypeAlign(Ty);
1316}
1317
1318// EntryNode could meaningfully have debug info if we can find it...
1320 : TM(tm), OptLevel(OL), EntryNode(ISD::EntryToken, 0, DebugLoc(),
1321 getVTList(MVT::Other, MVT::Glue)),
1322 Root(getEntryNode()) {
1323 InsertNode(&EntryNode);
1324 DbgInfo = new SDDbgInfo();
1325}
1326
1328 OptimizationRemarkEmitter &NewORE, Pass *PassPtr,
1329 const TargetLibraryInfo *LibraryInfo,
1330 UniformityInfo *NewUA, ProfileSummaryInfo *PSIin,
1331 BlockFrequencyInfo *BFIin,
1332 FunctionVarLocs const *VarLocs) {
1333 MF = &NewMF;
1334 SDAGISelPass = PassPtr;
1335 ORE = &NewORE;
1338 LibInfo = LibraryInfo;
1339 Context = &MF->getFunction().getContext();
1340 UA = NewUA;
1341 PSI = PSIin;
1342 BFI = BFIin;
1343 FnVarLocs = VarLocs;
1344}
1345
1347 assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
1348 allnodes_clear();
1349 OperandRecycler.clear(OperandAllocator);
1350 delete DbgInfo;
1351}
1352
1354 return MF->getFunction().hasOptSize() ||
1356}
1357
1358void SelectionDAG::allnodes_clear() {
1359 assert(&*AllNodes.begin() == &EntryNode);
1360 AllNodes.remove(AllNodes.begin());
1361 while (!AllNodes.empty())
1362 DeallocateNode(&AllNodes.front());
1363#ifndef NDEBUG
1364 NextPersistentId = 0;
1365#endif
1366}
1367
1368SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1369 void *&InsertPos) {
1370 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1371 if (N) {
1372 switch (N->getOpcode()) {
1373 default: break;
1374 case ISD::Constant:
1375 case ISD::ConstantFP:
1376 llvm_unreachable("Querying for Constant and ConstantFP nodes requires "
1377 "debug location. Use another overload.");
1378 }
1379 }
1380 return N;
1381}
1382
1383SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
1384 const SDLoc &DL, void *&InsertPos) {
1385 SDNode *N = CSEMap.FindNodeOrInsertPos(ID, InsertPos);
1386 if (N) {
1387 switch (N->getOpcode()) {
1388 case ISD::Constant:
1389 case ISD::ConstantFP:
1390 // Erase debug location from the node if the node is used at several
1391 // different places. Do not propagate one location to all uses as it
1392 // will cause a worse single stepping debugging experience.
1393 if (N->getDebugLoc() != DL.getDebugLoc())
1394 N->setDebugLoc(DebugLoc());
1395 break;
1396 default:
1397 // When the node's point of use is located earlier in the instruction
1398 // sequence than its prior point of use, update its debug info to the
1399 // earlier location.
1400 if (DL.getIROrder() && DL.getIROrder() < N->getIROrder())
1401 N->setDebugLoc(DL.getDebugLoc());
1402 break;
1403 }
1404 }
1405 return N;
1406}
1407
1409 allnodes_clear();
1410 OperandRecycler.clear(OperandAllocator);
1411 OperandAllocator.Reset();
1412 CSEMap.clear();
1413
1414 ExtendedValueTypeNodes.clear();
1415 ExternalSymbols.clear();
1416 TargetExternalSymbols.clear();
1417 MCSymbols.clear();
1418 SDEI.clear();
1419 std::fill(CondCodeNodes.begin(), CondCodeNodes.end(),
1420 static_cast<CondCodeSDNode*>(nullptr));
1421 std::fill(ValueTypeNodes.begin(), ValueTypeNodes.end(),
1422 static_cast<SDNode*>(nullptr));
1423
1424 EntryNode.UseList = nullptr;
1425 InsertNode(&EntryNode);
1426 Root = getEntryNode();
1427 DbgInfo->clear();
1428}
1429
1431 return VT.bitsGT(Op.getValueType())
1432 ? getNode(ISD::FP_EXTEND, DL, VT, Op)
1433 : getNode(ISD::FP_ROUND, DL, VT, Op,
1434 getIntPtrConstant(0, DL, /*isTarget=*/true));
1435}
1436
1437std::pair<SDValue, SDValue>
1439 const SDLoc &DL, EVT VT) {
1440 assert(!VT.bitsEq(Op.getValueType()) &&
1441 "Strict no-op FP extend/round not allowed.");
1442 SDValue Res =
1443 VT.bitsGT(Op.getValueType())
1444 ? getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op})
1445 : getNode(ISD::STRICT_FP_ROUND, DL, {VT, MVT::Other},
1446 {Chain, Op, getIntPtrConstant(0, DL)});
1447
1448 return std::pair<SDValue, SDValue>(Res, SDValue(Res.getNode(), 1));
1449}
1450
1452 return VT.bitsGT(Op.getValueType()) ?
1453 getNode(ISD::ANY_EXTEND, DL, VT, Op) :
1454 getNode(ISD::TRUNCATE, DL, VT, Op);
1455}
1456
1458 return VT.bitsGT(Op.getValueType()) ?
1459 getNode(ISD::SIGN_EXTEND, DL, VT, Op) :
1460 getNode(ISD::TRUNCATE, DL, VT, Op);
1461}
1462
1464 return VT.bitsGT(Op.getValueType()) ?
1465 getNode(ISD::ZERO_EXTEND, DL, VT, Op) :
1466 getNode(ISD::TRUNCATE, DL, VT, Op);
1467}
1468
1470 EVT VT) {
1471 assert(!VT.isVector());
1472 auto Type = Op.getValueType();
1473 SDValue DestOp;
1474 if (Type == VT)
1475 return Op;
1476 auto Size = Op.getValueSizeInBits();
1477 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1478 if (DestOp.getValueType() == VT)
1479 return DestOp;
1480
1481 return getAnyExtOrTrunc(DestOp, DL, VT);
1482}
1483
1485 EVT VT) {
1486 assert(!VT.isVector());
1487 auto Type = Op.getValueType();
1488 SDValue DestOp;
1489 if (Type == VT)
1490 return Op;
1491 auto Size = Op.getValueSizeInBits();
1492 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1493 if (DestOp.getValueType() == VT)
1494 return DestOp;
1495
1496 return getSExtOrTrunc(DestOp, DL, VT);
1497}
1498
1500 EVT VT) {
1501 assert(!VT.isVector());
1502 auto Type = Op.getValueType();
1503 SDValue DestOp;
1504 if (Type == VT)
1505 return Op;
1506 auto Size = Op.getValueSizeInBits();
1507 DestOp = getBitcast(MVT::getIntegerVT(Size), Op);
1508 if (DestOp.getValueType() == VT)
1509 return DestOp;
1510
1511 return getZExtOrTrunc(DestOp, DL, VT);
1512}
1513
1515 EVT OpVT) {
1516 if (VT.bitsLE(Op.getValueType()))
1517 return getNode(ISD::TRUNCATE, SL, VT, Op);
1518
1520 return getNode(TLI->getExtendForContent(BType), SL, VT, Op);
1521}
1522
1524 EVT OpVT = Op.getValueType();
1525 assert(VT.isInteger() && OpVT.isInteger() &&
1526 "Cannot getZeroExtendInReg FP types");
1527 assert(VT.isVector() == OpVT.isVector() &&
1528 "getZeroExtendInReg type should be vector iff the operand "
1529 "type is vector!");
1530 assert((!VT.isVector() ||
1532 "Vector element counts must match in getZeroExtendInReg");
1533 assert(VT.bitsLE(OpVT) && "Not extending!");
1534 if (OpVT == VT)
1535 return Op;
1537 VT.getScalarSizeInBits());
1538 return getNode(ISD::AND, DL, OpVT, Op, getConstant(Imm, DL, OpVT));
1539}
1540
1542 // Only unsigned pointer semantics are supported right now. In the future this
1543 // might delegate to TLI to check pointer signedness.
1544 return getZExtOrTrunc(Op, DL, VT);
1545}
1546
1548 // Only unsigned pointer semantics are supported right now. In the future this
1549 // might delegate to TLI to check pointer signedness.
1550 return getZeroExtendInReg(Op, DL, VT);
1551}
1552
1554 return getNode(ISD::SUB, DL, VT, getConstant(0, DL, VT), Val);
1555}
1556
1557/// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
1559 return getNode(ISD::XOR, DL, VT, Val, getAllOnesConstant(DL, VT));
1560}
1561
1563 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1564 return getNode(ISD::XOR, DL, VT, Val, TrueValue);
1565}
1566
1568 SDValue Mask, SDValue EVL, EVT VT) {
1569 SDValue TrueValue = getBoolConstant(true, DL, VT, VT);
1570 return getNode(ISD::VP_XOR, DL, VT, Val, TrueValue, Mask, EVL);
1571}
1572
1574 SDValue Mask, SDValue EVL) {
1575 return getVPZExtOrTrunc(DL, VT, Op, Mask, EVL);
1576}
1577
1579 SDValue Mask, SDValue EVL) {
1580 if (VT.bitsGT(Op.getValueType()))
1581 return getNode(ISD::VP_ZERO_EXTEND, DL, VT, Op, Mask, EVL);
1582 if (VT.bitsLT(Op.getValueType()))
1583 return getNode(ISD::VP_TRUNCATE, DL, VT, Op, Mask, EVL);
1584 return Op;
1585}
1586
1588 EVT OpVT) {
1589 if (!V)
1590 return getConstant(0, DL, VT);
1591
1592 switch (TLI->getBooleanContents(OpVT)) {
1595 return getConstant(1, DL, VT);
1597 return getAllOnesConstant(DL, VT);
1598 }
1599 llvm_unreachable("Unexpected boolean content enum!");
1600}
1601
1603 bool isT, bool isO) {
1604 EVT EltVT = VT.getScalarType();
1605 assert((EltVT.getSizeInBits() >= 64 ||
1606 (uint64_t)((int64_t)Val >> EltVT.getSizeInBits()) + 1 < 2) &&
1607 "getConstant with a uint64_t value that doesn't fit in the type!");
1608 return getConstant(APInt(EltVT.getSizeInBits(), Val), DL, VT, isT, isO);
1609}
1610
1612 bool isT, bool isO) {
1613 return getConstant(*ConstantInt::get(*Context, Val), DL, VT, isT, isO);
1614}
1615
1617 EVT VT, bool isT, bool isO) {
1618 assert(VT.isInteger() && "Cannot create FP integer constant!");
1619
1620 EVT EltVT = VT.getScalarType();
1621 const ConstantInt *Elt = &Val;
1622
1623 // In some cases the vector type is legal but the element type is illegal and
1624 // needs to be promoted, for example v8i8 on ARM. In this case, promote the
1625 // inserted value (the type does not need to match the vector element type).
1626 // Any extra bits introduced will be truncated away.
1627 if (VT.isVector() && TLI->getTypeAction(*getContext(), EltVT) ==
1629 EltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1630 APInt NewVal;
1631 if (TLI->isSExtCheaperThanZExt(VT.getScalarType(), EltVT))
1632 NewVal = Elt->getValue().sextOrTrunc(EltVT.getSizeInBits());
1633 else
1634 NewVal = Elt->getValue().zextOrTrunc(EltVT.getSizeInBits());
1635 Elt = ConstantInt::get(*getContext(), NewVal);
1636 }
1637 // In other cases the element type is illegal and needs to be expanded, for
1638 // example v2i64 on MIPS32. In this case, find the nearest legal type, split
1639 // the value into n parts and use a vector type with n-times the elements.
1640 // Then bitcast to the type requested.
1641 // Legalizing constants too early makes the DAGCombiner's job harder so we
1642 // only legalize if the DAG tells us we must produce legal types.
1643 else if (NewNodesMustHaveLegalTypes && VT.isVector() &&
1644 TLI->getTypeAction(*getContext(), EltVT) ==
1646 const APInt &NewVal = Elt->getValue();
1647 EVT ViaEltVT = TLI->getTypeToTransformTo(*getContext(), EltVT);
1648 unsigned ViaEltSizeInBits = ViaEltVT.getSizeInBits();
1649
1650 // For scalable vectors, try to use a SPLAT_VECTOR_PARTS node.
1651 if (VT.isScalableVector() ||
1653 assert(EltVT.getSizeInBits() % ViaEltSizeInBits == 0 &&
1654 "Can only handle an even split!");
1655 unsigned Parts = EltVT.getSizeInBits() / ViaEltSizeInBits;
1656
1657 SmallVector<SDValue, 2> ScalarParts;
1658 for (unsigned i = 0; i != Parts; ++i)
1659 ScalarParts.push_back(getConstant(
1660 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1661 ViaEltVT, isT, isO));
1662
1663 return getNode(ISD::SPLAT_VECTOR_PARTS, DL, VT, ScalarParts);
1664 }
1665
1666 unsigned ViaVecNumElts = VT.getSizeInBits() / ViaEltSizeInBits;
1667 EVT ViaVecVT = EVT::getVectorVT(*getContext(), ViaEltVT, ViaVecNumElts);
1668
1669 // Check the temporary vector is the correct size. If this fails then
1670 // getTypeToTransformTo() probably returned a type whose size (in bits)
1671 // isn't a power-of-2 factor of the requested type size.
1672 assert(ViaVecVT.getSizeInBits() == VT.getSizeInBits());
1673
1674 SmallVector<SDValue, 2> EltParts;
1675 for (unsigned i = 0; i < ViaVecNumElts / VT.getVectorNumElements(); ++i)
1676 EltParts.push_back(getConstant(
1677 NewVal.extractBits(ViaEltSizeInBits, i * ViaEltSizeInBits), DL,
1678 ViaEltVT, isT, isO));
1679
1680 // EltParts is currently in little endian order. If we actually want
1681 // big-endian order then reverse it now.
1682 if (getDataLayout().isBigEndian())
1683 std::reverse(EltParts.begin(), EltParts.end());
1684
1685 // The elements must be reversed when the element order is different
1686 // to the endianness of the elements (because the BITCAST is itself a
1687 // vector shuffle in this situation). However, we do not need any code to
1688 // perform this reversal because getConstant() is producing a vector
1689 // splat.
1690 // This situation occurs in MIPS MSA.
1691
1693 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1694 llvm::append_range(Ops, EltParts);
1695
1696 SDValue V =
1697 getNode(ISD::BITCAST, DL, VT, getBuildVector(ViaVecVT, DL, Ops));
1698 return V;
1699 }
1700
1701 assert(Elt->getBitWidth() == EltVT.getSizeInBits() &&
1702 "APInt size does not match type size!");
1703 unsigned Opc = isT ? ISD::TargetConstant : ISD::Constant;
1705 AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1706 ID.AddPointer(Elt);
1707 ID.AddBoolean(isO);
1708 void *IP = nullptr;
1709 SDNode *N = nullptr;
1710 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1711 if (!VT.isVector())
1712 return SDValue(N, 0);
1713
1714 if (!N) {
1715 N = newSDNode<ConstantSDNode>(isT, isO, Elt, EltVT);
1716 CSEMap.InsertNode(N, IP);
1717 InsertNode(N);
1718 NewSDValueDbgMsg(SDValue(N, 0), "Creating constant: ", this);
1719 }
1720
1721 SDValue Result(N, 0);
1722 if (VT.isVector())
1723 Result = getSplat(VT, DL, Result);
1724 return Result;
1725}
1726
1728 bool isTarget) {
1729 return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget);
1730}
1731
1733 const SDLoc &DL, bool LegalTypes) {
1734 assert(VT.isInteger() && "Shift amount is not an integer type!");
1735 EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes);
1736 return getConstant(Val, DL, ShiftVT);
1737}
1738
1740 const SDLoc &DL, bool LegalTypes) {
1741 assert(Val.ult(VT.getScalarSizeInBits()) && "Out of range shift");
1742 return getShiftAmountConstant(Val.getZExtValue(), VT, DL, LegalTypes);
1743}
1744
1746 bool isTarget) {
1747 return getConstant(Val, DL, TLI->getVectorIdxTy(getDataLayout()), isTarget);
1748}
1749
1751 bool isTarget) {
1752 return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget);
1753}
1754
1756 EVT VT, bool isTarget) {
1757 assert(VT.isFloatingPoint() && "Cannot create integer FP constant!");
1758
1759 EVT EltVT = VT.getScalarType();
1760
1761 // Do the map lookup using the actual bit pattern for the floating point
1762 // value, so that we don't have problems with 0.0 comparing equal to -0.0, and
1763 // we don't have issues with SNANs.
1764 unsigned Opc = isTarget ? ISD::TargetConstantFP : ISD::ConstantFP;
1766 AddNodeIDNode(ID, Opc, getVTList(EltVT), std::nullopt);
1767 ID.AddPointer(&V);
1768 void *IP = nullptr;
1769 SDNode *N = nullptr;
1770 if ((N = FindNodeOrInsertPos(ID, DL, IP)))
1771 if (!VT.isVector())
1772 return SDValue(N, 0);
1773
1774 if (!N) {
1775 N = newSDNode<ConstantFPSDNode>(isTarget, &V, EltVT);
1776 CSEMap.InsertNode(N, IP);
1777 InsertNode(N);
1778 }
1779
1780 SDValue Result(N, 0);
1781 if (VT.isVector())
1782 Result = getSplat(VT, DL, Result);
1783 NewSDValueDbgMsg(Result, "Creating fp constant: ", this);
1784 return Result;
1785}
1786
1788 bool isTarget) {
1789 EVT EltVT = VT.getScalarType();
1790 if (EltVT == MVT::f32)
1791 return getConstantFP(APFloat((float)Val), DL, VT, isTarget);
1792 if (EltVT == MVT::f64)
1793 return getConstantFP(APFloat(Val), DL, VT, isTarget);
1794 if (EltVT == MVT::f80 || EltVT == MVT::f128 || EltVT == MVT::ppcf128 ||
1795 EltVT == MVT::f16 || EltVT == MVT::bf16) {
1796 bool Ignored;
1797 APFloat APF = APFloat(Val);
1799 &Ignored);
1800 return getConstantFP(APF, DL, VT, isTarget);
1801 }
1802 llvm_unreachable("Unsupported type in getConstantFP");
1803}
1804
1806 EVT VT, int64_t Offset, bool isTargetGA,
1807 unsigned TargetFlags) {
1808 assert((TargetFlags == 0 || isTargetGA) &&
1809 "Cannot set target flags on target-independent globals");
1810
1811 // Truncate (with sign-extension) the offset value to the pointer size.
1813 if (BitWidth < 64)
1815
1816 unsigned Opc;
1817 if (GV->isThreadLocal())
1819 else
1820 Opc = isTargetGA ? ISD::TargetGlobalAddress : ISD::GlobalAddress;
1821
1823 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1824 ID.AddPointer(GV);
1825 ID.AddInteger(Offset);
1826 ID.AddInteger(TargetFlags);
1827 void *IP = nullptr;
1828 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
1829 return SDValue(E, 0);
1830
1831 auto *N = newSDNode<GlobalAddressSDNode>(
1832 Opc, DL.getIROrder(), DL.getDebugLoc(), GV, VT, Offset, TargetFlags);
1833 CSEMap.InsertNode(N, IP);
1834 InsertNode(N);
1835 return SDValue(N, 0);
1836}
1837
1838SDValue SelectionDAG::getFrameIndex(int FI, EVT VT, bool isTarget) {
1839 unsigned Opc = isTarget ? ISD::TargetFrameIndex : ISD::FrameIndex;
1841 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1842 ID.AddInteger(FI);
1843 void *IP = nullptr;
1844 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1845 return SDValue(E, 0);
1846
1847 auto *N = newSDNode<FrameIndexSDNode>(FI, VT, isTarget);
1848 CSEMap.InsertNode(N, IP);
1849 InsertNode(N);
1850 return SDValue(N, 0);
1851}
1852
1853SDValue SelectionDAG::getJumpTable(int JTI, EVT VT, bool isTarget,
1854 unsigned TargetFlags) {
1855 assert((TargetFlags == 0 || isTarget) &&
1856 "Cannot set target flags on target-independent jump tables");
1857 unsigned Opc = isTarget ? ISD::TargetJumpTable : ISD::JumpTable;
1859 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1860 ID.AddInteger(JTI);
1861 ID.AddInteger(TargetFlags);
1862 void *IP = nullptr;
1863 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1864 return SDValue(E, 0);
1865
1866 auto *N = newSDNode<JumpTableSDNode>(JTI, VT, isTarget, TargetFlags);
1867 CSEMap.InsertNode(N, IP);
1868 InsertNode(N);
1869 return SDValue(N, 0);
1870}
1871
1873 const SDLoc &DL) {
1875 return getNode(ISD::JUMP_TABLE_DEBUG_INFO, DL, MVT::Glue, Chain,
1876 getTargetConstant(static_cast<uint64_t>(JTI), DL, PTy, true));
1877}
1878
1880 MaybeAlign Alignment, int Offset,
1881 bool isTarget, unsigned TargetFlags) {
1882 assert((TargetFlags == 0 || isTarget) &&
1883 "Cannot set target flags on target-independent globals");
1884 if (!Alignment)
1885 Alignment = shouldOptForSize()
1886 ? getDataLayout().getABITypeAlign(C->getType())
1887 : getDataLayout().getPrefTypeAlign(C->getType());
1888 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1890 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1891 ID.AddInteger(Alignment->value());
1892 ID.AddInteger(Offset);
1893 ID.AddPointer(C);
1894 ID.AddInteger(TargetFlags);
1895 void *IP = nullptr;
1896 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1897 return SDValue(E, 0);
1898
1899 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1900 TargetFlags);
1901 CSEMap.InsertNode(N, IP);
1902 InsertNode(N);
1903 SDValue V = SDValue(N, 0);
1904 NewSDValueDbgMsg(V, "Creating new constant pool: ", this);
1905 return V;
1906}
1907
1909 MaybeAlign Alignment, int Offset,
1910 bool isTarget, unsigned TargetFlags) {
1911 assert((TargetFlags == 0 || isTarget) &&
1912 "Cannot set target flags on target-independent globals");
1913 if (!Alignment)
1914 Alignment = getDataLayout().getPrefTypeAlign(C->getType());
1915 unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
1917 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
1918 ID.AddInteger(Alignment->value());
1919 ID.AddInteger(Offset);
1920 C->addSelectionDAGCSEId(ID);
1921 ID.AddInteger(TargetFlags);
1922 void *IP = nullptr;
1923 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1924 return SDValue(E, 0);
1925
1926 auto *N = newSDNode<ConstantPoolSDNode>(isTarget, C, VT, Offset, *Alignment,
1927 TargetFlags);
1928 CSEMap.InsertNode(N, IP);
1929 InsertNode(N);
1930 return SDValue(N, 0);
1931}
1932
1935 AddNodeIDNode(ID, ISD::BasicBlock, getVTList(MVT::Other), std::nullopt);
1936 ID.AddPointer(MBB);
1937 void *IP = nullptr;
1938 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
1939 return SDValue(E, 0);
1940
1941 auto *N = newSDNode<BasicBlockSDNode>(MBB);
1942 CSEMap.InsertNode(N, IP);
1943 InsertNode(N);
1944 return SDValue(N, 0);
1945}
1946
1948 if (VT.isSimple() && (unsigned)VT.getSimpleVT().SimpleTy >=
1949 ValueTypeNodes.size())
1950 ValueTypeNodes.resize(VT.getSimpleVT().SimpleTy+1);
1951
1952 SDNode *&N = VT.isExtended() ?
1953 ExtendedValueTypeNodes[VT] : ValueTypeNodes[VT.getSimpleVT().SimpleTy];
1954
1955 if (N) return SDValue(N, 0);
1956 N = newSDNode<VTSDNode>(VT);
1957 InsertNode(N);
1958 return SDValue(N, 0);
1959}
1960
1962 SDNode *&N = ExternalSymbols[Sym];
1963 if (N) return SDValue(N, 0);
1964 N = newSDNode<ExternalSymbolSDNode>(false, Sym, 0, VT);
1965 InsertNode(N);
1966 return SDValue(N, 0);
1967}
1968
1970 SDNode *&N = MCSymbols[Sym];
1971 if (N)
1972 return SDValue(N, 0);
1973 N = newSDNode<MCSymbolSDNode>(Sym, VT);
1974 InsertNode(N);
1975 return SDValue(N, 0);
1976}
1977
1979 unsigned TargetFlags) {
1980 SDNode *&N =
1981 TargetExternalSymbols[std::pair<std::string, unsigned>(Sym, TargetFlags)];
1982 if (N) return SDValue(N, 0);
1983 N = newSDNode<ExternalSymbolSDNode>(true, Sym, TargetFlags, VT);
1984 InsertNode(N);
1985 return SDValue(N, 0);
1986}
1987
1989 if ((unsigned)Cond >= CondCodeNodes.size())
1990 CondCodeNodes.resize(Cond+1);
1991
1992 if (!CondCodeNodes[Cond]) {
1993 auto *N = newSDNode<CondCodeSDNode>(Cond);
1994 CondCodeNodes[Cond] = N;
1995 InsertNode(N);
1996 }
1997
1998 return SDValue(CondCodeNodes[Cond], 0);
1999}
2000
2002 bool ConstantFold) {
2003 assert(MulImm.getBitWidth() == VT.getSizeInBits() &&
2004 "APInt size does not match type size!");
2005
2006 if (MulImm == 0)
2007 return getConstant(0, DL, VT);
2008
2009 if (ConstantFold) {
2010 const MachineFunction &MF = getMachineFunction();
2011 const Function &F = MF.getFunction();
2012 ConstantRange CR = getVScaleRange(&F, 64);
2013 if (const APInt *C = CR.getSingleElement())
2014 return getConstant(MulImm * C->getZExtValue(), DL, VT);
2015 }
2016
2017 return getNode(ISD::VSCALE, DL, VT, getConstant(MulImm, DL, VT));
2018}
2019
2021 bool ConstantFold) {
2022 if (EC.isScalable())
2023 return getVScale(DL, VT,
2024 APInt(VT.getSizeInBits(), EC.getKnownMinValue()));
2025
2026 return getConstant(EC.getKnownMinValue(), DL, VT);
2027}
2028
2030 APInt One(ResVT.getScalarSizeInBits(), 1);
2031 return getStepVector(DL, ResVT, One);
2032}
2033
2035 const APInt &StepVal) {
2036 assert(ResVT.getScalarSizeInBits() == StepVal.getBitWidth());
2037 if (ResVT.isScalableVector())
2038 return getNode(
2039 ISD::STEP_VECTOR, DL, ResVT,
2040 getTargetConstant(StepVal, DL, ResVT.getVectorElementType()));
2041
2042 SmallVector<SDValue, 16> OpsStepConstants;
2043 for (uint64_t i = 0; i < ResVT.getVectorNumElements(); i++)
2044 OpsStepConstants.push_back(
2045 getConstant(StepVal * i, DL, ResVT.getVectorElementType()));
2046 return getBuildVector(ResVT, DL, OpsStepConstants);
2047}
2048
2049/// Swaps the values of N1 and N2. Swaps all indices in the shuffle mask M that
2050/// point at N1 to point at N2 and indices that point at N2 to point at N1.
2052 std::swap(N1, N2);
2054}
2055
2057 SDValue N2, ArrayRef<int> Mask) {
2058 assert(VT.getVectorNumElements() == Mask.size() &&
2059 "Must have the same number of vector elements as mask elements!");
2060 assert(VT == N1.getValueType() && VT == N2.getValueType() &&
2061 "Invalid VECTOR_SHUFFLE");
2062
2063 // Canonicalize shuffle undef, undef -> undef
2064 if (N1.isUndef() && N2.isUndef())
2065 return getUNDEF(VT);
2066
2067 // Validate that all indices in Mask are within the range of the elements
2068 // input to the shuffle.
2069 int NElts = Mask.size();
2070 assert(llvm::all_of(Mask,
2071 [&](int M) { return M < (NElts * 2) && M >= -1; }) &&
2072 "Index out of range");
2073
2074 // Copy the mask so we can do any needed cleanup.
2075 SmallVector<int, 8> MaskVec(Mask);
2076
2077 // Canonicalize shuffle v, v -> v, undef
2078 if (N1 == N2) {
2079 N2 = getUNDEF(VT);
2080 for (int i = 0; i != NElts; ++i)
2081 if (MaskVec[i] >= NElts) MaskVec[i] -= NElts;
2082 }
2083
2084 // Canonicalize shuffle undef, v -> v, undef. Commute the shuffle mask.
2085 if (N1.isUndef())
2086 commuteShuffle(N1, N2, MaskVec);
2087
2088 if (TLI->hasVectorBlend()) {
2089 // If shuffling a splat, try to blend the splat instead. We do this here so
2090 // that even when this arises during lowering we don't have to re-handle it.
2091 auto BlendSplat = [&](BuildVectorSDNode *BV, int Offset) {
2092 BitVector UndefElements;
2093 SDValue Splat = BV->getSplatValue(&UndefElements);
2094 if (!Splat)
2095 return;
2096
2097 for (int i = 0; i < NElts; ++i) {
2098 if (MaskVec[i] < Offset || MaskVec[i] >= (Offset + NElts))
2099 continue;
2100
2101 // If this input comes from undef, mark it as such.
2102 if (UndefElements[MaskVec[i] - Offset]) {
2103 MaskVec[i] = -1;
2104 continue;
2105 }
2106
2107 // If we can blend a non-undef lane, use that instead.
2108 if (!UndefElements[i])
2109 MaskVec[i] = i + Offset;
2110 }
2111 };
2112 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
2113 BlendSplat(N1BV, 0);
2114 if (auto *N2BV = dyn_cast<BuildVectorSDNode>(N2))
2115 BlendSplat(N2BV, NElts);
2116 }
2117
2118 // Canonicalize all index into lhs, -> shuffle lhs, undef
2119 // Canonicalize all index into rhs, -> shuffle rhs, undef
2120 bool AllLHS = true, AllRHS = true;
2121 bool N2Undef = N2.isUndef();
2122 for (int i = 0; i != NElts; ++i) {
2123 if (MaskVec[i] >= NElts) {
2124 if (N2Undef)
2125 MaskVec[i] = -1;
2126 else
2127 AllLHS = false;
2128 } else if (MaskVec[i] >= 0) {
2129 AllRHS = false;
2130 }
2131 }
2132 if (AllLHS && AllRHS)
2133 return getUNDEF(VT);
2134 if (AllLHS && !N2Undef)
2135 N2 = getUNDEF(VT);
2136 if (AllRHS) {
2137 N1 = getUNDEF(VT);
2138 commuteShuffle(N1, N2, MaskVec);
2139 }
2140 // Reset our undef status after accounting for the mask.
2141 N2Undef = N2.isUndef();
2142 // Re-check whether both sides ended up undef.
2143 if (N1.isUndef() && N2Undef)
2144 return getUNDEF(VT);
2145
2146 // If Identity shuffle return that node.
2147 bool Identity = true, AllSame = true;
2148 for (int i = 0; i != NElts; ++i) {
2149 if (MaskVec[i] >= 0 && MaskVec[i] != i) Identity = false;
2150 if (MaskVec[i] != MaskVec[0]) AllSame = false;
2151 }
2152 if (Identity && NElts)
2153 return N1;
2154
2155 // Shuffling a constant splat doesn't change the result.
2156 if (N2Undef) {
2157 SDValue V = N1;
2158
2159 // Look through any bitcasts. We check that these don't change the number
2160 // (and size) of elements and just changes their types.
2161 while (V.getOpcode() == ISD::BITCAST)
2162 V = V->getOperand(0);
2163
2164 // A splat should always show up as a build vector node.
2165 if (auto *BV = dyn_cast<BuildVectorSDNode>(V)) {
2166 BitVector UndefElements;
2167 SDValue Splat = BV->getSplatValue(&UndefElements);
2168 // If this is a splat of an undef, shuffling it is also undef.
2169 if (Splat && Splat.isUndef())
2170 return getUNDEF(VT);
2171
2172 bool SameNumElts =
2173 V.getValueType().getVectorNumElements() == VT.getVectorNumElements();
2174
2175 // We only have a splat which can skip shuffles if there is a splatted
2176 // value and no undef lanes rearranged by the shuffle.
2177 if (Splat && UndefElements.none()) {
2178 // Splat of <x, x, ..., x>, return <x, x, ..., x>, provided that the
2179 // number of elements match or the value splatted is a zero constant.
2180 if (SameNumElts || isNullConstant(Splat))
2181 return N1;
2182 }
2183
2184 // If the shuffle itself creates a splat, build the vector directly.
2185 if (AllSame && SameNumElts) {
2186 EVT BuildVT = BV->getValueType(0);
2187 const SDValue &Splatted = BV->getOperand(MaskVec[0]);
2188 SDValue NewBV = getSplatBuildVector(BuildVT, dl, Splatted);
2189
2190 // We may have jumped through bitcasts, so the type of the
2191 // BUILD_VECTOR may not match the type of the shuffle.
2192 if (BuildVT != VT)
2193 NewBV = getNode(ISD::BITCAST, dl, VT, NewBV);
2194 return NewBV;
2195 }
2196 }
2197 }
2198
2200 SDValue Ops[2] = { N1, N2 };
2202 for (int i = 0; i != NElts; ++i)
2203 ID.AddInteger(MaskVec[i]);
2204
2205 void* IP = nullptr;
2206 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2207 return SDValue(E, 0);
2208
2209 // Allocate the mask array for the node out of the BumpPtrAllocator, since
2210 // SDNode doesn't have access to it. This memory will be "leaked" when
2211 // the node is deallocated, but recovered when the NodeAllocator is released.
2212 int *MaskAlloc = OperandAllocator.Allocate<int>(NElts);
2213 llvm::copy(MaskVec, MaskAlloc);
2214
2215 auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
2216 dl.getDebugLoc(), MaskAlloc);
2217 createOperands(N, Ops);
2218
2219 CSEMap.InsertNode(N, IP);
2220 InsertNode(N);
2221 SDValue V = SDValue(N, 0);
2222 NewSDValueDbgMsg(V, "Creating new node: ", this);
2223 return V;
2224}
2225
2227 EVT VT = SV.getValueType(0);
2228 SmallVector<int, 8> MaskVec(SV.getMask());
2230
2231 SDValue Op0 = SV.getOperand(0);
2232 SDValue Op1 = SV.getOperand(1);
2233 return getVectorShuffle(VT, SDLoc(&SV), Op1, Op0, MaskVec);
2234}
2235
2238 AddNodeIDNode(ID, ISD::Register, getVTList(VT), std::nullopt);
2239 ID.AddInteger(RegNo);
2240 void *IP = nullptr;
2241 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2242 return SDValue(E, 0);
2243
2244 auto *N = newSDNode<RegisterSDNode>(RegNo, VT);
2245 N->SDNodeBits.IsDivergent = TLI->isSDNodeSourceOfDivergence(N, FLI, UA);
2246 CSEMap.InsertNode(N, IP);
2247 InsertNode(N);
2248 return SDValue(N, 0);
2249}
2250
2253 AddNodeIDNode(ID, ISD::RegisterMask, getVTList(MVT::Untyped), std::nullopt);
2254 ID.AddPointer(RegMask);
2255 void *IP = nullptr;
2256 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2257 return SDValue(E, 0);
2258
2259 auto *N = newSDNode<RegisterMaskSDNode>(RegMask);
2260 CSEMap.InsertNode(N, IP);
2261 InsertNode(N);
2262 return SDValue(N, 0);
2263}
2264
2266 MCSymbol *Label) {
2267 return getLabelNode(ISD::EH_LABEL, dl, Root, Label);
2268}
2269
2270SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl,
2271 SDValue Root, MCSymbol *Label) {
2273 SDValue Ops[] = { Root };
2274 AddNodeIDNode(ID, Opcode, getVTList(MVT::Other), Ops);
2275 ID.AddPointer(Label);
2276 void *IP = nullptr;
2277 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2278 return SDValue(E, 0);
2279
2280 auto *N =
2281 newSDNode<LabelSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label);
2282 createOperands(N, Ops);
2283
2284 CSEMap.InsertNode(N, IP);
2285 InsertNode(N);
2286 return SDValue(N, 0);
2287}
2288
2290 int64_t Offset, bool isTarget,
2291 unsigned TargetFlags) {
2292 unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
2293
2295 AddNodeIDNode(ID, Opc, getVTList(VT), std::nullopt);
2296 ID.AddPointer(BA);
2297 ID.AddInteger(Offset);
2298 ID.AddInteger(TargetFlags);
2299 void *IP = nullptr;
2300 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2301 return SDValue(E, 0);
2302
2303 auto *N = newSDNode<BlockAddressSDNode>(Opc, VT, BA, Offset, TargetFlags);
2304 CSEMap.InsertNode(N, IP);
2305 InsertNode(N);
2306 return SDValue(N, 0);
2307}
2308
2311 AddNodeIDNode(ID, ISD::SRCVALUE, getVTList(MVT::Other), std::nullopt);
2312 ID.AddPointer(V);
2313
2314 void *IP = nullptr;
2315 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2316 return SDValue(E, 0);
2317
2318 auto *N = newSDNode<SrcValueSDNode>(V);
2319 CSEMap.InsertNode(N, IP);
2320 InsertNode(N);
2321 return SDValue(N, 0);
2322}
2323
2326 AddNodeIDNode(ID, ISD::MDNODE_SDNODE, getVTList(MVT::Other), std::nullopt);
2327 ID.AddPointer(MD);
2328
2329 void *IP = nullptr;
2330 if (SDNode *E = FindNodeOrInsertPos(ID, IP))
2331 return SDValue(E, 0);
2332
2333 auto *N = newSDNode<MDNodeSDNode>(MD);
2334 CSEMap.InsertNode(N, IP);
2335 InsertNode(N);
2336 return SDValue(N, 0);
2337}
2338
2340 if (VT == V.getValueType())
2341 return V;
2342
2343 return getNode(ISD::BITCAST, SDLoc(V), VT, V);
2344}
2345
2347 unsigned SrcAS, unsigned DestAS) {
2348 SDValue Ops[] = {Ptr};
2351 ID.AddInteger(SrcAS);
2352 ID.AddInteger(DestAS);
2353
2354 void *IP = nullptr;
2355 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
2356 return SDValue(E, 0);
2357
2358 auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
2359 VT, SrcAS, DestAS);
2360 createOperands(N, Ops);
2361
2362 CSEMap.InsertNode(N, IP);
2363 InsertNode(N);
2364 return SDValue(N, 0);
2365}
2366
2368 return getNode(ISD::FREEZE, SDLoc(V), V.getValueType(), V);
2369}
2370
2371/// getShiftAmountOperand - Return the specified value casted to
2372/// the target's desired shift amount type.
2374 EVT OpTy = Op.getValueType();
2375 EVT ShTy = TLI->getShiftAmountTy(LHSTy, getDataLayout());
2376 if (OpTy == ShTy || OpTy.isVector()) return Op;
2377
2378 return getZExtOrTrunc(Op, SDLoc(Op), ShTy);
2379}
2380
2382 SDLoc dl(Node);
2384 const Value *V = cast<SrcValueSDNode>(Node->getOperand(2))->getValue();
2385 EVT VT = Node->getValueType(0);
2386 SDValue Tmp1 = Node->getOperand(0);
2387 SDValue Tmp2 = Node->getOperand(1);
2388 const MaybeAlign MA(Node->getConstantOperandVal(3));
2389
2390 SDValue VAListLoad = getLoad(TLI.getPointerTy(getDataLayout()), dl, Tmp1,
2391 Tmp2, MachinePointerInfo(V));
2392 SDValue VAList = VAListLoad;
2393
2394 if (MA && *MA > TLI.getMinStackArgumentAlignment()) {
2395 VAList = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2396 getConstant(MA->value() - 1, dl, VAList.getValueType()));
2397
2398 VAList =
2399 getNode(ISD::AND, dl, VAList.getValueType(), VAList,
2400 getConstant(-(int64_t)MA->value(), dl, VAList.getValueType()));
2401 }
2402
2403 // Increment the pointer, VAList, to the next vaarg
2404 Tmp1 = getNode(ISD::ADD, dl, VAList.getValueType(), VAList,
2405 getConstant(getDataLayout().getTypeAllocSize(
2406 VT.getTypeForEVT(*getContext())),
2407 dl, VAList.getValueType()));
2408 // Store the incremented VAList to the legalized pointer
2409 Tmp1 =
2410 getStore(VAListLoad.getValue(1), dl, Tmp1, Tmp2, MachinePointerInfo(V));
2411 // Load the actual argument out of the pointer VAList
2412 return getLoad(VT, dl, Tmp1, VAList, MachinePointerInfo());
2413}
2414
2416 SDLoc dl(Node);
2418 // This defaults to loading a pointer from the input and storing it to the
2419 // output, returning the chain.
2420 const Value *VD = cast<SrcValueSDNode>(Node->getOperand(3))->getValue();
2421 const Value *VS = cast<SrcValueSDNode>(Node->getOperand(4))->getValue();
2422 SDValue Tmp1 =
2423 getLoad(TLI.getPointerTy(getDataLayout()), dl, Node->getOperand(0),
2424 Node->getOperand(2), MachinePointerInfo(VS));
2425 return getStore(Tmp1.getValue(1), dl, Tmp1, Node->getOperand(1),
2426 MachinePointerInfo(VD));
2427}
2428
2430 const DataLayout &DL = getDataLayout();
2431 Type *Ty = VT.getTypeForEVT(*getContext());
2432 Align RedAlign = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2433
2434 if (TLI->isTypeLegal(VT) || !VT.isVector())
2435 return RedAlign;
2436
2438 const Align StackAlign = TFI->getStackAlign();
2439
2440 // See if we can choose a smaller ABI alignment in cases where it's an
2441 // illegal vector type that will get broken down.
2442 if (RedAlign > StackAlign) {
2443 EVT IntermediateVT;
2444 MVT RegisterVT;
2445 unsigned NumIntermediates;
2446 TLI->getVectorTypeBreakdown(*getContext(), VT, IntermediateVT,
2447 NumIntermediates, RegisterVT);
2448 Ty = IntermediateVT.getTypeForEVT(*getContext());
2449 Align RedAlign2 = UseABI ? DL.getABITypeAlign(Ty) : DL.getPrefTypeAlign(Ty);
2450 if (RedAlign2 < RedAlign)
2451 RedAlign = RedAlign2;
2452 }
2453
2454 return RedAlign;
2455}
2456
2458 MachineFrameInfo &MFI = MF->getFrameInfo();
2460 int StackID = 0;
2461 if (Bytes.isScalable())
2462 StackID = TFI->getStackIDForScalableVectors();
2463 // The stack id gives an indication of whether the object is scalable or
2464 // not, so it's safe to pass in the minimum size here.
2465 int FrameIdx = MFI.CreateStackObject(Bytes.getKnownMinValue(), Alignment,
2466 false, nullptr, StackID);
2467 return getFrameIndex(FrameIdx, TLI->getFrameIndexTy(getDataLayout()));
2468}
2469
2471 Type *Ty = VT.getTypeForEVT(*getContext());
2472 Align StackAlign =
2473 std::max(getDataLayout().getPrefTypeAlign(Ty), Align(minAlign));
2474 return CreateStackTemporary(VT.getStoreSize(), StackAlign);
2475}
2476
2478 TypeSize VT1Size = VT1.getStoreSize();
2479 TypeSize VT2Size = VT2.getStoreSize();
2480 assert(VT1Size.isScalable() == VT2Size.isScalable() &&
2481 "Don't know how to choose the maximum size when creating a stack "
2482 "temporary");
2483 TypeSize Bytes = VT1Size.getKnownMinValue() > VT2Size.getKnownMinValue()
2484 ? VT1Size
2485 : VT2Size;
2486
2487 Type *Ty1 = VT1.getTypeForEVT(*getContext());
2488 Type *Ty2 = VT2.getTypeForEVT(*getContext());
2489 const DataLayout &DL = getDataLayout();
2490 Align Align = std::max(DL.getPrefTypeAlign(Ty1), DL.getPrefTypeAlign(Ty2));
2491 return CreateStackTemporary(Bytes, Align);
2492}
2493
2495 ISD::CondCode Cond, const SDLoc &dl) {
2496 EVT OpVT = N1.getValueType();
2497
2498 auto GetUndefBooleanConstant = [&]() {
2499 if (VT.getScalarType() == MVT::i1 ||
2500 TLI->getBooleanContents(OpVT) ==
2502 return getUNDEF(VT);
2503 // ZeroOrOne / ZeroOrNegative require specific values for the high bits,
2504 // so we cannot use getUNDEF(). Return zero instead.
2505 return getConstant(0, dl, VT);
2506 };
2507
2508 // These setcc operations always fold.
2509 switch (Cond) {
2510 default: break;
2511 case ISD::SETFALSE:
2512 case ISD::SETFALSE2: return getBoolConstant(false, dl, VT, OpVT);
2513 case ISD::SETTRUE:
2514 case ISD::SETTRUE2: return getBoolConstant(true, dl, VT, OpVT);
2515
2516 case ISD::SETOEQ:
2517 case ISD::SETOGT:
2518 case ISD::SETOGE:
2519 case ISD::SETOLT:
2520 case ISD::SETOLE:
2521 case ISD::SETONE:
2522 case ISD::SETO:
2523 case ISD::SETUO:
2524 case ISD::SETUEQ:
2525 case ISD::SETUNE:
2526 assert(!OpVT.isInteger() && "Illegal setcc for integer!");
2527 break;
2528 }
2529
2530 if (OpVT.isInteger()) {
2531 // For EQ and NE, we can always pick a value for the undef to make the
2532 // predicate pass or fail, so we can return undef.
2533 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2534 // icmp eq/ne X, undef -> undef.
2535 if ((N1.isUndef() || N2.isUndef()) &&
2536 (Cond == ISD::SETEQ || Cond == ISD::SETNE))
2537 return GetUndefBooleanConstant();
2538
2539 // If both operands are undef, we can return undef for int comparison.
2540 // icmp undef, undef -> undef.
2541 if (N1.isUndef() && N2.isUndef())
2542 return GetUndefBooleanConstant();
2543
2544 // icmp X, X -> true/false
2545 // icmp X, undef -> true/false because undef could be X.
2546 if (N1.isUndef() || N2.isUndef() || N1 == N2)
2547 return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT);
2548 }
2549
2550 if (ConstantSDNode *N2C = dyn_cast<ConstantSDNode>(N2)) {
2551 const APInt &C2 = N2C->getAPIntValue();
2552 if (ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1)) {
2553 const APInt &C1 = N1C->getAPIntValue();
2554
2556 dl, VT, OpVT);
2557 }
2558 }
2559
2560 auto *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
2561 auto *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
2562
2563 if (N1CFP && N2CFP) {
2564 APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF());
2565 switch (Cond) {
2566 default: break;
2567 case ISD::SETEQ: if (R==APFloat::cmpUnordered)
2568 return GetUndefBooleanConstant();
2569 [[fallthrough]];
2570 case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT,
2571 OpVT);
2572 case ISD::SETNE: if (R==APFloat::cmpUnordered)
2573 return GetUndefBooleanConstant();
2574 [[fallthrough]];
2576 R==APFloat::cmpLessThan, dl, VT,
2577 OpVT);
2578 case ISD::SETLT: if (R==APFloat::cmpUnordered)
2579 return GetUndefBooleanConstant();
2580 [[fallthrough]];
2581 case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT,
2582 OpVT);
2583 case ISD::SETGT: if (R==APFloat::cmpUnordered)
2584 return GetUndefBooleanConstant();
2585 [[fallthrough]];
2587 VT, OpVT);
2588 case ISD::SETLE: if (R==APFloat::cmpUnordered)
2589 return GetUndefBooleanConstant();
2590 [[fallthrough]];
2592 R==APFloat::cmpEqual, dl, VT,
2593 OpVT);
2594 case ISD::SETGE: if (R==APFloat::cmpUnordered)
2595 return GetUndefBooleanConstant();
2596 [[fallthrough]];
2598 R==APFloat::cmpEqual, dl, VT, OpVT);
2599 case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT,
2600 OpVT);
2601 case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT,
2602 OpVT);
2604 R==APFloat::cmpEqual, dl, VT,
2605 OpVT);
2606 case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT,
2607 OpVT);
2609 R==APFloat::cmpLessThan, dl, VT,
2610 OpVT);
2612 R==APFloat::cmpUnordered, dl, VT,
2613 OpVT);
2615 VT, OpVT);
2616 case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT,
2617 OpVT);
2618 }
2619 } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) {
2620 // Ensure that the constant occurs on the RHS.
2622 if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT()))
2623 return SDValue();
2624 return getSetCC(dl, VT, N2, N1, SwappedCond);
2625 } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) ||
2626 (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) {
2627 // If an operand is known to be a nan (or undef that could be a nan), we can
2628 // fold it.
2629 // Choosing NaN for the undef will always make unordered comparison succeed
2630 // and ordered comparison fails.
2631 // Matches behavior in llvm::ConstantFoldCompareInstruction.
2632 switch (ISD::getUnorderedFlavor(Cond)) {
2633 default:
2634 llvm_unreachable("Unknown flavor!");
2635 case 0: // Known false.
2636 return getBoolConstant(false, dl, VT, OpVT);
2637 case 1: // Known true.
2638 return getBoolConstant(true, dl, VT, OpVT);
2639 case 2: // Undefined.
2640 return GetUndefBooleanConstant();
2641 }
2642 }
2643
2644 // Could not fold it.
2645 return SDValue();
2646}
2647
2648/// SignBitIsZero - Return true if the sign bit of Op is known to be zero. We
2649/// use this predicate to simplify operations downstream.
2651 unsigned BitWidth = Op.getScalarValueSizeInBits();
2653}
2654
2655/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use
2656/// this predicate to simplify operations downstream. Mask is known to be zero
2657/// for bits that V cannot have.
2659 unsigned Depth) const {
2660 return Mask.isSubsetOf(computeKnownBits(V, Depth).Zero);
2661}
2662
2663/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in
2664/// DemandedElts. We use this predicate to simplify operations downstream.
2665/// Mask is known to be zero for bits that V cannot have.
2667 const APInt &DemandedElts,
2668 unsigned Depth) const {
2669 return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero);
2670}
2671
2672/// MaskedVectorIsZero - Return true if 'Op' is known to be zero in
2673/// DemandedElts. We use this predicate to simplify operations downstream.
2675 unsigned Depth /* = 0 */) const {
2676 return computeKnownBits(V, DemandedElts, Depth).isZero();
2677}
2678
2679/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'.
2681 unsigned Depth) const {
2682 return Mask.isSubsetOf(computeKnownBits(V, Depth).One);
2683}
2684
2686 const APInt &DemandedElts,
2687 unsigned Depth) const {
2688 EVT VT = Op.getValueType();
2689 assert(VT.isVector() && !VT.isScalableVector() && "Only for fixed vectors!");
2690
2691 unsigned NumElts = VT.getVectorNumElements();
2692 assert(DemandedElts.getBitWidth() == NumElts && "Unexpected demanded mask.");
2693
2694 APInt KnownZeroElements = APInt::getZero(NumElts);
2695 for (unsigned EltIdx = 0; EltIdx != NumElts; ++EltIdx) {
2696 if (!DemandedElts[EltIdx])
2697 continue; // Don't query elements that are not demanded.
2698 APInt Mask = APInt::getOneBitSet(NumElts, EltIdx);
2699 if (MaskedVectorIsZero(Op, Mask, Depth))
2700 KnownZeroElements.setBit(EltIdx);
2701 }
2702 return KnownZeroElements;
2703}
2704
2705/// isSplatValue - Return true if the vector V has the same value
2706/// across all DemandedElts. For scalable vectors, we don't know the
2707/// number of lanes at compile time. Instead, we use a 1 bit APInt
2708/// to represent a conservative value for all lanes; that is, that
2709/// one bit value is implicitly splatted across all lanes.
2710bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts,
2711 APInt &UndefElts, unsigned Depth) const {
2712 unsigned Opcode = V.getOpcode();
2713 EVT VT = V.getValueType();
2714 assert(VT.isVector() && "Vector type expected");
2715 assert((!VT.isScalableVector() || DemandedElts.getBitWidth() == 1) &&
2716 "scalable demanded bits are ignored");
2717
2718 if (!DemandedElts)
2719 return false; // No demanded elts, better to assume we don't know anything.
2720
2721 if (Depth >= MaxRecursionDepth)
2722 return false; // Limit search depth.
2723
2724 // Deal with some common cases here that work for both fixed and scalable
2725 // vector types.
2726 switch (Opcode) {
2727 case ISD::SPLAT_VECTOR:
2728 UndefElts = V.getOperand(0).isUndef()
2729 ? APInt::getAllOnes(DemandedElts.getBitWidth())
2730 : APInt(DemandedElts.getBitWidth(), 0);
2731 return true;
2732 case ISD::ADD:
2733 case ISD::SUB:
2734 case ISD::AND:
2735 case ISD::XOR:
2736 case ISD::OR: {
2737 APInt UndefLHS, UndefRHS;
2738 SDValue LHS = V.getOperand(0);
2739 SDValue RHS = V.getOperand(1);
2740 if (isSplatValue(LHS, DemandedElts, UndefLHS, Depth + 1) &&
2741 isSplatValue(RHS, DemandedElts, UndefRHS, Depth + 1)) {
2742 UndefElts = UndefLHS | UndefRHS;
2743 return true;
2744 }
2745 return false;
2746 }
2747 case ISD::ABS:
2748 case ISD::TRUNCATE:
2749 case ISD::SIGN_EXTEND:
2750 case ISD::ZERO_EXTEND:
2751 return isSplatValue(V.getOperand(0), DemandedElts, UndefElts, Depth + 1);
2752 default:
2753 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
2754 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
2755 return TLI->isSplatValueForTargetNode(V, DemandedElts, UndefElts, *this,
2756 Depth);
2757 break;
2758}
2759
2760 // We don't support other cases than those above for scalable vectors at
2761 // the moment.
2762 if (VT.isScalableVector())
2763 return false;
2764
2765 unsigned NumElts = VT.getVectorNumElements();
2766 assert(NumElts == DemandedElts.getBitWidth() && "Vector size mismatch");
2767 UndefElts = APInt::getZero(NumElts);
2768
2769 switch (Opcode) {
2770 case ISD::BUILD_VECTOR: {
2771 SDValue Scl;
2772 for (unsigned i = 0; i != NumElts; ++i) {
2773 SDValue Op = V.getOperand(i);
2774 if (Op.isUndef()) {
2775 UndefElts.setBit(i);
2776 continue;
2777 }
2778 if (!DemandedElts[i])
2779 continue;
2780 if (Scl && Scl != Op)
2781 return false;
2782 Scl = Op;
2783 }
2784 return true;
2785 }
2786 case ISD::VECTOR_SHUFFLE: {
2787 // Check if this is a shuffle node doing a splat or a shuffle of a splat.
2788 APInt DemandedLHS = APInt::getZero(NumElts);
2789 APInt DemandedRHS = APInt::getZero(NumElts);
2790 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(V)->getMask();
2791 for (int i = 0; i != (int)NumElts; ++i) {
2792 int M = Mask[i];
2793 if (M < 0) {
2794 UndefElts.setBit(i);
2795 continue;
2796 }
2797 if (!DemandedElts[i])
2798 continue;
2799 if (M < (int)NumElts)
2800 DemandedLHS.setBit(M);
2801 else
2802 DemandedRHS.setBit(M - NumElts);
2803 }
2804
2805 // If we aren't demanding either op, assume there's no splat.
2806 // If we are demanding both ops, assume there's no splat.
2807 if ((DemandedLHS.isZero() && DemandedRHS.isZero()) ||
2808 (!DemandedLHS.isZero() && !DemandedRHS.isZero()))
2809 return false;
2810
2811 // See if the demanded elts of the source op is a splat or we only demand
2812 // one element, which should always be a splat.
2813 // TODO: Handle source ops splats with undefs.
2814 auto CheckSplatSrc = [&](SDValue Src, const APInt &SrcElts) {
2815 APInt SrcUndefs;
2816 return (SrcElts.popcount() == 1) ||
2817 (isSplatValue(Src, SrcElts, SrcUndefs, Depth + 1) &&
2818 (SrcElts & SrcUndefs).isZero());
2819 };
2820 if (!DemandedLHS.isZero())
2821 return CheckSplatSrc(V.getOperand(0), DemandedLHS);
2822 return CheckSplatSrc(V.getOperand(1), DemandedRHS);
2823 }
2825 // Offset the demanded elts by the subvector index.
2826 SDValue Src = V.getOperand(0);
2827 // We don't support scalable vectors at the moment.
2828 if (Src.getValueType().isScalableVector())
2829 return false;
2830 uint64_t Idx = V.getConstantOperandVal(1);
2831 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2832 APInt UndefSrcElts;
2833 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
2834 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2835 UndefElts = UndefSrcElts.extractBits(NumElts, Idx);
2836 return true;
2837 }
2838 break;
2839 }
2843 // Widen the demanded elts by the src element count.
2844 SDValue Src = V.getOperand(0);
2845 // We don't support scalable vectors at the moment.
2846 if (Src.getValueType().isScalableVector())
2847 return false;
2848 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
2849 APInt UndefSrcElts;
2850 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts);
2851 if (isSplatValue(Src, DemandedSrcElts, UndefSrcElts, Depth + 1)) {
2852 UndefElts = UndefSrcElts.trunc(NumElts);
2853 return true;
2854 }
2855 break;
2856 }
2857 case ISD::BITCAST: {
2858 SDValue Src = V.getOperand(0);
2859 EVT SrcVT = Src.getValueType();
2860 unsigned SrcBitWidth = SrcVT.getScalarSizeInBits();
2861 unsigned BitWidth = VT.getScalarSizeInBits();
2862
2863 // Ignore bitcasts from unsupported types.
2864 // TODO: Add fp support?
2865 if (!SrcVT.isVector() || !SrcVT.isInteger() || !VT.isInteger())
2866 break;
2867
2868 // Bitcast 'small element' vector to 'large element' vector.
2869 if ((BitWidth % SrcBitWidth) == 0) {
2870 // See if each sub element is a splat.
2871 unsigned Scale = BitWidth / SrcBitWidth;
2872 unsigned NumSrcElts = SrcVT.getVectorNumElements();
2873 APInt ScaledDemandedElts =
2874 APIntOps::ScaleBitMask(DemandedElts, NumSrcElts);
2875 for (unsigned I = 0; I != Scale; ++I) {
2876 APInt SubUndefElts;
2877 APInt SubDemandedElt = APInt::getOneBitSet(Scale, I);
2878 APInt SubDemandedElts = APInt::getSplat(NumSrcElts, SubDemandedElt);
2879 SubDemandedElts &= ScaledDemandedElts;
2880 if (!isSplatValue(Src, SubDemandedElts, SubUndefElts, Depth + 1))
2881 return false;
2882 // TODO: Add support for merging sub undef elements.
2883 if (!SubUndefElts.isZero())
2884 return false;
2885 }
2886 return true;
2887 }
2888 break;
2889 }
2890 }
2891
2892 return false;
2893}
2894
2895/// Helper wrapper to main isSplatValue function.
2896bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) const {
2897 EVT VT = V.getValueType();
2898 assert(VT.isVector() && "Vector type expected");
2899
2900 APInt UndefElts;
2901 // Since the number of lanes in a scalable vector is unknown at compile time,
2902 // we track one bit which is implicitly broadcast to all lanes. This means
2903 // that all lanes in a scalable vector are considered demanded.
2904 APInt DemandedElts
2906 return isSplatValue(V, DemandedElts, UndefElts) &&
2907 (AllowUndefs || !UndefElts);
2908}
2909
2912
2913 EVT VT = V.getValueType();
2914 unsigned Opcode = V.getOpcode();
2915 switch (Opcode) {
2916 default: {
2917 APInt UndefElts;
2918 // Since the number of lanes in a scalable vector is unknown at compile time,
2919 // we track one bit which is implicitly broadcast to all lanes. This means
2920 // that all lanes in a scalable vector are considered demanded.
2921 APInt DemandedElts
2923
2924 if (isSplatValue(V, DemandedElts, UndefElts)) {
2925 if (VT.isScalableVector()) {
2926 // DemandedElts and UndefElts are ignored for scalable vectors, since
2927 // the only supported cases are SPLAT_VECTOR nodes.
2928 SplatIdx = 0;
2929 } else {
2930 // Handle case where all demanded elements are UNDEF.
2931 if (DemandedElts.isSubsetOf(UndefElts)) {
2932 SplatIdx = 0;
2933 return getUNDEF(VT);
2934 }
2935 SplatIdx = (UndefElts & DemandedElts).countr_one();
2936 }
2937 return V;
2938 }
2939 break;
2940 }
2941 case ISD::SPLAT_VECTOR:
2942 SplatIdx = 0;
2943 return V;
2944 case ISD::VECTOR_SHUFFLE: {
2945 assert(!VT.isScalableVector());
2946 // Check if this is a shuffle node doing a splat.
2947 // TODO - remove this and rely purely on SelectionDAG::isSplatValue,
2948 // getTargetVShiftNode currently struggles without the splat source.
2949 auto *SVN = cast<ShuffleVectorSDNode>(V);
2950 if (!SVN->isSplat())
2951 break;
2952 int Idx = SVN->getSplatIndex();
2953 int NumElts = V.getValueType().getVectorNumElements();
2954 SplatIdx = Idx % NumElts;
2955 return V.getOperand(Idx / NumElts);
2956 }
2957 }
2958
2959 return SDValue();
2960}
2961
2963 int SplatIdx;
2964 if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) {
2965 EVT SVT = SrcVector.getValueType().getScalarType();
2966 EVT LegalSVT = SVT;
2967 if (LegalTypes && !TLI->isTypeLegal(SVT)) {
2968 if (!SVT.isInteger())
2969 return SDValue();
2970 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
2971 if (LegalSVT.bitsLT(SVT))
2972 return SDValue();
2973 }
2974 return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), LegalSVT, SrcVector,
2975 getVectorIdxConstant(SplatIdx, SDLoc(V)));
2976 }
2977 return SDValue();
2978}
2979
2980const APInt *
2982 const APInt &DemandedElts) const {
2983 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2984 V.getOpcode() == ISD::SRA) &&
2985 "Unknown shift node");
2986 unsigned BitWidth = V.getScalarValueSizeInBits();
2987 if (ConstantSDNode *SA = isConstOrConstSplat(V.getOperand(1), DemandedElts)) {
2988 // Shifting more than the bitwidth is not valid.
2989 const APInt &ShAmt = SA->getAPIntValue();
2990 if (ShAmt.ult(BitWidth))
2991 return &ShAmt;
2992 }
2993 return nullptr;
2994}
2995
2997 SDValue V, const APInt &DemandedElts) const {
2998 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
2999 V.getOpcode() == ISD::SRA) &&
3000 "Unknown shift node");
3001 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3002 return ValidAmt;
3003 unsigned BitWidth = V.getScalarValueSizeInBits();
3004 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3005 if (!BV)
3006 return nullptr;
3007 const APInt *MinShAmt = nullptr;
3008 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3009 if (!DemandedElts[i])
3010 continue;
3011 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3012 if (!SA)
3013 return nullptr;
3014 // Shifting more than the bitwidth is not valid.
3015 const APInt &ShAmt = SA->getAPIntValue();
3016 if (ShAmt.uge(BitWidth))
3017 return nullptr;
3018 if (MinShAmt && MinShAmt->ule(ShAmt))
3019 continue;
3020 MinShAmt = &ShAmt;
3021 }
3022 return MinShAmt;
3023}
3024
3026 SDValue V, const APInt &DemandedElts) const {
3027 assert((V.getOpcode() == ISD::SHL || V.getOpcode() == ISD::SRL ||
3028 V.getOpcode() == ISD::SRA) &&
3029 "Unknown shift node");
3030 if (const APInt *ValidAmt = getValidShiftAmountConstant(V, DemandedElts))
3031 return ValidAmt;
3032 unsigned BitWidth = V.getScalarValueSizeInBits();
3033 auto *BV = dyn_cast<BuildVectorSDNode>(V.getOperand(1));
3034 if (!BV)
3035 return nullptr;
3036 const APInt *MaxShAmt = nullptr;
3037 for (unsigned i = 0, e = BV->getNumOperands(); i != e; ++i) {
3038 if (!DemandedElts[i])
3039 continue;
3040 auto *SA = dyn_cast<ConstantSDNode>(BV->getOperand(i));
3041 if (!SA)
3042 return nullptr;
3043 // Shifting more than the bitwidth is not valid.
3044 const APInt &ShAmt = SA->getAPIntValue();
3045 if (ShAmt.uge(BitWidth))
3046 return nullptr;
3047 if (MaxShAmt && MaxShAmt->uge(ShAmt))
3048 continue;
3049 MaxShAmt = &ShAmt;
3050 }
3051 return MaxShAmt;
3052}
3053
3054/// Determine which bits of Op are known to be either zero or one and return
3055/// them in Known. For vectors, the known bits are those that are shared by
3056/// every vector element.
3058 EVT VT = Op.getValueType();
3059
3060 // Since the number of lanes in a scalable vector is unknown at compile time,
3061 // we track one bit which is implicitly broadcast to all lanes. This means
3062 // that all lanes in a scalable vector are considered demanded.
3063 APInt DemandedElts = VT.isFixedLengthVector()
3065 : APInt(1, 1);
3066 return computeKnownBits(Op, DemandedElts, Depth);
3067}
3068
3069/// Determine which bits of Op are known to be either zero or one and return
3070/// them in Known. The DemandedElts argument allows us to only collect the known
3071/// bits that are shared by the requested vector elements.
3073 unsigned Depth) const {
3074 unsigned BitWidth = Op.getScalarValueSizeInBits();
3075
3076 KnownBits Known(BitWidth); // Don't know anything.
3077
3078 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
3079 // We know all of the bits for a constant!
3080 return KnownBits::makeConstant(C->getAPIntValue());
3081 }
3082 if (auto *C = dyn_cast<ConstantFPSDNode>(Op)) {
3083 // We know all of the bits for a constant fp!
3084 return KnownBits::makeConstant(C->getValueAPF().bitcastToAPInt());
3085 }
3086
3087 if (Depth >= MaxRecursionDepth)
3088 return Known; // Limit search depth.
3089
3090 KnownBits Known2;
3091 unsigned NumElts = DemandedElts.getBitWidth();
3092 assert((!Op.getValueType().isFixedLengthVector() ||
3093 NumElts == Op.getValueType().getVectorNumElements()) &&
3094 "Unexpected vector size");
3095
3096 if (!DemandedElts)
3097 return Known; // No demanded elts, better to assume we don't know anything.
3098
3099 unsigned Opcode = Op.getOpcode();
3100 switch (Opcode) {
3101 case ISD::MERGE_VALUES:
3102 return computeKnownBits(Op.getOperand(Op.getResNo()), DemandedElts,
3103 Depth + 1);
3104 case ISD::SPLAT_VECTOR: {
3105 SDValue SrcOp = Op.getOperand(0);
3106 assert(SrcOp.getValueSizeInBits() >= BitWidth &&
3107 "Expected SPLAT_VECTOR implicit truncation");
3108 // Implicitly truncate the bits to match the official semantics of
3109 // SPLAT_VECTOR.
3110 Known = computeKnownBits(SrcOp, Depth + 1).trunc(BitWidth);
3111 break;
3112 }
3114 unsigned ScalarSize = Op.getOperand(0).getScalarValueSizeInBits();
3115 assert(ScalarSize * Op.getNumOperands() == BitWidth &&
3116 "Expected SPLAT_VECTOR_PARTS scalars to cover element width");
3117 for (auto [I, SrcOp] : enumerate(Op->ops())) {
3118 Known.insertBits(computeKnownBits(SrcOp, Depth + 1), ScalarSize * I);
3119 }
3120 break;
3121 }
3122 case ISD::STEP_VECTOR: {
3123 const APInt &Step = Op.getConstantOperandAPInt(0);
3124
3125 if (Step.isPowerOf2())
3126 Known.Zero.setLowBits(Step.logBase2());
3127
3129
3130 if (!isUIntN(BitWidth, Op.getValueType().getVectorMinNumElements()))
3131 break;
3132 const APInt MinNumElts =
3133 APInt(BitWidth, Op.getValueType().getVectorMinNumElements());
3134
3135 bool Overflow;
3136 const APInt MaxNumElts = getVScaleRange(&F, BitWidth)
3138 .umul_ov(MinNumElts, Overflow);
3139 if (Overflow)
3140 break;
3141
3142 const APInt MaxValue = (MaxNumElts - 1).umul_ov(Step, Overflow);
3143 if (Overflow)
3144 break;
3145
3146 Known.Zero.setHighBits(MaxValue.countl_zero());
3147 break;
3148 }
3149 case ISD::BUILD_VECTOR:
3150 assert(!Op.getValueType().isScalableVector());
3151 // Collect the known bits that are shared by every demanded vector element.
3152 Known.Zero.setAllBits(); Known.One.setAllBits();
3153 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
3154 if (!DemandedElts[i])
3155 continue;
3156
3157 SDValue SrcOp = Op.getOperand(i);
3158 Known2 = computeKnownBits(SrcOp, Depth + 1);
3159
3160 // BUILD_VECTOR can implicitly truncate sources, we must handle this.
3161 if (SrcOp.getValueSizeInBits() != BitWidth) {
3162 assert(SrcOp.getValueSizeInBits() > BitWidth &&
3163 "Expected BUILD_VECTOR implicit truncation");
3164 Known2 = Known2.trunc(BitWidth);
3165 }
3166
3167 // Known bits are the values that are shared by every demanded element.
3168 Known = Known.intersectWith(Known2);
3169
3170 // If we don't know any bits, early out.
3171 if (Known.isUnknown())
3172 break;
3173 }
3174 break;
3175 case ISD::VECTOR_SHUFFLE: {
3176 assert(!Op.getValueType().isScalableVector());
3177 // Collect the known bits that are shared by every vector element referenced
3178 // by the shuffle.
3179 APInt DemandedLHS, DemandedRHS;
3180 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
3181 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
3182 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
3183 DemandedLHS, DemandedRHS))
3184 break;
3185
3186 // Known bits are the values that are shared by every demanded element.
3187 Known.Zero.setAllBits(); Known.One.setAllBits();
3188 if (!!DemandedLHS) {
3189 SDValue LHS = Op.getOperand(0);
3190 Known2 = computeKnownBits(LHS, DemandedLHS, Depth + 1);
3191 Known = Known.intersectWith(Known2);
3192 }
3193 // If we don't know any bits, early out.
3194 if (Known.isUnknown())
3195 break;
3196 if (!!DemandedRHS) {
3197 SDValue RHS = Op.getOperand(1);
3198 Known2 = computeKnownBits(RHS, DemandedRHS, Depth + 1);
3199 Known = Known.intersectWith(Known2);
3200 }
3201 break;
3202 }
3203 case ISD::VSCALE: {
3205 const APInt &Multiplier = Op.getConstantOperandAPInt(0);
3206 Known = getVScaleRange(&F, BitWidth).multiply(Multiplier).toKnownBits();
3207 break;
3208 }
3209 case ISD::CONCAT_VECTORS: {
3210 if (Op.getValueType().isScalableVector())
3211 break;
3212 // Split DemandedElts and test each of the demanded subvectors.
3213 Known.Zero.setAllBits(); Known.One.setAllBits();
3214 EVT SubVectorVT = Op.getOperand(0).getValueType();
3215 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
3216 unsigned NumSubVectors = Op.getNumOperands();
3217 for (unsigned i = 0; i != NumSubVectors; ++i) {
3218 APInt DemandedSub =
3219 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
3220 if (!!DemandedSub) {
3221 SDValue Sub = Op.getOperand(i);
3222 Known2 = computeKnownBits(Sub, DemandedSub, Depth + 1);
3223 Known = Known.intersectWith(Known2);
3224 }
3225 // If we don't know any bits, early out.
3226 if (Known.isUnknown())
3227 break;
3228 }
3229 break;
3230 }
3231 case ISD::INSERT_SUBVECTOR: {
3232 if (Op.getValueType().isScalableVector())
3233 break;
3234 // Demand any elements from the subvector and the remainder from the src its
3235 // inserted into.
3236 SDValue Src = Op.getOperand(0);
3237 SDValue Sub = Op.getOperand(1);
3238 uint64_t Idx = Op.getConstantOperandVal(2);
3239 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
3240 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
3241 APInt DemandedSrcElts = DemandedElts;
3242 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
3243
3244 Known.One.setAllBits();
3245 Known.Zero.setAllBits();
3246 if (!!DemandedSubElts) {
3247 Known = computeKnownBits(Sub, DemandedSubElts, Depth + 1);
3248 if (Known.isUnknown())
3249 break; // early-out.
3250 }
3251 if (!!DemandedSrcElts) {
3252 Known2 = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3253 Known = Known.intersectWith(Known2);
3254 }
3255 break;
3256 }
3258 // Offset the demanded elts by the subvector index.
3259 SDValue Src = Op.getOperand(0);
3260 // Bail until we can represent demanded elements for scalable vectors.
3261 if (Op.getValueType().isScalableVector() || Src.getValueType().isScalableVector())
3262 break;
3263 uint64_t Idx = Op.getConstantOperandVal(1);
3264 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
3265 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
3266 Known = computeKnownBits(Src, DemandedSrcElts, Depth + 1);
3267 break;
3268 }
3269 case ISD::SCALAR_TO_VECTOR: {
3270 if (Op.getValueType().isScalableVector())
3271 break;
3272 // We know about scalar_to_vector as much as we know about it source,
3273 // which becomes the first element of otherwise unknown vector.
3274 if (DemandedElts != 1)
3275 break;
3276
3277 SDValue N0 = Op.getOperand(0);
3278 Known = computeKnownBits(N0, Depth + 1);
3279 if (N0.getValueSizeInBits() != BitWidth)
3280 Known = Known.trunc(BitWidth);
3281
3282 break;
3283 }
3284 case ISD::BITCAST: {
3285 if (Op.getValueType().isScalableVector())
3286 break;
3287
3288 SDValue N0 = Op.getOperand(0);
3289 EVT SubVT = N0.getValueType();
3290 unsigned SubBitWidth = SubVT.getScalarSizeInBits();
3291
3292 // Ignore bitcasts from unsupported types.
3293 if (!(SubVT.isInteger() || SubVT.isFloatingPoint()))
3294 break;
3295
3296 // Fast handling of 'identity' bitcasts.
3297 if (BitWidth == SubBitWidth) {
3298 Known = computeKnownBits(N0, DemandedElts, Depth + 1);
3299 break;
3300 }
3301
3302 bool IsLE = getDataLayout().isLittleEndian();
3303
3304 // Bitcast 'small element' vector to 'large element' scalar/vector.
3305 if ((BitWidth % SubBitWidth) == 0) {
3306 assert(N0.getValueType().isVector() && "Expected bitcast from vector");
3307
3308 // Collect known bits for the (larger) output by collecting the known
3309 // bits from each set of sub elements and shift these into place.
3310 // We need to separately call computeKnownBits for each set of
3311 // sub elements as the knownbits for each is likely to be different.
3312 unsigned SubScale = BitWidth / SubBitWidth;
3313 APInt SubDemandedElts(NumElts * SubScale, 0);
3314 for (unsigned i = 0; i != NumElts; ++i)
3315 if (DemandedElts[i])
3316 SubDemandedElts.setBit(i * SubScale);
3317
3318 for (unsigned i = 0; i != SubScale; ++i) {
3319 Known2 = computeKnownBits(N0, SubDemandedElts.shl(i),
3320 Depth + 1);
3321 unsigned Shifts = IsLE ? i : SubScale - 1 - i;
3322 Known.insertBits(Known2, SubBitWidth * Shifts);
3323 }
3324 }
3325
3326 // Bitcast 'large element' scalar/vector to 'small element' vector.
3327 if ((SubBitWidth % BitWidth) == 0) {
3328 assert(Op.getValueType().isVector() && "Expected bitcast to vector");
3329
3330 // Collect known bits for the (smaller) output by collecting the known
3331 // bits from the overlapping larger input elements and extracting the
3332 // sub sections we actually care about.
3333 unsigned SubScale = SubBitWidth / BitWidth;
3334 APInt SubDemandedElts =
3335 APIntOps::ScaleBitMask(DemandedElts, NumElts / SubScale);
3336 Known2 = computeKnownBits(N0, SubDemandedElts, Depth + 1);
3337
3338 Known.Zero.setAllBits(); Known.One.setAllBits();
3339 for (unsigned i = 0; i != NumElts; ++i)
3340 if (DemandedElts[i]) {
3341 unsigned Shifts = IsLE ? i : NumElts - 1 - i;
3342 unsigned Offset = (Shifts % SubScale) * BitWidth;
3343 Known = Known.intersectWith(Known2.extractBits(BitWidth, Offset));
3344 // If we don't know any bits, early out.
3345 if (Known.isUnknown())
3346 break;
3347 }
3348 }
3349 break;
3350 }
3351 case ISD::AND:
3352 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3353 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3354
3355 Known &= Known2;
3356 break;
3357 case ISD::OR:
3358 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3359 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3360
3361 Known |= Known2;
3362 break;
3363 case ISD::XOR:
3364 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3365 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3366
3367 Known ^= Known2;
3368 break;
3369 case ISD::MUL: {
3370 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3371 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3372 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3373 // TODO: SelfMultiply can be poison, but not undef.
3374 if (SelfMultiply)
3375 SelfMultiply &= isGuaranteedNotToBeUndefOrPoison(
3376 Op.getOperand(0), DemandedElts, false, Depth + 1);
3377 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3378
3379 // If the multiplication is known not to overflow, the product of a number
3380 // with itself is non-negative. Only do this if we didn't already computed
3381 // the opposite value for the sign bit.
3382 if (Op->getFlags().hasNoSignedWrap() &&
3383 Op.getOperand(0) == Op.getOperand(1) &&
3384 !Known.isNegative())
3385 Known.makeNonNegative();
3386 break;
3387 }
3388 case ISD::MULHU: {
3389 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3390 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3391 Known = KnownBits::mulhu(Known, Known2);
3392 break;
3393 }
3394 case ISD::MULHS: {
3395 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3396 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3397 Known = KnownBits::mulhs(Known, Known2);
3398 break;
3399 }
3400 case ISD::UMUL_LOHI: {
3401 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3402 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3403 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3404 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3405 if (Op.getResNo() == 0)
3406 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3407 else
3408 Known = KnownBits::mulhu(Known, Known2);
3409 break;
3410 }
3411 case ISD::SMUL_LOHI: {
3412 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3413 Known = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3414 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3415 bool SelfMultiply = Op.getOperand(0) == Op.getOperand(1);
3416 if (Op.getResNo() == 0)
3417 Known = KnownBits::mul(Known, Known2, SelfMultiply);
3418 else
3419 Known = KnownBits::mulhs(Known, Known2);
3420 break;
3421 }
3422 case ISD::AVGCEILU: {
3423 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3424 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3425 Known = Known.zext(BitWidth + 1);
3426 Known2 = Known2.zext(BitWidth + 1);
3428 Known = KnownBits::computeForAddCarry(Known, Known2, One);
3429 Known = Known.extractBits(BitWidth, 1);
3430 break;
3431 }
3432 case ISD::SELECT:
3433 case ISD::VSELECT:
3434 Known = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3435 // If we don't know any bits, early out.
3436 if (Known.isUnknown())
3437 break;
3438 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth+1);
3439
3440 // Only known if known in both the LHS and RHS.
3441 Known = Known.intersectWith(Known2);
3442 break;
3443 case ISD::SELECT_CC:
3444 Known = computeKnownBits(Op.getOperand(3), DemandedElts, Depth+1);
3445 // If we don't know any bits, early out.
3446 if (Known.isUnknown())
3447 break;
3448 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth+1);
3449
3450 // Only known if known in both the LHS and RHS.
3451 Known = Known.intersectWith(Known2);
3452 break;
3453 case ISD::SMULO:
3454 case ISD::UMULO:
3455 if (Op.getResNo() != 1)
3456 break;
3457 // The boolean result conforms to getBooleanContents.
3458 // If we know the result of a setcc has the top bits zero, use this info.
3459 // We know that we have an integer-based boolean since these operations
3460 // are only available for integer.
3461 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
3463 BitWidth > 1)
3464 Known.Zero.setBitsFrom(1);
3465 break;
3466 case ISD::SETCC:
3467 case ISD::SETCCCARRY:
3468 case ISD::STRICT_FSETCC:
3469 case ISD::STRICT_FSETCCS: {
3470 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
3471 // If we know the result of a setcc has the top bits zero, use this info.
3472 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
3474 BitWidth > 1)
3475 Known.Zero.setBitsFrom(1);
3476 break;
3477 }
3478 case ISD::SHL:
3479 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3480 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3481 Known = KnownBits::shl(Known, Known2);
3482
3483 // Minimum shift low bits are known zero.
3484 if (const APInt *ShMinAmt =
3486 Known.Zero.setLowBits(ShMinAmt->getZExtValue());
3487 break;
3488 case ISD::SRL:
3489 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3490 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3491 Known = KnownBits::lshr(Known, Known2, /*ShAmtNonZero=*/false,
3492 Op->getFlags().hasExact());
3493
3494 // Minimum shift high bits are known zero.
3495 if (const APInt *ShMinAmt =
3497 Known.Zero.setHighBits(ShMinAmt->getZExtValue());
3498 break;
3499 case ISD::SRA:
3500 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3501 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3502 Known = KnownBits::ashr(Known, Known2, /*ShAmtNonZero=*/false,
3503 Op->getFlags().hasExact());
3504 break;
3505 case ISD::FSHL:
3506 case ISD::FSHR:
3507 if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) {
3508 unsigned Amt = C->getAPIntValue().urem(BitWidth);
3509
3510 // For fshl, 0-shift returns the 1st arg.
3511 // For fshr, 0-shift returns the 2nd arg.
3512 if (Amt == 0) {
3513 Known = computeKnownBits(Op.getOperand(Opcode == ISD::FSHL ? 0 : 1),
3514 DemandedElts, Depth + 1);
3515 break;
3516 }
3517
3518 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
3519 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
3520 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3521 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3522 if (Opcode == ISD::FSHL) {
3523 Known.One <<= Amt;
3524 Known.Zero <<= Amt;
3525 Known2.One.lshrInPlace(BitWidth - Amt);
3526 Known2.Zero.lshrInPlace(BitWidth - Amt);
3527 } else {
3528 Known.One <<= BitWidth - Amt;
3529 Known.Zero <<= BitWidth - Amt;
3530 Known2.One.lshrInPlace(Amt);
3531 Known2.Zero.lshrInPlace(Amt);
3532 }
3533 Known = Known.unionWith(Known2);
3534 }
3535 break;
3536 case ISD::SHL_PARTS:
3537 case ISD::SRA_PARTS:
3538 case ISD::SRL_PARTS: {
3539 assert((Op.getResNo() == 0 || Op.getResNo() == 1) && "Unknown result");
3540
3541 // Collect lo/hi source values and concatenate.
3542 unsigned LoBits = Op.getOperand(0).getScalarValueSizeInBits();
3543 unsigned HiBits = Op.getOperand(1).getScalarValueSizeInBits();
3544 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3545 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3546 Known = Known2.concat(Known);
3547
3548 // Collect shift amount.
3549 Known2 = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3550
3551 if (Opcode == ISD::SHL_PARTS)
3552 Known = KnownBits::shl(Known, Known2);
3553 else if (Opcode == ISD::SRA_PARTS)
3554 Known = KnownBits::ashr(Known, Known2);
3555 else // if (Opcode == ISD::SRL_PARTS)
3556 Known = KnownBits::lshr(Known, Known2);
3557
3558 // TODO: Minimum shift low/high bits are known zero.
3559
3560 if (Op.getResNo() == 0)
3561 Known = Known.extractBits(LoBits, 0);
3562 else
3563 Known = Known.extractBits(HiBits, LoBits);
3564 break;
3565 }
3567 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3568 EVT EVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3569 Known = Known.sextInReg(EVT.getScalarSizeInBits());
3570 break;
3571 }
3572 case ISD::CTTZ:
3573 case ISD::CTTZ_ZERO_UNDEF: {
3574 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3575 // If we have a known 1, its position is our upper bound.
3576 unsigned PossibleTZ = Known2.countMaxTrailingZeros();
3577 unsigned LowBits = llvm::bit_width(PossibleTZ);
3578 Known.Zero.setBitsFrom(LowBits);
3579 break;
3580 }
3581 case ISD::CTLZ:
3582 case ISD::CTLZ_ZERO_UNDEF: {
3583 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3584 // If we have a known 1, its position is our upper bound.
3585 unsigned PossibleLZ = Known2.countMaxLeadingZeros();
3586 unsigned LowBits = llvm::bit_width(PossibleLZ);
3587 Known.Zero.setBitsFrom(LowBits);
3588 break;
3589 }
3590 case ISD::CTPOP: {
3591 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3592 // If we know some of the bits are zero, they can't be one.
3593 unsigned PossibleOnes = Known2.countMaxPopulation();
3594 Known.Zero.setBitsFrom(llvm::bit_width(PossibleOnes));
3595 break;
3596 }
3597 case ISD::PARITY: {
3598 // Parity returns 0 everywhere but the LSB.
3599 Known.Zero.setBitsFrom(1);
3600 break;
3601 }
3602 case ISD::LOAD: {
3603 LoadSDNode *LD = cast<LoadSDNode>(Op);
3604 const Constant *Cst = TLI->getTargetConstantFromLoad(LD);
3605 if (ISD::isNON_EXTLoad(LD) && Cst) {
3606 // Determine any common known bits from the loaded constant pool value.
3607 Type *CstTy = Cst->getType();
3608 if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits() &&
3609 !Op.getValueType().isScalableVector()) {
3610 // If its a vector splat, then we can (quickly) reuse the scalar path.
3611 // NOTE: We assume all elements match and none are UNDEF.
3612 if (CstTy->isVectorTy()) {
3613 if (const Constant *Splat = Cst->getSplatValue()) {
3614 Cst = Splat;
3615 CstTy = Cst->getType();
3616 }
3617 }
3618 // TODO - do we need to handle different bitwidths?
3619 if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) {
3620 // Iterate across all vector elements finding common known bits.
3621 Known.One.setAllBits();
3622 Known.Zero.setAllBits();
3623 for (unsigned i = 0; i != NumElts; ++i) {
3624 if (!DemandedElts[i])
3625 continue;
3626 if (Constant *Elt = Cst->getAggregateElement(i)) {
3627 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
3628 const APInt &Value = CInt->getValue();
3629 Known.One &= Value;
3630 Known.Zero &= ~Value;
3631 continue;
3632 }
3633 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
3634 APInt Value = CFP->getValueAPF().bitcastToAPInt();
3635 Known.One &= Value;
3636 Known.Zero &= ~Value;
3637 continue;
3638 }
3639 }
3640 Known.One.clearAllBits();
3641 Known.Zero.clearAllBits();
3642 break;
3643 }
3644 } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) {
3645 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
3646 Known = KnownBits::makeConstant(CInt->getValue());
3647 } else if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
3648 Known =
3649 KnownBits::makeConstant(CFP->getValueAPF().bitcastToAPInt());
3650 }
3651 }
3652 }
3653 } else if (Op.getResNo() == 0) {
3654 KnownBits Known0(!LD->getMemoryVT().isScalableVT()
3655 ? LD->getMemoryVT().getFixedSizeInBits()
3656 : BitWidth);
3657 EVT VT = Op.getValueType();
3658 // Fill in any known bits from range information. There are 3 types being
3659 // used. The results VT (same vector elt size as BitWidth), the loaded
3660 // MemoryVT (which may or may not be vector) and the range VTs original
3661 // type. The range matadata needs the full range (i.e
3662 // MemoryVT().getSizeInBits()), which is truncated to the correct elt size
3663 // if it is know. These are then extended to the original VT sizes below.
3664 if (const MDNode *MD = LD->getRanges()) {
3666 if (VT.isVector()) {
3667 // Handle truncation to the first demanded element.
3668 // TODO: Figure out which demanded elements are covered
3669 if (DemandedElts != 1 || !getDataLayout().isLittleEndian())
3670 break;
3671 Known0 = Known0.trunc(BitWidth);
3672 }
3673 }
3674
3675 if (LD->getMemoryVT().isVector())
3676 Known0 = Known0.trunc(LD->getMemoryVT().getScalarSizeInBits());
3677
3678 // Extend the Known bits from memory to the size of the result.
3679 if (ISD::isZEXTLoad(Op.getNode()))
3680 Known = Known0.zext(BitWidth);
3681 else if (ISD::isSEXTLoad(Op.getNode()))
3682 Known = Known0.sext(BitWidth);
3683 else if (ISD::isEXTLoad(Op.getNode()))
3684 Known = Known0.anyext(BitWidth);
3685 else
3686 Known = Known0;
3687 assert(Known.getBitWidth() == BitWidth);
3688 return Known;
3689 }
3690 break;
3691 }
3693 if (Op.getValueType().isScalableVector())
3694 break;
3695 EVT InVT = Op.getOperand(0).getValueType();
3696 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3697 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3698 Known = Known.zext(BitWidth);
3699 break;
3700 }
3701 case ISD::ZERO_EXTEND: {
3702 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3703 Known = Known.zext(BitWidth);
3704 break;
3705 }
3707 if (Op.getValueType().isScalableVector())
3708 break;
3709 EVT InVT = Op.getOperand(0).getValueType();
3710 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3711 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3712 // If the sign bit is known to be zero or one, then sext will extend
3713 // it to the top bits, else it will just zext.
3714 Known = Known.sext(BitWidth);
3715 break;
3716 }
3717 case ISD::SIGN_EXTEND: {
3718 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3719 // If the sign bit is known to be zero or one, then sext will extend
3720 // it to the top bits, else it will just zext.
3721 Known = Known.sext(BitWidth);
3722 break;
3723 }
3725 if (Op.getValueType().isScalableVector())
3726 break;
3727 EVT InVT = Op.getOperand(0).getValueType();
3728 APInt InDemandedElts = DemandedElts.zext(InVT.getVectorNumElements());
3729 Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1);
3730 Known = Known.anyext(BitWidth);
3731 break;
3732 }
3733 case ISD::ANY_EXTEND: {
3734 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3735 Known = Known.anyext(BitWidth);
3736 break;
3737 }
3738 case ISD::TRUNCATE: {
3739 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3740 Known = Known.trunc(BitWidth);
3741 break;
3742 }
3743 case ISD::AssertZext: {
3744 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
3746 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3747 Known.Zero |= (~InMask);
3748 Known.One &= (~Known.Zero);
3749 break;
3750 }
3751 case ISD::AssertAlign: {
3752 unsigned LogOfAlign = Log2(cast<AssertAlignSDNode>(Op)->getAlign());
3753 assert(LogOfAlign != 0);
3754
3755 // TODO: Should use maximum with source
3756 // If a node is guaranteed to be aligned, set low zero bits accordingly as
3757 // well as clearing one bits.
3758 Known.Zero.setLowBits(LogOfAlign);
3759 Known.One.clearLowBits(LogOfAlign);
3760 break;
3761 }
3762 case ISD::FGETSIGN:
3763 // All bits are zero except the low bit.
3764 Known.Zero.setBitsFrom(1);
3765 break;
3766 case ISD::ADD:
3767 case ISD::SUB: {
3768 SDNodeFlags Flags = Op.getNode()->getFlags();
3769 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3770 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3772 Op.getOpcode() == ISD::ADD, Flags.hasNoSignedWrap(),
3773 Flags.hasNoUnsignedWrap(), Known, Known2);
3774 break;
3775 }
3776 case ISD::USUBO:
3777 case ISD::SSUBO:
3778 case ISD::USUBO_CARRY:
3779 case ISD::SSUBO_CARRY:
3780 if (Op.getResNo() == 1) {
3781 // If we know the result of a setcc has the top bits zero, use this info.
3782 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3784 BitWidth > 1)
3785 Known.Zero.setBitsFrom(1);
3786 break;
3787 }
3788 [[fallthrough]];
3789 case ISD::SUBC: {
3790 assert(Op.getResNo() == 0 &&
3791 "We only compute knownbits for the difference here.");
3792
3793 // With USUBO_CARRY and SSUBO_CARRY a borrow bit may be added in.
3794 KnownBits Borrow(1);
3795 if (Opcode == ISD::USUBO_CARRY || Opcode == ISD::SSUBO_CARRY) {
3796 Borrow = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3797 // Borrow has bit width 1
3798 Borrow = Borrow.trunc(1);
3799 } else {
3800 Borrow.setAllZero();
3801 }
3802
3803 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3804 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3805 Known = KnownBits::computeForSubBorrow(Known, Known2, Borrow);
3806 break;
3807 }
3808 case ISD::UADDO:
3809 case ISD::SADDO:
3810 case ISD::UADDO_CARRY:
3811 case ISD::SADDO_CARRY:
3812 if (Op.getResNo() == 1) {
3813 // If we know the result of a setcc has the top bits zero, use this info.
3814 if (TLI->getBooleanContents(Op.getOperand(0).getValueType()) ==
3816 BitWidth > 1)
3817 Known.Zero.setBitsFrom(1);
3818 break;
3819 }
3820 [[fallthrough]];
3821 case ISD::ADDC:
3822 case ISD::ADDE: {
3823 assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here.");
3824
3825 // With ADDE and UADDO_CARRY, a carry bit may be added in.
3826 KnownBits Carry(1);
3827 if (Opcode == ISD::ADDE)
3828 // Can't track carry from glue, set carry to unknown.
3829 Carry.resetAll();
3830 else if (Opcode == ISD::UADDO_CARRY || Opcode == ISD::SADDO_CARRY) {
3831 Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1);
3832 // Carry has bit width 1
3833 Carry = Carry.trunc(1);
3834 } else {
3835 Carry.setAllZero();
3836 }
3837
3838 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3839 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3840 Known = KnownBits::computeForAddCarry(Known, Known2, Carry);
3841 break;
3842 }
3843 case ISD::UDIV: {
3844 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3845 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3846 Known = KnownBits::udiv(Known, Known2, Op->getFlags().hasExact());
3847 break;
3848 }
3849 case ISD::SDIV: {
3850 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3851 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3852 Known = KnownBits::sdiv(Known, Known2, Op->getFlags().hasExact());
3853 break;
3854 }
3855 case ISD::SREM: {
3856 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3857 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3858 Known = KnownBits::srem(Known, Known2);
3859 break;
3860 }
3861 case ISD::UREM: {
3862 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3863 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3864 Known = KnownBits::urem(Known, Known2);
3865 break;
3866 }
3867 case ISD::EXTRACT_ELEMENT: {
3868 Known = computeKnownBits(Op.getOperand(0), Depth+1);
3869 const unsigned Index = Op.getConstantOperandVal(1);
3870 const unsigned EltBitWidth = Op.getValueSizeInBits();
3871
3872 // Remove low part of known bits mask
3873 Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3874 Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth);
3875
3876 // Remove high part of known bit mask
3877 Known = Known.trunc(EltBitWidth);
3878 break;
3879 }
3881 SDValue InVec = Op.getOperand(0);
3882 SDValue EltNo = Op.getOperand(1);
3883 EVT VecVT = InVec.getValueType();
3884 // computeKnownBits not yet implemented for scalable vectors.
3885 if (VecVT.isScalableVector())
3886 break;
3887 const unsigned EltBitWidth = VecVT.getScalarSizeInBits();
3888 const unsigned NumSrcElts = VecVT.getVectorNumElements();
3889
3890 // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know
3891 // anything about the extended bits.
3892 if (BitWidth > EltBitWidth)
3893 Known = Known.trunc(EltBitWidth);
3894
3895 // If we know the element index, just demand that vector element, else for
3896 // an unknown element index, ignore DemandedElts and demand them all.
3897 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
3898 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
3899 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
3900 DemandedSrcElts =
3901 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
3902
3903 Known = computeKnownBits(InVec, DemandedSrcElts, Depth + 1);
3904 if (BitWidth > EltBitWidth)
3905 Known = Known.anyext(BitWidth);
3906 break;
3907 }
3909 if (Op.getValueType().isScalableVector())
3910 break;
3911
3912 // If we know the element index, split the demand between the
3913 // source vector and the inserted element, otherwise assume we need
3914 // the original demanded vector elements and the value.
3915 SDValue InVec = Op.getOperand(0);
3916 SDValue InVal = Op.getOperand(1);
3917 SDValue EltNo = Op.getOperand(2);
3918 bool DemandedVal = true;
3919 APInt DemandedVecElts = DemandedElts;
3920 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
3921 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
3922 unsigned EltIdx = CEltNo->getZExtValue();
3923 DemandedVal = !!DemandedElts[EltIdx];
3924 DemandedVecElts.clearBit(EltIdx);
3925 }
3926 Known.One.setAllBits();
3927 Known.Zero.setAllBits();
3928 if (DemandedVal) {
3929 Known2 = computeKnownBits(InVal, Depth + 1);
3930 Known = Known.intersectWith(Known2.zextOrTrunc(BitWidth));
3931 }
3932 if (!!DemandedVecElts) {
3933 Known2 = computeKnownBits(InVec, DemandedVecElts, Depth + 1);
3934 Known = Known.intersectWith(Known2);
3935 }
3936 break;
3937 }
3938 case ISD::BITREVERSE: {
3939 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3940 Known = Known2.reverseBits();
3941 break;
3942 }
3943 case ISD::BSWAP: {
3944 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3945 Known = Known2.byteSwap();
3946 break;
3947 }
3948 case ISD::ABS: {
3949 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3950 Known = Known2.abs();
3951 break;
3952 }
3953 case ISD::USUBSAT: {
3954 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3955 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3956 Known = KnownBits::usub_sat(Known, Known2);
3957 break;
3958 }
3959 case ISD::UMIN: {
3960 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3961 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3962 Known = KnownBits::umin(Known, Known2);
3963 break;
3964 }
3965 case ISD::UMAX: {
3966 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
3967 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
3968 Known = KnownBits::umax(Known, Known2);
3969 break;
3970 }
3971 case ISD::SMIN:
3972 case ISD::SMAX: {
3973 // If we have a clamp pattern, we know that the number of sign bits will be
3974 // the minimum of the clamp min/max range.
3975 bool IsMax = (Opcode == ISD::SMAX);
3976 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
3977 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
3978 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
3979 CstHigh =
3980 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
3981 if (CstLow && CstHigh) {
3982 if (!IsMax)
3983 std::swap(CstLow, CstHigh);
3984
3985 const APInt &ValueLow = CstLow->getAPIntValue();
3986 const APInt &ValueHigh = CstHigh->getAPIntValue();
3987 if (ValueLow.sle(ValueHigh)) {
3988 unsigned LowSignBits = ValueLow.getNumSignBits();
3989 unsigned HighSignBits = ValueHigh.getNumSignBits();
3990 unsigned MinSignBits = std::min(LowSignBits, HighSignBits);
3991 if (ValueLow.isNegative() && ValueHigh.isNegative()) {
3992 Known.One.setHighBits(MinSignBits);
3993 break;
3994 }
3995 if (ValueLow.isNonNegative() && ValueHigh.isNonNegative()) {
3996 Known.Zero.setHighBits(MinSignBits);
3997 break;
3998 }
3999 }
4000 }
4001
4002 Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4003 Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4004 if (IsMax)
4005 Known = KnownBits::smax(Known, Known2);
4006 else
4007 Known = KnownBits::smin(Known, Known2);
4008
4009 // For SMAX, if CstLow is non-negative we know the result will be
4010 // non-negative and thus all sign bits are 0.
4011 // TODO: There's an equivalent of this for smin with negative constant for
4012 // known ones.
4013 if (IsMax && CstLow) {
4014 const APInt &ValueLow = CstLow->getAPIntValue();
4015 if (ValueLow.isNonNegative()) {
4016 unsigned SignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4017 Known.Zero.setHighBits(std::min(SignBits, ValueLow.getNumSignBits()));
4018 }
4019 }
4020
4021 break;
4022 }
4023 case ISD::UINT_TO_FP: {
4024 Known.makeNonNegative();
4025 break;
4026 }
4027 case ISD::SINT_TO_FP: {
4028 Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4029 if (Known2.isNonNegative())
4030 Known.makeNonNegative();
4031 else if (Known2.isNegative())
4032 Known.makeNegative();
4033 break;
4034 }
4035 case ISD::FP_TO_UINT_SAT: {
4036 // FP_TO_UINT_SAT produces an unsigned value that fits in the saturating VT.
4037 EVT VT = cast<VTSDNode>(Op.getOperand(1))->getVT();
4039 break;
4040 }
4042 if (Op.getResNo() == 1) {
4043 // The boolean result conforms to getBooleanContents.
4044 // If we know the result of a setcc has the top bits zero, use this info.
4045 // We know that we have an integer-based boolean since these operations
4046 // are only available for integer.
4047 if (TLI->getBooleanContents(Op.getValueType().isVector(), false) ==
4049 BitWidth > 1)
4050 Known.Zero.setBitsFrom(1);
4051 break;
4052 }
4053 [[fallthrough]];
4055 case ISD::ATOMIC_SWAP:
4067 case ISD::ATOMIC_LOAD: {
4068 unsigned MemBits =
4069 cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4070 // If we are looking at the loaded value.
4071 if (Op.getResNo() == 0) {
4073 Known.Zero.setBitsFrom(MemBits);
4074 else if (Op->getOpcode() == ISD::ATOMIC_LOAD &&
4075 cast<AtomicSDNode>(Op)->getExtensionType() == ISD::ZEXTLOAD)
4076 Known.Zero.setBitsFrom(MemBits);
4077 }
4078 break;
4079 }
4080 case ISD::FrameIndex:
4082 TLI->computeKnownBitsForFrameIndex(cast<FrameIndexSDNode>(Op)->getIndex(),
4083 Known, getMachineFunction());
4084 break;
4085
4086 default:
4087 if (Opcode < ISD::BUILTIN_OP_END)
4088 break;
4089 [[fallthrough]];
4093 // TODO: Probably okay to remove after audit; here to reduce change size
4094 // in initial enablement patch for scalable vectors
4095 if (Op.getValueType().isScalableVector())
4096 break;
4097
4098 // Allow the target to implement this method for its nodes.
4099 TLI->computeKnownBitsForTargetNode(Op, Known, DemandedElts, *this, Depth);
4100 break;
4101 }
4102
4103 assert(!Known.hasConflict() && "Bits known to be one AND zero?");
4104 return Known;
4105}
4106
4107/// Convert ConstantRange OverflowResult into SelectionDAG::OverflowKind.
4109 switch (OR) {
4117 }
4118 llvm_unreachable("Unknown OverflowResult");
4119}
4120
4123 // X + 0 never overflow
4124 if (isNullConstant(N1))
4125 return OFK_Never;
4126
4127 // If both operands each have at least two sign bits, the addition
4128 // cannot overflow.
4129 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4130 return OFK_Never;
4131
4132 // TODO: Add ConstantRange::signedAddMayOverflow handling.
4133 return OFK_Sometime;
4134}
4135
4138 // X + 0 never overflow
4139 if (isNullConstant(N1))
4140 return OFK_Never;
4141
4142 // mulhi + 1 never overflow
4143 KnownBits N1Known = computeKnownBits(N1);
4144 if (N0.getOpcode() == ISD::UMUL_LOHI && N0.getResNo() == 1 &&
4145 N1Known.getMaxValue().ult(2))
4146 return OFK_Never;
4147
4148 KnownBits N0Known = computeKnownBits(N0);
4149 if (N1.getOpcode() == ISD::UMUL_LOHI && N1.getResNo() == 1 &&
4150 N0Known.getMaxValue().ult(2))
4151 return OFK_Never;
4152
4153 // Fallback to ConstantRange::unsignedAddMayOverflow handling.
4154 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4155 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4156 return mapOverflowResult(N0Range.unsignedAddMayOverflow(N1Range));
4157}
4158
4161 // X - 0 never overflow
4162 if (isNullConstant(N1))
4163 return OFK_Never;
4164
4165 // If both operands each have at least two sign bits, the subtraction
4166 // cannot overflow.
4167 if (ComputeNumSignBits(N0) > 1 && ComputeNumSignBits(N1) > 1)
4168 return OFK_Never;
4169
4170 KnownBits N0Known = computeKnownBits(N0);
4171 KnownBits N1Known = computeKnownBits(N1);
4172 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, true);
4173 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, true);
4174 return mapOverflowResult(N0Range.signedSubMayOverflow(N1Range));
4175}
4176
4179 // X - 0 never overflow
4180 if (isNullConstant(N1))
4181 return OFK_Never;
4182
4183 KnownBits N0Known = computeKnownBits(N0);
4184 KnownBits N1Known = computeKnownBits(N1);
4185 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4186 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4187 return mapOverflowResult(N0Range.unsignedSubMayOverflow(N1Range));
4188}
4189
4192 // X * 0 and X * 1 never overflow.
4193 if (isNullConstant(N1) || isOneConstant(N1))
4194 return OFK_Never;
4195
4196 KnownBits N0Known = computeKnownBits(N0);
4197 KnownBits N1Known = computeKnownBits(N1);
4198 ConstantRange N0Range = ConstantRange::fromKnownBits(N0Known, false);
4199 ConstantRange N1Range = ConstantRange::fromKnownBits(N1Known, false);
4200 return mapOverflowResult(N0Range.unsignedMulMayOverflow(N1Range));
4201}
4202
4205 // X * 0 and X * 1 never overflow.
4206 if (isNullConstant(N1) || isOneConstant(N1))
4207 return OFK_Never;
4208
4209 // Get the size of the result.
4210 unsigned BitWidth = N0.getScalarValueSizeInBits();
4211
4212 // Sum of the sign bits.
4213 unsigned SignBits = ComputeNumSignBits(N0) + ComputeNumSignBits(N1);
4214
4215 // If we have enough sign bits, then there's no overflow.
4216 if (SignBits > BitWidth + 1)
4217 return OFK_Never;
4218
4219 if (SignBits == BitWidth + 1) {
4220 // The overflow occurs when the true multiplication of the
4221 // the operands is the minimum negative number.
4222 KnownBits N0Known = computeKnownBits(N0);
4223 KnownBits N1Known = computeKnownBits(N1);
4224 // If one of the operands is non-negative, then there's no
4225 // overflow.
4226 if (N0Known.isNonNegative() || N1Known.isNonNegative())
4227 return OFK_Never;
4228 }
4229
4230 return OFK_Sometime;
4231}
4232
4234 if (Depth >= MaxRecursionDepth)
4235 return false; // Limit search depth.
4236
4237 EVT OpVT = Val.getValueType();
4238 unsigned BitWidth = OpVT.getScalarSizeInBits();
4239
4240 // Is the constant a known power of 2?
4242 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4243 }))
4244 return true;
4245
4246 // A left-shift of a constant one will have exactly one bit set because
4247 // shifting the bit off the end is undefined.
4248 if (Val.getOpcode() == ISD::SHL) {
4249 auto *C = isConstOrConstSplat(Val.getOperand(0));
4250 if (C && C->getAPIntValue() == 1)
4251 return true;
4252 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4253 isKnownNeverZero(Val, Depth);
4254 }
4255
4256 // Similarly, a logical right-shift of a constant sign-bit will have exactly
4257 // one bit set.
4258 if (Val.getOpcode() == ISD::SRL) {
4259 auto *C = isConstOrConstSplat(Val.getOperand(0));
4260 if (C && C->getAPIntValue().isSignMask())
4261 return true;
4262 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1) &&
4263 isKnownNeverZero(Val, Depth);
4264 }
4265
4266 if (Val.getOpcode() == ISD::ROTL || Val.getOpcode() == ISD::ROTR)
4267 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4268
4269 // Are all operands of a build vector constant powers of two?
4270 if (Val.getOpcode() == ISD::BUILD_VECTOR)
4271 if (llvm::all_of(Val->ops(), [BitWidth](SDValue E) {
4272 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(E))
4273 return C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2();
4274 return false;
4275 }))
4276 return true;
4277
4278 // Is the operand of a splat vector a constant power of two?
4279 if (Val.getOpcode() == ISD::SPLAT_VECTOR)
4280 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val->getOperand(0)))
4281 if (C->getAPIntValue().zextOrTrunc(BitWidth).isPowerOf2())
4282 return true;
4283
4284 // vscale(power-of-two) is a power-of-two for some targets
4285 if (Val.getOpcode() == ISD::VSCALE &&
4286 getTargetLoweringInfo().isVScaleKnownToBeAPowerOfTwo() &&
4288 return true;
4289
4290 if (Val.getOpcode() == ISD::SMIN || Val.getOpcode() == ISD::SMAX ||
4291 Val.getOpcode() == ISD::UMIN || Val.getOpcode() == ISD::UMAX)
4292 return isKnownToBeAPowerOfTwo(Val.getOperand(1), Depth + 1) &&
4294
4295 if (Val.getOpcode() == ISD::SELECT || Val.getOpcode() == ISD::VSELECT)
4296 return isKnownToBeAPowerOfTwo(Val.getOperand(2), Depth + 1) &&
4298
4299 // Looking for `x & -x` pattern:
4300 // If x == 0:
4301 // x & -x -> 0
4302 // If x != 0:
4303 // x & -x -> non-zero pow2
4304 // so if we find the pattern return whether we know `x` is non-zero.
4305 SDValue X;
4306 if (sd_match(Val, m_And(m_Value(X), m_Sub(m_Zero(), m_Deferred(X)))))
4307 return isKnownNeverZero(X, Depth);
4308
4309 if (Val.getOpcode() == ISD::ZERO_EXTEND)
4310 return isKnownToBeAPowerOfTwo(Val.getOperand(0), Depth + 1);
4311
4312 // More could be done here, though the above checks are enough
4313 // to handle some common cases.
4314 return false;
4315}
4316
4318 EVT VT = Op.getValueType();
4319
4320 // Since the number of lanes in a scalable vector is unknown at compile time,
4321 // we track one bit which is implicitly broadcast to all lanes. This means
4322 // that all lanes in a scalable vector are considered demanded.
4323 APInt DemandedElts = VT.isFixedLengthVector()
4325 : APInt(1, 1);
4326 return ComputeNumSignBits(Op, DemandedElts, Depth);
4327}
4328
4329unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
4330 unsigned Depth) const {
4331 EVT VT = Op.getValueType();
4332 assert((VT.isInteger() || VT.isFloatingPoint()) && "Invalid VT!");
4333 unsigned VTBits = VT.getScalarSizeInBits();
4334 unsigned NumElts = DemandedElts.getBitWidth();
4335 unsigned Tmp, Tmp2;
4336 unsigned FirstAnswer = 1;
4337
4338 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
4339 const APInt &Val = C->getAPIntValue();
4340 return Val.getNumSignBits();
4341 }
4342
4343 if (Depth >= MaxRecursionDepth)
4344 return 1; // Limit search depth.
4345
4346 if (!DemandedElts)
4347 return 1; // No demanded elts, better to assume we don't know anything.
4348
4349 unsigned Opcode = Op.getOpcode();
4350 switch (Opcode) {
4351 default: break;
4352 case ISD::AssertSext:
4353 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4354 return VTBits-Tmp+1;
4355 case ISD::AssertZext:
4356 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4357 return VTBits-Tmp;
4358 case ISD::MERGE_VALUES:
4359 return ComputeNumSignBits(Op.getOperand(Op.getResNo()), DemandedElts,
4360 Depth + 1);
4361 case ISD::SPLAT_VECTOR: {
4362 // Check if the sign bits of source go down as far as the truncated value.
4363 unsigned NumSrcBits = Op.getOperand(0).getValueSizeInBits();
4364 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4365 if (NumSrcSignBits > (NumSrcBits - VTBits))
4366 return NumSrcSignBits - (NumSrcBits - VTBits);
4367 break;
4368 }
4369 case ISD::BUILD_VECTOR:
4370 assert(!VT.isScalableVector());
4371 Tmp = VTBits;
4372 for (unsigned i = 0, e = Op.getNumOperands(); (i < e) && (Tmp > 1); ++i) {
4373 if (!DemandedElts[i])
4374 continue;
4375
4376 SDValue SrcOp = Op.getOperand(i);
4377 // BUILD_VECTOR can implicitly truncate sources, we handle this specially
4378 // for constant nodes to ensure we only look at the sign bits.
4379 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(SrcOp)) {
4380 APInt T = C->getAPIntValue().trunc(VTBits);
4381 Tmp2 = T.getNumSignBits();
4382 } else {
4383 Tmp2 = ComputeNumSignBits(SrcOp, Depth + 1);
4384
4385 if (SrcOp.getValueSizeInBits() != VTBits) {
4386 assert(SrcOp.getValueSizeInBits() > VTBits &&
4387 "Expected BUILD_VECTOR implicit truncation");
4388 unsigned ExtraBits = SrcOp.getValueSizeInBits() - VTBits;
4389 Tmp2 = (Tmp2 > ExtraBits ? Tmp2 - ExtraBits : 1);
4390 }
4391 }
4392 Tmp = std::min(Tmp, Tmp2);
4393 }
4394 return Tmp;
4395
4396 case ISD::VECTOR_SHUFFLE: {
4397 // Collect the minimum number of sign bits that are shared by every vector
4398 // element referenced by the shuffle.
4399 APInt DemandedLHS, DemandedRHS;
4400 const ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Op);
4401 assert(NumElts == SVN->getMask().size() && "Unexpected vector size");
4402 if (!getShuffleDemandedElts(NumElts, SVN->getMask(), DemandedElts,
4403 DemandedLHS, DemandedRHS))
4404 return 1;
4405
4406 Tmp = std::numeric_limits<unsigned>::max();
4407 if (!!DemandedLHS)
4408 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
4409 if (!!DemandedRHS) {
4410 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
4411 Tmp = std::min(Tmp, Tmp2);
4412 }
4413 // If we don't know anything, early out and try computeKnownBits fall-back.
4414 if (Tmp == 1)
4415 break;
4416 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4417 return Tmp;
4418 }
4419
4420 case ISD::BITCAST: {
4421 if (VT.isScalableVector())
4422 break;
4423 SDValue N0 = Op.getOperand(0);
4424 EVT SrcVT = N0.getValueType();
4425 unsigned SrcBits = SrcVT.getScalarSizeInBits();
4426
4427 // Ignore bitcasts from unsupported types..
4428 if (!(SrcVT.isInteger() || SrcVT.isFloatingPoint()))
4429 break;
4430
4431 // Fast handling of 'identity' bitcasts.
4432 if (VTBits == SrcBits)
4433 return ComputeNumSignBits(N0, DemandedElts, Depth + 1);
4434
4435 bool IsLE = getDataLayout().isLittleEndian();
4436
4437 // Bitcast 'large element' scalar/vector to 'small element' vector.
4438 if ((SrcBits % VTBits) == 0) {
4439 assert(VT.isVector() && "Expected bitcast to vector");
4440
4441 unsigned Scale = SrcBits / VTBits;
4442 APInt SrcDemandedElts =
4443 APIntOps::ScaleBitMask(DemandedElts, NumElts / Scale);
4444
4445 // Fast case - sign splat can be simply split across the small elements.
4446 Tmp = ComputeNumSignBits(N0, SrcDemandedElts, Depth + 1);
4447 if (Tmp == SrcBits)
4448 return VTBits;
4449
4450 // Slow case - determine how far the sign extends into each sub-element.
4451 Tmp2 = VTBits;
4452 for (unsigned i = 0; i != NumElts; ++i)
4453 if (DemandedElts[i]) {
4454 unsigned SubOffset = i % Scale;
4455 SubOffset = (IsLE ? ((Scale - 1) - SubOffset) : SubOffset);
4456 SubOffset = SubOffset * VTBits;
4457 if (Tmp <= SubOffset)
4458 return 1;
4459 Tmp2 = std::min(Tmp2, Tmp - SubOffset);
4460 }
4461 return Tmp2;
4462 }
4463 break;
4464 }
4465
4467 // FP_TO_SINT_SAT produces a signed value that fits in the saturating VT.
4468 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4469 return VTBits - Tmp + 1;
4470 case ISD::SIGN_EXTEND:
4471 Tmp = VTBits - Op.getOperand(0).getScalarValueSizeInBits();
4472 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1) + Tmp;
4474 // Max of the input and what this extends.
4475 Tmp = cast<VTSDNode>(Op.getOperand(1))->getVT().getScalarSizeInBits();
4476 Tmp = VTBits-Tmp+1;
4477 Tmp2 = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4478 return std::max(Tmp, Tmp2);
4480 if (VT.isScalableVector())
4481 break;
4482 SDValue Src = Op.getOperand(0);
4483 EVT SrcVT = Src.getValueType();
4484 APInt DemandedSrcElts = DemandedElts.zext(SrcVT.getVectorNumElements());
4485 Tmp = VTBits - SrcVT.getScalarSizeInBits();
4486 return ComputeNumSignBits(Src, DemandedSrcElts, Depth+1) + Tmp;
4487 }
4488 case ISD::SRA:
4489 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4490 // SRA X, C -> adds C sign bits.
4491 if (const APInt *ShAmt =
4493 Tmp = std::min<uint64_t>(Tmp + ShAmt->getZExtValue(), VTBits);
4494 return Tmp;
4495 case ISD::SHL:
4496 if (const APInt *ShAmt =
4497 getValidMaximumShiftAmountConstant(Op, DemandedElts)) {
4498 // shl destroys sign bits, ensure it doesn't shift out all sign bits.
4499 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4500 if (ShAmt->ult(Tmp))
4501 return Tmp - ShAmt->getZExtValue();
4502 }
4503 break;
4504 case ISD::AND:
4505 case ISD::OR:
4506 case ISD::XOR: // NOT is handled here.
4507 // Logical binary ops preserve the number of sign bits at the worst.
4508 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1);
4509 if (Tmp != 1) {
4510 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4511 FirstAnswer = std::min(Tmp, Tmp2);
4512 // We computed what we know about the sign bits as our first
4513 // answer. Now proceed to the generic code that uses
4514 // computeKnownBits, and pick whichever answer is better.
4515 }
4516 break;
4517
4518 case ISD::SELECT:
4519 case ISD::VSELECT:
4520 Tmp = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth+1);
4521 if (Tmp == 1) return 1; // Early out.
4522 Tmp2 = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4523 return std::min(Tmp, Tmp2);
4524 case ISD::SELECT_CC:
4525 Tmp = ComputeNumSignBits(Op.getOperand(2), DemandedElts, Depth+1);
4526 if (Tmp == 1) return 1; // Early out.
4527 Tmp2 = ComputeNumSignBits(Op.getOperand(3), DemandedElts, Depth+1);
4528 return std::min(Tmp, Tmp2);
4529
4530 case ISD::SMIN:
4531 case ISD::SMAX: {
4532 // If we have a clamp pattern, we know that the number of sign bits will be
4533 // the minimum of the clamp min/max range.
4534 bool IsMax = (Opcode == ISD::SMAX);
4535 ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr;
4536 if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts)))
4537 if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX))
4538 CstHigh =
4539 isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts);
4540 if (CstLow && CstHigh) {
4541 if (!IsMax)
4542 std::swap(CstLow, CstHigh);
4543 if (CstLow->getAPIntValue().sle(CstHigh->getAPIntValue())) {
4544 Tmp = CstLow->getAPIntValue().getNumSignBits();
4545 Tmp2 = CstHigh->getAPIntValue().getNumSignBits();
4546 return std::min(Tmp, Tmp2);
4547 }
4548 }
4549
4550 // Fallback - just get the minimum number of sign bits of the operands.
4551 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4552 if (Tmp == 1)
4553 return 1; // Early out.
4554 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4555 return std::min(Tmp, Tmp2);
4556 }
4557 case ISD::UMIN:
4558 case ISD::UMAX:
4559 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4560 if (Tmp == 1)
4561 return 1; // Early out.
4562 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4563 return std::min(Tmp, Tmp2);
4564 case ISD::SADDO:
4565 case ISD::UADDO:
4566 case ISD::SADDO_CARRY:
4567 case ISD::UADDO_CARRY:
4568 case ISD::SSUBO:
4569 case ISD::USUBO:
4570 case ISD::SSUBO_CARRY:
4571 case ISD::USUBO_CARRY:
4572 case ISD::SMULO:
4573 case ISD::UMULO:
4574 if (Op.getResNo() != 1)
4575 break;
4576 // The boolean result conforms to getBooleanContents. Fall through.
4577 // If setcc returns 0/-1, all bits are sign bits.
4578 // We know that we have an integer-based boolean since these operations
4579 // are only available for integer.
4580 if (TLI->getBooleanContents(VT.isVector(), false) ==
4582 return VTBits;
4583 break;
4584 case ISD::SETCC:
4585 case ISD::SETCCCARRY:
4586 case ISD::STRICT_FSETCC:
4587 case ISD::STRICT_FSETCCS: {
4588 unsigned OpNo = Op->isStrictFPOpcode() ? 1 : 0;
4589 // If setcc returns 0/-1, all bits are sign bits.
4590 if (TLI->getBooleanContents(Op.getOperand(OpNo).getValueType()) ==
4592 return VTBits;
4593 break;
4594 }
4595 case ISD::ROTL:
4596 case ISD::ROTR:
4597 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4598
4599 // If we're rotating an 0/-1 value, then it stays an 0/-1 value.
4600 if (Tmp == VTBits)
4601 return VTBits;
4602
4603 if (ConstantSDNode *C =
4604 isConstOrConstSplat(Op.getOperand(1), DemandedElts)) {
4605 unsigned RotAmt = C->getAPIntValue().urem(VTBits);
4606
4607 // Handle rotate right by N like a rotate left by 32-N.
4608 if (Opcode == ISD::ROTR)
4609 RotAmt = (VTBits - RotAmt) % VTBits;
4610
4611 // If we aren't rotating out all of the known-in sign bits, return the
4612 // number that are left. This handles rotl(sext(x), 1) for example.
4613 if (Tmp > (RotAmt + 1)) return (Tmp - RotAmt);
4614 }
4615 break;
4616 case ISD::ADD:
4617 case ISD::ADDC:
4618 // Add can have at most one carry bit. Thus we know that the output
4619 // is, at worst, one more bit than the inputs.
4620 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4621 if (Tmp == 1) return 1; // Early out.
4622
4623 // Special case decrementing a value (ADD X, -1):
4624 if (ConstantSDNode *CRHS =
4625 isConstOrConstSplat(Op.getOperand(1), DemandedElts))
4626 if (CRHS->isAllOnes()) {
4627 KnownBits Known =
4628 computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
4629
4630 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4631 // sign bits set.
4632 if ((Known.Zero | 1).isAllOnes())
4633 return VTBits;
4634
4635 // If we are subtracting one from a positive number, there is no carry
4636 // out of the result.
4637 if (Known.isNonNegative())
4638 return Tmp;
4639 }
4640
4641 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4642 if (Tmp2 == 1) return 1; // Early out.
4643 return std::min(Tmp, Tmp2) - 1;
4644 case ISD::SUB:
4645 Tmp2 = ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
4646 if (Tmp2 == 1) return 1; // Early out.
4647
4648 // Handle NEG.
4649 if (ConstantSDNode *CLHS =
4650 isConstOrConstSplat(Op.getOperand(0), DemandedElts))
4651 if (CLHS->isZero()) {
4652 KnownBits Known =
4653 computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
4654 // If the input is known to be 0 or 1, the output is 0/-1, which is all
4655 // sign bits set.
4656 if ((Known.Zero | 1).isAllOnes())
4657 return VTBits;
4658
4659 // If the input is known to be positive (the sign bit is known clear),
4660 // the output of the NEG has the same number of sign bits as the input.
4661 if (Known.isNonNegative())
4662 return Tmp2;
4663
4664 // Otherwise, we treat this like a SUB.
4665 }
4666
4667 // Sub can have at most one carry bit. Thus we know that the output
4668 // is, at worst, one more bit than the inputs.
4669 Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4670 if (Tmp == 1) return 1; // Early out.
4671 return std::min(Tmp, Tmp2) - 1;
4672 case ISD::MUL: {
4673 // The output of the Mul can be at most twice the valid bits in the inputs.
4674 unsigned SignBitsOp0 = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4675 if (SignBitsOp0 == 1)
4676 break;
4677 unsigned SignBitsOp1 = ComputeNumSignBits(Op.getOperand(1), Depth + 1);
4678 if (SignBitsOp1 == 1)
4679 break;
4680 unsigned OutValidBits =
4681 (VTBits - SignBitsOp0 + 1) + (VTBits - SignBitsOp1 + 1);
4682 return OutValidBits > VTBits ? 1 : VTBits - OutValidBits + 1;
4683 }
4684 case ISD::SREM:
4685 // The sign bit is the LHS's sign bit, except when the result of the
4686 // remainder is zero. The magnitude of the result should be less than or
4687 // equal to the magnitude of the LHS. Therefore, the result should have
4688 // at least as many sign bits as the left hand side.
4689 return ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
4690 case ISD::TRUNCATE: {
4691 // Check if the sign bits of source go down as far as the truncated value.
4692 unsigned NumSrcBits = Op.getOperand(0).getScalarValueSizeInBits();
4693 unsigned NumSrcSignBits = ComputeNumSignBits(Op.getOperand(0), Depth + 1);
4694 if (NumSrcSignBits > (NumSrcBits - VTBits))
4695 return NumSrcSignBits - (NumSrcBits - VTBits);
4696 break;
4697 }
4698 case ISD::EXTRACT_ELEMENT: {
4699 if (VT.isScalableVector())
4700 break;
4701 const int KnownSign = ComputeNumSignBits(Op.getOperand(0), Depth+1);
4702 const int BitWidth = Op.getValueSizeInBits();
4703 const int Items = Op.getOperand(0).getValueSizeInBits() / BitWidth;
4704
4705 // Get reverse index (starting from 1), Op1 value indexes elements from
4706 // little end. Sign starts at big end.
4707 const int rIndex = Items - 1 - Op.getConstantOperandVal(1);
4708
4709 // If the sign portion ends in our element the subtraction gives correct
4710 // result. Otherwise it gives either negative or > bitwidth result
4711 return std::clamp(KnownSign - rIndex * BitWidth, 0, BitWidth);
4712 }
4714 if (VT.isScalableVector())
4715 break;
4716 // If we know the element index, split the demand between the
4717 // source vector and the inserted element, otherwise assume we need
4718 // the original demanded vector elements and the value.
4719 SDValue InVec = Op.getOperand(0);
4720 SDValue InVal = Op.getOperand(1);
4721 SDValue EltNo = Op.getOperand(2);
4722 bool DemandedVal = true;
4723 APInt DemandedVecElts = DemandedElts;
4724 auto *CEltNo = dyn_cast<ConstantSDNode>(EltNo);
4725 if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) {
4726 unsigned EltIdx = CEltNo->getZExtValue();
4727 DemandedVal = !!DemandedElts[EltIdx];
4728 DemandedVecElts.clearBit(EltIdx);
4729 }
4730 Tmp = std::numeric_limits<unsigned>::max();
4731 if (DemandedVal) {
4732 // TODO - handle implicit truncation of inserted elements.
4733 if (InVal.getScalarValueSizeInBits() != VTBits)
4734 break;
4735 Tmp2 = ComputeNumSignBits(InVal, Depth + 1);
4736 Tmp = std::min(Tmp, Tmp2);
4737 }
4738 if (!!DemandedVecElts) {
4739 Tmp2 = ComputeNumSignBits(InVec, DemandedVecElts, Depth + 1);
4740 Tmp = std::min(Tmp, Tmp2);
4741 }
4742 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4743 return Tmp;
4744 }
4746 assert(!VT.isScalableVector());
4747 SDValue InVec = Op.getOperand(0);
4748 SDValue EltNo = Op.getOperand(1);
4749 EVT VecVT = InVec.getValueType();
4750 // ComputeNumSignBits not yet implemented for scalable vectors.
4751 if (VecVT.isScalableVector())
4752 break;
4753 const unsigned BitWidth = Op.getValueSizeInBits();
4754 const unsigned EltBitWidth = Op.getOperand(0).getScalarValueSizeInBits();
4755 const unsigned NumSrcElts = VecVT.getVectorNumElements();
4756
4757 // If BitWidth > EltBitWidth the value is anyext:ed, and we do not know
4758 // anything about sign bits. But if the sizes match we can derive knowledge
4759 // about sign bits from the vector operand.
4760 if (BitWidth != EltBitWidth)
4761 break;
4762
4763 // If we know the element index, just demand that vector element, else for
4764 // an unknown element index, ignore DemandedElts and demand them all.
4765 APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
4766 auto *ConstEltNo = dyn_cast<ConstantSDNode>(EltNo);
4767 if (ConstEltNo && ConstEltNo->getAPIntValue().ult(NumSrcElts))
4768 DemandedSrcElts =
4769 APInt::getOneBitSet(NumSrcElts, ConstEltNo->getZExtValue());
4770
4771 return ComputeNumSignBits(InVec, DemandedSrcElts, Depth + 1);
4772 }
4774 // Offset the demanded elts by the subvector index.
4775 SDValue Src = Op.getOperand(0);
4776 // Bail until we can represent demanded elements for scalable vectors.
4777 if (Src.getValueType().isScalableVector())
4778 break;
4779 uint64_t Idx = Op.getConstantOperandVal(1);
4780 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
4781 APInt DemandedSrcElts = DemandedElts.zext(NumSrcElts).shl(Idx);
4782 return ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4783 }
4784 case ISD::CONCAT_VECTORS: {
4785 if (VT.isScalableVector())
4786 break;
4787 // Determine the minimum number of sign bits across all demanded
4788 // elts of the input vectors. Early out if the result is already 1.
4789 Tmp = std::numeric_limits<unsigned>::max();
4790 EVT SubVectorVT = Op.getOperand(0).getValueType();
4791 unsigned NumSubVectorElts = SubVectorVT.getVectorNumElements();
4792 unsigned NumSubVectors = Op.getNumOperands();
4793 for (unsigned i = 0; (i < NumSubVectors) && (Tmp > 1); ++i) {
4794 APInt DemandedSub =
4795 DemandedElts.extractBits(NumSubVectorElts, i * NumSubVectorElts);
4796 if (!DemandedSub)
4797 continue;
4798 Tmp2 = ComputeNumSignBits(Op.getOperand(i), DemandedSub, Depth + 1);
4799 Tmp = std::min(Tmp, Tmp2);
4800 }
4801 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4802 return Tmp;
4803 }
4804 case ISD::INSERT_SUBVECTOR: {
4805 if (VT.isScalableVector())
4806 break;
4807 // Demand any elements from the subvector and the remainder from the src its
4808 // inserted into.
4809 SDValue Src = Op.getOperand(0);
4810 SDValue Sub = Op.getOperand(1);
4811 uint64_t Idx = Op.getConstantOperandVal(2);
4812 unsigned NumSubElts = Sub.getValueType().getVectorNumElements();
4813 APInt DemandedSubElts = DemandedElts.extractBits(NumSubElts, Idx);
4814 APInt DemandedSrcElts = DemandedElts;
4815 DemandedSrcElts.insertBits(APInt::getZero(NumSubElts), Idx);
4816
4817 Tmp = std::numeric_limits<unsigned>::max();
4818 if (!!DemandedSubElts) {
4819 Tmp = ComputeNumSignBits(Sub, DemandedSubElts, Depth + 1);
4820 if (Tmp == 1)
4821 return 1; // early-out
4822 }
4823 if (!!DemandedSrcElts) {
4824 Tmp2 = ComputeNumSignBits(Src, DemandedSrcElts, Depth + 1);
4825 Tmp = std::min(Tmp, Tmp2);
4826 }
4827 assert(Tmp <= VTBits && "Failed to determine minimum sign bits");
4828 return Tmp;
4829 }
4830 case ISD::LOAD: {
4831 LoadSDNode *LD = cast<LoadSDNode>(Op);
4832 if (const MDNode *Ranges = LD->getRanges()) {
4833 if (DemandedElts != 1)
4834 break;
4835
4837 if (VTBits > CR.getBitWidth()) {
4838 switch (LD->getExtensionType()) {
4839 case ISD::SEXTLOAD:
4840 CR = CR.signExtend(VTBits);
4841 break;
4842 case ISD::ZEXTLOAD:
4843 CR = CR.zeroExtend(VTBits);
4844 break;
4845 default:
4846 break;
4847 }
4848 }
4849
4850 if (VTBits != CR.getBitWidth())
4851 break;
4852 return std::min(CR.getSignedMin().getNumSignBits(),
4854 }
4855
4856 break;
4857 }
4860 case ISD::ATOMIC_SWAP:
4872 case ISD::ATOMIC_LOAD: {
4873 Tmp = cast<AtomicSDNode>(Op)->getMemoryVT().getScalarSizeInBits();
4874 // If we are looking at the loaded value.
4875 if (Op.getResNo() == 0) {
4876 if (Tmp == VTBits)
4877 return 1; // early-out
4879 return VTBits - Tmp + 1;
4881 return VTBits - Tmp;
4882 if (Op->getOpcode() == ISD::ATOMIC_LOAD) {
4883 ISD::LoadExtType ETy = cast<AtomicSDNode>(Op)->getExtensionType();
4884 if (ETy == ISD::SEXTLOAD)
4885 return VTBits - Tmp + 1;
4886 if (ETy == ISD::ZEXTLOAD)
4887 return VTBits - Tmp;
4888 }
4889 }
4890 break;
4891 }
4892 }
4893
4894 // If we are looking at the loaded value of the SDNode.
4895 if (Op.getResNo() == 0) {
4896 // Handle LOADX separately here. EXTLOAD case will fallthrough.
4897 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Op)) {
4898 unsigned ExtType = LD->getExtensionType();
4899 switch (ExtType) {
4900 default: break;
4901 case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known.
4902 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4903 return VTBits - Tmp + 1;
4904 case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known.
4905 Tmp = LD->getMemoryVT().getScalarSizeInBits();
4906 return VTBits - Tmp;
4907 case ISD::NON_EXTLOAD:
4908 if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) {
4909 // We only need to handle vectors - computeKnownBits should handle
4910 // scalar cases.
4911 Type *CstTy = Cst->getType();
4912 if (CstTy->isVectorTy() && !VT.isScalableVector() &&
4913 (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits() &&
4914 VTBits == CstTy->getScalarSizeInBits()) {
4915 Tmp = VTBits;
4916 for (unsigned i = 0; i != NumElts; ++i) {
4917 if (!DemandedElts[i])
4918 continue;
4919 if (Constant *Elt = Cst->getAggregateElement(i)) {
4920 if (auto *CInt = dyn_cast<ConstantInt>(Elt)) {
4921 const APInt &Value = CInt->getValue();
4922 Tmp = std::min(Tmp, Value.getNumSignBits());
4923 continue;
4924 }
4925 if (auto *CFP = dyn_cast<ConstantFP>(Elt)) {
4926 APInt Value = CFP->getValueAPF().bitcastToAPInt();
4927 Tmp = std::min(Tmp, Value.getNumSignBits());
4928 continue;
4929 }
4930 }
4931 // Unknown type. Conservatively assume no bits match sign bit.
4932 return 1;
4933 }
4934 return Tmp;
4935 }
4936 }
4937 break;
4938 }
4939 }
4940 }
4941
4942 // Allow the target to implement this method for its nodes.
4943 if (Opcode >= ISD::BUILTIN_OP_END ||
4944 Opcode == ISD::INTRINSIC_WO_CHAIN ||
4945 Opcode == ISD::INTRINSIC_W_CHAIN ||
4946 Opcode == ISD::INTRINSIC_VOID) {
4947 // TODO: This can probably be removed once target code is audited. This
4948 // is here purely to reduce patch size and review complexity.
4949 if (!VT.isScalableVector()) {
4950 unsigned NumBits =
4951 TLI->ComputeNumSignBitsForTargetNode(Op, DemandedElts, *this, Depth);
4952 if (NumBits > 1)
4953 FirstAnswer = std::max(FirstAnswer, NumBits);
4954 }
4955 }
4956
4957 // Finally, if we can prove that the top bits of the result are 0's or 1's,
4958 // use this information.
4959 KnownBits Known = computeKnownBits(Op, DemandedElts, Depth);
4960 return std::max(FirstAnswer, Known.countMinSignBits());
4961}
4962
4964 unsigned Depth) const {
4965 unsigned SignBits = ComputeNumSignBits(Op, Depth);
4966 return Op.getScalarValueSizeInBits() - SignBits + 1;
4967}
4968
4970 const APInt &DemandedElts,
4971 unsigned Depth) const {
4972 unsigned SignBits = ComputeNumSignBits(Op, DemandedElts, Depth);
4973 return Op.getScalarValueSizeInBits() - SignBits + 1;
4974}
4975
4977 unsigned Depth) const {
4978 // Early out for FREEZE.
4979 if (Op.getOpcode() == ISD::FREEZE)
4980 return true;
4981
4982 // TODO: Assume we don't know anything for now.
4983 EVT VT = Op.getValueType();
4984 if (VT.isScalableVector())
4985 return false;
4986
4987 APInt DemandedElts = VT.isVector()
4989 : APInt(1, 1);
4990 return isGuaranteedNotToBeUndefOrPoison(Op, DemandedElts, PoisonOnly, Depth);
4991}
4992
4994 const APInt &DemandedElts,
4995 bool PoisonOnly,
4996 unsigned Depth) const {
4997 unsigned Opcode = Op.getOpcode();
4998
4999 // Early out for FREEZE.
5000 if (Opcode == ISD::FREEZE)
5001 return true;
5002
5003 if (Depth >= MaxRecursionDepth)
5004 return false; // Limit search depth.
5005
5006 if (isIntOrFPConstant(Op))
5007 return true;
5008
5009 switch (Opcode) {
5010 case ISD::CONDCODE:
5011 case ISD::VALUETYPE:
5012 case ISD::FrameIndex:
5014 return true;
5015
5016 case ISD::UNDEF:
5017 return PoisonOnly;
5018
5019 case ISD::BUILD_VECTOR:
5020 // NOTE: BUILD_VECTOR has implicit truncation of wider scalar elements -
5021 // this shouldn't affect the result.
5022 for (unsigned i = 0, e = Op.getNumOperands(); i < e; ++i) {
5023 if (!DemandedElts[i])
5024 continue;
5026 Depth + 1))
5027 return false;
5028 }
5029 return true;
5030
5031 // TODO: Search for noundef attributes from library functions.
5032
5033 // TODO: Pointers dereferenced by ISD::LOAD/STORE ops are noundef.
5034
5035 default:
5036 // Allow the target to implement this method for its nodes.
5037 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5038 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5040 Op, DemandedElts, *this, PoisonOnly, Depth);
5041 break;
5042 }
5043
5044 // If Op can't create undef/poison and none of its operands are undef/poison
5045 // then Op is never undef/poison.
5046 // NOTE: TargetNodes can handle this in themselves in
5047 // isGuaranteedNotToBeUndefOrPoisonForTargetNode or let
5048 // TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode handle it.
5049 return !canCreateUndefOrPoison(Op, PoisonOnly, /*ConsiderFlags*/ true,
5050 Depth) &&
5051 all_of(Op->ops(), [&](SDValue V) {
5052 return isGuaranteedNotToBeUndefOrPoison(V, PoisonOnly, Depth + 1);
5053 });
5054}
5055
5057 bool ConsiderFlags,
5058 unsigned Depth) const {
5059 // TODO: Assume we don't know anything for now.
5060 EVT VT = Op.getValueType();
5061 if (VT.isScalableVector())
5062 return true;
5063
5064 APInt DemandedElts = VT.isVector()
5066 : APInt(1, 1);
5067 return canCreateUndefOrPoison(Op, DemandedElts, PoisonOnly, ConsiderFlags,
5068 Depth);
5069}
5070
5072 bool PoisonOnly, bool ConsiderFlags,
5073 unsigned Depth) const {
5074 // TODO: Assume we don't know anything for now.
5075 EVT VT = Op.getValueType();
5076 if (VT.isScalableVector())
5077 return true;
5078
5079 unsigned Opcode = Op.getOpcode();
5080 switch (Opcode) {
5081 case ISD::FREEZE:
5084 case ISD::AND:
5085 case ISD::XOR:
5086 case ISD::ROTL:
5087 case ISD::ROTR:
5088 case ISD::FSHL:
5089 case ISD::FSHR:
5090 case ISD::BSWAP:
5091 case ISD::CTPOP:
5092 case ISD::BITREVERSE:
5093 case ISD::PARITY:
5094 case ISD::SIGN_EXTEND:
5095 case ISD::TRUNCATE:
5099 case ISD::BITCAST:
5100 case ISD::BUILD_VECTOR:
5101 case ISD::BUILD_PAIR:
5102 return false;
5103
5104 case ISD::SETCC: {
5105 // Integer setcc cannot create undef or poison.
5106 if (Op.getOperand(0).getValueType().isInteger())
5107 return false;
5108
5109 // FP compares are more complicated. They can create poison for nan/infinity
5110 // based on options and flags. The options and flags also cause special
5111 // nonan condition codes to be used. Those condition codes may be preserved
5112 // even if the nonan flag is dropped somewhere.
5113 ISD::CondCode CCCode = cast<CondCodeSDNode>(Op.getOperand(2))->get();
5114 if (((unsigned)CCCode & 0x10U))
5115 return true;
5116
5118 return Options.NoNaNsFPMath || Options.NoInfsFPMath ||
5119 (ConsiderFlags &&
5120 (Op->getFlags().hasNoNaNs() || Op->getFlags().hasNoInfs()));
5121 }
5122
5123 // Matches hasPoisonGeneratingFlags().
5124 case ISD::ZERO_EXTEND:
5125 return ConsiderFlags && Op->getFlags().hasNonNeg();
5126
5127 case ISD::ADD:
5128 case ISD::SUB:
5129 case ISD::MUL:
5130 // Matches hasPoisonGeneratingFlags().
5131 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5132 Op->getFlags().hasNoUnsignedWrap());
5133
5134 case ISD::SHL:
5135 // If the max shift amount isn't in range, then the shift can create poison.
5136 if (!getValidMaximumShiftAmountConstant(Op, DemandedElts))
5137 return true;
5138
5139 // Matches hasPoisonGeneratingFlags().
5140 return ConsiderFlags && (Op->getFlags().hasNoSignedWrap() ||
5141 Op->getFlags().hasNoUnsignedWrap());
5142
5143 // Matches hasPoisonGeneratingFlags().
5144 case ISD::OR:
5145 return ConsiderFlags && Op->getFlags().hasDisjoint();
5146
5148 // Ensure that the element index is in bounds.
5149 EVT VecVT = Op.getOperand(0).getValueType();
5150 KnownBits KnownIdx = computeKnownBits(Op.getOperand(2), Depth + 1);
5151 return KnownIdx.getMaxValue().uge(VecVT.getVectorMinNumElements());
5152 }
5153
5154 default:
5155 // Allow the target to implement this method for its nodes.
5156 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::INTRINSIC_WO_CHAIN ||
5157 Opcode == ISD::INTRINSIC_W_CHAIN || Opcode == ISD::INTRINSIC_VOID)
5159 Op, DemandedElts, *this, PoisonOnly, ConsiderFlags, Depth);
5160 break;
5161 }
5162
5163 // Be conservative and return true.
5164 return true;
5165}
5166
5168 unsigned Opcode = Op.getOpcode();
5169 if (Opcode == ISD::OR)
5170 return Op->getFlags().hasDisjoint() ||
5171 haveNoCommonBitsSet(Op.getOperand(0), Op.getOperand(1));
5172 if (Opcode == ISD::XOR)
5173 return isMinSignedConstant(Op.getOperand(1));
5174 return false;
5175}
5176
5178 if ((Op.getOpcode() != ISD::ADD && Op.getOpcode() != ISD::OR) ||
5179 !isa<ConstantSDNode>(Op.getOperand(1)))
5180 return false;
5181
5182 if (Op.getOpcode() == ISD::OR &&
5183 !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1)))
5184 return false;
5185
5186 return true;
5187}
5188
5189bool SelectionDAG::isKnownNeverNaN(SDValue Op, bool SNaN, unsigned Depth) const {
5190 // If we're told that NaNs won't happen, assume they won't.
5191 if (getTarget().Options.NoNaNsFPMath || Op->getFlags().hasNoNaNs())
5192 return true;
5193
5194 if (Depth >= MaxRecursionDepth)
5195 return false; // Limit search depth.
5196
5197 // If the value is a constant, we can obviously see if it is a NaN or not.
5198 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op)) {
5199 return !C->getValueAPF().isNaN() ||
5200 (SNaN && !C->getValueAPF().isSignaling());
5201 }
5202
5203 unsigned Opcode = Op.getOpcode();
5204 switch (Opcode) {
5205 case ISD::FADD:
5206 case ISD::FSUB:
5207 case ISD::FMUL:
5208 case ISD::FDIV:
5209 case ISD::FREM:
5210 case ISD::FSIN:
5211 case ISD::FCOS:
5212 case ISD::FMA:
5213 case ISD::FMAD: {
5214 if (SNaN)
5215 return true;
5216 // TODO: Need isKnownNeverInfinity
5217 return false;
5218 }
5219 case ISD::FCANONICALIZE:
5220 case ISD::FEXP:
5221 case ISD::FEXP2:
5222 case ISD::FEXP10:
5223 case ISD::FTRUNC:
5224 case ISD::FFLOOR:
5225 case ISD::FCEIL:
5226 case ISD::FROUND:
5227 case ISD::FROUNDEVEN:
5228 case ISD::FRINT:
5229 case ISD::LRINT:
5230 case ISD::LLRINT:
5231 case ISD::FNEARBYINT:
5232 case ISD::FLDEXP: {
5233 if (SNaN)
5234 return true;
5235 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5236 }
5237 case ISD::FABS:
5238 case ISD::FNEG:
5239 case ISD::FCOPYSIGN: {
5240 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5241 }
5242 case ISD::SELECT:
5243 return isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1) &&
5244 isKnownNeverNaN(Op.getOperand(2), SNaN, Depth + 1);
5245 case ISD::FP_EXTEND:
5246 case ISD::FP_ROUND: {
5247 if (SNaN)
5248 return true;
5249 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5250 }
5251 case ISD::SINT_TO_FP:
5252 case ISD::UINT_TO_FP:
5253 return true;
5254 case ISD::FSQRT: // Need is known positive
5255 case ISD::FLOG:
5256 case ISD::FLOG2:
5257 case ISD::FLOG10:
5258 case ISD::FPOWI:
5259 case ISD::FPOW: {
5260 if (SNaN)
5261 return true;
5262 // TODO: Refine on operand
5263 return false;
5264 }
5265 case ISD::FMINNUM:
5266 case ISD::FMAXNUM: {
5267 // Only one needs to be known not-nan, since it will be returned if the
5268 // other ends up being one.
5269 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) ||
5270 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5271 }
5272 case ISD::FMINNUM_IEEE:
5273 case ISD::FMAXNUM_IEEE: {
5274 if (SNaN)
5275 return true;
5276 // This can return a NaN if either operand is an sNaN, or if both operands
5277 // are NaN.
5278 return (isKnownNeverNaN(Op.getOperand(0), false, Depth + 1) &&
5279 isKnownNeverSNaN(Op.getOperand(1), Depth + 1)) ||
5280 (isKnownNeverNaN(Op.getOperand(1), false, Depth + 1) &&
5281 isKnownNeverSNaN(Op.getOperand(0), Depth + 1));
5282 }
5283 case ISD::FMINIMUM:
5284 case ISD::FMAXIMUM: {
5285 // TODO: Does this quiet or return the origina NaN as-is?
5286 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1) &&
5287 isKnownNeverNaN(Op.getOperand(1), SNaN, Depth + 1);
5288 }
5290 return isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1);
5291 }
5292 case ISD::BUILD_VECTOR: {
5293 for (const SDValue &Opnd : Op->ops())
5294 if (!isKnownNeverNaN(Opnd, SNaN, Depth + 1))
5295 return false;
5296 return true;
5297 }
5298 default:
5299 if (Opcode >= ISD::BUILTIN_OP_END ||
5300 Opcode == ISD::INTRINSIC_WO_CHAIN ||
5301 Opcode == ISD::INTRINSIC_W_CHAIN ||
5302 Opcode == ISD::INTRINSIC_VOID) {
5303 return TLI->isKnownNeverNaNForTargetNode(Op, *this, SNaN, Depth);
5304 }
5305
5306 return false;
5307 }
5308}
5309
5311 assert(Op.getValueType().isFloatingPoint() &&
5312 "Floating point type expected");
5313
5314 // If the value is a constant, we can obviously see if it is a zero or not.
5316 Op, [](ConstantFPSDNode *C) { return !C->isZero(); });
5317}
5318
5320 if (Depth >= MaxRecursionDepth)
5321 return false; // Limit search depth.
5322
5323 assert(!Op.getValueType().isFloatingPoint() &&
5324 "Floating point types unsupported - use isKnownNeverZeroFloat");
5325
5326 // If the value is a constant, we can obviously see if it is a zero or not.
5328 [](ConstantSDNode *C) { return !C->isZero(); }))
5329 return true;
5330
5331 // TODO: Recognize more cases here. Most of the cases are also incomplete to
5332 // some degree.
5333 switch (Op.getOpcode()) {
5334 default:
5335 break;
5336
5337 case ISD::OR:
5338 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5339 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5340
5341 case ISD::VSELECT:
5342 case ISD::SELECT:
5343 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5344 isKnownNeverZero(Op.getOperand(2), Depth + 1);
5345
5346 case ISD::SHL: {
5347 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5348 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5349 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5350 // 1 << X is never zero.
5351 if (ValKnown.One[0])
5352 return true;
5353 // If max shift cnt of known ones is non-zero, result is non-zero.
5354 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5355 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5356 !ValKnown.One.shl(MaxCnt).isZero())
5357 return true;
5358 break;
5359 }
5360 case ISD::UADDSAT:
5361 case ISD::UMAX:
5362 return isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5363 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5364
5365 // For smin/smax: If either operand is known negative/positive
5366 // respectively we don't need the other to be known at all.
5367 case ISD::SMAX: {
5368 KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);
5369 if (Op1.isStrictlyPositive())
5370 return true;
5371
5372 KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);
5373 if (Op0.isStrictlyPositive())
5374 return true;
5375
5376 if (Op1.isNonZero() && Op0.isNonZero())
5377 return true;
5378
5379 if (KnownBits::smax(Op0, Op1).isNonZero())
5380 return true;
5381
5382 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5383 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5384 }
5385 case ISD::SMIN: {
5386 KnownBits Op1 = computeKnownBits(Op.getOperand(1), Depth + 1);
5387 if (Op1.isNegative())
5388 return true;
5389
5390 KnownBits Op0 = computeKnownBits(Op.getOperand(0), Depth + 1);
5391 if (Op0.isNegative())
5392 return true;
5393
5394 if (Op1.isNonZero() && Op0.isNonZero())
5395 return true;
5396
5397 if (KnownBits::smin(Op0, Op1).isNonZero())
5398 return true;
5399
5400 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5401 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5402 }
5403 case ISD::UMIN:
5404 return isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5405 isKnownNeverZero(Op.getOperand(0), Depth + 1);
5406
5407 case ISD::ROTL:
5408 case ISD::ROTR:
5409 case ISD::BITREVERSE:
5410 case ISD::BSWAP:
5411 case ISD::CTPOP:
5412 case ISD::ABS:
5413 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5414
5415 case ISD::SRA:
5416 case ISD::SRL: {
5417 if (Op->getFlags().hasExact())
5418 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5419 KnownBits ValKnown = computeKnownBits(Op.getOperand(0), Depth + 1);
5420 if (ValKnown.isNegative())
5421 return true;
5422 // If max shift cnt of known ones is non-zero, result is non-zero.
5423 APInt MaxCnt = computeKnownBits(Op.getOperand(1), Depth + 1).getMaxValue();
5424 if (MaxCnt.ult(ValKnown.getBitWidth()) &&
5425 !ValKnown.One.lshr(MaxCnt).isZero())
5426 return true;
5427 break;
5428 }
5429 case ISD::UDIV:
5430 case ISD::SDIV:
5431 // div exact can only produce a zero if the dividend is zero.
5432 // TODO: For udiv this is also true if Op1 u<= Op0
5433 if (Op->getFlags().hasExact())
5434 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5435 break;
5436
5437 case ISD::ADD:
5438 if (Op->getFlags().hasNoUnsignedWrap())
5439 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) ||
5440 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5441 return true;
5442 // TODO: There are a lot more cases we can prove for add.
5443 break;
5444
5445 case ISD::SUB: {
5446 if (isNullConstant(Op.getOperand(0)))
5447 return isKnownNeverZero(Op.getOperand(1), Depth + 1);
5448
5449 std::optional<bool> ne =
5450 KnownBits::ne(computeKnownBits(Op.getOperand(0), Depth + 1),
5451 computeKnownBits(Op.getOperand(1), Depth + 1));
5452 return ne && *ne;
5453 }
5454
5455 case ISD::MUL:
5456 if (Op->getFlags().hasNoSignedWrap() || Op->getFlags().hasNoUnsignedWrap())
5457 if (isKnownNeverZero(Op.getOperand(1), Depth + 1) &&
5458 isKnownNeverZero(Op.getOperand(0), Depth + 1))
5459 return true;
5460 break;
5461
5462 case ISD::ZERO_EXTEND:
5463 case ISD::SIGN_EXTEND:
5464 return isKnownNeverZero(Op.getOperand(0), Depth + 1);
5465 }
5466
5468}
5469
5471 // Check the obvious case.
5472 if (A == B) return true;
5473
5474 // For negative and positive zero.
5475 if (const ConstantFPSDNode *CA = dyn_cast<ConstantFPSDNode>(A))
5476 if (const ConstantFPSDNode *CB = dyn_cast<ConstantFPSDNode>(B))
5477 if (CA->isZero() && CB->isZero()) return true;
5478
5479 // Otherwise they may not be equal.
5480 return false;
5481}
5482
5483// Only bits set in Mask must be negated, other bits may be arbitrary.
5485 if (isBitwiseNot(V, AllowUndefs))
5486 return V.getOperand(0);
5487
5488 // Handle any_extend (not (truncate X)) pattern, where Mask only sets
5489 // bits in the non-extended part.
5490 ConstantSDNode *MaskC = isConstOrConstSplat(Mask);
5491 if (!MaskC || V.getOpcode() != ISD::ANY_EXTEND)
5492 return SDValue();
5493 SDValue ExtArg = V.getOperand(0);
5494 if (ExtArg.getScalarValueSizeInBits() >=
5495 MaskC->getAPIntValue().getActiveBits() &&
5496 isBitwiseNot(ExtArg, AllowUndefs) &&
5497 ExtArg.getOperand(0).getOpcode() == ISD::TRUNCATE &&
5498 ExtArg.getOperand(0).getOperand(0).getValueType() == V.getValueType())
5499 return ExtArg.getOperand(0).getOperand(0);
5500 return SDValue();
5501}
5502
5504 // Match masked merge pattern (X & ~M) op (Y & M)
5505 // Including degenerate case (X & ~M) op M
5506 auto MatchNoCommonBitsPattern = [&](SDValue Not, SDValue Mask,
5507 SDValue Other) {
5508 if (SDValue NotOperand =
5509 getBitwiseNotOperand(Not, Mask, /* AllowUndefs */ true)) {
5510 if (NotOperand->getOpcode() == ISD::ZERO_EXTEND ||
5511 NotOperand->getOpcode() == ISD::TRUNCATE)
5512 NotOperand = NotOperand->getOperand(0);
5513
5514 if (Other == NotOperand)
5515 return true;
5516 if (Other->getOpcode() == ISD::AND)
5517 return NotOperand == Other->getOperand(0) ||
5518 NotOperand == Other->getOperand(1);
5519 }
5520 return false;
5521 };
5522
5523 if (A->getOpcode() == ISD::ZERO_EXTEND || A->getOpcode() == ISD::TRUNCATE)
5524 A = A->getOperand(0);
5525
5526 if (B->getOpcode() == ISD::ZERO_EXTEND || B->getOpcode() == ISD::TRUNCATE)
5527 B = B->getOperand(0);
5528
5529 if (A->getOpcode() == ISD::AND)
5530 return MatchNoCommonBitsPattern(A->getOperand(0), A->getOperand(1), B) ||
5531 MatchNoCommonBitsPattern(A->getOperand(1), A->getOperand(0), B);
5532 return false;
5533}
5534
5535// FIXME: unify with llvm::haveNoCommonBitsSet.
5537 assert(A.getValueType() == B.getValueType() &&
5538 "Values must have the same type");
5541 return true;
5544}
5545
5546static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step,
5547 SelectionDAG &DAG) {
5548 if (cast<ConstantSDNode>(Step)->isZero())
5549 return DAG.getConstant(0, DL, VT);
5550
5551 return SDValue();
5552}
5553
5556 SelectionDAG &DAG) {
5557 int NumOps = Ops.size();
5558 assert(NumOps != 0 && "Can't build an empty vector!");
5559 assert(!VT.isScalableVector() &&
5560 "BUILD_VECTOR cannot be used with scalable types");
5561 assert(VT.getVectorNumElements() == (unsigned)NumOps &&
5562 "Incorrect element count in BUILD_VECTOR!");
5563
5564 // BUILD_VECTOR of UNDEFs is UNDEF.
5565 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5566 return DAG.getUNDEF(VT);
5567
5568 // BUILD_VECTOR of seq extract/insert from the same vector + type is Identity.
5569 SDValue IdentitySrc;
5570 bool IsIdentity = true;
5571 for (int i = 0; i != NumOps; ++i) {
5572 if (Ops[i].getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
5573 Ops[i].getOperand(0).getValueType() != VT ||
5574 (IdentitySrc && Ops[i].getOperand(0) != IdentitySrc) ||
5575 !isa<ConstantSDNode>(Ops[i].getOperand(1)) ||
5576 Ops[i].getConstantOperandAPInt(1) != i) {
5577 IsIdentity = false;
5578 break;
5579 }
5580 IdentitySrc = Ops[i].getOperand(0);
5581 }
5582 if (IsIdentity)
5583 return IdentitySrc;
5584
5585 return SDValue();
5586}
5587
5588/// Try to simplify vector concatenation to an input value, undef, or build
5589/// vector.
5592 SelectionDAG &DAG) {
5593 assert(!Ops.empty() && "Can't concatenate an empty list of vectors!");
5594 assert(llvm::all_of(Ops,
5595 [Ops](SDValue Op) {
5596 return Ops[0].getValueType() == Op.getValueType();
5597 }) &&
5598 "Concatenation of vectors with inconsistent value types!");
5599 assert((Ops[0].getValueType().getVectorElementCount() * Ops.size()) ==
5600 VT.getVectorElementCount() &&
5601 "Incorrect element count in vector concatenation!");
5602
5603 if (Ops.size() == 1)
5604 return Ops[0];
5605
5606 // Concat of UNDEFs is UNDEF.
5607 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
5608 return DAG.getUNDEF(VT);
5609
5610 // Scan the operands and look for extract operations from a single source
5611 // that correspond to insertion at the same location via this concatenation:
5612 // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ...
5613 SDValue IdentitySrc;
5614 bool IsIdentity = true;
5615 for (unsigned i = 0, e = Ops.size(); i != e; ++i) {
5616 SDValue Op = Ops[i];
5617 unsigned IdentityIndex = i * Op.getValueType().getVectorMinNumElements();
5618 if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5619 Op.getOperand(0).getValueType() != VT ||
5620 (IdentitySrc && Op.getOperand(0) != IdentitySrc) ||
5621 Op.getConstantOperandVal(1) != IdentityIndex) {
5622 IsIdentity = false;
5623 break;
5624 }
5625 assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) &&
5626 "Unexpected identity source vector for concat of extracts");
5627 IdentitySrc = Op.getOperand(0);
5628 }
5629 if (IsIdentity) {
5630 assert(IdentitySrc && "Failed to set source vector of extracts");
5631 return IdentitySrc;
5632 }
5633
5634 // The code below this point is only designed to work for fixed width
5635 // vectors, so we bail out for now.
5636 if (VT.isScalableVector())
5637 return SDValue();
5638
5639 // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be
5640 // simplified to one big BUILD_VECTOR.
5641 // FIXME: Add support for SCALAR_TO_VECTOR as well.
5642 EVT SVT = VT.getScalarType();
5644 for (SDValue Op : Ops) {
5645 EVT OpVT = Op.getValueType();
5646 if (Op.isUndef())
5647 Elts.append(OpVT.getVectorNumElements(), DAG.getUNDEF(SVT));
5648 else if (Op.getOpcode() == ISD::BUILD_VECTOR)
5649 Elts.append(Op->op_begin(), Op->op_end());
5650 else
5651 return SDValue();
5652 }
5653
5654 // BUILD_VECTOR requires all inputs to be of the same type, find the
5655 // maximum type and extend them all.
5656 for (SDValue Op : Elts)
5657 SVT = (SVT.bitsLT(Op.getValueType()) ? Op.getValueType() : SVT);
5658
5659 if (SVT.bitsGT(VT.getScalarType())) {
5660 for (SDValue &Op : Elts) {
5661 if (Op.isUndef())
5662 Op = DAG.getUNDEF(SVT);
5663 else
5664 Op = DAG.getTargetLoweringInfo().isZExtFree(Op.getValueType(), SVT)
5665 ? DAG.getZExtOrTrunc(Op, DL, SVT)
5666 : DAG.getSExtOrTrunc(Op, DL, SVT);
5667 }
5668 }
5669
5670 SDValue V = DAG.getBuildVector(VT, DL, Elts);
5671 NewSDValueDbgMsg(V, "New node fold concat vectors: ", &DAG);
5672 return V;
5673}
5674
5675/// Gets or creates the specified node.
5676SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT) {
5678 AddNodeIDNode(ID, Opcode, getVTList(VT), std::nullopt);
5679 void *IP = nullptr;
5680 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
5681 return SDValue(E, 0);
5682
5683 auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
5684 getVTList(VT));
5685 CSEMap.InsertNode(N, IP);
5686
5687 InsertNode(N);
5688 SDValue V = SDValue(N, 0);
5689 NewSDValueDbgMsg(V, "Creating new node: ", this);
5690 return V;
5691}
5692
5693SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5694 SDValue N1) {
5695 SDNodeFlags Flags;
5696 if (Inserter)
5697 Flags = Inserter->getFlags();
5698 return getNode(Opcode, DL, VT, N1, Flags);
5699}
5700
5701SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
5702 SDValue N1, const SDNodeFlags Flags) {
5703 assert(N1.getOpcode() != ISD::DELETED_NODE && "Operand is DELETED_NODE!");
5704
5705 // Constant fold unary operations with a vector integer or float operand.
5706 switch (Opcode) {
5707 default:
5708 // FIXME: Entirely reasonable to perform folding of other unary
5709 // operations here as the need arises.
5710 break;
5711 case ISD::FNEG:
5712 case ISD::FABS:
5713 case ISD::FCEIL:
5714 case ISD::FTRUNC:
5715 case ISD::FFLOOR:
5716 case ISD::FP_EXTEND:
5717 case ISD::FP_TO_SINT:
5718 case ISD::FP_TO_UINT:
5719 case ISD::FP_TO_FP16:
5720 case ISD::FP_TO_BF16:
5721 case ISD::TRUNCATE:
5722 case ISD::ANY_EXTEND:
5723 case ISD::ZERO_EXTEND:
5724 case ISD::SIGN_EXTEND:
5725 case ISD::UINT_TO_FP:
5726 case ISD::SINT_TO_FP:
5727 case ISD::FP16_TO_FP:
5728 case ISD::BF16_TO_FP:
5729 case ISD::BITCAST:
5730 case ISD::ABS:
5731 case ISD::BITREVERSE:
5732 case ISD::BSWAP:
5733 case ISD::CTLZ:
5735 case ISD::CTTZ:
5737 case ISD::CTPOP:
5738 case ISD::STEP_VECTOR: {
5739 SDValue Ops = {N1};
5740 if (SDValue Fold = FoldConstantArithmetic(Opcode, DL, VT, Ops))
5741 return Fold;
5742 }
5743 }
5744
5745 unsigned OpOpcode = N1.getNode()->getOpcode();
5746 switch (Opcode) {
5747 case ISD::STEP_VECTOR:
5748 assert(VT.isScalableVector() &&
5749 "STEP_VECTOR can only be used with scalable types");
5750 assert(OpOpcode == ISD::TargetConstant &&
5751 VT.getVectorElementType() == N1.getValueType() &&
5752 "Unexpected step operand");
5753 break;
5754 case ISD::FREEZE:
5755 assert(VT == N1.getValueType() && "Unexpected VT!");
5756 if (isGuaranteedNotToBeUndefOrPoison(N1, /*PoisonOnly*/ false,
5757 /*Depth*/ 1))
5758 return N1;
5759 break;
5760 case ISD::TokenFactor:
5761 case ISD::MERGE_VALUES:
5763 return N1; // Factor, merge or concat of one node? No need.
5764 case ISD::BUILD_VECTOR: {
5765 // Attempt to simplify BUILD_VECTOR.
5766 SDValue Ops[] = {N1};
5767 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
5768 return V;
5769 break;
5770 }
5771 case ISD::FP_ROUND: llvm_unreachable("Invalid method to make FP_ROUND node");
5772 case ISD::FP_EXTEND:
5774 "Invalid FP cast!");
5775 if (N1.getValueType() == VT) return N1; // noop conversion.
5776 assert((!VT.isVector() || VT.getVectorElementCount() ==
5778 "Vector element count mismatch!");
5779 assert(N1.getValueType().bitsLT(VT) && "Invalid fpext node, dst < src!");
5780 if (N1.isUndef())
5781 return getUNDEF(VT);
5782 break;
5783 case ISD::FP_TO_SINT:
5784 case ISD::FP_TO_UINT:
5785 if (N1.isUndef())
5786 return getUNDEF(VT);
5787 break;
5788 case ISD::SINT_TO_FP:
5789 case ISD::UINT_TO_FP:
5790 // [us]itofp(undef) = 0, because the result value is bounded.
5791 if (N1.isUndef())
5792 return getConstantFP(0.0, DL, VT);
5793 break;
5794 case ISD::SIGN_EXTEND:
5795 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5796 "Invalid SIGN_EXTEND!");
5797 assert(VT.isVector() == N1.getValueType().isVector() &&
5798 "SIGN_EXTEND result type type should be vector iff the operand "
5799 "type is vector!");
5800 if (N1.getValueType() == VT) return N1; // noop extension
5801 assert((!VT.isVector() || VT.getVectorElementCount() ==
5803 "Vector element count mismatch!");
5804 assert(N1.getValueType().bitsLT(VT) && "Invalid sext node, dst < src!");
5805 if (OpOpcode == ISD::SIGN_EXTEND || OpOpcode == ISD::ZERO_EXTEND) {
5806 SDNodeFlags Flags;
5807 if (OpOpcode == ISD::ZERO_EXTEND)
5808 Flags.setNonNeg(N1->getFlags().hasNonNeg());
5809 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
5810 }
5811 if (OpOpcode == ISD::UNDEF)
5812 // sext(undef) = 0, because the top bits will all be the same.
5813 return getConstant(0, DL, VT);
5814 break;
5815 case ISD::ZERO_EXTEND:
5816 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5817 "Invalid ZERO_EXTEND!");
5818 assert(VT.isVector() == N1.getValueType().isVector() &&
5819 "ZERO_EXTEND result type type should be vector iff the operand "
5820 "type is vector!");
5821 if (N1.getValueType() == VT) return N1; // noop extension
5822 assert((!VT.isVector() || VT.getVectorElementCount() ==
5824 "Vector element count mismatch!");
5825 assert(N1.getValueType().bitsLT(VT) && "Invalid zext node, dst < src!");
5826 if (OpOpcode == ISD::ZERO_EXTEND) { // (zext (zext x)) -> (zext x)
5827 SDNodeFlags Flags;
5828 Flags.setNonNeg(N1->getFlags().hasNonNeg());
5829 return getNode(ISD::ZERO_EXTEND, DL, VT, N1.getOperand(0), Flags);
5830 }
5831 if (OpOpcode == ISD::UNDEF)
5832 // zext(undef) = 0, because the top bits will be zero.
5833 return getConstant(0, DL, VT);
5834
5835 // Skip unnecessary zext_inreg pattern:
5836 // (zext (trunc x)) -> x iff the upper bits are known zero.
5837 // TODO: Remove (zext (trunc (and x, c))) exception which some targets
5838 // use to recognise zext_inreg patterns.
5839 if (OpOpcode == ISD::TRUNCATE) {
5840 SDValue OpOp = N1.getOperand(0);
5841 if (OpOp.getValueType() == VT) {
5842 if (OpOp.getOpcode() != ISD::AND) {
5845 if (MaskedValueIsZero(OpOp, HiBits)) {
5846 transferDbgValues(N1, OpOp);
5847 return OpOp;
5848 }
5849 }
5850 }
5851 }
5852 break;
5853 case ISD::ANY_EXTEND:
5854 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5855 "Invalid ANY_EXTEND!");
5856 assert(VT.isVector() == N1.getValueType().isVector() &&
5857 "ANY_EXTEND result type type should be vector iff the operand "
5858 "type is vector!");
5859 if (N1.getValueType() == VT) return N1; // noop extension
5860 assert((!VT.isVector() || VT.getVectorElementCount() ==
5862 "Vector element count mismatch!");
5863 assert(N1.getValueType().bitsLT(VT) && "Invalid anyext node, dst < src!");
5864
5865 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5866 OpOpcode == ISD::ANY_EXTEND) {
5867 SDNodeFlags Flags;
5868 if (OpOpcode == ISD::ZERO_EXTEND)
5869 Flags.setNonNeg(N1->getFlags().hasNonNeg());
5870 // (ext (zext x)) -> (zext x) and (ext (sext x)) -> (sext x)
5871 return getNode(OpOpcode, DL, VT, N1.getOperand(0), Flags);
5872 }
5873 if (OpOpcode == ISD::UNDEF)
5874 return getUNDEF(VT);
5875
5876 // (ext (trunc x)) -> x
5877 if (OpOpcode == ISD::TRUNCATE) {
5878 SDValue OpOp = N1.getOperand(0);
5879 if (OpOp.getValueType() == VT) {
5880 transferDbgValues(N1, OpOp);
5881 return OpOp;
5882 }
5883 }
5884 break;
5885 case ISD::TRUNCATE:
5886 assert(VT.isInteger() && N1.getValueType().isInteger() &&
5887 "Invalid TRUNCATE!");
5888 assert(VT.isVector() == N1.getValueType().isVector() &&
5889 "TRUNCATE result type type should be vector iff the operand "
5890 "type is vector!");
5891 if (N1.getValueType() == VT) return N1; // noop truncate
5892 assert((!VT.isVector() || VT.getVectorElementCount() ==
5894 "Vector element count mismatch!");
5895 assert(N1.getValueType().bitsGT(VT) && "Invalid truncate node, src < dst!");
5896 if (OpOpcode == ISD::TRUNCATE)
5897 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5898 if (OpOpcode == ISD::ZERO_EXTEND || OpOpcode == ISD::SIGN_EXTEND ||
5899 OpOpcode == ISD::ANY_EXTEND) {
5900 // If the source is smaller than the dest, we still need an extend.
5902 VT.getScalarType()))
5903 return getNode(OpOpcode, DL, VT, N1.getOperand(0));
5904 if (N1.getOperand(0).getValueType().bitsGT(VT))
5905 return getNode(ISD::TRUNCATE, DL, VT, N1.getOperand(0));
5906 return N1.getOperand(0);
5907 }
5908 if (OpOpcode == ISD::UNDEF)
5909 return getUNDEF(VT);
5910 if (OpOpcode == ISD::VSCALE && !NewNodesMustHaveLegalTypes)
5911 return getVScale(DL, VT,
5913 break;
5917 assert(VT.isVector() && "This DAG node is restricted to vector types.");
5918 assert(N1.getValueType().bitsLE(VT) &&
5919 "The input must be the same size or smaller than the result.");
5922 "The destination vector type must have fewer lanes than the input.");
5923 break;
5924 case ISD::ABS:
5925 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid ABS!");
5926 if (OpOpcode == ISD::UNDEF)
5927 return getConstant(0, DL, VT);
5928 break;
5929 case ISD::BSWAP:
5930 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BSWAP!");
5931 assert((VT.getScalarSizeInBits() % 16 == 0) &&
5932 "BSWAP types must be a multiple of 16 bits!");
5933 if (OpOpcode == ISD::UNDEF)
5934 return getUNDEF(VT);
5935 // bswap(bswap(X)) -> X.
5936 if (OpOpcode == ISD::BSWAP)
5937 return N1.getOperand(0);
5938 break;
5939 case ISD::BITREVERSE:
5940 assert(VT.isInteger() && VT == N1.getValueType() && "Invalid BITREVERSE!");
5941 if (OpOpcode == ISD::UNDEF)
5942 return getUNDEF(VT);
5943 break;
5944 case ISD::BITCAST:
5946 "Cannot BITCAST between types of different sizes!");
5947 if (VT == N1.getValueType()) return N1; // noop conversion.
5948 if (OpOpcode == ISD::BITCAST) // bitconv(bitconv(x)) -> bitconv(x)
5949 return getNode(ISD::BITCAST, DL, VT, N1.getOperand(0));
5950 if (OpOpcode == ISD::UNDEF)
5951 return getUNDEF(VT);
5952 break;
5954 assert(VT.isVector() && !N1.getValueType().isVector() &&
5955 (VT.getVectorElementType() == N1.getValueType() ||
5957 N1.getValueType().isInteger() &&
5959 "Illegal SCALAR_TO_VECTOR node!");
5960 if (OpOpcode == ISD::UNDEF)
5961 return getUNDEF(VT);
5962 // scalar_to_vector(extract_vector_elt V, 0) -> V, top bits are undefined.
5963 if (OpOpcode == ISD::EXTRACT_VECTOR_ELT &&
5964 isa<ConstantSDNode>(N1.getOperand(1)) &&
5965 N1.getConstantOperandVal(1) == 0 &&
5966 N1.getOperand(0).getValueType() == VT)
5967 return N1.getOperand(0);
5968 break;
5969 case ISD::FNEG:
5970 // Negation of an unknown bag of bits is still completely undefined.
5971 if (OpOpcode == ISD::UNDEF)
5972 return getUNDEF(VT);
5973
5974 if (OpOpcode == ISD::FNEG) // --X -> X
5975 return N1.getOperand(0);
5976 break;
5977 case ISD::FABS:
5978 if (OpOpcode == ISD::FNEG) // abs(-X) -> abs(X)
5979 return getNode(ISD::FABS, DL, VT, N1.getOperand(0));
5980 break;
5981 case ISD::VSCALE:
5982 assert(VT == N1.getValueType() && "Unexpected VT!");
5983 break;
5984 case ISD::CTPOP:
5985 if (N1.getValueType().getScalarType() == MVT::i1)
5986 return N1;
5987 break;
5988 case ISD::CTLZ:
5989 case ISD::CTTZ:
5990 if (N1.getValueType().getScalarType() == MVT::i1)
5991 return getNOT(DL, N1, N1.getValueType());
5992 break;
5993 case ISD::VECREDUCE_ADD:
5994 if (N1.getValueType().getScalarType() == MVT::i1)
5995 return getNode(ISD::VECREDUCE_XOR, DL, VT, N1);
5996 break;
5999 if (N1.getValueType().getScalarType() == MVT::i1)
6000 return getNode(ISD::VECREDUCE_OR, DL, VT, N1);
6001 break;
6004 if (N1.getValueType().getScalarType() == MVT::i1)
6005 return getNode(ISD::VECREDUCE_AND, DL, VT, N1);
6006 break;
6007 }
6008
6009 SDNode *N;
6010 SDVTList VTs = getVTList(VT);
6011 SDValue Ops[] = {N1};
6012 if (VT != MVT::Glue) { // Don't CSE glue producing nodes
6014 AddNodeIDNode(ID, Opcode, VTs, Ops);
6015 void *IP = nullptr;
6016 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
6017 E->intersectFlagsWith(Flags);
6018 return SDValue(E, 0);
6019 }
6020
6021 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6022 N->setFlags(Flags);
6023 createOperands(N, Ops);
6024 CSEMap.InsertNode(N, IP);
6025 } else {
6026 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
6027 createOperands(N, Ops);
6028 }
6029
6030 InsertNode(N);
6031 SDValue V = SDValue(N, 0);
6032 NewSDValueDbgMsg(V, "Creating new node: ", this);
6033 return V;
6034}
6035
6036static std::optional<APInt> FoldValue(unsigned Opcode, const APInt &C1,
6037 const APInt &C2) {
6038 switch (Opcode) {
6039 case ISD::ADD: return C1 + C2;
6040 case ISD::SUB: return C1 - C2;
6041 case ISD::MUL: return C1 * C2;
6042 case ISD::AND: return C1 & C2;
6043 case ISD::OR: return C1 | C2;
6044 case ISD::XOR: return C1 ^ C2;
6045 case ISD::SHL: return C1 << C2;
6046 case ISD::SRL: return C1.lshr(C2);
6047 case ISD::SRA: return C1.ashr(C2);
6048 case ISD::ROTL: return C1.rotl(C2);
6049 case ISD::ROTR: return C1.rotr(C2);
6050 case ISD::SMIN: return C1.sle(C2) ? C1 : C2;
6051 case ISD::SMAX: return C1.sge(C2) ? C1 : C2;
6052 case ISD::UMIN: return C1.ule(C2) ? C1 : C2;
6053 case ISD::UMAX: return C1.uge(C2) ? C1 : C2;
6054 case ISD::SADDSAT: return C1.sadd_sat(C2);
6055 case ISD::UADDSAT: return C1.uadd_sat(C2);
6056 case ISD::SSUBSAT: return C1.ssub_sat(C2);
6057 case ISD::USUBSAT: return C1.usub_sat(C2);
6058 case ISD::SSHLSAT: return C1.sshl_sat(C2);
6059 case ISD::USHLSAT: return C1.ushl_sat(C2);
6060 case ISD::UDIV:
6061 if (!C2.getBoolValue())
6062 break;
6063 return C1.udiv(C2);
6064 case ISD::UREM:
6065 if (!C2.getBoolValue())
6066 break;
6067 return C1.urem(C2);
6068 case ISD::SDIV:
6069 if (!C2.getBoolValue())
6070 break;
6071 return C1.sdiv(C2);
6072 case ISD::SREM:
6073 if (!C2.getBoolValue())
6074 break;
6075 return C1.srem(C2);
6076 case ISD::MULHS: {
6077 unsigned FullWidth = C1.getBitWidth() * 2;
6078 APInt C1Ext = C1.sext(FullWidth);
6079 APInt C2Ext = C2.sext(FullWidth);
6080 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
6081 }
6082 case ISD::MULHU: {
6083 unsigned FullWidth = C1.getBitWidth() * 2;
6084 APInt C1Ext = C1.zext(FullWidth);
6085 APInt C2Ext = C2.zext(FullWidth);
6086 return (C1Ext * C2Ext).extractBits(C1.getBitWidth(), C1.getBitWidth());
6087 }
6088 case ISD::AVGFLOORS:
6089 return APIntOps::avgFloorS(C1, C2);
6090 case ISD::AVGFLOORU:
6091 return APIntOps::avgFloorU(C1, C2);
6092 case ISD::AVGCEILS:
6093 return APIntOps::avgCeilS(C1, C2);
6094 case ISD::AVGCEILU:
6095 return APIntOps::avgCeilU(C1, C2);
6096 case ISD::ABDS:
6097 return APIntOps::abds(C1, C2);
6098 case ISD::ABDU:
6099 return APIntOps::abdu(C1, C2);
6100 }
6101 return std::nullopt;
6102}
6103
6104// Handle constant folding with UNDEF.
6105// TODO: Handle more cases.
6106static std::optional<APInt> FoldValueWithUndef(unsigned Opcode, const APInt &C1,
6107 bool IsUndef1, const APInt &C2,
6108 bool IsUndef2) {
6109 if (!(IsUndef1 || IsUndef2))
6110 return FoldValue(Opcode, C1, C2);
6111
6112 // Fold and(x, undef) -> 0
6113 // Fold mul(x, undef) -> 0
6114 if (Opcode == ISD::AND || Opcode == ISD::MUL)
6115 return APInt::getZero(C1.getBitWidth());
6116
6117 return std::nullopt;
6118}
6119
6121 const GlobalAddressSDNode *GA,
6122 const SDNode *N2) {
6123 if (GA->getOpcode() != ISD::GlobalAddress)
6124 return SDValue();
6125 if (!TLI->isOffsetFoldingLegal(GA))
6126 return SDValue();
6127 auto *C2 = dyn_cast<ConstantSDNode>(N2);
6128 if (!C2)
6129 return SDValue();
6130 int64_t Offset = C2->getSExtValue();
6131 switch (Opcode) {
6132 case ISD::ADD: break;
6133 case ISD::SUB: Offset = -uint64_t(Offset); break;
6134 default: return SDValue();
6135 }
6136 return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT,
6137 GA->getOffset() + uint64_t(Offset));
6138}
6139
6140bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef<SDValue> Ops) {
6141 switch (Opcode) {
6142 case ISD::SDIV:
6143 case ISD::UDIV:
6144 case ISD::SREM:
6145 case ISD::UREM: {
6146 // If a divisor is zero/undef or any element of a divisor vector is
6147 // zero/undef, the whole op is undef.
6148 assert(Ops.size() == 2 && "Div/rem should have 2 operands");
6149 SDValue Divisor = Ops[1];
6150 if (Divisor.isUndef() || isNullConstant(Divisor))
6151 return true;
6152
6153 return ISD::isBuildVectorOfConstantSDNodes(Divisor.getNode()) &&
6154 llvm::any_of(Divisor->op_values(),
6155 [](SDValue V) { return V.isUndef() ||
6156 isNullConstant(V); });
6157 // TODO: Handle signed overflow.
6158 }
6159 // TODO: Handle oversized shifts.
6160 default:
6161 return false;
6162 }
6163}
6164
6166 EVT VT, ArrayRef<SDValue> Ops) {
6167 // If the opcode is a target-specific ISD node, there's nothing we can
6168 // do here and the operand rules may not line up with the below, so
6169 // bail early.
6170 // We can't create a scalar CONCAT_VECTORS so skip it. It will break
6171 // for concats involving SPLAT_VECTOR. Concats of BUILD_VECTORS are handled by
6172 // foldCONCAT_VECTORS in getNode before this is called.
6173 if (Opcode >= ISD::BUILTIN_OP_END || Opcode == ISD::CONCAT_VECTORS)
6174 return SDValue();
6175
6176 unsigned NumOps = Ops.size();
6177 if (NumOps == 0)
6178 return SDValue();
6179
6180 if (isUndef(Opcode, Ops))
6181 return getUNDEF(VT);
6182
6183 // Handle unary special cases.
6184 if (NumOps == 1) {
6185 SDValue N1 = Ops[0];
6186
6187 // Constant fold unary operations with an integer constant operand. Even
6188 // opaque constant will be folded, because the folding of unary operations
6189 // doesn't create new constants with different values. Nevertheless, the
6190 // opaque flag is preserved during folding to prevent future folding with
6191 // other constants.
6192 if (auto *C = dyn_cast<ConstantSDNode>(N1)) {
6193 const APInt &Val = C->getAPIntValue();
6194 switch (Opcode) {
6195 case ISD::SIGN_EXTEND:
6196 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6197 C->isTargetOpcode(), C->isOpaque());
6198 case ISD::TRUNCATE:
6199 if (C->isOpaque())
6200 break;
6201 [[fallthrough]];
6202 case ISD::ZERO_EXTEND:
6203 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6204 C->isTargetOpcode(), C->isOpaque());
6205 case ISD::ANY_EXTEND:
6206 // Some targets like RISCV prefer to sign extend some types.
6207 if (TLI->isSExtCheaperThanZExt(N1.getValueType(), VT))
6208 return getConstant(Val.sextOrTrunc(VT.getSizeInBits()), DL, VT,
6209 C->isTargetOpcode(), C->isOpaque());
6210 return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), DL, VT,
6211 C->isTargetOpcode(), C->isOpaque());
6212 case ISD::ABS:
6213 return getConstant(Val.abs(), DL, VT, C->isTargetOpcode(),
6214 C->isOpaque());
6215 case ISD::BITREVERSE:
6216 return getConstant(Val.reverseBits(), DL, VT, C->isTargetOpcode(),
6217 C->isOpaque());
6218 case ISD::BSWAP:
6219 return getConstant(Val.byteSwap(), DL, VT, C->isTargetOpcode(),
6220 C->isOpaque());
6221 case ISD::CTPOP:
6222 return getConstant(Val.popcount(), DL, VT, C->isTargetOpcode(),
6223 C->isOpaque());
6224 case ISD::CTLZ:
6226 return getConstant(Val.countl_zero(), DL, VT, C->isTargetOpcode(),
6227 C->isOpaque());
6228 case ISD::CTTZ:
6230 return getConstant(Val.countr_zero(), DL, VT, C->isTargetOpcode(),
6231 C->isOpaque());
6232 case ISD::UINT_TO_FP:
6233 case ISD::SINT_TO_FP: {
6236 (void)apf.convertFromAPInt(Val, Opcode == ISD::SINT_TO_FP,
6238 return getConstantFP(apf, DL, VT);
6239 }
6240 case ISD::FP16_TO_FP:
6241 case ISD::BF16_TO_FP: {
6242 bool Ignored;
6243 APFloat FPV(Opcode == ISD::FP16_TO_FP ? APFloat::IEEEhalf()
6244 : APFloat::BFloat(),
6245 (Val.getBitWidth() == 16) ? Val : Val.trunc(16));
6246
6247 // This can return overflow, underflow, or inexact; we don't care.
6248 // FIXME need to be more flexible about rounding mode.
6249 (void)FPV.convert(EVTToAPFloatSemantics(VT),
6251 return getConstantFP(FPV, DL, VT);
6252 }
6253 case ISD::STEP_VECTOR:
6254 if (SDValue V = FoldSTEP_VECTOR(DL, VT, N1, *this))
6255 return V;
6256 break;
6257 case ISD::BITCAST:
6258 if (VT == MVT::f16 && C->getValueType(0) == MVT::i16)
6259 return getConstantFP(APFloat(APFloat::IEEEhalf(), Val), DL, VT);
6260 if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
6261 return getConstantFP(APFloat(APFloat::IEEEsingle(), Val), DL, VT);
6262 if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
6263 return getConstantFP(APFloat(APFloat::IEEEdouble(), Val), DL, VT);
6264 if (VT == MVT::f128 && C->getValueType(0) == MVT::i128)
6265 return getConstantFP(APFloat(APFloat::IEEEquad(), Val), DL, VT);
6266 break;
6267 }
6268 }
6269
6270 // Constant fold unary operations with a floating point constant operand.
6271 if (auto *C = dyn_cast<ConstantFPSDNode>(N1)) {
6272 APFloat V = C->getValueAPF(); // make copy
6273 switch (Opcode) {
6274 case ISD::FNEG:
6275 V.changeSign();
6276 return getConstantFP(V, DL, VT);
6277 case ISD::FABS:
6278 V.clearSign();
6279 return getConstantFP(V, DL, VT);
6280 case ISD::FCEIL: {
6281 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
6282 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6283 return getConstantFP(V, DL, VT);
6284 return SDValue();
6285 }
6286 case ISD::FTRUNC: {
6287 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
6288 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6289 return getConstantFP(V, DL, VT);
6290 return SDValue();
6291 }
6292 case ISD::FFLOOR: {
6293 APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
6294 if (fs == APFloat::opOK || fs == APFloat::opInexact)
6295 return getConstantFP(V, DL, VT);
6296 return SDValue();
6297 }
6298 case ISD::FP_EXTEND: {
6299 bool ignored;
6300 // This can return overflow, underflow, or inexact; we don't care.
6301 // FIXME need to be more flexible about rounding mode.
6303 &ignored);
6304 return getConstantFP(V, DL, VT);
6305 }
6306 case ISD::FP_TO_SINT:
6307 case ISD::FP_TO_UINT: {
6308 bool ignored;
6309 APSInt IntVal(VT.getSizeInBits(), Opcode == ISD::FP_TO_UINT);
6310 // FIXME need to be more flexible about rounding mode.
6312 V.convertToInteger(IntVal, APFloat::rmTowardZero, &ignored);
6313 if (s == APFloat::opInvalidOp) // inexact is OK, in fact usual
6314 break;
6315 return getConstant(IntVal, DL, VT);
6316 }
6317 case ISD::FP_TO_FP16:
6318 case ISD::FP_TO_BF16: {
6319 bool Ignored;
6320 // This can return overflow, underflow, or inexact; we don't care.
6321 // FIXME need to be more flexible about rounding mode.
6322 (void)V.convert(Opcode == ISD::FP_TO_FP16 ? APFloat::IEEEhalf()
6323 : APFloat::BFloat(),
6325 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6326 }
6327 case ISD::BITCAST:
6328 if (VT == MVT::i16 && C->getValueType(0) == MVT::f16)
6329 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6330 VT);
6331 if (VT == MVT::i16 && C->getValueType(0) == MVT::bf16)
6332 return getConstant((uint16_t)V.bitcastToAPInt().getZExtValue(), DL,
6333 VT);
6334 if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
6335 return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), DL,
6336 VT);
6337 if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
6338 return getConstant(V.bitcastToAPInt().getZExtValue(), DL, VT);
6339 break;
6340 }
6341 }
6342
6343 // Early-out if we failed to constant fold a bitcast.
6344 if (Opcode == ISD::BITCAST)
6345 return SDValue();
6346 }
6347
6348 // Handle binops special cases.
6349 if (NumOps == 2) {
6350 if (SDValue CFP = foldConstantFPMath(Opcode, DL, VT, Ops))
6351 return CFP;
6352
6353 if (auto *C1 = dyn_cast<ConstantSDNode>(Ops[0])) {
6354 if (auto *C2 = dyn_cast<ConstantSDNode>(Ops[1])) {
6355 if (C1->isOpaque() || C2->isOpaque())
6356 return SDValue();
6357
6358 std::optional<APInt> FoldAttempt =
6359 FoldValue(Opcode, C1->getAPIntValue(), C2->getAPIntValue());
6360 if (!FoldAttempt)
6361 return SDValue();
6362
6363 SDValue Folded = getConstant(*FoldAttempt, DL, VT);
6364 assert((!Folded || !VT.isVector()) &&
6365 "Can't fold vectors ops with scalar operands");
6366 return Folded;
6367 }
6368 }
6369
6370 // fold (add Sym, c) -> Sym+c
6371 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[0]))
6372 return FoldSymbolOffset(Opcode, VT, GA, Ops[1].getNode());
6373 if (TLI->isCommutativeBinOp(Opcode))
6374 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Ops[1]))
6375 return FoldSymbolOffset(Opcode, VT, GA, Ops[0].getNode());
6376 }
6377
6378 // This is for vector folding only from here on.
6379 if (!VT.isVector())
6380 return SDValue();
6381
6382 ElementCount NumElts = VT.getVectorElementCount();
6383
6384 // See if we can fold through bitcasted integer ops.
6385 if (NumOps == 2 && VT.isFixedLengthVector() && VT.isInteger() &&
6386 Ops[0].getValueType() == VT && Ops[1].getValueType() == VT &&
6387 Ops[0].getOpcode() == ISD::BITCAST &&
6388 Ops[1].getOpcode() == ISD::BITCAST) {
6389 SDValue N1 = peekThroughBitcasts(Ops[0]);
6390 SDValue N2 = peekThroughBitcasts(Ops[1]);
6391 auto *BV1 = dyn_cast<BuildVectorSDNode>(N1);
6392 auto *BV2 = dyn_cast<BuildVectorSDNode>(N2);
6393 EVT BVVT = N1.getValueType();
6394 if (BV1 && BV2 && BVVT.isInteger() && BVVT == N2.getValueType()) {
6395 bool IsLE = getDataLayout().isLittleEndian();
6396 unsigned EltBits = VT.getScalarSizeInBits();
6397 SmallVector<APInt> RawBits1, RawBits2;
6398 BitVector UndefElts1, UndefElts2;
6399 if (BV1->getConstantRawBits(IsLE, EltBits, RawBits1, UndefElts1) &&
6400 BV2->getConstantRawBits(IsLE, EltBits, RawBits2, UndefElts2)) {
6401 SmallVector<APInt> RawBits;
6402 for (unsigned I = 0, E = NumElts.getFixedValue(); I != E; ++I) {
6403 std::optional<APInt> Fold = FoldValueWithUndef(
6404 Opcode, RawBits1[I], UndefElts1[I], RawBits2[I], UndefElts2[I]);
6405 if (!Fold)
6406 break;
6407 RawBits.push_back(*Fold);
6408 }
6409 if (RawBits.size() == NumElts.getFixedValue()) {
6410 // We have constant folded, but we need to cast this again back to
6411 // the original (possibly legalized) type.
6412 SmallVector<APInt> DstBits;
6413 BitVector DstUndefs;
6415 DstBits, RawBits, DstUndefs,
6416 BitVector(RawBits.size(), false));
6417 EVT BVEltVT = BV1->getOperand(0).getValueType();
6418 unsigned BVEltBits = BVEltVT.getSizeInBits();
6419 SmallVector<SDValue> Ops(DstBits.size(), getUNDEF(BVEltVT));
6420 for (unsigned I = 0, E = DstBits.size(); I != E; ++I) {
6421 if (DstUndefs[I])
6422 continue;
6423 Ops[I] = getConstant(DstBits[I].sext(BVEltBits), DL, BVEltVT);
6424 }
6425 return getBitcast(VT, getBuildVector(BVVT, DL, Ops));
6426 }
6427 }
6428 }
6429 }
6430
6431 // Fold (mul step_vector(C0), C1) to (step_vector(C0 * C1)).
6432 // (shl step_vector(C0), C1) -> (step_vector(C0 << C1))
6433 if ((Opcode == ISD::MUL || Opcode == ISD::SHL) &&
6434 Ops[0].getOpcode() == ISD::STEP_VECTOR) {
6435 APInt RHSVal;
6436 if (ISD::isConstantSplatVector(Ops[1].getNode(), RHSVal)) {
6437 APInt NewStep = Opcode == ISD::MUL
6438 ? Ops[0].getConstantOperandAPInt(0) * RHSVal
6439 : Ops[0].getConstantOperandAPInt(0) << RHSVal;
6440 return getStepVector(DL, VT, NewStep);
6441 }
6442 }
6443
6444 auto IsScalarOrSameVectorSize = [NumElts](const SDValue &Op) {
6445 return !Op.getValueType().isVector() ||
6446 Op.getValueType().getVectorElementCount() == NumElts;
6447 };
6448
6449 auto IsBuildVectorSplatVectorOrUndef = [](const SDValue &Op) {
6450 return Op.isUndef() || Op.getOpcode() == ISD::CONDCODE ||
6451 Op.getOpcode() == ISD::BUILD_VECTOR ||
6452 Op.getOpcode() == ISD::SPLAT_VECTOR;
6453 };
6454
6455 // All operands must be vector types with the same number of elements as
6456 // the result type and must be either UNDEF or a build/splat vector
6457 // or UNDEF scalars.
6458 if (!llvm::all_of(Ops, IsBuildVectorSplatVectorOrUndef) ||
6459 !llvm::all_of(Ops, IsScalarOrSameVectorSize))
6460 return SDValue();
6461
6462 // If we are comparing vectors, then the result needs to be a i1 boolean that
6463 // is then extended back to the legal result type depending on how booleans
6464 // are represented.
6465 EVT SVT = (Opcode == ISD::SETCC ? MVT::i1 : VT.getScalarType());
6466 ISD::NodeType ExtendCode =
6467 (Opcode == ISD::SETCC && SVT != VT.getScalarType())
6470
6471 // Find legal integer scalar type for constant promotion and
6472 // ensure that its scalar size is at least as large as source.
6473 EVT LegalSVT = VT.getScalarType();
6474 if (NewNodesMustHaveLegalTypes && LegalSVT.isInteger()) {
6475 LegalSVT = TLI->getTypeToTransformTo(*getContext(), LegalSVT);
6476 if (LegalSVT.bitsLT(VT.getScalarType()))
6477 return SDValue();
6478 }
6479
6480 // For scalable vector types we know we're dealing with SPLAT_VECTORs. We
6481 // only have one operand to check. For fixed-length vector types we may have
6482 // a combination of BUILD_VECTOR and SPLAT_VECTOR.
6483 unsigned NumVectorElts = NumElts.isScalable() ? 1 : NumElts.getFixedValue();
6484
6485 // Constant fold each scalar lane separately.
6486 SmallVector<SDValue, 4> ScalarResults;
6487 for (unsigned I = 0; I != NumVectorElts; I++) {
6488 SmallVector<SDValue, 4> ScalarOps;
6489 for (SDValue Op : Ops) {
6490 EVT InSVT = Op.getValueType().getScalarType();
6491 if (Op.getOpcode() != ISD::BUILD_VECTOR &&
6492 Op.getOpcode() != ISD::SPLAT_VECTOR) {
6493 if (Op.isUndef())
6494 ScalarOps.push_back(getUNDEF(InSVT));
6495 else
6496 ScalarOps.push_back(Op);
6497 continue;
6498 }
6499
6500 SDValue ScalarOp =
6501 Op.getOperand(Op.getOpcode() == ISD::SPLAT_VECTOR ? 0 : I);
6502 EVT ScalarVT = ScalarOp.getValueType();
6503
6504 // Build vector (integer) scalar operands may need implicit
6505 // truncation - do this before constant folding.
6506 if (ScalarVT.isInteger() && ScalarVT.bitsGT(InSVT)) {
6507 // Don't create illegally-typed nodes unless they're constants or undef
6508 // - if we fail to constant fold we can't guarantee the (dead) nodes
6509 // we're creating will be cleaned up before being visited for
6510 // legalization.
6511 if (NewNodesMustHaveLegalTypes && !ScalarOp.isUndef() &&
6512 !isa<ConstantSDNode>(ScalarOp) &&
6513 TLI->getTypeAction(*getContext(), InSVT) !=
6515 return SDValue();
6516 ScalarOp = getNode(ISD::TRUNCATE, DL, InSVT, ScalarOp);
6517 }
6518
6519 ScalarOps.push_back(ScalarOp);
6520 }
6521
6522 // Constant fold the scalar operands.
6523 SDValue ScalarResult = getNode(Opcode, DL, SVT, ScalarOps);
6524
6525 // Legalize the (integer) scalar constant if necessary.
6526 if (LegalSVT != SVT)
6527 ScalarResult = getNode(ExtendCode, DL, LegalSVT, ScalarResult);
6528
6529 // Scalar folding only succeeded if the result is a constant or UNDEF.
6530 if (!ScalarResult.isUndef() && ScalarResult.getOpcode() != ISD::Constant &&
6531 ScalarResult.getOpcode() != ISD::ConstantFP)
6532 return SDValue();
6533 ScalarResults.push_back(ScalarResult);
6534 }
6535
6536 SDValue V = NumElts.isScalable() ? getSplatVector(VT, DL, ScalarResults[0])
6537 : getBuildVector(VT, DL, ScalarResults);
6538 NewSDValueDbgMsg(V, "New node fold constant vector: ", this);
6539 return V;
6540}
6541
6543 EVT VT, ArrayRef<SDValue> Ops) {
6544 // TODO: Add support for unary/ternary fp opcodes.
6545 if (Ops.size() != 2)
6546 return SDValue();
6547
6548 // TODO: We don't do any constant folding for strict FP opcodes here, but we
6549 // should. That will require dealing with a potentially non-default
6550 // rounding mode, checking the "opStatus" return value from the APFloat
6551 // math calculations, and possibly other variations.
6552 SDValue N1 = Ops[0];
6553 SDValue N2 = Ops[1];
6554 ConstantFPSDNode *N1CFP = isConstOrConstSplatFP(N1, /*AllowUndefs*/ false);
6555 ConstantFPSDNode *N2CFP = isConstOrConstSplatFP(N2, /*AllowUndefs*/ false);
6556 if (N1CFP && N2CFP) {
6557 APFloat C1 = N1CFP->getValueAPF(); // make copy
6558 const APFloat &C2 = N2CFP->getValueAPF();
6559 switch (Opcode) {
6560 case ISD::FADD:
6562 return getConstantFP(C1, DL, VT);
6563 case ISD::FSUB:
6565 return getConstantFP(C1, DL, VT);
6566 case ISD::FMUL:
6568 return getConstantFP(C1, DL, VT);
6569 case ISD::FDIV:
6571 return getConstantFP(C1, DL, VT);
6572 case ISD::FREM:
6573 C1.mod(C2);
6574 return getConstantFP(C1, DL, VT);
6575 case ISD::FCOPYSIGN:
6576 C1.copySign(C2);
6577 return getConstantFP(C1, DL, VT);
6578 case ISD::FMINNUM:
6579 return getConstantFP(minnum(C1, C2), DL, VT);
6580 case ISD::FMAXNUM:
6581 return getConstantFP(maxnum(C1, C2), DL, VT);
6582 case ISD::FMINIMUM:
6583 return getConstantFP(minimum(C1, C2), DL, VT);
6584 case ISD::FMAXIMUM:
6585 return getConstantFP(maximum(C1, C2), DL, VT);
6586 default: break;
6587 }
6588 }
6589 if (N1CFP && Opcode == ISD::FP_ROUND) {
6590 APFloat C1 = N1CFP->getValueAPF(); // make copy
6591 bool Unused;
6592 // This can return overflow, underflow, or inexact; we don't care.
6593 // FIXME need to be more flexible about rounding mode.
6595 &Unused);
6596 return getConstantFP(C1, DL, VT);
6597 }
6598
6599 switch (Opcode) {
6600 case ISD::FSUB:
6601 // -0.0 - undef --> undef (consistent with "fneg undef")
6602 if (ConstantFPSDNode *N1C = isConstOrConstSplatFP(N1, /*AllowUndefs*/ true))
6603 if (N1C && N1C->getValueAPF().isNegZero() && N2.isUndef())
6604 return getUNDEF(VT);
6605 [[fallthrough]];
6606
6607 case ISD::FADD:
6608 case ISD::FMUL:
6609 case ISD::FDIV:
6610 case ISD::FREM:
6611 // If both operands are undef, the result is undef. If 1 operand is undef,
6612 // the result is NaN. This should match the behavior of the IR optimizer.
6613 if (N1.isUndef() && N2.isUndef())
6614 return getUNDEF(VT);
6615 if (N1.isUndef() || N2.isUndef())
6617 }
6618 return SDValue();
6619}
6620
6622 assert(Val.getValueType().isInteger() && "Invalid AssertAlign!");
6623
6624 // There's no need to assert on a byte-aligned pointer. All pointers are at
6625 // least byte aligned.
6626 if (A == Align(1))
6627 return Val;
6628
6631 ID.AddInteger(A.value());
6632
6633 void *IP = nullptr;
6634 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
6635 return SDValue(E, 0);
6636
6637 auto *N = newSDNode<AssertAlignSDNode>(DL.getIROrder(), DL.getDebugLoc(),
6638 Val.getValueType(), A);
6639 createOperands(N, {Val});
6640
6641 CSEMap.InsertNode(N, IP);
6642 InsertNode(N);
6643
6644 SDValue V(N, 0);
6645 NewSDValueDbgMsg(V, "Creating new node: ", this);
6646 return V;
6647}
6648
6649SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6650 SDValue N1, SDValue N2) {
6651 SDNodeFlags Flags;
6652 if (Inserter)
6653 Flags = Inserter->getFlags();
6654 return getNode(Opcode, DL, VT, N1, N2, Flags);
6655}
6656
6658 SDValue &N2) const {
6659 if (!TLI->isCommutativeBinOp(Opcode))
6660 return;
6661
6662 // Canonicalize:
6663 // binop(const, nonconst) -> binop(nonconst, const)
6668 if ((N1C && !N2C) || (N1CFP && !N2CFP))
6669 std::swap(N1, N2);
6670
6671 // Canonicalize:
6672 // binop(splat(x), step_vector) -> binop(step_vector, splat(x))
6673 else if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6675 std::swap(N1, N2);
6676}
6677
6678SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
6679 SDValue N1, SDValue N2, const SDNodeFlags Flags) {
6681 N2.getOpcode() != ISD::DELETED_NODE &&
6682 "Operand is DELETED_NODE!");
6683
6684 canonicalizeCommutativeBinop(Opcode, N1, N2);
6685
6686 auto *N1C = dyn_cast<ConstantSDNode>(N1);
6687 auto *N2C = dyn_cast<ConstantSDNode>(N2);
6688
6689 // Don't allow undefs in vector splats - we might be returning N2 when folding
6690 // to zero etc.
6691 ConstantSDNode *N2CV =
6692 isConstOrConstSplat(N2, /*AllowUndefs*/ false, /*AllowTruncation*/ true);
6693
6694 switch (Opcode) {
6695 default: break;
6696 case ISD::TokenFactor:
6697 assert(VT == MVT::Other && N1.getValueType() == MVT::Other &&
6698 N2.getValueType() == MVT::Other && "Invalid token factor!");
6699 // Fold trivial token factors.
6700 if (N1.getOpcode() == ISD::EntryToken) return N2;
6701 if (N2.getOpcode() == ISD::EntryToken) return N1;
6702 if (N1 == N2) return N1;
6703 break;
6704 case ISD::BUILD_VECTOR: {
6705 // Attempt to simplify BUILD_VECTOR.
6706 SDValue Ops[] = {N1, N2};
6707 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
6708 return V;
6709 break;
6710 }
6711 case ISD::CONCAT_VECTORS: {
6712 SDValue Ops[] = {N1, N2};
6713 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
6714 return V;
6715 break;
6716 }
6717 case ISD::AND:
6718 assert(VT.isInteger() && "This operator does not apply to FP types!");
6719 assert(N1.getValueType() == N2.getValueType() &&
6720 N1.getValueType() == VT && "Binary operator types must match!");
6721 // (X & 0) -> 0. This commonly occurs when legalizing i64 values, so it's
6722 // worth handling here.
6723 if (N2CV && N2CV->isZero())
6724 return N2;
6725 if (N2CV && N2CV->isAllOnes()) // X & -1 -> X
6726 return N1;
6727 break;
6728 case ISD::OR:
6729 case ISD::XOR:
6730 case ISD::ADD:
6731 case ISD::SUB:
6732 assert(VT.isInteger() && "This operator does not apply to FP types!");
6733 assert(N1.getValueType() == N2.getValueType() &&
6734 N1.getValueType() == VT && "Binary operator types must match!");
6735 // (X ^|+- 0) -> X. This commonly occurs when legalizing i64 values, so
6736 // it's worth handling here.
6737 if (N2CV && N2CV->isZero())
6738 return N1;
6739 if ((Opcode == ISD::ADD || Opcode == ISD::SUB) && VT.isVector() &&
6740 VT.getVectorElementType() == MVT::i1)
6741 return getNode(ISD::XOR, DL, VT, N1, N2);
6742 break;
6743 case ISD::MUL:
6744 assert(VT.isInteger() && "This operator does not apply to FP types!");
6745 assert(N1.getValueType() == N2.getValueType() &&
6746 N1.getValueType() == VT && "Binary operator types must match!");
6747 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6748 return getNode(ISD::AND, DL, VT, N1, N2);
6749 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6750 const APInt &MulImm = N1->getConstantOperandAPInt(0);
6751 const APInt &N2CImm = N2C->getAPIntValue();
6752 return getVScale(DL, VT, MulImm * N2CImm);
6753 }
6754 break;
6755 case ISD::UDIV:
6756 case ISD::UREM:
6757 case ISD::MULHU:
6758 case ISD::MULHS:
6759 case ISD::SDIV:
6760 case ISD::SREM:
6761 case ISD::SADDSAT:
6762 case ISD::SSUBSAT:
6763 case ISD::UADDSAT:
6764 case ISD::USUBSAT:
6765 assert(VT.isInteger() && "This operator does not apply to FP types!");
6766 assert(N1.getValueType() == N2.getValueType() &&
6767 N1.getValueType() == VT && "Binary operator types must match!");
6768 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
6769 // fold (add_sat x, y) -> (or x, y) for bool types.
6770 if (Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT)
6771 return getNode(ISD::OR, DL, VT, N1, N2);
6772 // fold (sub_sat x, y) -> (and x, ~y) for bool types.
6773 if (Opcode == ISD::SSUBSAT || Opcode == ISD::USUBSAT)
6774 return getNode(ISD::AND, DL, VT, N1, getNOT(DL, N2, VT));
6775 }
6776 break;
6777 case ISD::ABDS:
6778 case ISD::ABDU:
6779 assert(VT.isInteger() && "This operator does not apply to FP types!");
6780 assert(N1.getValueType() == N2.getValueType() &&
6781 N1.getValueType() == VT && "Binary operator types must match!");
6782 break;
6783 case ISD::SMIN:
6784 case ISD::UMAX:
6785 assert(VT.isInteger() && "This operator does not apply to FP types!");
6786 assert(N1.getValueType() == N2.getValueType() &&
6787 N1.getValueType() == VT && "Binary operator types must match!");
6788 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6789 return getNode(ISD::OR, DL, VT, N1, N2);
6790 break;
6791 case ISD::SMAX:
6792 case ISD::UMIN:
6793 assert(VT.isInteger() && "This operator does not apply to FP types!");
6794 assert(N1.getValueType() == N2.getValueType() &&
6795 N1.getValueType() == VT && "Binary operator types must match!");
6796 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
6797 return getNode(ISD::AND, DL, VT, N1, N2);
6798 break;
6799 case ISD::FADD:
6800 case ISD::FSUB:
6801 case ISD::FMUL:
6802 case ISD::FDIV:
6803 case ISD::FREM:
6804 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
6805 assert(N1.getValueType() == N2.getValueType() &&
6806 N1.getValueType() == VT && "Binary operator types must match!");
6807 if (SDValue V = simplifyFPBinop(Opcode, N1, N2, Flags))
6808 return V;
6809 break;
6810 case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match.
6811 assert(N1.getValueType() == VT &&
6814 "Invalid FCOPYSIGN!");
6815 break;
6816 case ISD::SHL:
6817 if (N2C && (N1.getOpcode() == ISD::VSCALE) && Flags.hasNoSignedWrap()) {
6818 const APInt &MulImm = N1->getConstantOperandAPInt(0);
6819 const APInt &ShiftImm = N2C->getAPIntValue();
6820 return getVScale(DL, VT, MulImm << ShiftImm);
6821 }
6822 [[fallthrough]];
6823 case ISD::SRA:
6824 case ISD::SRL:
6825 if (SDValue V = simplifyShift(N1, N2))
6826 return V;
6827 [[fallthrough]];
6828 case ISD::ROTL:
6829 case ISD::ROTR:
6830 assert(VT == N1.getValueType() &&
6831 "Shift operators return type must be the same as their first arg");
6832 assert(VT.isInteger() && N2.getValueType().isInteger() &&
6833 "Shifts only work on integers");
6834 assert((!VT.isVector() || VT == N2.getValueType()) &&
6835 "Vector shift amounts must be in the same as their first arg");
6836 // Verify that the shift amount VT is big enough to hold valid shift
6837 // amounts. This catches things like trying to shift an i1024 value by an
6838 // i8, which is easy to fall into in generic code that uses
6839 // TLI.getShiftAmount().
6842 "Invalid use of small shift amount with oversized value!");
6843
6844 // Always fold shifts of i1 values so the code generator doesn't need to
6845 // handle them. Since we know the size of the shift has to be less than the
6846 // size of the value, the shift/rotate count is guaranteed to be zero.
6847 if (VT == MVT::i1)
6848 return N1;
6849 if (N2CV && N2CV->isZero())
6850 return N1;
6851 break;
6852 case ISD::FP_ROUND:
6853 assert(VT.isFloatingPoint() &&
6855 VT.bitsLE(N1.getValueType()) &&
6856 N2C && (N2C->getZExtValue() == 0 || N2C->getZExtValue() == 1) &&
6857 "Invalid FP_ROUND!");
6858 if (N1.getValueType() == VT) return N1; // noop conversion.
6859 break;
6860 case ISD::AssertSext:
6861 case ISD::AssertZext: {
6862 EVT EVT = cast<VTSDNode>(N2)->getVT();
6863 assert(VT == N1.getValueType() && "Not an inreg extend!");
6864 assert(VT.isInteger() && EVT.isInteger() &&
6865 "Cannot *_EXTEND_INREG FP types");
6866 assert(!EVT.isVector() &&
6867 "AssertSExt/AssertZExt type should be the vector element type "
6868 "rather than the vector type!");
6869 assert(EVT.bitsLE(VT.getScalarType()) && "Not extending!");
6870 if (VT.getScalarType() == EVT) return N1; // noop assertion.
6871 break;
6872 }
6874 EVT EVT = cast<VTSDNode>(N2)->getVT();
6875 assert(VT == N1.getValueType() && "Not an inreg extend!");
6876 assert(VT.isInteger() && EVT.isInteger() &&
6877 "Cannot *_EXTEND_INREG FP types");
6878 assert(EVT.isVector() == VT.isVector() &&
6879 "SIGN_EXTEND_INREG type should be vector iff the operand "
6880 "type is vector!");
6881 assert((!EVT.isVector() ||
6883 "Vector element counts must match in SIGN_EXTEND_INREG");
6884 assert(EVT.bitsLE(VT) && "Not extending!");
6885 if (EVT == VT) return N1; // Not actually extending
6886
6887 auto SignExtendInReg = [&](APInt Val, llvm::EVT ConstantVT) {
6888 unsigned FromBits = EVT.getScalarSizeInBits();
6889 Val <<= Val.getBitWidth() - FromBits;
6890 Val.ashrInPlace(Val.getBitWidth() - FromBits);
6891 return getConstant(Val, DL, ConstantVT);
6892 };
6893
6894 if (N1C) {
6895 const APInt &Val = N1C->getAPIntValue();
6896 return SignExtendInReg(Val, VT);
6897 }
6898
6901 llvm::EVT OpVT = N1.getOperand(0).getValueType();
6902 for (int i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
6903 SDValue Op = N1.getOperand(i);
6904 if (Op.isUndef()) {
6905 Ops.push_back(getUNDEF(OpVT));
6906 continue;
6907 }
6908 ConstantSDNode *C = cast<ConstantSDNode>(Op);
6909 APInt Val = C->getAPIntValue();
6910 Ops.push_back(SignExtendInReg(Val, OpVT));
6911 }
6912 return getBuildVector(VT, DL, Ops);
6913 }
6914
6915 if (N1.getOpcode() == ISD::SPLAT_VECTOR &&
6916 isa<ConstantSDNode>(N1.getOperand(0)))
6917 return getNode(
6918 ISD::SPLAT_VECTOR, DL, VT,
6919 SignExtendInReg(N1.getConstantOperandAPInt(0),
6920 N1.getOperand(0).getValueType()));
6921 break;
6922 }
6924 case ISD::FP_TO_UINT_SAT: {
6925 assert(VT.isInteger() && cast<VTSDNode>(N2)->getVT().isInteger() &&
6926 N1.getValueType().isFloatingPoint() && "Invalid FP_TO_*INT_SAT");
6927 assert(N1.getValueType().isVector() == VT.isVector() &&
6928 "FP_TO_*INT_SAT type should be vector iff the operand type is "
6929 "vector!");
6930 assert((!VT.isVector() || VT.getVectorElementCount() ==
6932 "Vector element counts must match in FP_TO_*INT_SAT");
6933 assert(!cast<VTSDNode>(N2)->getVT().isVector() &&
6934 "Type to saturate to must be a scalar.");
6935 assert(cast<VTSDNode>(N2)->getVT().bitsLE(VT.getScalarType()) &&
6936 "Not extending!");
6937 break;
6938 }
6941 "The result of EXTRACT_VECTOR_ELT must be at least as wide as the \
6942 element type of the vector.");
6943
6944 // Extract from an undefined value or using an undefined index is undefined.
6945 if (N1.isUndef() || N2.isUndef())
6946 return getUNDEF(VT);
6947
6948 // EXTRACT_VECTOR_ELT of out-of-bounds element is an UNDEF for fixed length
6949 // vectors. For scalable vectors we will provide appropriate support for
6950 // dealing with arbitrary indices.
6951 if (N2C && N1.getValueType().isFixedLengthVector() &&
6952 N2C->getAPIntValue().uge(N1.getValueType().getVectorNumElements()))
6953 return getUNDEF(VT);
6954
6955 // EXTRACT_VECTOR_ELT of CONCAT_VECTORS is often formed while lowering is
6956 // expanding copies of large vectors from registers. This only works for
6957 // fixed length vectors, since we need to know the exact number of
6958 // elements.
6959 if (N2C && N1.getOpcode() == ISD::CONCAT_VECTORS &&
6961 unsigned Factor =
6964 N1.getOperand(N2C->getZExtValue() / Factor),
6965 getVectorIdxConstant(N2C->getZExtValue() % Factor, DL));
6966 }
6967
6968 // EXTRACT_VECTOR_ELT of BUILD_VECTOR or SPLAT_VECTOR is often formed while
6969 // lowering is expanding large vector constants.
6970 if (N2C && (N1.getOpcode() == ISD::BUILD_VECTOR ||
6971 N1.getOpcode() == ISD::SPLAT_VECTOR)) {
6974 "BUILD_VECTOR used for scalable vectors");
6975 unsigned Index =
6976 N1.getOpcode() == ISD::BUILD_VECTOR ? N2C->getZExtValue() : 0;
6977 SDValue Elt = N1.getOperand(Index);
6978
6979 if (VT != Elt.getValueType())
6980 // If the vector element type is not legal, the BUILD_VECTOR operands
6981 // are promoted and implicitly truncated, and the result implicitly
6982 // extended. Make that explicit here.
6983 Elt = getAnyExtOrTrunc(Elt, DL, VT);
6984
6985 return Elt;
6986 }
6987
6988 // EXTRACT_VECTOR_ELT of INSERT_VECTOR_ELT is often formed when vector
6989 // operations are lowered to scalars.
6990 if (N1.getOpcode() == ISD::INSERT_VECTOR_ELT) {
6991 // If the indices are the same, return the inserted element else
6992 // if the indices are known different, extract the element from
6993 // the original vector.
6994 SDValue N1Op2 = N1.getOperand(2);
6995 ConstantSDNode *N1Op2C = dyn_cast<ConstantSDNode>(N1Op2);
6996
6997 if (N1Op2C && N2C) {
6998 if (N1Op2C->getZExtValue() == N2C->getZExtValue()) {
6999 if (VT == N1.getOperand(1).getValueType())
7000 return N1.getOperand(1);
7001 if (VT.isFloatingPoint()) {
7003 return getFPExtendOrRound(N1.getOperand(1), DL, VT);
7004 }
7005 return getSExtOrTrunc(N1.getOperand(1), DL, VT);
7006 }
7007 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0), N2);
7008 }
7009 }
7010
7011 // EXTRACT_VECTOR_ELT of v1iX EXTRACT_SUBVECTOR could be formed
7012 // when vector types are scalarized and v1iX is legal.
7013 // vextract (v1iX extract_subvector(vNiX, Idx)) -> vextract(vNiX,Idx).
7014 // Here we are completely ignoring the extract element index (N2),
7015 // which is fine for fixed width vectors, since any index other than 0
7016 // is undefined anyway. However, this cannot be ignored for scalable
7017 // vectors - in theory we could support this, but we don't want to do this
7018 // without a profitability check.
7019 if (N1.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7021 N1.getValueType().getVectorNumElements() == 1) {
7022 return getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, N1.getOperand(0),
7023 N1.getOperand(1));
7024 }
7025 break;
7027 assert(N2C && (unsigned)N2C->getZExtValue() < 2 && "Bad EXTRACT_ELEMENT!");
7028 assert(!N1.getValueType().isVector() && !VT.isVector() &&
7029 (N1.getValueType().isInteger() == VT.isInteger()) &&
7030 N1.getValueType() != VT &&
7031 "Wrong types for EXTRACT_ELEMENT!");
7032
7033 // EXTRACT_ELEMENT of BUILD_PAIR is often formed while legalize is expanding
7034 // 64-bit integers into 32-bit parts. Instead of building the extract of
7035 // the BUILD_PAIR, only to have legalize rip it apart, just do it now.
7036 if (N1.getOpcode() == ISD::BUILD_PAIR)
7037 return N1.getOperand(N2C->getZExtValue());
7038
7039 // EXTRACT_ELEMENT of a constant int is also very common.
7040 if (N1C) {
7041 unsigned ElementSize = VT.getSizeInBits();
7042 unsigned Shift = ElementSize * N2C->getZExtValue();
7043 const APInt &Val = N1C->getAPIntValue();
7044 return getConstant(Val.extractBits(ElementSize, Shift), DL, VT);
7045 }
7046 break;
7048 EVT N1VT = N1.getValueType();
7049 assert(VT.isVector() && N1VT.isVector() &&
7050 "Extract subvector VTs must be vectors!");
7052 "Extract subvector VTs must have the same element type!");
7053 assert((VT.isFixedLengthVector() || N1VT.isScalableVector()) &&
7054 "Cannot extract a scalable vector from a fixed length vector!");
7055 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7057 "Extract subvector must be from larger vector to smaller vector!");
7058 assert(N2C && "Extract subvector index must be a constant");
7059 assert((VT.isScalableVector() != N1VT.isScalableVector() ||
7060 (VT.getVectorMinNumElements() + N2C->getZExtValue()) <=
7061 N1VT.getVectorMinNumElements()) &&
7062 "Extract subvector overflow!");
7063 assert(N2C->getAPIntValue().getBitWidth() ==
7064 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7065 "Constant index for EXTRACT_SUBVECTOR has an invalid size");
7066
7067 // Trivial extraction.
7068 if (VT == N1VT)
7069 return N1;
7070
7071 // EXTRACT_SUBVECTOR of an UNDEF is an UNDEF.
7072 if (N1.isUndef())
7073 return getUNDEF(VT);
7074
7075 // EXTRACT_SUBVECTOR of CONCAT_VECTOR can be simplified if the pieces of
7076 // the concat have the same type as the extract.
7077 if (N1.getOpcode() == ISD::CONCAT_VECTORS &&
7078 VT == N1.getOperand(0).getValueType()) {
7079 unsigned Factor = VT.getVectorMinNumElements();
7080 return N1.getOperand(N2C->getZExtValue() / Factor);
7081 }
7082
7083 // EXTRACT_SUBVECTOR of INSERT_SUBVECTOR is often created
7084 // during shuffle legalization.
7085 if (N1.getOpcode() == ISD::INSERT_SUBVECTOR && N2 == N1.getOperand(2) &&
7086 VT == N1.getOperand(1).getValueType())
7087 return N1.getOperand(1);
7088 break;
7089 }
7090 }
7091
7092 // Perform trivial constant folding.
7093 if (SDValue SV = FoldConstantArithmetic(Opcode, DL, VT, {N1, N2}))
7094 return SV;
7095
7096 // Canonicalize an UNDEF to the RHS, even over a constant.
7097 if (N1.isUndef()) {
7098 if (TLI->isCommutativeBinOp(Opcode)) {
7099 std::swap(N1, N2);
7100 } else {
7101 switch (Opcode) {
7102 case ISD::SUB:
7103 return getUNDEF(VT); // fold op(undef, arg2) -> undef
7105 case ISD::UDIV:
7106 case ISD::SDIV:
7107 case ISD::UREM:
7108 case ISD::SREM:
7109 case ISD::SSUBSAT:
7110 case ISD::USUBSAT:
7111 return getConstant(0, DL, VT); // fold op(undef, arg2) -> 0
7112 }
7113 }
7114 }
7115
7116 // Fold a bunch of operators when the RHS is undef.
7117 if (N2.isUndef()) {
7118 switch (Opcode) {
7119 case ISD::XOR:
7120 if (N1.isUndef())
7121 // Handle undef ^ undef -> 0 special case. This is a common
7122 // idiom (misuse).
7123 return getConstant(0, DL, VT);
7124 [[fallthrough]];
7125 case ISD::ADD:
7126 case ISD::SUB:
7127 case ISD::UDIV:
7128 case ISD::SDIV:
7129 case ISD::UREM:
7130 case ISD::SREM:
7131 return getUNDEF(VT); // fold op(arg1, undef) -> undef
7132 case ISD::MUL:
7133 case ISD::AND:
7134 case ISD::SSUBSAT:
7135 case ISD::USUBSAT:
7136 return getConstant(0, DL, VT); // fold op(arg1, undef) -> 0
7137 case ISD::OR:
7138 case ISD::SADDSAT:
7139 case ISD::UADDSAT:
7140 return getAllOnesConstant(DL, VT);
7141 }
7142 }
7143
7144 // Memoize this node if possible.
7145 SDNode *N;
7146 SDVTList VTs = getVTList(VT);
7147 SDValue Ops[] = {N1, N2};
7148 if (VT != MVT::Glue) {
7150 AddNodeIDNode(ID, Opcode, VTs, Ops);
7151 void *IP = nullptr;
7152 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7153 E->intersectFlagsWith(Flags);
7154 return SDValue(E, 0);
7155 }
7156
7157 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7158 N->setFlags(Flags);
7159 createOperands(N, Ops);
7160 CSEMap.InsertNode(N, IP);
7161 } else {
7162 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7163 createOperands(N, Ops);
7164 }
7165
7166 InsertNode(N);
7167 SDValue V = SDValue(N, 0);
7168 NewSDValueDbgMsg(V, "Creating new node: ", this);
7169 return V;
7170}
7171
7172SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7173 SDValue N1, SDValue N2, SDValue N3) {
7174 SDNodeFlags Flags;
7175 if (Inserter)
7176 Flags = Inserter->getFlags();
7177 return getNode(Opcode, DL, VT, N1, N2, N3, Flags);
7178}
7179
7180SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7181 SDValue N1, SDValue N2, SDValue N3,
7182 const SDNodeFlags Flags) {
7184 N2.getOpcode() != ISD::DELETED_NODE &&
7185 N3.getOpcode() != ISD::DELETED_NODE &&
7186 "Operand is DELETED_NODE!");
7187 // Perform various simplifications.
7188 switch (Opcode) {
7189 case ISD::FMA:
7190 case ISD::FMAD: {
7191 assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
7192 assert(N1.getValueType() == VT && N2.getValueType() == VT &&
7193 N3.getValueType() == VT && "FMA types must match!");
7194 ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
7195 ConstantFPSDNode *N2CFP = dyn_cast<ConstantFPSDNode>(N2);
7196 ConstantFPSDNode *N3CFP = dyn_cast<ConstantFPSDNode>(N3);
7197 if (N1CFP && N2CFP && N3CFP) {
7198 APFloat V1 = N1CFP->getValueAPF();
7199 const APFloat &V2 = N2CFP->getValueAPF();
7200 const APFloat &V3 = N3CFP->getValueAPF();
7201 if (Opcode == ISD::FMAD) {
7204 } else
7206 return getConstantFP(V1, DL, VT);
7207 }
7208 break;
7209 }
7210 case ISD::BUILD_VECTOR: {
7211 // Attempt to simplify BUILD_VECTOR.
7212 SDValue Ops[] = {N1, N2, N3};
7213 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
7214 return V;
7215 break;
7216 }
7217 case ISD::CONCAT_VECTORS: {
7218 SDValue Ops[] = {N1, N2, N3};
7219 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
7220 return V;
7221 break;
7222 }
7223 case ISD::SETCC: {
7224 assert(VT.isInteger() && "SETCC result type must be an integer!");
7225 assert(N1.getValueType() == N2.getValueType() &&
7226 "SETCC operands must have the same type!");
7227 assert(VT.isVector() == N1.getValueType().isVector() &&
7228 "SETCC type should be vector iff the operand type is vector!");
7229 assert((!VT.isVector() || VT.getVectorElementCount() ==
7231 "SETCC vector element counts must match!");
7232 // Use FoldSetCC to simplify SETCC's.
7233 if (SDValue V = FoldSetCC(VT, N1, N2, cast<CondCodeSDNode>(N3)->get(), DL))
7234 return V;
7235 // Vector constant folding.
7236 SDValue Ops[] = {N1, N2, N3};
7237 if (SDValue V = FoldConstantArithmetic(Opcode, DL, VT, Ops)) {
7238 NewSDValueDbgMsg(V, "New node vector constant folding: ", this);
7239 return V;
7240 }
7241 break;
7242 }
7243 case ISD::SELECT:
7244 case ISD::VSELECT:
7245 if (SDValue V = simplifySelect(N1, N2, N3))
7246 return V;
7247 break;
7249 llvm_unreachable("should use getVectorShuffle constructor!");
7250 case ISD::VECTOR_SPLICE: {
7251 if (cast<ConstantSDNode>(N3)->isZero())
7252 return N1;
7253 break;
7254 }
7256 ConstantSDNode *N3C = dyn_cast<ConstantSDNode>(N3);
7257 // INSERT_VECTOR_ELT into out-of-bounds element is an UNDEF, except
7258 // for scalable vectors where we will generate appropriate code to
7259 // deal with out-of-bounds cases correctly.
7260 if (N3C && N1.getValueType().isFixedLengthVector() &&
7262 return getUNDEF(VT);
7263
7264 // Undefined index can be assumed out-of-bounds, so that's UNDEF too.
7265 if (N3.isUndef())
7266 return getUNDEF(VT);
7267
7268 // If the inserted element is an UNDEF, just use the input vector.
7269 if (N2.isUndef())
7270 return N1;
7271
7272 break;
7273 }
7274 case ISD::INSERT_SUBVECTOR: {
7275 // Inserting undef into undef is still undef.
7276 if (N1.isUndef() && N2.isUndef())
7277 return getUNDEF(VT);
7278
7279 EVT N2VT = N2.getValueType();
7280 assert(VT == N1.getValueType() &&
7281 "Dest and insert subvector source types must match!");
7282 assert(VT.isVector() && N2VT.isVector() &&
7283 "Insert subvector VTs must be vectors!");
7285 "Insert subvector VTs must have the same element type!");
7286 assert((VT.isScalableVector() || N2VT.isFixedLengthVector()) &&
7287 "Cannot insert a scalable vector into a fixed length vector!");
7288 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7290 "Insert subvector must be from smaller vector to larger vector!");
7291 assert(isa<ConstantSDNode>(N3) &&
7292 "Insert subvector index must be constant");
7293 assert((VT.isScalableVector() != N2VT.isScalableVector() ||
7294 (N2VT.getVectorMinNumElements() + N3->getAsZExtVal()) <=
7296 "Insert subvector overflow!");
7298 TLI->getVectorIdxTy(getDataLayout()).getFixedSizeInBits() &&
7299 "Constant index for INSERT_SUBVECTOR has an invalid size");
7300
7301 // Trivial insertion.
7302 if (VT == N2VT)
7303 return N2;
7304
7305 // If this is an insert of an extracted vector into an undef vector, we
7306 // can just use the input to the extract.
7307 if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7308 N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT)
7309 return N2.getOperand(0);
7310 break;
7311 }
7312 case ISD::BITCAST:
7313 // Fold bit_convert nodes from a type to themselves.
7314 if (N1.getValueType() == VT)
7315 return N1;
7316 break;
7317 case ISD::VP_TRUNCATE:
7318 case ISD::VP_SIGN_EXTEND:
7319 case ISD::VP_ZERO_EXTEND:
7320 // Don't create noop casts.
7321 if (N1.getValueType() == VT)
7322 return N1;
7323 break;
7324 }
7325
7326 // Memoize node if it doesn't produce a glue result.
7327 SDNode *N;
7328 SDVTList VTs = getVTList(VT);
7329 SDValue Ops[] = {N1, N2, N3};
7330 if (VT != MVT::Glue) {
7332 AddNodeIDNode(ID, Opcode, VTs, Ops);
7333 void *IP = nullptr;
7334 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
7335 E->intersectFlagsWith(Flags);
7336 return SDValue(E, 0);
7337 }
7338
7339 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7340 N->setFlags(Flags);
7341 createOperands(N, Ops);
7342 CSEMap.InsertNode(N, IP);
7343 } else {
7344 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
7345 createOperands(N, Ops);
7346 }
7347
7348 InsertNode(N);
7349 SDValue V = SDValue(N, 0);
7350 NewSDValueDbgMsg(V, "Creating new node: ", this);
7351 return V;
7352}
7353
7354SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7355 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
7356 SDValue Ops[] = { N1, N2, N3, N4 };
7357 return getNode(Opcode, DL, VT, Ops);
7358}
7359
7360SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
7361 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
7362 SDValue N5) {
7363 SDValue Ops[] = { N1, N2, N3, N4, N5 };
7364 return getNode(Opcode, DL, VT, Ops);
7365}
7366
7367/// getStackArgumentTokenFactor - Compute a TokenFactor to force all
7368/// the incoming stack arguments to be loaded from the stack.
7370 SmallVector<SDValue, 8> ArgChains;
7371
7372 // Include the original chain at the beginning of the list. When this is
7373 // used by target LowerCall hooks, this helps legalize find the
7374 // CALLSEQ_BEGIN node.
7375 ArgChains.push_back(Chain);
7376
7377 // Add a chain value for each stack argument.
7378 for (SDNode *U : getEntryNode().getNode()->uses())
7379 if (LoadSDNode *L = dyn_cast<LoadSDNode>(U))
7380 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(L->getBasePtr()))
7381 if (FI->getIndex() < 0)
7382 ArgChains.push_back(SDValue(L, 1));
7383
7384 // Build a tokenfactor for all the chains.
7385 return getNode(ISD::TokenFactor, SDLoc(Chain), MVT::Other, ArgChains);
7386}
7387
7388/// getMemsetValue - Vectorized representation of the memset value
7389/// operand.
7391 const SDLoc &dl) {
7392 assert(!Value.isUndef());
7393
7394 unsigned NumBits = VT.getScalarSizeInBits();
7395 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Value)) {
7396 assert(C->getAPIntValue().getBitWidth() == 8);
7397 APInt Val = APInt::getSplat(NumBits, C->getAPIntValue());
7398 if (VT.isInteger()) {
7399 bool IsOpaque = VT.getSizeInBits() > 64 ||
7400 !DAG.getTargetLoweringInfo().isLegalStoreImmediate(C->getSExtValue());
7401 return DAG.getConstant(Val, dl, VT, false, IsOpaque);
7402 }
7403 return DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(VT), Val), dl,
7404 VT);
7405 }
7406
7407 assert(Value.getValueType() == MVT::i8 && "memset with non-byte fill value?");
7408 EVT IntVT = VT.getScalarType();
7409 if (!IntVT.isInteger())
7410 IntVT = EVT::getIntegerVT(*DAG.getContext(), IntVT.getSizeInBits());
7411
7412 Value = DAG.getNode(ISD::ZERO_EXTEND, dl, IntVT, Value);
7413 if (NumBits > 8) {
7414 // Use a multiplication with 0x010101... to extend the input to the
7415 // required length.
7416 APInt Magic = APInt::getSplat(NumBits, APInt(8, 0x01));
7417 Value = DAG.getNode(ISD::MUL, dl, IntVT, Value,
7418 DAG.getConstant(Magic, dl, IntVT));
7419 }
7420
7421 if (VT != Value.getValueType() && !VT.isInteger())
7422 Value = DAG.getBitcast(VT.getScalarType(), Value);
7423 if (VT != Value.getValueType())
7424 Value = DAG.getSplatBuildVector(VT, dl, Value);
7425
7426 return Value;
7427}
7428
7429/// getMemsetStringVal - Similar to getMemsetValue. Except this is only
7430/// used when a memcpy is turned into a memset when the source is a constant
7431/// string ptr.
7433 const TargetLowering &TLI,
7434 const ConstantDataArraySlice &Slice) {
7435 // Handle vector with all elements zero.
7436 if (Slice.Array == nullptr) {
7437 if (VT.isInteger())
7438 return DAG.getConstant(0, dl, VT);
7439 if (VT == MVT::f32 || VT == MVT::f64 || VT == MVT::f128)
7440 return DAG.getConstantFP(0.0, dl, VT);
7441 if (VT.isVector()) {
7442 unsigned NumElts = VT.getVectorNumElements();
7443 MVT EltVT = (VT.getVectorElementType() == MVT::f32) ? MVT::i32 : MVT::i64;
7444 return DAG.getNode(ISD::BITCAST, dl, VT,
7445 DAG.getConstant(0, dl,
7447 EltVT, NumElts)));
7448 }
7449 llvm_unreachable("Expected type!");
7450 }
7451
7452 assert(!VT.isVector() && "Can't handle vector type here!");
7453 unsigned NumVTBits = VT.getSizeInBits();
7454 unsigned NumVTBytes = NumVTBits / 8;
7455 unsigned NumBytes = std::min(NumVTBytes, unsigned(Slice.Length));
7456
7457 APInt Val(NumVTBits, 0);
7458 if (DAG.getDataLayout().isLittleEndian()) {
7459 for (unsigned i = 0; i != NumBytes; ++i)
7460 Val |= (uint64_t)(unsigned char)Slice[i] << i*8;
7461 } else {
7462 for (unsigned i = 0; i != NumBytes; ++i)
7463 Val |= (uint64_t)(unsigned char)Slice[i] << (NumVTBytes-i-1)*8;
7464 }
7465
7466 // If the "cost" of materializing the integer immediate is less than the cost
7467 // of a load, then it is cost effective to turn the load into the immediate.
7468 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
7469 if (TLI.shouldConvertConstantLoadToIntImm(Val, Ty))
7470 return DAG.getConstant(Val, dl, VT);
7471 return SDValue();
7472}
7473
7475 const SDLoc &DL,
7476 const SDNodeFlags Flags) {
7477 EVT VT = Base.getValueType();
7478 SDValue Index;
7479
7480 if (Offset.isScalable())
7481 Index = getVScale(DL, Base.getValueType(),
7482 APInt(Base.getValueSizeInBits().getFixedValue(),
7483 Offset.getKnownMinValue()));
7484 else
7485 Index = getConstant(Offset.getFixedValue(), DL, VT);
7486
7487 return getMemBasePlusOffset(Base, Index, DL, Flags);
7488}
7489
7491 const SDLoc &DL,
7492 const SDNodeFlags Flags) {
7493 assert(Offset.getValueType().isInteger());
7494 EVT BasePtrVT = Ptr.getValueType();
7495 return getNode(ISD::ADD, DL, BasePtrVT, Ptr, Offset, Flags);
7496}
7497
7498/// Returns true if memcpy source is constant data.
7500 uint64_t SrcDelta = 0;
7501 GlobalAddressSDNode *G = nullptr;
7502 if (Src.getOpcode() == ISD::GlobalAddress)
7503 G = cast<GlobalAddressSDNode>(Src);
7504 else if (Src.getOpcode() == ISD::ADD &&
7505 Src.getOperand(0).getOpcode() == ISD::GlobalAddress &&
7506 Src.getOperand(1).getOpcode() == ISD::Constant) {
7507 G = cast<GlobalAddressSDNode>(Src.getOperand(0));
7508 SrcDelta = Src.getConstantOperandVal(1);
7509 }
7510 if (!G)
7511 return false;
7512
7513 return getConstantDataArrayInfo(G->getGlobal(), Slice, 8,
7514 SrcDelta + G->getOffset());
7515}
7516
7518 SelectionDAG &DAG) {
7519 // On Darwin, -Os means optimize for size without hurting performance, so
7520 // only really optimize for size when -Oz (MinSize) is used.
7522 return MF.getFunction().hasMinSize();
7523 return DAG.shouldOptForSize();
7524}
7525
7527 SmallVector<SDValue, 32> &OutChains, unsigned From,
7528 unsigned To, SmallVector<SDValue, 16> &OutLoadChains,
7529 SmallVector<SDValue, 16> &OutStoreChains) {
7530 assert(OutLoadChains.size() && "Missing loads in memcpy inlining");
7531 assert(OutStoreChains.size() && "Missing stores in memcpy inlining");
7532 SmallVector<SDValue, 16> GluedLoadChains;
7533 for (unsigned i = From; i < To; ++i) {
7534 OutChains.push_back(OutLoadChains[i]);
7535 GluedLoadChains.push_back(OutLoadChains[i]);
7536 }
7537
7538 // Chain for all loads.
7539 SDValue LoadToken = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
7540 GluedLoadChains);
7541
7542 for (unsigned i = From; i < To; ++i) {
7543 StoreSDNode *ST = dyn_cast<StoreSDNode>(OutStoreChains[i]);
7544 SDValue NewStore = DAG.getTruncStore(LoadToken, dl, ST->getValue(),
7545 ST->getBasePtr(), ST->getMemoryVT(),
7546 ST->getMemOperand());
7547 OutChains.push_back(NewStore);
7548 }
7549}
7550
7552 SDValue Chain, SDValue Dst, SDValue Src,
7553 uint64_t Size, Align Alignment,
7554 bool isVol, bool AlwaysInline,
7555 MachinePointerInfo DstPtrInfo,
7556 MachinePointerInfo SrcPtrInfo,
7557 const AAMDNodes &AAInfo, AAResults *AA) {
7558 // Turn a memcpy of undef to nop.
7559 // FIXME: We need to honor volatile even is Src is undef.
7560 if (Src.isUndef())
7561 return Chain;
7562
7563 // Expand memcpy to a series of load and store ops if the size operand falls
7564 // below a certain threshold.
7565 // TODO: In the AlwaysInline case, if the size is big then generate a loop
7566 // rather than maybe a humongous number of loads and stores.
7567 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7568 const DataLayout &DL = DAG.getDataLayout();
7569 LLVMContext &C = *DAG.getContext();
7570 std::vector<EVT> MemOps;
7571 bool DstAlignCanChange = false;
7573 MachineFrameInfo &MFI = MF.getFrameInfo();
7574 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7575 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7576 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7577 DstAlignCanChange = true;
7578 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7579 if (!SrcAlign || Alignment > *SrcAlign)
7580 SrcAlign = Alignment;
7581 assert(SrcAlign && "SrcAlign must be set");
7583 // If marked as volatile, perform a copy even when marked as constant.
7584 bool CopyFromConstant = !isVol && isMemSrcFromConstant(Src, Slice);
7585 bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr;
7586 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize);
7587 const MemOp Op = isZeroConstant
7588 ? MemOp::Set(Size, DstAlignCanChange, Alignment,
7589 /*IsZeroMemset*/ true, isVol)
7590 : MemOp::Copy(Size, DstAlignCanChange, Alignment,
7591 *SrcAlign, isVol, CopyFromConstant);
7592 if (!TLI.findOptimalMemOpLowering(
7593 MemOps, Limit, Op, DstPtrInfo.getAddrSpace(),
7594 SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes()))
7595 return SDValue();
7596
7597 if (DstAlignCanChange) {
7598 Type *Ty = MemOps[0].getTypeForEVT(C);
7599 Align NewAlign = DL.getABITypeAlign(Ty);
7600
7601 // Don't promote to an alignment that would require dynamic stack
7602 // realignment which may conflict with optimizations such as tail call
7603 // optimization.
7605 if (!TRI->hasStackRealignment(MF))
7606 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7607 NewAlign = NewAlign.previous();
7608
7609 if (NewAlign > Alignment) {
7610 // Give the stack frame object a larger alignment if needed.
7611 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7612 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7613 Alignment = NewAlign;
7614 }
7615 }
7616
7617 // Prepare AAInfo for loads/stores after lowering this memcpy.
7618 AAMDNodes NewAAInfo = AAInfo;
7619 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7620
7621 const Value *SrcVal = dyn_cast_if_present<const Value *>(SrcPtrInfo.V);
7622 bool isConstant =
7623 AA && SrcVal &&
7624 AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo));
7625
7626 MachineMemOperand::Flags MMOFlags =
7628 SmallVector<SDValue, 16> OutLoadChains;
7629 SmallVector<SDValue, 16> OutStoreChains;
7630 SmallVector<SDValue, 32> OutChains;
7631 unsigned NumMemOps = MemOps.size();
7632 uint64_t SrcOff = 0, DstOff = 0;
7633 for (unsigned i = 0; i != NumMemOps; ++i) {
7634 EVT VT = MemOps[i];
7635 unsigned VTSize = VT.getSizeInBits() / 8;
7636 SDValue Value, Store;
7637
7638 if (VTSize > Size) {
7639 // Issuing an unaligned load / store pair that overlaps with the previous
7640 // pair. Adjust the offset accordingly.
7641 assert(i == NumMemOps-1 && i != 0);
7642 SrcOff -= VTSize - Size;
7643 DstOff -= VTSize - Size;
7644 }
7645
7646 if (CopyFromConstant &&
7647 (isZeroConstant || (VT.isInteger() && !VT.isVector()))) {
7648 // It's unlikely a store of a vector immediate can be done in a single
7649 // instruction. It would require a load from a constantpool first.
7650 // We only handle zero vectors here.
7651 // FIXME: Handle other cases where store of vector immediate is done in
7652 // a single instruction.
7653 ConstantDataArraySlice SubSlice;
7654 if (SrcOff < Slice.Length) {
7655 SubSlice = Slice;
7656 SubSlice.move(SrcOff);
7657 } else {
7658 // This is an out-of-bounds access and hence UB. Pretend we read zero.
7659 SubSlice.Array = nullptr;
7660 SubSlice.Offset = 0;
7661 SubSlice.Length = VTSize;
7662 }
7663 Value = getMemsetStringVal(VT, dl, DAG, TLI, SubSlice);
7664 if (Value.getNode()) {
7665 Store = DAG.getStore(
7666 Chain, dl, Value,
7667 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7668 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7669 OutChains.push_back(Store);
7670 }
7671 }
7672
7673 if (!Store.getNode()) {
7674 // The type might not be legal for the target. This should only happen
7675 // if the type is smaller than a legal type, as on PPC, so the right
7676 // thing to do is generate a LoadExt/StoreTrunc pair. These simplify
7677 // to Load/Store if NVT==VT.
7678 // FIXME does the case above also need this?
7679 EVT NVT = TLI.getTypeToTransformTo(C, VT);
7680 assert(NVT.bitsGE(VT));
7681
7682 bool isDereferenceable =
7683 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7684 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7685 if (isDereferenceable)
7687 if (isConstant)
7688 SrcMMOFlags |= MachineMemOperand::MOInvariant;
7689
7690 Value = DAG.getExtLoad(
7691 ISD::EXTLOAD, dl, NVT, Chain,
7692 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7693 SrcPtrInfo.getWithOffset(SrcOff), VT,
7694 commonAlignment(*SrcAlign, SrcOff), SrcMMOFlags, NewAAInfo);
7695 OutLoadChains.push_back(Value.getValue(1));
7696
7697 Store = DAG.getTruncStore(
7698 Chain, dl, Value,
7699 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7700 DstPtrInfo.getWithOffset(DstOff), VT, Alignment, MMOFlags, NewAAInfo);
7701 OutStoreChains.push_back(Store);
7702 }
7703 SrcOff += VTSize;
7704 DstOff += VTSize;
7705 Size -= VTSize;
7706 }
7707
7708 unsigned GluedLdStLimit = MaxLdStGlue == 0 ?
7710 unsigned NumLdStInMemcpy = OutStoreChains.size();
7711
7712 if (NumLdStInMemcpy) {
7713 // It may be that memcpy might be converted to memset if it's memcpy
7714 // of constants. In such a case, we won't have loads and stores, but
7715 // just stores. In the absence of loads, there is nothing to gang up.
7716 if ((GluedLdStLimit <= 1) || !EnableMemCpyDAGOpt) {
7717 // If target does not care, just leave as it.
7718 for (unsigned i = 0; i < NumLdStInMemcpy; ++i) {
7719 OutChains.push_back(OutLoadChains[i]);
7720 OutChains.push_back(OutStoreChains[i]);
7721 }
7722 } else {
7723 // Ld/St less than/equal limit set by target.
7724 if (NumLdStInMemcpy <= GluedLdStLimit) {
7725 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7726 NumLdStInMemcpy, OutLoadChains,
7727 OutStoreChains);
7728 } else {
7729 unsigned NumberLdChain = NumLdStInMemcpy / GluedLdStLimit;
7730 unsigned RemainingLdStInMemcpy = NumLdStInMemcpy % GluedLdStLimit;
7731 unsigned GlueIter = 0;
7732
7733 for (unsigned cnt = 0; cnt < NumberLdChain; ++cnt) {
7734 unsigned IndexFrom = NumLdStInMemcpy - GlueIter - GluedLdStLimit;
7735 unsigned IndexTo = NumLdStInMemcpy - GlueIter;
7736
7737 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, IndexFrom, IndexTo,
7738 OutLoadChains, OutStoreChains);
7739 GlueIter += GluedLdStLimit;
7740 }
7741
7742 // Residual ld/st.
7743 if (RemainingLdStInMemcpy) {
7744 chainLoadsAndStoresForMemcpy(DAG, dl, OutChains, 0,
7745 RemainingLdStInMemcpy, OutLoadChains,
7746 OutStoreChains);
7747 }
7748 }
7749 }
7750 }
7751 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7752}
7753
7755 SDValue Chain, SDValue Dst, SDValue Src,
7756 uint64_t Size, Align Alignment,
7757 bool isVol, bool AlwaysInline,
7758 MachinePointerInfo DstPtrInfo,
7759 MachinePointerInfo SrcPtrInfo,
7760 const AAMDNodes &AAInfo) {
7761 // Turn a memmove of undef to nop.
7762 // FIXME: We need to honor volatile even is Src is undef.
7763 if (Src.isUndef())
7764 return Chain;
7765
7766 // Expand memmove to a series of load and store ops if the size operand falls
7767 // below a certain threshold.
7768 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7769 const DataLayout &DL = DAG.getDataLayout();
7770 LLVMContext &C = *DAG.getContext();
7771 std::vector<EVT> MemOps;
7772 bool DstAlignCanChange = false;
7774 MachineFrameInfo &MFI = MF.getFrameInfo();
7775 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7776 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7777 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7778 DstAlignCanChange = true;
7779 MaybeAlign SrcAlign = DAG.InferPtrAlign(Src);
7780 if (!SrcAlign || Alignment > *SrcAlign)
7781 SrcAlign = Alignment;
7782 assert(SrcAlign && "SrcAlign must be set");
7783 unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize);
7784 if (!TLI.findOptimalMemOpLowering(
7785 MemOps, Limit,
7786 MemOp::Copy(Size, DstAlignCanChange, Alignment, *SrcAlign,
7787 /*IsVolatile*/ true),
7788 DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(),
7789 MF.getFunction().getAttributes()))
7790 return SDValue();
7791
7792 if (DstAlignCanChange) {
7793 Type *Ty = MemOps[0].getTypeForEVT(C);
7794 Align NewAlign = DL.getABITypeAlign(Ty);
7795
7796 // Don't promote to an alignment that would require dynamic stack
7797 // realignment which may conflict with optimizations such as tail call
7798 // optimization.
7800 if (!TRI->hasStackRealignment(MF))
7801 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7802 NewAlign = NewAlign.previous();
7803
7804 if (NewAlign > Alignment) {
7805 // Give the stack frame object a larger alignment if needed.
7806 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7807 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7808 Alignment = NewAlign;
7809 }
7810 }
7811
7812 // Prepare AAInfo for loads/stores after lowering this memmove.
7813 AAMDNodes NewAAInfo = AAInfo;
7814 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7815
7816 MachineMemOperand::Flags MMOFlags =
7818 uint64_t SrcOff = 0, DstOff = 0;
7819 SmallVector<SDValue, 8> LoadValues;
7820 SmallVector<SDValue, 8> LoadChains;
7821 SmallVector<SDValue, 8> OutChains;
7822 unsigned NumMemOps = MemOps.size();
7823 for (unsigned i = 0; i < NumMemOps; i++) {
7824 EVT VT = MemOps[i];
7825 unsigned VTSize = VT.getSizeInBits() / 8;
7826 SDValue Value;
7827
7828 bool isDereferenceable =
7829 SrcPtrInfo.getWithOffset(SrcOff).isDereferenceable(VTSize, C, DL);
7830 MachineMemOperand::Flags SrcMMOFlags = MMOFlags;
7831 if (isDereferenceable)
7833
7834 Value = DAG.getLoad(
7835 VT, dl, Chain,
7836 DAG.getMemBasePlusOffset(Src, TypeSize::getFixed(SrcOff), dl),
7837 SrcPtrInfo.getWithOffset(SrcOff), *SrcAlign, SrcMMOFlags, NewAAInfo);
7838 LoadValues.push_back(Value);
7839 LoadChains.push_back(Value.getValue(1));
7840 SrcOff += VTSize;
7841 }
7842 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, LoadChains);
7843 OutChains.clear();
7844 for (unsigned i = 0; i < NumMemOps; i++) {
7845 EVT VT = MemOps[i];
7846 unsigned VTSize = VT.getSizeInBits() / 8;
7847 SDValue Store;
7848
7849 Store = DAG.getStore(
7850 Chain, dl, LoadValues[i],
7851 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7852 DstPtrInfo.getWithOffset(DstOff), Alignment, MMOFlags, NewAAInfo);
7853 OutChains.push_back(Store);
7854 DstOff += VTSize;
7855 }
7856
7857 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7858}
7859
7860/// Lower the call to 'memset' intrinsic function into a series of store
7861/// operations.
7862///
7863/// \param DAG Selection DAG where lowered code is placed.
7864/// \param dl Link to corresponding IR location.
7865/// \param Chain Control flow dependency.
7866/// \param Dst Pointer to destination memory location.
7867/// \param Src Value of byte to write into the memory.
7868/// \param Size Number of bytes to write.
7869/// \param Alignment Alignment of the destination in bytes.
7870/// \param isVol True if destination is volatile.
7871/// \param AlwaysInline Makes sure no function call is generated.
7872/// \param DstPtrInfo IR information on the memory pointer.
7873/// \returns New head in the control flow, if lowering was successful, empty
7874/// SDValue otherwise.
7875///
7876/// The function tries to replace 'llvm.memset' intrinsic with several store
7877/// operations and value calculation code. This is usually profitable for small
7878/// memory size or when the semantic requires inlining.
7880 SDValue Chain, SDValue Dst, SDValue Src,
7881 uint64_t Size, Align Alignment, bool isVol,
7882 bool AlwaysInline, MachinePointerInfo DstPtrInfo,
7883 const AAMDNodes &AAInfo) {
7884 // Turn a memset of undef to nop.
7885 // FIXME: We need to honor volatile even is Src is undef.
7886 if (Src.isUndef())
7887 return Chain;
7888
7889 // Expand memset to a series of load/store ops if the size operand
7890 // falls below a certain threshold.
7891 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
7892 std::vector<EVT> MemOps;
7893 bool DstAlignCanChange = false;
7895 MachineFrameInfo &MFI = MF.getFrameInfo();
7896 bool OptSize = shouldLowerMemFuncForSize(MF, DAG);
7897 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
7898 if (FI && !MFI.isFixedObjectIndex(FI->getIndex()))
7899 DstAlignCanChange = true;
7900 bool IsZeroVal = isNullConstant(Src);
7901 unsigned Limit = AlwaysInline ? ~0 : TLI.getMaxStoresPerMemset(OptSize);
7902
7903 if (!TLI.findOptimalMemOpLowering(
7904 MemOps, Limit,
7905 MemOp::Set(Size, DstAlignCanChange, Alignment, IsZeroVal, isVol),
7906 DstPtrInfo.getAddrSpace(), ~0u, MF.getFunction().getAttributes()))
7907 return SDValue();
7908
7909 if (DstAlignCanChange) {
7910 Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
7911 const DataLayout &DL = DAG.getDataLayout();
7912 Align NewAlign = DL.getABITypeAlign(Ty);
7913
7914 // Don't promote to an alignment that would require dynamic stack
7915 // realignment which may conflict with optimizations such as tail call
7916 // optimization.
7918 if (!TRI->hasStackRealignment(MF))
7919 while (NewAlign > Alignment && DL.exceedsNaturalStackAlignment(NewAlign))
7920 NewAlign = NewAlign.previous();
7921
7922 if (NewAlign > Alignment) {
7923 // Give the stack frame object a larger alignment if needed.
7924 if (MFI.getObjectAlign(FI->getIndex()) < NewAlign)
7925 MFI.setObjectAlignment(FI->getIndex(), NewAlign);
7926 Alignment = NewAlign;
7927 }
7928 }
7929
7930 SmallVector<SDValue, 8> OutChains;
7931 uint64_t DstOff = 0;
7932 unsigned NumMemOps = MemOps.size();
7933
7934 // Find the largest store and generate the bit pattern for it.
7935 EVT LargestVT = MemOps[0];
7936 for (unsigned i = 1; i < NumMemOps; i++)
7937 if (MemOps[i].bitsGT(LargestVT))
7938 LargestVT = MemOps[i];
7939 SDValue MemSetValue = getMemsetValue(Src, LargestVT, DAG, dl);
7940
7941 // Prepare AAInfo for loads/stores after lowering this memset.
7942 AAMDNodes NewAAInfo = AAInfo;
7943 NewAAInfo.TBAA = NewAAInfo.TBAAStruct = nullptr;
7944
7945 for (unsigned i = 0; i < NumMemOps; i++) {
7946 EVT VT = MemOps[i];
7947 unsigned VTSize = VT.getSizeInBits() / 8;
7948 if (VTSize > Size) {
7949 // Issuing an unaligned load / store pair that overlaps with the previous
7950 // pair. Adjust the offset accordingly.
7951 assert(i == NumMemOps-1 && i != 0);
7952 DstOff -= VTSize - Size;
7953 }
7954
7955 // If this store is smaller than the largest store see whether we can get
7956 // the smaller value for free with a truncate or extract vector element and
7957 // then store.
7958 SDValue Value = MemSetValue;
7959 if (VT.bitsLT(LargestVT)) {
7960 unsigned Index;
7961 unsigned NElts = LargestVT.getSizeInBits() / VT.getSizeInBits();
7962 EVT SVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), NElts);
7963 if (!LargestVT.isVector() && !VT.isVector() &&
7964 TLI.isTruncateFree(LargestVT, VT))
7965 Value = DAG.getNode(ISD::TRUNCATE, dl, VT, MemSetValue);
7966 else if (LargestVT.isVector() && !VT.isVector() &&
7968 LargestVT.getTypeForEVT(*DAG.getContext()),
7969 VT.getSizeInBits(), Index) &&
7970 TLI.isTypeLegal(SVT) &&
7971 LargestVT.getSizeInBits() == SVT.getSizeInBits()) {
7972 // Target which can combine store(extractelement VectorTy, Idx) can get
7973 // the smaller value for free.
7974 SDValue TailValue = DAG.getNode(ISD::BITCAST, dl, SVT, MemSetValue);
7975 Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, TailValue,
7976 DAG.getVectorIdxConstant(Index, dl));
7977 } else
7978 Value = getMemsetValue(Src, VT, DAG, dl);
7979 }
7980 assert(Value.getValueType() == VT && "Value with wrong type.");
7981 SDValue Store = DAG.getStore(
7982 Chain, dl, Value,
7983 DAG.getMemBasePlusOffset(Dst, TypeSize::getFixed(DstOff), dl),
7984 DstPtrInfo.getWithOffset(DstOff), Alignment,
7986 NewAAInfo);
7987 OutChains.push_back(Store);
7988 DstOff += VT.getSizeInBits() / 8;
7989 Size -= VTSize;
7990 }
7991
7992 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
7993}
7994
7996 unsigned AS) {
7997 // Lowering memcpy / memset / memmove intrinsics to calls is only valid if all
7998 // pointer operands can be losslessly bitcasted to pointers of address space 0
7999 if (AS != 0 && !TLI->getTargetMachine().isNoopAddrSpaceCast(AS, 0)) {
8000 report_fatal_error("cannot lower memory intrinsic in address space " +
8001 Twine(AS));
8002 }
8003}
8004
8006 SDValue Src, SDValue Size, Align Alignment,
8007 bool isVol, bool AlwaysInline, bool isTailCall,
8008 MachinePointerInfo DstPtrInfo,
8009 MachinePointerInfo SrcPtrInfo,
8010 const AAMDNodes &AAInfo, AAResults *AA) {
8011 // Check to see if we should lower the memcpy to loads and stores first.
8012 // For cases within the target-specified limits, this is the best choice.
8013 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8014 if (ConstantSize) {
8015 // Memcpy with size zero? Just return the original chain.
8016 if (ConstantSize->isZero())
8017 return Chain;
8018
8020 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8021 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8022 if (Result.getNode())
8023 return Result;
8024 }
8025
8026 // Then check to see if we should lower the memcpy with target-specific
8027 // code. If the target chooses to do this, this is the next best.
8028 if (TSI) {
8029 SDValue Result = TSI->EmitTargetCodeForMemcpy(
8030 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline,
8031 DstPtrInfo, SrcPtrInfo);
8032 if (Result.getNode())
8033 return Result;
8034 }
8035
8036 // If we really need inline code and the target declined to provide it,
8037 // use a (potentially long) sequence of loads and stores.
8038 if (AlwaysInline) {
8039 assert(ConstantSize && "AlwaysInline requires a constant size!");
8041 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8042 isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA);
8043 }
8044
8047
8048 // FIXME: If the memcpy is volatile (isVol), lowering it to a plain libc
8049 // memcpy is not guaranteed to be safe. libc memcpys aren't required to
8050 // respect volatile, so they may do things like read or write memory
8051 // beyond the given memory regions. But fixing this isn't easy, and most
8052 // people don't care.
8053
8054 // Emit a library call.
8057 Entry.Ty = PointerType::getUnqual(*getContext());
8058 Entry.Node = Dst; Args.push_back(Entry);
8059 Entry.Node = Src; Args.push_back(Entry);
8060
8061 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8062 Entry.Node = Size; Args.push_back(Entry);
8063 // FIXME: pass in SDLoc
8065 CLI.setDebugLoc(dl)
8066 .setChain(Chain)
8067 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMCPY),
8068 Dst.getValueType().getTypeForEVT(*getContext()),
8069 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMCPY),
8070 TLI->getPointerTy(getDataLayout())),
8071 std::move(Args))
8073 .setTailCall(isTailCall);
8074
8075 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8076 return CallResult.second;
8077}
8078
8080 SDValue Dst, SDValue Src, SDValue Size,
8081 Type *SizeTy, unsigned ElemSz,
8082 bool isTailCall,
8083 MachinePointerInfo DstPtrInfo,
8084 MachinePointerInfo SrcPtrInfo) {
8085 // Emit a library call.
8088 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8089 Entry.Node = Dst;
8090 Args.push_back(Entry);
8091
8092 Entry.Node = Src;
8093 Args.push_back(Entry);
8094
8095 Entry.Ty = SizeTy;
8096 Entry.Node = Size;
8097 Args.push_back(Entry);
8098
8099 RTLIB::Libcall LibraryCall =
8101 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8102 report_fatal_error("Unsupported element size");
8103
8105 CLI.setDebugLoc(dl)
8106 .setChain(Chain)
8107 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8109 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8110 TLI->getPointerTy(getDataLayout())),
8111 std::move(Args))
8113 .setTailCall(isTailCall);
8114
8115 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8116 return CallResult.second;
8117}
8118
8120 SDValue Src, SDValue Size, Align Alignment,
8121 bool isVol, bool isTailCall,
8122 MachinePointerInfo DstPtrInfo,
8123 MachinePointerInfo SrcPtrInfo,
8124 const AAMDNodes &AAInfo, AAResults *AA) {
8125 // Check to see if we should lower the memmove to loads and stores first.
8126 // For cases within the target-specified limits, this is the best choice.
8127 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8128 if (ConstantSize) {
8129 // Memmove with size zero? Just return the original chain.
8130 if (ConstantSize->isZero())
8131 return Chain;
8132
8134 *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment,
8135 isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo);
8136 if (Result.getNode())
8137 return Result;
8138 }
8139
8140 // Then check to see if we should lower the memmove with target-specific
8141 // code. If the target chooses to do this, this is the next best.
8142 if (TSI) {
8143 SDValue Result =
8144 TSI->EmitTargetCodeForMemmove(*this, dl, Chain, Dst, Src, Size,
8145 Alignment, isVol, DstPtrInfo, SrcPtrInfo);
8146 if (Result.getNode())
8147 return Result;
8148 }
8149
8152
8153 // FIXME: If the memmove is volatile, lowering it to plain libc memmove may
8154 // not be safe. See memcpy above for more details.
8155
8156 // Emit a library call.
8159 Entry.Ty = PointerType::getUnqual(*getContext());
8160 Entry.Node = Dst; Args.push_back(Entry);
8161 Entry.Node = Src; Args.push_back(Entry);
8162
8163 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8164 Entry.Node = Size; Args.push_back(Entry);
8165 // FIXME: pass in SDLoc
8167 CLI.setDebugLoc(dl)
8168 .setChain(Chain)
8169 .setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMMOVE),
8170 Dst.getValueType().getTypeForEVT(*getContext()),
8171 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMMOVE),
8172 TLI->getPointerTy(getDataLayout())),
8173 std::move(Args))
8175 .setTailCall(isTailCall);
8176
8177 std::pair<SDValue,SDValue> CallResult = TLI->LowerCallTo(CLI);
8178 return CallResult.second;
8179}
8180
8182 SDValue Dst, SDValue Src, SDValue Size,
8183 Type *SizeTy, unsigned ElemSz,
8184 bool isTailCall,
8185 MachinePointerInfo DstPtrInfo,
8186 MachinePointerInfo SrcPtrInfo) {
8187 // Emit a library call.
8190 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8191 Entry.Node = Dst;
8192 Args.push_back(Entry);
8193
8194 Entry.Node = Src;
8195 Args.push_back(Entry);
8196
8197 Entry.Ty = SizeTy;
8198 Entry.Node = Size;
8199 Args.push_back(Entry);
8200
8201 RTLIB::Libcall LibraryCall =
8203 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8204 report_fatal_error("Unsupported element size");
8205
8207 CLI.setDebugLoc(dl)
8208 .setChain(Chain)
8209 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8211 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8212 TLI->getPointerTy(getDataLayout())),
8213 std::move(Args))
8215 .setTailCall(isTailCall);
8216
8217 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8218 return CallResult.second;
8219}
8220
8222 SDValue Src, SDValue Size, Align Alignment,
8223 bool isVol, bool AlwaysInline, bool isTailCall,
8224 MachinePointerInfo DstPtrInfo,
8225 const AAMDNodes &AAInfo) {
8226 // Check to see if we should lower the memset to stores first.
8227 // For cases within the target-specified limits, this is the best choice.
8228 ConstantSDNode *ConstantSize = dyn_cast<ConstantSDNode>(Size);
8229 if (ConstantSize) {
8230 // Memset with size zero? Just return the original chain.
8231 if (ConstantSize->isZero())
8232 return Chain;
8233
8234 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8235 ConstantSize->getZExtValue(), Alignment,
8236 isVol, false, DstPtrInfo, AAInfo);
8237
8238 if (Result.getNode())
8239 return Result;
8240 }
8241
8242 // Then check to see if we should lower the memset with target-specific
8243 // code. If the target chooses to do this, this is the next best.
8244 if (TSI) {
8245 SDValue Result = TSI->EmitTargetCodeForMemset(
8246 *this, dl, Chain, Dst, Src, Size, Alignment, isVol, AlwaysInline, DstPtrInfo);
8247 if (Result.getNode())
8248 return Result;
8249 }
8250
8251 // If we really need inline code and the target declined to provide it,
8252 // use a (potentially long) sequence of loads and stores.
8253 if (AlwaysInline) {
8254 assert(ConstantSize && "AlwaysInline requires a constant size!");
8255 SDValue Result = getMemsetStores(*this, dl, Chain, Dst, Src,
8256 ConstantSize->getZExtValue(), Alignment,
8257 isVol, true, DstPtrInfo, AAInfo);
8258 assert(Result &&
8259 "getMemsetStores must return a valid sequence when AlwaysInline");
8260 return Result;
8261 }
8262
8264
8265 // Emit a library call.
8266 auto &Ctx = *getContext();
8267 const auto& DL = getDataLayout();
8268
8270 // FIXME: pass in SDLoc
8271 CLI.setDebugLoc(dl).setChain(Chain);
8272
8273 const char *BzeroName = getTargetLoweringInfo().getLibcallName(RTLIB::BZERO);
8274
8275 // Helper function to create an Entry from Node and Type.
8276 const auto CreateEntry = [](SDValue Node, Type *Ty) {
8278 Entry.Node = Node;
8279 Entry.Ty = Ty;
8280 return Entry;
8281 };
8282
8283 // If zeroing out and bzero is present, use it.
8284 if (isNullConstant(Src) && BzeroName) {
8286 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8287 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8288 CLI.setLibCallee(
8289 TLI->getLibcallCallingConv(RTLIB::BZERO), Type::getVoidTy(Ctx),
8290 getExternalSymbol(BzeroName, TLI->getPointerTy(DL)), std::move(Args));
8291 } else {
8293 Args.push_back(CreateEntry(Dst, PointerType::getUnqual(Ctx)));
8294 Args.push_back(CreateEntry(Src, Src.getValueType().getTypeForEVT(Ctx)));
8295 Args.push_back(CreateEntry(Size, DL.getIntPtrType(Ctx)));
8296 CLI.setLibCallee(TLI->getLibcallCallingConv(RTLIB::MEMSET),
8297 Dst.getValueType().getTypeForEVT(Ctx),
8298 getExternalSymbol(TLI->getLibcallName(RTLIB::MEMSET),
8299 TLI->getPointerTy(DL)),
8300 std::move(Args));
8301 }
8302
8303 CLI.setDiscardResult().setTailCall(isTailCall);
8304
8305 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8306 return CallResult.second;
8307}
8308
8311 Type *SizeTy, unsigned ElemSz,
8312 bool isTailCall,
8313 MachinePointerInfo DstPtrInfo) {
8314 // Emit a library call.
8317 Entry.Ty = getDataLayout().getIntPtrType(*getContext());
8318 Entry.Node = Dst;
8319 Args.push_back(Entry);
8320
8321 Entry.Ty = Type::getInt8Ty(*getContext());
8322 Entry.Node = Value;
8323 Args.push_back(Entry);
8324
8325 Entry.Ty = SizeTy;
8326 Entry.Node = Size;
8327 Args.push_back(Entry);
8328
8329 RTLIB::Libcall LibraryCall =
8331 if (LibraryCall == RTLIB::UNKNOWN_LIBCALL)
8332 report_fatal_error("Unsupported element size");
8333
8335 CLI.setDebugLoc(dl)
8336 .setChain(Chain)
8337 .setLibCallee(TLI->getLibcallCallingConv(LibraryCall),
8339 getExternalSymbol(TLI->getLibcallName(LibraryCall),
8340 TLI->getPointerTy(getDataLayout())),
8341 std::move(Args))
8343 .setTailCall(isTailCall);
8344
8345 std::pair<SDValue, SDValue> CallResult = TLI->LowerCallTo(CLI);
8346 return CallResult.second;
8347}
8348
8349SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8350 SDVTList VTList, ArrayRef<SDValue> Ops,
8351 MachineMemOperand *MMO) {
8353 ID.AddInteger(MemVT.getRawBits());
8354 AddNodeIDNode(ID, Opcode, VTList, Ops);
8355 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8356 ID.AddInteger(MMO->getFlags());
8357 void* IP = nullptr;
8358 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8359 cast<AtomicSDNode>(E)->refineAlignment(MMO);
8360 return SDValue(E, 0);
8361 }
8362
8363 auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8364 VTList, MemVT, MMO);
8365 createOperands(N, Ops);
8366
8367 CSEMap.InsertNode(N, IP);
8368 InsertNode(N);
8369 return SDValue(N, 0);
8370}
8371
8373 EVT MemVT, SDVTList VTs, SDValue Chain,
8374 SDValue Ptr, SDValue Cmp, SDValue Swp,
8375 MachineMemOperand *MMO) {
8376 assert(Opcode == ISD::ATOMIC_CMP_SWAP ||
8378 assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
8379
8380 SDValue Ops[] = {Chain, Ptr, Cmp, Swp};
8381 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8382}
8383
8384SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8385 SDValue Chain, SDValue Ptr, SDValue Val,
8386 MachineMemOperand *MMO) {
8387 assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
8388 Opcode == ISD::ATOMIC_LOAD_SUB ||
8389 Opcode == ISD::ATOMIC_LOAD_AND ||
8390 Opcode == ISD::ATOMIC_LOAD_CLR ||
8391 Opcode == ISD::ATOMIC_LOAD_OR ||
8392 Opcode == ISD::ATOMIC_LOAD_XOR ||
8393 Opcode == ISD::ATOMIC_LOAD_NAND ||
8394 Opcode == ISD::ATOMIC_LOAD_MIN ||
8395 Opcode == ISD::ATOMIC_LOAD_MAX ||
8396 Opcode == ISD::ATOMIC_LOAD_UMIN ||
8397 Opcode == ISD::ATOMIC_LOAD_UMAX ||
8398 Opcode == ISD::ATOMIC_LOAD_FADD ||
8399 Opcode == ISD::ATOMIC_LOAD_FSUB ||
8400 Opcode == ISD::ATOMIC_LOAD_FMAX ||
8401 Opcode == ISD::ATOMIC_LOAD_FMIN ||
8402 Opcode == ISD::ATOMIC_LOAD_UINC_WRAP ||
8403 Opcode == ISD::ATOMIC_LOAD_UDEC_WRAP ||
8404 Opcode == ISD::ATOMIC_SWAP ||
8405 Opcode == ISD::ATOMIC_STORE) &&
8406 "Invalid Atomic Op");
8407
8408 EVT VT = Val.getValueType();
8409
8410 SDVTList VTs = Opcode == ISD::ATOMIC_STORE ? getVTList(MVT::Other) :
8411 getVTList(VT, MVT::Other);
8412 SDValue Ops[] = {Chain, Ptr, Val};
8413 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8414}
8415
8416SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
8417 EVT VT, SDValue Chain, SDValue Ptr,
8418 MachineMemOperand *MMO) {
8419 assert(Opcode == ISD::ATOMIC_LOAD && "Invalid Atomic Op");
8420
8421 SDVTList VTs = getVTList(VT, MVT::Other);
8422 SDValue Ops[] = {Chain, Ptr};
8423 return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO);
8424}
8425
8426/// getMergeValues - Create a MERGE_VALUES node from the given operands.
8428 if (Ops.size() == 1)
8429 return Ops[0];
8430
8432 VTs.reserve(Ops.size());
8433 for (const SDValue &Op : Ops)
8434 VTs.push_back(Op.getValueType());
8435 return getNode(ISD::MERGE_VALUES, dl, getVTList(VTs), Ops);
8436}
8437
8439 unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef<SDValue> Ops,
8440 EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment,
8442 const AAMDNodes &AAInfo) {
8443 if (Size.hasValue() && !Size.getValue())
8445
8447 MachineMemOperand *MMO =
8448 MF.getMachineMemOperand(PtrInfo, Flags, Size, Alignment, AAInfo);
8449
8450 return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO);
8451}
8452
8454 SDVTList VTList,
8455 ArrayRef<SDValue> Ops, EVT MemVT,
8456 MachineMemOperand *MMO) {
8457 assert((Opcode == ISD::INTRINSIC_VOID ||
8458 Opcode == ISD::INTRINSIC_W_CHAIN ||
8459 Opcode == ISD::PREFETCH ||
8460 (Opcode <= (unsigned)std::numeric_limits<int>::max() &&
8461 (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
8462 "Opcode is not a memory-accessing opcode!");
8463
8464 // Memoize the node unless it returns a glue result.
8466 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
8468 AddNodeIDNode(ID, Opcode, VTList, Ops);
8469 ID.AddInteger(getSyntheticNodeSubclassData<MemIntrinsicSDNode>(
8470 Opcode, dl.getIROrder(), VTList, MemVT, MMO));
8471 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8472 ID.AddInteger(MMO->getFlags());
8473 ID.AddInteger(MemVT.getRawBits());
8474 void *IP = nullptr;
8475 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8476 cast<MemIntrinsicSDNode>(E)->refineAlignment(MMO);
8477 return SDValue(E, 0);
8478 }
8479
8480 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8481 VTList, MemVT, MMO);
8482 createOperands(N, Ops);
8483
8484 CSEMap.InsertNode(N, IP);
8485 } else {
8486 N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
8487 VTList, MemVT, MMO);
8488 createOperands(N, Ops);
8489 }
8490 InsertNode(N);
8491 SDValue V(N, 0);
8492 NewSDValueDbgMsg(V, "Creating new node: ", this);
8493 return V;
8494}
8495
8497 SDValue Chain, int FrameIndex,
8498 int64_t Size, int64_t Offset) {
8499 const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END;
8500 const auto VTs = getVTList(MVT::Other);
8501 SDValue Ops[2] = {
8502 Chain,
8503 getFrameIndex(FrameIndex,
8504 getTargetLoweringInfo().getFrameIndexTy(getDataLayout()),
8505 true)};
8506
8508 AddNodeIDNode(ID, Opcode, VTs, Ops);
8509 ID.AddInteger(FrameIndex);
8510 ID.AddInteger(Size);
8511 ID.AddInteger(Offset);
8512 void *IP = nullptr;
8513 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8514 return SDValue(E, 0);
8515
8516 LifetimeSDNode *N = newSDNode<LifetimeSDNode>(
8517 Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset);
8518 createOperands(N, Ops);
8519 CSEMap.InsertNode(N, IP);
8520 InsertNode(N);
8521 SDValue V(N, 0);
8522 NewSDValueDbgMsg(V, "Creating new node: ", this);
8523 return V;
8524}
8525
8527 uint64_t Guid, uint64_t Index,
8528 uint32_t Attr) {
8529 const unsigned Opcode = ISD::PSEUDO_PROBE;
8530 const auto VTs = getVTList(MVT::Other);
8531 SDValue Ops[] = {Chain};
8533 AddNodeIDNode(ID, Opcode, VTs, Ops);
8534 ID.AddInteger(Guid);
8535 ID.AddInteger(Index);
8536 void *IP = nullptr;
8537 if (SDNode *E = FindNodeOrInsertPos(ID, Dl, IP))
8538 return SDValue(E, 0);
8539
8540 auto *N = newSDNode<PseudoProbeSDNode>(
8541 Opcode, Dl.getIROrder(), Dl.getDebugLoc(), VTs, Guid, Index, Attr);
8542 createOperands(N, Ops);
8543 CSEMap.InsertNode(N, IP);
8544 InsertNode(N);
8545 SDValue V(N, 0);
8546 NewSDValueDbgMsg(V, "Creating new node: ", this);
8547 return V;
8548}
8549
8550/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8551/// MachinePointerInfo record from it. This is particularly useful because the
8552/// code generator has many cases where it doesn't bother passing in a
8553/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8555 SelectionDAG &DAG, SDValue Ptr,
8556 int64_t Offset = 0) {
8557 // If this is FI+Offset, we can model it.
8558 if (const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr))
8560 FI->getIndex(), Offset);
8561
8562 // If this is (FI+Offset1)+Offset2, we can model it.
8563 if (Ptr.getOpcode() != ISD::ADD ||
8564 !isa<ConstantSDNode>(Ptr.getOperand(1)) ||
8565 !isa<FrameIndexSDNode>(Ptr.getOperand(0)))
8566 return Info;
8567
8568 int FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
8570 DAG.getMachineFunction(), FI,
8571 Offset + cast<ConstantSDNode>(Ptr.getOperand(1))->getSExtValue());
8572}
8573
8574/// InferPointerInfo - If the specified ptr/offset is a frame index, infer a
8575/// MachinePointerInfo record from it. This is particularly useful because the
8576/// code generator has many cases where it doesn't bother passing in a
8577/// MachinePointerInfo to getLoad or getStore when it has "FI+Cst".
8579 SelectionDAG &DAG, SDValue Ptr,
8580 SDValue OffsetOp) {
8581 // If the 'Offset' value isn't a constant, we can't handle this.
8582 if (ConstantSDNode *OffsetNode = dyn_cast<ConstantSDNode>(OffsetOp))
8583 return InferPointerInfo(Info, DAG, Ptr, OffsetNode->getSExtValue());
8584 if (OffsetOp.isUndef())
8585 return InferPointerInfo(Info, DAG, Ptr);
8586 return Info;
8587}
8588
8590 EVT VT, const SDLoc &dl, SDValue Chain,
8592 MachinePointerInfo PtrInfo, EVT MemVT,
8593 Align Alignment,
8594 MachineMemOperand::Flags MMOFlags,
8595 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8596 assert(Chain.getValueType() == MVT::Other &&
8597 "Invalid chain type");
8598
8599 MMOFlags |= MachineMemOperand::MOLoad;
8600 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8601 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8602 // clients.
8603 if (PtrInfo.V.isNull())
8604 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8605
8608 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8609 Alignment, AAInfo, Ranges);
8610 return getLoad(AM, ExtType, VT, dl, Chain, Ptr, Offset, MemVT, MMO);
8611}
8612
8614 EVT VT, const SDLoc &dl, SDValue Chain,
8615 SDValue Ptr, SDValue Offset, EVT MemVT,
8616 MachineMemOperand *MMO) {
8617 if (VT == MemVT) {
8618 ExtType = ISD::NON_EXTLOAD;
8619 } else if (ExtType == ISD::NON_EXTLOAD) {
8620 assert(VT == MemVT && "Non-extending load from different memory type!");
8621 } else {
8622 // Extending load.
8623 assert(MemVT.getScalarType().bitsLT(VT.getScalarType()) &&
8624 "Should only be an extending load, not truncating!");
8625 assert(VT.isInteger() == MemVT.isInteger() &&
8626 "Cannot convert from FP to Int or Int -> FP!");
8627 assert(VT.isVector() == MemVT.isVector() &&
8628 "Cannot use an ext load to convert to or from a vector!");
8629 assert((!VT.isVector() ||
8631 "Cannot use an ext load to change the number of vector elements!");
8632 }
8633
8634 bool Indexed = AM != ISD::UNINDEXED;
8635 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8636
8637 SDVTList VTs = Indexed ?
8638 getVTList(VT, Ptr.getValueType(), MVT::Other) : getVTList(VT, MVT::Other);
8639 SDValue Ops[] = { Chain, Ptr, Offset };
8641 AddNodeIDNode(ID, ISD::LOAD, VTs, Ops);
8642 ID.AddInteger(MemVT.getRawBits());
8643 ID.AddInteger(getSyntheticNodeSubclassData<LoadSDNode>(
8644 dl.getIROrder(), VTs, AM, ExtType, MemVT, MMO));
8645 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8646 ID.AddInteger(MMO->getFlags());
8647 void *IP = nullptr;
8648 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8649 cast<LoadSDNode>(E)->refineAlignment(MMO);
8650 return SDValue(E, 0);
8651 }
8652 auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8653 ExtType, MemVT, MMO);
8654 createOperands(N, Ops);
8655
8656 CSEMap.InsertNode(N, IP);
8657 InsertNode(N);
8658 SDValue V(N, 0);
8659 NewSDValueDbgMsg(V, "Creating new node: ", this);
8660 return V;
8661}
8662
8665 MaybeAlign Alignment,
8666 MachineMemOperand::Flags MMOFlags,
8667 const AAMDNodes &AAInfo, const MDNode *Ranges) {
8668 SDValue Undef = getUNDEF(Ptr.getValueType());
8669 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8670 PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges);
8671}
8672
8675 SDValue Undef = getUNDEF(Ptr.getValueType());
8676 return getLoad(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8677 VT, MMO);
8678}
8679
8681 EVT VT, SDValue Chain, SDValue Ptr,
8682 MachinePointerInfo PtrInfo, EVT MemVT,
8683 MaybeAlign Alignment,
8684 MachineMemOperand::Flags MMOFlags,
8685 const AAMDNodes &AAInfo) {
8686 SDValue Undef = getUNDEF(Ptr.getValueType());
8687 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, PtrInfo,
8688 MemVT, Alignment, MMOFlags, AAInfo);
8689}
8690
8692 EVT VT, SDValue Chain, SDValue Ptr, EVT MemVT,
8693 MachineMemOperand *MMO) {
8694 SDValue Undef = getUNDEF(Ptr.getValueType());
8695 return getLoad(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef,
8696 MemVT, MMO);
8697}
8698
8702 LoadSDNode *LD = cast<LoadSDNode>(OrigLoad);
8703 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8704 // Don't propagate the invariant or dereferenceable flags.
8705 auto MMOFlags =
8706 LD->getMemOperand()->getFlags() &
8708 return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8709 LD->getChain(), Base, Offset, LD->getPointerInfo(),
8710 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo());
8711}
8712
8715 Align Alignment,
8716 MachineMemOperand::Flags MMOFlags,
8717 const AAMDNodes &AAInfo) {
8718 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8719
8720 MMOFlags |= MachineMemOperand::MOStore;
8721 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8722
8723 if (PtrInfo.V.isNull())
8724 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8725
8728 MachineMemOperand *MMO =
8729 MF.getMachineMemOperand(PtrInfo, MMOFlags, Size, Alignment, AAInfo);
8730 return getStore(Chain, dl, Val, Ptr, MMO);
8731}
8732
8735 assert(Chain.getValueType() == MVT::Other &&
8736 "Invalid chain type");
8737 EVT VT = Val.getValueType();
8738 SDVTList VTs = getVTList(MVT::Other);
8739 SDValue Undef = getUNDEF(Ptr.getValueType());
8740 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8742 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8743 ID.AddInteger(VT.getRawBits());
8744 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8745 dl.getIROrder(), VTs, ISD::UNINDEXED, false, VT, MMO));
8746 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8747 ID.AddInteger(MMO->getFlags());
8748 void *IP = nullptr;
8749 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8750 cast<StoreSDNode>(E)->refineAlignment(MMO);
8751 return SDValue(E, 0);
8752 }
8753 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8754 ISD::UNINDEXED, false, VT, MMO);
8755 createOperands(N, Ops);
8756
8757 CSEMap.InsertNode(N, IP);
8758 InsertNode(N);
8759 SDValue V(N, 0);
8760 NewSDValueDbgMsg(V, "Creating new node: ", this);
8761 return V;
8762}
8763
8766 EVT SVT, Align Alignment,
8767 MachineMemOperand::Flags MMOFlags,
8768 const AAMDNodes &AAInfo) {
8769 assert(Chain.getValueType() == MVT::Other &&
8770 "Invalid chain type");
8771
8772 MMOFlags |= MachineMemOperand::MOStore;
8773 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
8774
8775 if (PtrInfo.V.isNull())
8776 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
8777
8780 PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
8781 AAInfo);
8782 return getTruncStore(Chain, dl, Val, Ptr, SVT, MMO);
8783}
8784
8786 SDValue Ptr, EVT SVT,
8787 MachineMemOperand *MMO) {
8788 EVT VT = Val.getValueType();
8789
8790 assert(Chain.getValueType() == MVT::Other &&
8791 "Invalid chain type");
8792 if (VT == SVT)
8793 return getStore(Chain, dl, Val, Ptr, MMO);
8794
8796 "Should only be a truncating store, not extending!");
8797 assert(VT.isInteger() == SVT.isInteger() &&
8798 "Can't do FP-INT conversion!");
8799 assert(VT.isVector() == SVT.isVector() &&
8800 "Cannot use trunc store to convert to or from a vector!");
8801 assert((!VT.isVector() ||
8803 "Cannot use trunc store to change the number of vector elements!");
8804
8805 SDVTList VTs = getVTList(MVT::Other);
8806 SDValue Undef = getUNDEF(Ptr.getValueType());
8807 SDValue Ops[] = { Chain, Val, Ptr, Undef };
8809 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8810 ID.AddInteger(SVT.getRawBits());
8811 ID.AddInteger(getSyntheticNodeSubclassData<StoreSDNode>(
8812 dl.getIROrder(), VTs, ISD::UNINDEXED, true, SVT, MMO));
8813 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8814 ID.AddInteger(MMO->getFlags());
8815 void *IP = nullptr;
8816 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8817 cast<StoreSDNode>(E)->refineAlignment(MMO);
8818 return SDValue(E, 0);
8819 }
8820 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
8821 ISD::UNINDEXED, true, SVT, MMO);
8822 createOperands(N, Ops);
8823
8824 CSEMap.InsertNode(N, IP);
8825 InsertNode(N);
8826 SDValue V(N, 0);
8827 NewSDValueDbgMsg(V, "Creating new node: ", this);
8828 return V;
8829}
8830
8834 StoreSDNode *ST = cast<StoreSDNode>(OrigStore);
8835 assert(ST->getOffset().isUndef() && "Store is already a indexed store!");
8836 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
8837 SDValue Ops[] = { ST->getChain(), ST->getValue(), Base, Offset };
8839 AddNodeIDNode(ID, ISD::STORE, VTs, Ops);
8840 ID.AddInteger(ST->getMemoryVT().getRawBits());
8841 ID.AddInteger(ST->getRawSubclassData());
8842 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
8843 ID.AddInteger(ST->getMemOperand()->getFlags());
8844 void *IP = nullptr;
8845 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
8846 return SDValue(E, 0);
8847
8848 auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8849 ST->isTruncatingStore(), ST->getMemoryVT(),
8850 ST->getMemOperand());
8851 createOperands(N, Ops);
8852
8853 CSEMap.InsertNode(N, IP);
8854 InsertNode(N);
8855 SDValue V(N, 0);
8856 NewSDValueDbgMsg(V, "Creating new node: ", this);
8857 return V;
8858}
8859
8861 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl,
8862 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL,
8863 MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment,
8864 MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo,
8865 const MDNode *Ranges, bool IsExpanding) {
8866 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8867
8868 MMOFlags |= MachineMemOperand::MOLoad;
8869 assert((MMOFlags & MachineMemOperand::MOStore) == 0);
8870 // If we don't have a PtrInfo, infer the trivial frame index case to simplify
8871 // clients.
8872 if (PtrInfo.V.isNull())
8873 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr, Offset);
8874
8877 MachineMemOperand *MMO = MF.getMachineMemOperand(PtrInfo, MMOFlags, Size,
8878 Alignment, AAInfo, Ranges);
8879 return getLoadVP(AM, ExtType, VT, dl, Chain, Ptr, Offset, Mask, EVL, MemVT,
8880 MMO, IsExpanding);
8881}
8882
8884 ISD::LoadExtType ExtType, EVT VT,
8885 const SDLoc &dl, SDValue Chain, SDValue Ptr,
8886 SDValue Offset, SDValue Mask, SDValue EVL,
8887 EVT MemVT, MachineMemOperand *MMO,
8888 bool IsExpanding) {
8889 bool Indexed = AM != ISD::UNINDEXED;
8890 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
8891
8892 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
8893 : getVTList(VT, MVT::Other);
8894 SDValue Ops[] = {Chain, Ptr, Offset, Mask, EVL};
8896 AddNodeIDNode(ID, ISD::VP_LOAD, VTs, Ops);
8897 ID.AddInteger(MemVT.getRawBits());
8898 ID.AddInteger(getSyntheticNodeSubclassData<VPLoadSDNode>(
8899 dl.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
8900 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8901 ID.AddInteger(MMO->getFlags());
8902 void *IP = nullptr;
8903 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8904 cast<VPLoadSDNode>(E)->refineAlignment(MMO);
8905 return SDValue(E, 0);
8906 }
8907 auto *N = newSDNode<VPLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
8908 ExtType, IsExpanding, MemVT, MMO);
8909 createOperands(N, Ops);
8910
8911 CSEMap.InsertNode(N, IP);
8912 InsertNode(N);
8913 SDValue V(N, 0);
8914 NewSDValueDbgMsg(V, "Creating new node: ", this);
8915 return V;
8916}
8917
8919 SDValue Ptr, SDValue Mask, SDValue EVL,
8920 MachinePointerInfo PtrInfo,
8921 MaybeAlign Alignment,
8922 MachineMemOperand::Flags MMOFlags,
8923 const AAMDNodes &AAInfo, const MDNode *Ranges,
8924 bool IsExpanding) {
8925 SDValue Undef = getUNDEF(Ptr.getValueType());
8926 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8927 Mask, EVL, PtrInfo, VT, Alignment, MMOFlags, AAInfo, Ranges,
8928 IsExpanding);
8929}
8930
8932 SDValue Ptr, SDValue Mask, SDValue EVL,
8933 MachineMemOperand *MMO, bool IsExpanding) {
8934 SDValue Undef = getUNDEF(Ptr.getValueType());
8935 return getLoadVP(ISD::UNINDEXED, ISD::NON_EXTLOAD, VT, dl, Chain, Ptr, Undef,
8936 Mask, EVL, VT, MMO, IsExpanding);
8937}
8938
8940 EVT VT, SDValue Chain, SDValue Ptr,
8941 SDValue Mask, SDValue EVL,
8942 MachinePointerInfo PtrInfo, EVT MemVT,
8943 MaybeAlign Alignment,
8944 MachineMemOperand::Flags MMOFlags,
8945 const AAMDNodes &AAInfo, bool IsExpanding) {
8946 SDValue Undef = getUNDEF(Ptr.getValueType());
8947 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8948 EVL, PtrInfo, MemVT, Alignment, MMOFlags, AAInfo, nullptr,
8949 IsExpanding);
8950}
8951
8953 EVT VT, SDValue Chain, SDValue Ptr,
8954 SDValue Mask, SDValue EVL, EVT MemVT,
8955 MachineMemOperand *MMO, bool IsExpanding) {
8956 SDValue Undef = getUNDEF(Ptr.getValueType());
8957 return getLoadVP(ISD::UNINDEXED, ExtType, VT, dl, Chain, Ptr, Undef, Mask,
8958 EVL, MemVT, MMO, IsExpanding);
8959}
8960
8964 auto *LD = cast<VPLoadSDNode>(OrigLoad);
8965 assert(LD->getOffset().isUndef() && "Load is already a indexed load!");
8966 // Don't propagate the invariant or dereferenceable flags.
8967 auto MMOFlags =
8968 LD->getMemOperand()->getFlags() &
8970 return getLoadVP(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
8971 LD->getChain(), Base, Offset, LD->getMask(),
8972 LD->getVectorLength(), LD->getPointerInfo(),
8973 LD->getMemoryVT(), LD->getAlign(), MMOFlags, LD->getAAInfo(),
8974 nullptr, LD->isExpandingLoad());
8975}
8976
8979 SDValue EVL, EVT MemVT, MachineMemOperand *MMO,
8980 ISD::MemIndexedMode AM, bool IsTruncating,
8981 bool IsCompressing) {
8982 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
8983 bool Indexed = AM != ISD::UNINDEXED;
8984 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
8985 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
8986 : getVTList(MVT::Other);
8987 SDValue Ops[] = {Chain, Val, Ptr, Offset, Mask, EVL};
8989 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
8990 ID.AddInteger(MemVT.getRawBits());
8991 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
8992 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
8993 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
8994 ID.AddInteger(MMO->getFlags());
8995 void *IP = nullptr;
8996 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
8997 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
8998 return SDValue(E, 0);
8999 }
9000 auto *N = newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9001 IsTruncating, IsCompressing, MemVT, MMO);
9002 createOperands(N, Ops);
9003
9004 CSEMap.InsertNode(N, IP);
9005 InsertNode(N);
9006 SDValue V(N, 0);
9007 NewSDValueDbgMsg(V, "Creating new node: ", this);
9008 return V;
9009}
9010
9012 SDValue Val, SDValue Ptr, SDValue Mask,
9013 SDValue EVL, MachinePointerInfo PtrInfo,
9014 EVT SVT, Align Alignment,
9015 MachineMemOperand::Flags MMOFlags,
9016 const AAMDNodes &AAInfo,
9017 bool IsCompressing) {
9018 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9019
9020 MMOFlags |= MachineMemOperand::MOStore;
9021 assert((MMOFlags & MachineMemOperand::MOLoad) == 0);
9022
9023 if (PtrInfo.V.isNull())
9024 PtrInfo = InferPointerInfo(PtrInfo, *this, Ptr);
9025
9028 PtrInfo, MMOFlags, LocationSize::precise(SVT.getStoreSize()), Alignment,
9029 AAInfo);
9030 return getTruncStoreVP(Chain, dl, Val, Ptr, Mask, EVL, SVT, MMO,
9031 IsCompressing);
9032}
9033
9035 SDValue Val, SDValue Ptr, SDValue Mask,
9036 SDValue EVL, EVT SVT,
9037 MachineMemOperand *MMO,
9038 bool IsCompressing) {
9039 EVT VT = Val.getValueType();
9040
9041 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9042 if (VT == SVT)
9043 return getStoreVP(Chain, dl, Val, Ptr, getUNDEF(Ptr.getValueType()), Mask,
9044 EVL, VT, MMO, ISD::UNINDEXED,
9045 /*IsTruncating*/ false, IsCompressing);
9046
9048 "Should only be a truncating store, not extending!");
9049 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9050 assert(VT.isVector() == SVT.isVector() &&
9051 "Cannot use trunc store to convert to or from a vector!");
9052 assert((!VT.isVector() ||
9054 "Cannot use trunc store to change the number of vector elements!");
9055
9056 SDVTList VTs = getVTList(MVT::Other);
9057 SDValue Undef = getUNDEF(Ptr.getValueType());
9058 SDValue Ops[] = {Chain, Val, Ptr, Undef, Mask, EVL};
9060 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9061 ID.AddInteger(SVT.getRawBits());
9062 ID.AddInteger(getSyntheticNodeSubclassData<VPStoreSDNode>(
9063 dl.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9064 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9065 ID.AddInteger(MMO->getFlags());
9066 void *IP = nullptr;
9067 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9068 cast<VPStoreSDNode>(E)->refineAlignment(MMO);
9069 return SDValue(E, 0);
9070 }
9071 auto *N =
9072 newSDNode<VPStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9073 ISD::UNINDEXED, true, IsCompressing, SVT, MMO);
9074 createOperands(N, Ops);
9075
9076 CSEMap.InsertNode(N, IP);
9077 InsertNode(N);
9078 SDValue V(N, 0);
9079 NewSDValueDbgMsg(V, "Creating new node: ", this);
9080 return V;
9081}
9082
9086 auto *ST = cast<VPStoreSDNode>(OrigStore);
9087 assert(ST->getOffset().isUndef() && "Store is already an indexed store!");
9088 SDVTList VTs = getVTList(Base.getValueType(), MVT::Other);
9089 SDValue Ops[] = {ST->getChain(), ST->getValue(), Base,
9090 Offset, ST->getMask(), ST->getVectorLength()};
9092 AddNodeIDNode(ID, ISD::VP_STORE, VTs, Ops);
9093 ID.AddInteger(ST->getMemoryVT().getRawBits());
9094 ID.AddInteger(ST->getRawSubclassData());
9095 ID.AddInteger(ST->getPointerInfo().getAddrSpace());
9096 ID.AddInteger(ST->getMemOperand()->getFlags());
9097 void *IP = nullptr;
9098 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9099 return SDValue(E, 0);
9100
9101 auto *N = newSDNode<VPStoreSDNode>(
9102 dl.getIROrder(), dl.getDebugLoc(), VTs, AM, ST->isTruncatingStore(),
9103 ST->isCompressingStore(), ST->getMemoryVT(), ST->getMemOperand());
9104 createOperands(N, Ops);
9105
9106 CSEMap.InsertNode(N, IP);
9107 InsertNode(N);
9108 SDValue V(N, 0);
9109 NewSDValueDbgMsg(V, "Creating new node: ", this);
9110 return V;
9111}
9112
9114 ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL,
9115 SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask,
9116 SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding) {
9117 bool Indexed = AM != ISD::UNINDEXED;
9118 assert((Indexed || Offset.isUndef()) && "Unindexed load with an offset!");
9119
9120 SDValue Ops[] = {Chain, Ptr, Offset, Stride, Mask, EVL};
9121 SDVTList VTs = Indexed ? getVTList(VT, Ptr.getValueType(), MVT::Other)
9122 : getVTList(VT, MVT::Other);
9124 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_LOAD, VTs, Ops);
9125 ID.AddInteger(VT.getRawBits());
9126 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedLoadSDNode>(
9127 DL.getIROrder(), VTs, AM, ExtType, IsExpanding, MemVT, MMO));
9128 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9129
9130 void *IP = nullptr;
9131 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9132 cast<VPStridedLoadSDNode>(E)->refineAlignment(MMO);
9133 return SDValue(E, 0);
9134 }
9135
9136 auto *N =
9137 newSDNode<VPStridedLoadSDNode>(DL.getIROrder(), DL.getDebugLoc(), VTs, AM,
9138 ExtType, IsExpanding, MemVT, MMO);
9139 createOperands(N, Ops);
9140 CSEMap.InsertNode(N, IP);
9141 InsertNode(N);
9142 SDValue V(N, 0);
9143 NewSDValueDbgMsg(V, "Creating new node: ", this);
9144 return V;
9145}
9146
9148 SDValue Ptr, SDValue Stride,
9149 SDValue Mask, SDValue EVL,
9150 MachineMemOperand *MMO,
9151 bool IsExpanding) {
9152 SDValue Undef = getUNDEF(Ptr.getValueType());
9154 Undef, Stride, Mask, EVL, VT, MMO, IsExpanding);
9155}
9156
9158 ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain,
9159 SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT,
9160 MachineMemOperand *MMO, bool IsExpanding) {
9161 SDValue Undef = getUNDEF(Ptr.getValueType());
9162 return getStridedLoadVP(ISD::UNINDEXED, ExtType, VT, DL, Chain, Ptr, Undef,
9163 Stride, Mask, EVL, MemVT, MMO, IsExpanding);
9164}
9165
9167 SDValue Val, SDValue Ptr,
9168 SDValue Offset, SDValue Stride,
9169 SDValue Mask, SDValue EVL, EVT MemVT,
9170 MachineMemOperand *MMO,
9172 bool IsTruncating, bool IsCompressing) {
9173 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9174 bool Indexed = AM != ISD::UNINDEXED;
9175 assert((Indexed || Offset.isUndef()) && "Unindexed vp_store with an offset!");
9176 SDVTList VTs = Indexed ? getVTList(Ptr.getValueType(), MVT::Other)
9177 : getVTList(MVT::Other);
9178 SDValue Ops[] = {Chain, Val, Ptr, Offset, Stride, Mask, EVL};
9180 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9181 ID.AddInteger(MemVT.getRawBits());
9182 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9183 DL.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9184 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9185 void *IP = nullptr;
9186 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9187 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9188 return SDValue(E, 0);
9189 }
9190 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9191 VTs, AM, IsTruncating,
9192 IsCompressing, MemVT, MMO);
9193 createOperands(N, Ops);
9194
9195 CSEMap.InsertNode(N, IP);
9196 InsertNode(N);
9197 SDValue V(N, 0);
9198 NewSDValueDbgMsg(V, "Creating new node: ", this);
9199 return V;
9200}
9201
9203 SDValue Val, SDValue Ptr,
9204 SDValue Stride, SDValue Mask,
9205 SDValue EVL, EVT SVT,
9206 MachineMemOperand *MMO,
9207 bool IsCompressing) {
9208 EVT VT = Val.getValueType();
9209
9210 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9211 if (VT == SVT)
9212 return getStridedStoreVP(Chain, DL, Val, Ptr, getUNDEF(Ptr.getValueType()),
9213 Stride, Mask, EVL, VT, MMO, ISD::UNINDEXED,
9214 /*IsTruncating*/ false, IsCompressing);
9215
9217 "Should only be a truncating store, not extending!");
9218 assert(VT.isInteger() == SVT.isInteger() && "Can't do FP-INT conversion!");
9219 assert(VT.isVector() == SVT.isVector() &&
9220 "Cannot use trunc store to convert to or from a vector!");
9221 assert((!VT.isVector() ||
9223 "Cannot use trunc store to change the number of vector elements!");
9224
9225 SDVTList VTs = getVTList(MVT::Other);
9226 SDValue Undef = getUNDEF(Ptr.getValueType());
9227 SDValue Ops[] = {Chain, Val, Ptr, Undef, Stride, Mask, EVL};
9229 AddNodeIDNode(ID, ISD::EXPERIMENTAL_VP_STRIDED_STORE, VTs, Ops);
9230 ID.AddInteger(SVT.getRawBits());
9231 ID.AddInteger(getSyntheticNodeSubclassData<VPStridedStoreSDNode>(
9232 DL.getIROrder(), VTs, ISD::UNINDEXED, true, IsCompressing, SVT, MMO));
9233 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9234 void *IP = nullptr;
9235 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
9236 cast<VPStridedStoreSDNode>(E)->refineAlignment(MMO);
9237 return SDValue(E, 0);
9238 }
9239 auto *N = newSDNode<VPStridedStoreSDNode>(DL.getIROrder(), DL.getDebugLoc(),
9240 VTs, ISD::UNINDEXED, true,
9241 IsCompressing, SVT, MMO);
9242 createOperands(N, Ops);
9243
9244 CSEMap.InsertNode(N, IP);
9245 InsertNode(N);
9246 SDValue V(N, 0);
9247 NewSDValueDbgMsg(V, "Creating new node: ", this);
9248 return V;
9249}
9250
9253 ISD::MemIndexType IndexType) {
9254 assert(Ops.size() == 6 && "Incompatible number of operands");
9255
9257 AddNodeIDNode(ID, ISD::VP_GATHER, VTs, Ops);
9258 ID.AddInteger(VT.getRawBits());
9259 ID.AddInteger(getSyntheticNodeSubclassData<VPGatherSDNode>(
9260 dl.getIROrder(), VTs, VT, MMO, IndexType));
9261 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9262 ID.AddInteger(MMO->getFlags());
9263 void *IP = nullptr;
9264 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9265 cast<VPGatherSDNode>(E)->refineAlignment(MMO);
9266 return SDValue(E, 0);
9267 }
9268
9269 auto *N = newSDNode<VPGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9270 VT, MMO, IndexType);
9271 createOperands(N, Ops);
9272
9273 assert(N->getMask().getValueType().getVectorElementCount() ==
9274 N->getValueType(0).getVectorElementCount() &&
9275 "Vector width mismatch between mask and data");
9276 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9277 N->getValueType(0).getVectorElementCount().isScalable() &&
9278 "Scalable flags of index and data do not match");
9280 N->getIndex().getValueType().getVectorElementCount(),
9281 N->getValueType(0).getVectorElementCount()) &&
9282 "Vector width mismatch between index and data");
9283 assert(isa<ConstantSDNode>(N->getScale()) &&
9284 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9285 "Scale should be a constant power of 2");
9286
9287 CSEMap.InsertNode(N, IP);
9288 InsertNode(N);
9289 SDValue V(N, 0);
9290 NewSDValueDbgMsg(V, "Creating new node: ", this);
9291 return V;
9292}
9293
9296 MachineMemOperand *MMO,
9297 ISD::MemIndexType IndexType) {
9298 assert(Ops.size() == 7 && "Incompatible number of operands");
9299
9301 AddNodeIDNode(ID, ISD::VP_SCATTER, VTs, Ops);
9302 ID.AddInteger(VT.getRawBits());
9303 ID.AddInteger(getSyntheticNodeSubclassData<VPScatterSDNode>(
9304 dl.getIROrder(), VTs, VT, MMO, IndexType));
9305 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9306 ID.AddInteger(MMO->getFlags());
9307 void *IP = nullptr;
9308 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9309 cast<VPScatterSDNode>(E)->refineAlignment(MMO);
9310 return SDValue(E, 0);
9311 }
9312 auto *N = newSDNode<VPScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9313 VT, MMO, IndexType);
9314 createOperands(N, Ops);
9315
9316 assert(N->getMask().getValueType().getVectorElementCount() ==
9317 N->getValue().getValueType().getVectorElementCount() &&
9318 "Vector width mismatch between mask and data");
9319 assert(
9320 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9321 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9322 "Scalable flags of index and data do not match");
9324 N->getIndex().getValueType().getVectorElementCount(),
9325 N->getValue().getValueType().getVectorElementCount()) &&
9326 "Vector width mismatch between index and data");
9327 assert(isa<ConstantSDNode>(N->getScale()) &&
9328 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9329 "Scale should be a constant power of 2");
9330
9331 CSEMap.InsertNode(N, IP);
9332 InsertNode(N);
9333 SDValue V(N, 0);
9334 NewSDValueDbgMsg(V, "Creating new node: ", this);
9335 return V;
9336}
9337
9340 SDValue PassThru, EVT MemVT,
9341 MachineMemOperand *MMO,
9343 ISD::LoadExtType ExtTy, bool isExpanding) {
9344 bool Indexed = AM != ISD::UNINDEXED;
9345 assert((Indexed || Offset.isUndef()) &&
9346 "Unindexed masked load with an offset!");
9347 SDVTList VTs = Indexed ? getVTList(VT, Base.getValueType(), MVT::Other)
9348 : getVTList(VT, MVT::Other);
9349 SDValue Ops[] = {Chain, Base, Offset, Mask, PassThru};
9351 AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops);
9352 ID.AddInteger(MemVT.getRawBits());
9353 ID.AddInteger(getSyntheticNodeSubclassData<MaskedLoadSDNode>(
9354 dl.getIROrder(), VTs, AM, ExtTy, isExpanding, MemVT, MMO));
9355 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9356 ID.AddInteger(MMO->getFlags());
9357 void *IP = nullptr;
9358 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9359 cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
9360 return SDValue(E, 0);
9361 }
9362 auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
9363 AM, ExtTy, isExpanding, MemVT, MMO);
9364 createOperands(N, Ops);
9365
9366 CSEMap.InsertNode(N, IP);
9367 InsertNode(N);
9368 SDValue V(N, 0);
9369 NewSDValueDbgMsg(V, "Creating new node: ", this);
9370 return V;
9371}
9372
9376 MaskedLoadSDNode *LD = cast<MaskedLoadSDNode>(OrigLoad);
9377 assert(LD->getOffset().isUndef() && "Masked load is already a indexed load!");
9378 return getMaskedLoad(OrigLoad.getValueType(), dl, LD->getChain(), Base,
9379 Offset, LD->getMask(), LD->getPassThru(),
9380 LD->getMemoryVT(), LD->getMemOperand(), AM,
9381 LD->getExtensionType(), LD->isExpandingLoad());
9382}
9383
9386 SDValue Mask, EVT MemVT,
9387 MachineMemOperand *MMO,
9388 ISD::MemIndexedMode AM, bool IsTruncating,
9389 bool IsCompressing) {
9390 assert(Chain.getValueType() == MVT::Other &&
9391 "Invalid chain type");
9392 bool Indexed = AM != ISD::UNINDEXED;
9393 assert((Indexed || Offset.isUndef()) &&
9394 "Unindexed masked store with an offset!");
9395 SDVTList VTs = Indexed ? getVTList(Base.getValueType(), MVT::Other)
9396 : getVTList(MVT::Other);
9397 SDValue Ops[] = {Chain, Val, Base, Offset, Mask};
9399 AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops);
9400 ID.AddInteger(MemVT.getRawBits());
9401 ID.AddInteger(getSyntheticNodeSubclassData<MaskedStoreSDNode>(
9402 dl.getIROrder(), VTs, AM, IsTruncating, IsCompressing, MemVT, MMO));
9403 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9404 ID.AddInteger(MMO->getFlags());
9405 void *IP = nullptr;
9406 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9407 cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
9408 return SDValue(E, 0);
9409 }
9410 auto *N =
9411 newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
9412 IsTruncating, IsCompressing, MemVT, MMO);
9413 createOperands(N, Ops);
9414
9415 CSEMap.InsertNode(N, IP);
9416 InsertNode(N);
9417 SDValue V(N, 0);
9418 NewSDValueDbgMsg(V, "Creating new node: ", this);
9419 return V;
9420}
9421
9425 MaskedStoreSDNode *ST = cast<MaskedStoreSDNode>(OrigStore);
9426 assert(ST->getOffset().isUndef() &&
9427 "Masked store is already a indexed store!");
9428 return getMaskedStore(ST->getChain(), dl, ST->getValue(), Base, Offset,
9429 ST->getMask(), ST->getMemoryVT(), ST->getMemOperand(),
9430 AM, ST->isTruncatingStore(), ST->isCompressingStore());
9431}
9432
9435 MachineMemOperand *MMO,
9436 ISD::MemIndexType IndexType,
9437 ISD::LoadExtType ExtTy) {
9438 assert(Ops.size() == 6 && "Incompatible number of operands");
9439
9441 AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
9442 ID.AddInteger(MemVT.getRawBits());
9443 ID.AddInteger(getSyntheticNodeSubclassData<MaskedGatherSDNode>(
9444 dl.getIROrder(), VTs, MemVT, MMO, IndexType, ExtTy));
9445 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9446 ID.AddInteger(MMO->getFlags());
9447 void *IP = nullptr;
9448 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9449 cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
9450 return SDValue(E, 0);
9451 }
9452
9453 auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9454 VTs, MemVT, MMO, IndexType, ExtTy);
9455 createOperands(N, Ops);
9456
9457 assert(N->getPassThru().getValueType() == N->getValueType(0) &&
9458 "Incompatible type of the PassThru value in MaskedGatherSDNode");
9459 assert(N->getMask().getValueType().getVectorElementCount() ==
9460 N->getValueType(0).getVectorElementCount() &&
9461 "Vector width mismatch between mask and data");
9462 assert(N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9463 N->getValueType(0).getVectorElementCount().isScalable() &&
9464 "Scalable flags of index and data do not match");
9466 N->getIndex().getValueType().getVectorElementCount(),
9467 N->getValueType(0).getVectorElementCount()) &&
9468 "Vector width mismatch between index and data");
9469 assert(isa<ConstantSDNode>(N->getScale()) &&
9470 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9471 "Scale should be a constant power of 2");
9472
9473 CSEMap.InsertNode(N, IP);
9474 InsertNode(N);
9475 SDValue V(N, 0);
9476 NewSDValueDbgMsg(V, "Creating new node: ", this);
9477 return V;
9478}
9479
9482 MachineMemOperand *MMO,
9483 ISD::MemIndexType IndexType,
9484 bool IsTrunc) {
9485 assert(Ops.size() == 6 && "Incompatible number of operands");
9486
9488 AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
9489 ID.AddInteger(MemVT.getRawBits());
9490 ID.AddInteger(getSyntheticNodeSubclassData<MaskedScatterSDNode>(
9491 dl.getIROrder(), VTs, MemVT, MMO, IndexType, IsTrunc));
9492 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9493 ID.AddInteger(MMO->getFlags());
9494 void *IP = nullptr;
9495 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
9496 cast<MaskedScatterSDNode>(E)->refineAlignment(MMO);
9497 return SDValue(E, 0);
9498 }
9499
9500 auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
9501 VTs, MemVT, MMO, IndexType, IsTrunc);
9502 createOperands(N, Ops);
9503
9504 assert(N->getMask().getValueType().getVectorElementCount() ==
9505 N->getValue().getValueType().getVectorElementCount() &&
9506 "Vector width mismatch between mask and data");
9507 assert(
9508 N->getIndex().getValueType().getVectorElementCount().isScalable() ==
9509 N->getValue().getValueType().getVectorElementCount().isScalable() &&
9510 "Scalable flags of index and data do not match");
9512 N->getIndex().getValueType().getVectorElementCount(),
9513 N->getValue().getValueType().getVectorElementCount()) &&
9514 "Vector width mismatch between index and data");
9515 assert(isa<ConstantSDNode>(N->getScale()) &&
9516 N->getScale()->getAsAPIntVal().isPowerOf2() &&
9517 "Scale should be a constant power of 2");
9518
9519 CSEMap.InsertNode(N, IP);
9520 InsertNode(N);
9521 SDValue V(N, 0);
9522 NewSDValueDbgMsg(V, "Creating new node: ", this);
9523 return V;
9524}
9525
9527 EVT MemVT, MachineMemOperand *MMO) {
9528 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9529 SDVTList VTs = getVTList(MVT::Other);
9530 SDValue Ops[] = {Chain, Ptr};
9533 ID.AddInteger(MemVT.getRawBits());
9534 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9535 ISD::GET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9536 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9537 ID.AddInteger(MMO->getFlags());
9538 void *IP = nullptr;
9539 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9540 return SDValue(E, 0);
9541
9542 auto *N = newSDNode<FPStateAccessSDNode>(ISD::GET_FPENV_MEM, dl.getIROrder(),
9543 dl.getDebugLoc(), VTs, MemVT, MMO);
9544 createOperands(N, Ops);
9545
9546 CSEMap.InsertNode(N, IP);
9547 InsertNode(N);
9548 SDValue V(N, 0);
9549 NewSDValueDbgMsg(V, "Creating new node: ", this);
9550 return V;
9551}
9552
9554 EVT MemVT, MachineMemOperand *MMO) {
9555 assert(Chain.getValueType() == MVT::Other && "Invalid chain type");
9556 SDVTList VTs = getVTList(MVT::Other);
9557 SDValue Ops[] = {Chain, Ptr};
9560 ID.AddInteger(MemVT.getRawBits());
9561 ID.AddInteger(getSyntheticNodeSubclassData<FPStateAccessSDNode>(
9562 ISD::SET_FPENV_MEM, dl.getIROrder(), VTs, MemVT, MMO));
9563 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
9564 ID.AddInteger(MMO->getFlags());
9565 void *IP = nullptr;
9566 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP))
9567 return SDValue(E, 0);
9568
9569 auto *N = newSDNode<FPStateAccessSDNode>(ISD::SET_FPENV_MEM, dl.getIROrder(),
9570 dl.getDebugLoc(), VTs, MemVT, MMO);
9571 createOperands(N, Ops);
9572
9573 CSEMap.InsertNode(N, IP);
9574 InsertNode(N);
9575 SDValue V(N, 0);
9576 NewSDValueDbgMsg(V, "Creating new node: ", this);
9577 return V;
9578}
9579
9581 // select undef, T, F --> T (if T is a constant), otherwise F
9582 // select, ?, undef, F --> F
9583 // select, ?, T, undef --> T
9584 if (Cond.isUndef())
9585 return isConstantValueOfAnyType(T) ? T : F;
9586 if (T.isUndef())
9587 return F;
9588 if (F.isUndef())
9589 return T;
9590
9591 // select true, T, F --> T
9592 // select false, T, F --> F
9593 if (auto *CondC = dyn_cast<ConstantSDNode>(Cond))
9594 return CondC->isZero() ? F : T;
9595
9596 // TODO: This should simplify VSELECT with non-zero constant condition using
9597 // something like this (but check boolean contents to be complete?):
9598 if (ConstantSDNode *CondC = isConstOrConstSplat(Cond, /*AllowUndefs*/ false,
9599 /*AllowTruncation*/ true))
9600 if (CondC->isZero())
9601 return F;
9602
9603 // select ?, T, T --> T
9604 if (T == F)
9605 return T;
9606
9607 return SDValue();
9608}
9609
9611 // shift undef, Y --> 0 (can always assume that the undef value is 0)
9612 if (X.isUndef())
9613 return getConstant(0, SDLoc(X.getNode()), X.getValueType());
9614 // shift X, undef --> undef (because it may shift by the bitwidth)
9615 if (Y.isUndef())
9616 return getUNDEF(X.getValueType());
9617
9618 // shift 0, Y --> 0
9619 // shift X, 0 --> X
9621 return X;
9622
9623 // shift X, C >= bitwidth(X) --> undef
9624 // All vector elements must be too big (or undef) to avoid partial undefs.
9625 auto isShiftTooBig = [X](ConstantSDNode *Val) {
9626 return !Val || Val->getAPIntValue().uge(X.getScalarValueSizeInBits());
9627 };
9628 if (ISD::matchUnaryPredicate(Y, isShiftTooBig, true))
9629 return getUNDEF(X.getValueType());
9630
9631 // shift i1/vXi1 X, Y --> X (any non-zero shift amount is undefined).
9632 if (X.getValueType().getScalarType() == MVT::i1)
9633 return X;
9634
9635 return SDValue();
9636}
9637
9639 SDNodeFlags Flags) {
9640 // If this operation has 'nnan' or 'ninf' and at least 1 disallowed operand
9641 // (an undef operand can be chosen to be Nan/Inf), then the result of this
9642 // operation is poison. That result can be relaxed to undef.
9643 ConstantFPSDNode *XC = isConstOrConstSplatFP(X, /* AllowUndefs */ true);
9644 ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true);
9645 bool HasNan = (XC && XC->getValueAPF().isNaN()) ||
9646 (YC && YC->getValueAPF().isNaN());
9647 bool HasInf = (XC && XC->getValueAPF().isInfinity()) ||
9648 (YC && YC->getValueAPF().isInfinity());
9649
9650 if (Flags.hasNoNaNs() && (HasNan || X.isUndef() || Y.isUndef()))
9651 return getUNDEF(X.getValueType());
9652
9653 if (Flags.hasNoInfs() && (HasInf || X.isUndef() || Y.isUndef()))
9654 return getUNDEF(X.getValueType());
9655
9656 if (!YC)
9657 return SDValue();
9658
9659 // X + -0.0 --> X
9660 if (Opcode == ISD::FADD)
9661 if (YC->getValueAPF().isNegZero())
9662 return X;
9663
9664 // X - +0.0 --> X
9665 if (Opcode == ISD::FSUB)
9666 if (YC->getValueAPF().isPosZero())
9667 return X;
9668
9669 // X * 1.0 --> X
9670 // X / 1.0 --> X
9671 if (Opcode == ISD::FMUL || Opcode == ISD::FDIV)
9672 if (YC->getValueAPF().isExactlyValue(1.0))
9673 return X;
9674
9675 // X * 0.0 --> 0.0
9676 if (Opcode == ISD::FMUL && Flags.hasNoNaNs() && Flags.hasNoSignedZeros())
9677 if (YC->getValueAPF().isZero())
9678 return getConstantFP(0.0, SDLoc(Y), Y.getValueType());
9679
9680 return SDValue();
9681}
9682
9684 SDValue Ptr, SDValue SV, unsigned Align) {
9685 SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) };
9686 return getNode(ISD::VAARG, dl, getVTList(VT, MVT::Other), Ops);
9687}
9688
9689SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9690 ArrayRef<SDUse> Ops) {
9691 switch (Ops.size()) {
9692 case 0: return getNode(Opcode, DL, VT);
9693 case 1: return getNode(Opcode, DL, VT, static_cast<const SDValue>(Ops[0]));
9694 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1]);
9695 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2]);
9696 default: break;
9697 }
9698
9699 // Copy from an SDUse array into an SDValue array for use with
9700 // the regular getNode logic.
9701 SmallVector<SDValue, 8> NewOps(Ops.begin(), Ops.end());
9702 return getNode(Opcode, DL, VT, NewOps);
9703}
9704
9705SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9706 ArrayRef<SDValue> Ops) {
9707 SDNodeFlags Flags;
9708 if (Inserter)
9709 Flags = Inserter->getFlags();
9710 return getNode(Opcode, DL, VT, Ops, Flags);
9711}
9712
9713SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
9714 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9715 unsigned NumOps = Ops.size();
9716 switch (NumOps) {
9717 case 0: return getNode(Opcode, DL, VT);
9718 case 1: return getNode(Opcode, DL, VT, Ops[0], Flags);
9719 case 2: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Flags);
9720 case 3: return getNode(Opcode, DL, VT, Ops[0], Ops[1], Ops[2], Flags);
9721 default: break;
9722 }
9723
9724#ifndef NDEBUG
9725 for (const auto &Op : Ops)
9726 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9727 "Operand is DELETED_NODE!");
9728#endif
9729
9730 switch (Opcode) {
9731 default: break;
9732 case ISD::BUILD_VECTOR:
9733 // Attempt to simplify BUILD_VECTOR.
9734 if (SDValue V = FoldBUILD_VECTOR(DL, VT, Ops, *this))
9735 return V;
9736 break;
9738 if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this))
9739 return V;
9740 break;
9741 case ISD::SELECT_CC:
9742 assert(NumOps == 5 && "SELECT_CC takes 5 operands!");
9743 assert(Ops[0].getValueType() == Ops[1].getValueType() &&
9744 "LHS and RHS of condition must have same type!");
9745 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9746 "True and False arms of SelectCC must have same type!");
9747 assert(Ops[2].getValueType() == VT &&
9748 "select_cc node must be of same type as true and false value!");
9749 assert((!Ops[0].getValueType().isVector() ||
9750 Ops[0].getValueType().getVectorElementCount() ==
9751 VT.getVectorElementCount()) &&
9752 "Expected select_cc with vector result to have the same sized "
9753 "comparison type!");
9754 break;
9755 case ISD::BR_CC:
9756 assert(NumOps == 5 && "BR_CC takes 5 operands!");
9757 assert(Ops[2].getValueType() == Ops[3].getValueType() &&
9758 "LHS/RHS of comparison should match types!");
9759 break;
9760 case ISD::VP_ADD:
9761 case ISD::VP_SUB:
9762 // If it is VP_ADD/VP_SUB mask operation then turn it to VP_XOR
9763 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9764 Opcode = ISD::VP_XOR;
9765 break;
9766 case ISD::VP_MUL:
9767 // If it is VP_MUL mask operation then turn it to VP_AND
9768 if (VT.isVector() && VT.getVectorElementType() == MVT::i1)
9769 Opcode = ISD::VP_AND;
9770 break;
9771 case ISD::VP_REDUCE_MUL:
9772 // If it is VP_REDUCE_MUL mask operation then turn it to VP_REDUCE_AND
9773 if (VT == MVT::i1)
9774 Opcode = ISD::VP_REDUCE_AND;
9775 break;
9776 case ISD::VP_REDUCE_ADD:
9777 // If it is VP_REDUCE_ADD mask operation then turn it to VP_REDUCE_XOR
9778 if (VT == MVT::i1)
9779 Opcode = ISD::VP_REDUCE_XOR;
9780 break;
9781 case ISD::VP_REDUCE_SMAX:
9782 case ISD::VP_REDUCE_UMIN:
9783 // If it is VP_REDUCE_SMAX/VP_REDUCE_UMIN mask operation then turn it to
9784 // VP_REDUCE_AND.
9785 if (VT == MVT::i1)
9786 Opcode = ISD::VP_REDUCE_AND;
9787 break;
9788 case ISD::VP_REDUCE_SMIN:
9789 case ISD::VP_REDUCE_UMAX:
9790 // If it is VP_REDUCE_SMIN/VP_REDUCE_UMAX mask operation then turn it to
9791 // VP_REDUCE_OR.
9792 if (VT == MVT::i1)
9793 Opcode = ISD::VP_REDUCE_OR;
9794 break;
9795 }
9796
9797 // Memoize nodes.
9798 SDNode *N;
9799 SDVTList VTs = getVTList(VT);
9800
9801 if (VT != MVT::Glue) {
9803 AddNodeIDNode(ID, Opcode, VTs, Ops);
9804 void *IP = nullptr;
9805
9806 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
9807 return SDValue(E, 0);
9808
9809 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9810 createOperands(N, Ops);
9811
9812 CSEMap.InsertNode(N, IP);
9813 } else {
9814 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
9815 createOperands(N, Ops);
9816 }
9817
9818 N->setFlags(Flags);
9819 InsertNode(N);
9820 SDValue V(N, 0);
9821 NewSDValueDbgMsg(V, "Creating new node: ", this);
9822 return V;
9823}
9824
9825SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
9826 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops) {
9827 return getNode(Opcode, DL, getVTList(ResultTys), Ops);
9828}
9829
9830SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9831 ArrayRef<SDValue> Ops) {
9832 SDNodeFlags Flags;
9833 if (Inserter)
9834 Flags = Inserter->getFlags();
9835 return getNode(Opcode, DL, VTList, Ops, Flags);
9836}
9837
9838SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
9839 ArrayRef<SDValue> Ops, const SDNodeFlags Flags) {
9840 if (VTList.NumVTs == 1)
9841 return getNode(Opcode, DL, VTList.VTs[0], Ops, Flags);
9842
9843#ifndef NDEBUG
9844 for (const auto &Op : Ops)
9845 assert(Op.getOpcode() != ISD::DELETED_NODE &&
9846 "Operand is DELETED_NODE!");
9847#endif
9848
9849 switch (Opcode) {
9850 case ISD::SADDO:
9851 case ISD::UADDO:
9852 case ISD::SSUBO:
9853 case ISD::USUBO: {
9854 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9855 "Invalid add/sub overflow op!");
9856 assert(VTList.VTs[0].isInteger() && VTList.VTs[1].isInteger() &&
9857 Ops[0].getValueType() == Ops[1].getValueType() &&
9858 Ops[0].getValueType() == VTList.VTs[0] &&
9859 "Binary operator types must match!");
9860 SDValue N1 = Ops[0], N2 = Ops[1];
9861 canonicalizeCommutativeBinop(Opcode, N1, N2);
9862
9863 // (X +- 0) -> X with zero-overflow.
9864 ConstantSDNode *N2CV = isConstOrConstSplat(N2, /*AllowUndefs*/ false,
9865 /*AllowTruncation*/ true);
9866 if (N2CV && N2CV->isZero()) {
9867 SDValue ZeroOverFlow = getConstant(0, DL, VTList.VTs[1]);
9868 return getNode(ISD::MERGE_VALUES, DL, VTList, {N1, ZeroOverFlow}, Flags);
9869 }
9870
9871 if (VTList.VTs[0].isVector() &&
9872 VTList.VTs[0].getVectorElementType() == MVT::i1 &&
9873 VTList.VTs[1].getVectorElementType() == MVT::i1) {
9874 SDValue F1 = getFreeze(N1);
9875 SDValue F2 = getFreeze(N2);
9876 // {vXi1,vXi1} (u/s)addo(vXi1 x, vXi1y) -> {xor(x,y),and(x,y)}
9877 if (Opcode == ISD::UADDO || Opcode == ISD::SADDO)
9878 return getNode(ISD::MERGE_VALUES, DL, VTList,
9879 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9880 getNode(ISD::AND, DL, VTList.VTs[1], F1, F2)},
9881 Flags);
9882 // {vXi1,vXi1} (u/s)subo(vXi1 x, vXi1y) -> {xor(x,y),and(~x,y)}
9883 if (Opcode == ISD::USUBO || Opcode == ISD::SSUBO) {
9884 SDValue NotF1 = getNOT(DL, F1, VTList.VTs[0]);
9885 return getNode(ISD::MERGE_VALUES, DL, VTList,
9886 {getNode(ISD::XOR, DL, VTList.VTs[0], F1, F2),
9887 getNode(ISD::AND, DL, VTList.VTs[1], NotF1, F2)},
9888 Flags);
9889 }
9890 }
9891 break;
9892 }
9893 case ISD::SMUL_LOHI:
9894 case ISD::UMUL_LOHI: {
9895 assert(VTList.NumVTs == 2 && Ops.size() == 2 && "Invalid mul lo/hi op!");
9896 assert(VTList.VTs[0].isInteger() && VTList.VTs[0] == VTList.VTs[1] &&
9897 VTList.VTs[0] == Ops[0].getValueType() &&
9898 VTList.VTs[0] == Ops[1].getValueType() &&
9899 "Binary operator types must match!");
9900 // Constant fold.
9901 ConstantSDNode *LHS = dyn_cast<ConstantSDNode>(Ops[0]);
9902 ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Ops[1]);
9903 if (LHS && RHS) {
9904 unsigned Width = VTList.VTs[0].getScalarSizeInBits();
9905 unsigned OutWidth = Width * 2;
9906 APInt Val = LHS->getAPIntValue();
9907 APInt Mul = RHS->getAPIntValue();
9908 if (Opcode == ISD::SMUL_LOHI) {
9909 Val = Val.sext(OutWidth);
9910 Mul = Mul.sext(OutWidth);
9911 } else {
9912 Val = Val.zext(OutWidth);
9913 Mul = Mul.zext(OutWidth);
9914 }
9915 Val *= Mul;
9916
9917 SDValue Hi =
9918 getConstant(Val.extractBits(Width, Width), DL, VTList.VTs[0]);
9919 SDValue Lo = getConstant(Val.trunc(Width), DL, VTList.VTs[0]);
9920 return getNode(ISD::MERGE_VALUES, DL, VTList, {Lo, Hi}, Flags);
9921 }
9922 break;
9923 }
9924 case ISD::FFREXP: {
9925 assert(VTList.NumVTs == 2 && Ops.size() == 1 && "Invalid ffrexp op!");
9926 assert(VTList.VTs[0].isFloatingPoint() && VTList.VTs[1].isInteger() &&
9927 VTList.VTs[0] == Ops[0].getValueType() && "frexp type mismatch");
9928
9929 if (const ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Ops[0])) {
9930 int FrexpExp;
9931 APFloat FrexpMant =
9932 frexp(C->getValueAPF(), FrexpExp, APFloat::rmNearestTiesToEven);
9933 SDValue Result0 = getConstantFP(FrexpMant, DL, VTList.VTs[0]);
9934 SDValue Result1 =
9935 getConstant(FrexpMant.isFinite() ? FrexpExp : 0, DL, VTList.VTs[1]);
9936 return getNode(ISD::MERGE_VALUES, DL, VTList, {Result0, Result1}, Flags);
9937 }
9938
9939 break;
9940 }
9942 assert(VTList.NumVTs == 2 && Ops.size() == 2 &&
9943 "Invalid STRICT_FP_EXTEND!");
9944 assert(VTList.VTs[0].isFloatingPoint() &&
9945 Ops[1].getValueType().isFloatingPoint() && "Invalid FP cast!");
9946 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9947 "STRICT_FP_EXTEND result type should be vector iff the operand "
9948 "type is vector!");
9949 assert((!VTList.VTs[0].isVector() ||
9950 VTList.VTs[0].getVectorElementCount() ==
9951 Ops[1].getValueType().getVectorElementCount()) &&
9952 "Vector element count mismatch!");
9953 assert(Ops[1].getValueType().bitsLT(VTList.VTs[0]) &&
9954 "Invalid fpext node, dst <= src!");
9955 break;
9957 assert(VTList.NumVTs == 2 && Ops.size() == 3 && "Invalid STRICT_FP_ROUND!");
9958 assert(VTList.VTs[0].isVector() == Ops[1].getValueType().isVector() &&
9959 "STRICT_FP_ROUND result type should be vector iff the operand "
9960 "type is vector!");
9961 assert((!VTList.VTs[0].isVector() ||
9962 VTList.VTs[0].getVectorElementCount() ==
9963 Ops[1].getValueType().getVectorElementCount()) &&
9964 "Vector element count mismatch!");
9965 assert(VTList.VTs[0].isFloatingPoint() &&
9966 Ops[1].getValueType().isFloatingPoint() &&
9967 VTList.VTs[0].bitsLT(Ops[1].getValueType()) &&
9968 isa<ConstantSDNode>(Ops[2]) &&
9969 (Ops[2]->getAsZExtVal() == 0 || Ops[2]->getAsZExtVal() == 1) &&
9970 "Invalid STRICT_FP_ROUND!");
9971 break;
9972#if 0
9973 // FIXME: figure out how to safely handle things like
9974 // int foo(int x) { return 1 << (x & 255); }
9975 // int bar() { return foo(256); }
9976 case ISD::SRA_PARTS:
9977 case ISD::SRL_PARTS:
9978 case ISD::SHL_PARTS:
9979 if (N3.getOpcode() == ISD::SIGN_EXTEND_INREG &&
9980 cast<VTSDNode>(N3.getOperand(1))->getVT() != MVT::i1)
9981 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9982 else if (N3.getOpcode() == ISD::AND)
9983 if (ConstantSDNode *AndRHS = dyn_cast<ConstantSDNode>(N3.getOperand(1))) {
9984 // If the and is only masking out bits that cannot effect the shift,
9985 // eliminate the and.
9986 unsigned NumBits = VT.getScalarSizeInBits()*2;
9987 if ((AndRHS->getValue() & (NumBits-1)) == NumBits-1)
9988 return getNode(Opcode, DL, VT, N1, N2, N3.getOperand(0));
9989 }
9990 break;
9991#endif
9992 }
9993
9994 // Memoize the node unless it returns a glue result.
9995 SDNode *N;
9996 if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
9998 AddNodeIDNode(ID, Opcode, VTList, Ops);
9999 void *IP = nullptr;
10000 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP))
10001 return SDValue(E, 0);
10002
10003 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10004 createOperands(N, Ops);
10005 CSEMap.InsertNode(N, IP);
10006 } else {
10007 N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
10008 createOperands(N, Ops);
10009 }
10010
10011 N->setFlags(Flags);
10012 InsertNode(N);
10013 SDValue V(N, 0);
10014 NewSDValueDbgMsg(V, "Creating new node: ", this);
10015 return V;
10016}
10017
10018SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL,
10019 SDVTList VTList) {
10020 return getNode(Opcode, DL, VTList, std::nullopt);
10021}
10022
10023SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10024 SDValue N1) {
10025 SDValue Ops[] = { N1 };
10026 return getNode(Opcode, DL, VTList, Ops);
10027}
10028
10029SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10030 SDValue N1, SDValue N2) {
10031 SDValue Ops[] = { N1, N2 };
10032 return getNode(Opcode, DL, VTList, Ops);
10033}
10034
10035SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10036 SDValue N1, SDValue N2, SDValue N3) {
10037 SDValue Ops[] = { N1, N2, N3 };
10038 return getNode(Opcode, DL, VTList, Ops);
10039}
10040
10041SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10042 SDValue N1, SDValue N2, SDValue N3, SDValue N4) {
10043 SDValue Ops[] = { N1, N2, N3, N4 };
10044 return getNode(Opcode, DL, VTList, Ops);
10045}
10046
10047SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
10048 SDValue N1, SDValue N2, SDValue N3, SDValue N4,
10049 SDValue N5) {
10050 SDValue Ops[] = { N1, N2, N3, N4, N5 };
10051 return getNode(Opcode, DL, VTList, Ops);
10052}
10053
10055 return makeVTList(SDNode::getValueTypeList(VT), 1);
10056}
10057
10060 ID.AddInteger(2U);
10061 ID.AddInteger(VT1.getRawBits());
10062 ID.AddInteger(VT2.getRawBits());
10063
10064 void *IP = nullptr;
10065 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10066 if (!Result) {
10067 EVT *Array = Allocator.Allocate<EVT>(2);
10068 Array[0] = VT1;
10069 Array[1] = VT2;
10070 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 2);
10071 VTListMap.InsertNode(Result, IP);
10072 }
10073 return Result->getSDVTList();
10074}
10075
10078 ID.AddInteger(3U);
10079 ID.AddInteger(VT1.getRawBits());
10080 ID.AddInteger(VT2.getRawBits());
10081 ID.AddInteger(VT3.getRawBits());
10082
10083 void *IP = nullptr;
10084 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10085 if (!Result) {
10086 EVT *Array = Allocator.Allocate<EVT>(3);
10087 Array[0] = VT1;
10088 Array[1] = VT2;
10089 Array[2] = VT3;
10090 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 3);
10091 VTListMap.InsertNode(Result, IP);
10092 }
10093 return Result->getSDVTList();
10094}
10095
10098 ID.AddInteger(4U);
10099 ID.AddInteger(VT1.getRawBits());
10100 ID.AddInteger(VT2.getRawBits());
10101 ID.AddInteger(VT3.getRawBits());
10102 ID.AddInteger(VT4.getRawBits());
10103
10104 void *IP = nullptr;
10105 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10106 if (!Result) {
10107 EVT *Array = Allocator.Allocate<EVT>(4);
10108 Array[0] = VT1;
10109 Array[1] = VT2;
10110 Array[2] = VT3;
10111 Array[3] = VT4;
10112 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, 4);
10113 VTListMap.InsertNode(Result, IP);
10114 }
10115 return Result->getSDVTList();
10116}
10117
10119 unsigned NumVTs = VTs.size();
10121 ID.AddInteger(NumVTs);
10122 for (unsigned index = 0; index < NumVTs; index++) {
10123 ID.AddInteger(VTs[index].getRawBits());
10124 }
10125
10126 void *IP = nullptr;
10127 SDVTListNode *Result = VTListMap.FindNodeOrInsertPos(ID, IP);
10128 if (!Result) {
10129 EVT *Array = Allocator.Allocate<EVT>(NumVTs);
10130 llvm::copy(VTs, Array);
10131 Result = new (Allocator) SDVTListNode(ID.Intern(Allocator), Array, NumVTs);
10132 VTListMap.InsertNode(Result, IP);
10133 }
10134 return Result->getSDVTList();
10135}
10136
10137
10138/// UpdateNodeOperands - *Mutate* the specified node in-place to have the
10139/// specified operands. If the resultant node already exists in the DAG,
10140/// this does not modify the specified node, instead it returns the node that
10141/// already exists. If the resultant node does not exist in the DAG, the
10142/// input node is returned. As a degenerate case, if you specify the same
10143/// input operands as the node already has, the input node is returned.
10145 assert(N->getNumOperands() == 1 && "Update with wrong number of operands");
10146
10147 // Check to see if there is no change.
10148 if (Op == N->getOperand(0)) return N;
10149
10150 // See if the modified node already exists.
10151 void *InsertPos = nullptr;
10152 if (SDNode *Existing = FindModifiedNodeSlot(N, Op, InsertPos))
10153 return Existing;
10154
10155 // Nope it doesn't. Remove the node from its current place in the maps.
10156 if (InsertPos)
10157 if (!RemoveNodeFromCSEMaps(N))
10158 InsertPos = nullptr;
10159
10160 // Now we update the operands.
10161 N->OperandList[0].set(Op);
10162
10164 // If this gets put into a CSE map, add it.
10165 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10166 return N;
10167}
10168
10170 assert(N->getNumOperands() == 2 && "Update with wrong number of operands");
10171
10172 // Check to see if there is no change.
10173 if (Op1 == N->getOperand(0) && Op2 == N->getOperand(1))
10174 return N; // No operands changed, just return the input node.
10175
10176 // See if the modified node already exists.
10177 void *InsertPos = nullptr;
10178 if (SDNode *Existing = FindModifiedNodeSlot(N, Op1, Op2, InsertPos))
10179 return Existing;
10180
10181 // Nope it doesn't. Remove the node from its current place in the maps.
10182 if (InsertPos)
10183 if (!RemoveNodeFromCSEMaps(N))
10184 InsertPos = nullptr;
10185
10186 // Now we update the operands.
10187 if (N->OperandList[0] != Op1)
10188 N->OperandList[0].set(Op1);
10189 if (N->OperandList[1] != Op2)
10190 N->OperandList[1].set(Op2);
10191
10193 // If this gets put into a CSE map, add it.
10194 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10195 return N;
10196}
10197
10200 SDValue Ops[] = { Op1, Op2, Op3 };
10201 return UpdateNodeOperands(N, Ops);
10202}
10203
10206 SDValue Op3, SDValue Op4) {
10207 SDValue Ops[] = { Op1, Op2, Op3, Op4 };
10208 return UpdateNodeOperands(N, Ops);
10209}
10210
10213 SDValue Op3, SDValue Op4, SDValue Op5) {
10214 SDValue Ops[] = { Op1, Op2, Op3, Op4, Op5 };
10215 return UpdateNodeOperands(N, Ops);
10216}
10217
10220 unsigned NumOps = Ops.size();
10221 assert(N->getNumOperands() == NumOps &&
10222 "Update with wrong number of operands");
10223
10224 // If no operands changed just return the input node.
10225 if (std::equal(Ops.begin(), Ops.end(), N->op_begin()))
10226 return N;
10227
10228 // See if the modified node already exists.
10229 void *InsertPos = nullptr;
10230 if (SDNode *Existing = FindModifiedNodeSlot(N, Ops, InsertPos))
10231 return Existing;
10232
10233 // Nope it doesn't. Remove the node from its current place in the maps.
10234 if (InsertPos)
10235 if (!RemoveNodeFromCSEMaps(N))
10236 InsertPos = nullptr;
10237
10238 // Now we update the operands.
10239 for (unsigned i = 0; i != NumOps; ++i)
10240 if (N->OperandList[i] != Ops[i])
10241 N->OperandList[i].set(Ops[i]);
10242
10244 // If this gets put into a CSE map, add it.
10245 if (InsertPos) CSEMap.InsertNode(N, InsertPos);
10246 return N;
10247}
10248
10249/// DropOperands - Release the operands and set this node to have
10250/// zero operands.
10252 // Unlike the code in MorphNodeTo that does this, we don't need to
10253 // watch for dead nodes here.
10254 for (op_iterator I = op_begin(), E = op_end(); I != E; ) {
10255 SDUse &Use = *I++;
10256 Use.set(SDValue());
10257 }
10258}
10259
10261 ArrayRef<MachineMemOperand *> NewMemRefs) {
10262 if (NewMemRefs.empty()) {
10263 N->clearMemRefs();
10264 return;
10265 }
10266
10267 // Check if we can avoid allocating by storing a single reference directly.
10268 if (NewMemRefs.size() == 1) {
10269 N->MemRefs = NewMemRefs[0];
10270 N->NumMemRefs = 1;
10271 return;
10272 }
10273
10274 MachineMemOperand **MemRefsBuffer =
10275 Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size());
10276 llvm::copy(NewMemRefs, MemRefsBuffer);
10277 N->MemRefs = MemRefsBuffer;
10278 N->NumMemRefs = static_cast<int>(NewMemRefs.size());
10279}
10280
10281/// SelectNodeTo - These are wrappers around MorphNodeTo that accept a
10282/// machine opcode.
10283///
10285 EVT VT) {
10286 SDVTList VTs = getVTList(VT);
10287 return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10288}
10289
10291 EVT VT, SDValue Op1) {
10292 SDVTList VTs = getVTList(VT);
10293 SDValue Ops[] = { Op1 };
10294 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10295}
10296
10298 EVT VT, SDValue Op1,
10299 SDValue Op2) {
10300 SDVTList VTs = getVTList(VT);
10301 SDValue Ops[] = { Op1, Op2 };
10302 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10303}
10304
10306 EVT VT, SDValue Op1,
10307 SDValue Op2, SDValue Op3) {
10308 SDVTList VTs = getVTList(VT);
10309 SDValue Ops[] = { Op1, Op2, Op3 };
10310 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10311}
10312
10314 EVT VT, ArrayRef<SDValue> Ops) {
10315 SDVTList VTs = getVTList(VT);
10316 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10317}
10318
10320 EVT VT1, EVT VT2, ArrayRef<SDValue> Ops) {
10321 SDVTList VTs = getVTList(VT1, VT2);
10322 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10323}
10324
10326 EVT VT1, EVT VT2) {
10327 SDVTList VTs = getVTList(VT1, VT2);
10328 return SelectNodeTo(N, MachineOpc, VTs, std::nullopt);
10329}
10330
10332 EVT VT1, EVT VT2, EVT VT3,
10333 ArrayRef<SDValue> Ops) {
10334 SDVTList VTs = getVTList(VT1, VT2, VT3);
10335 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10336}
10337
10339 EVT VT1, EVT VT2,
10340 SDValue Op1, SDValue Op2) {
10341 SDVTList VTs = getVTList(VT1, VT2);
10342 SDValue Ops[] = { Op1, Op2 };
10343 return SelectNodeTo(N, MachineOpc, VTs, Ops);
10344}
10345
10347 SDVTList VTs,ArrayRef<SDValue> Ops) {
10348 SDNode *New = MorphNodeTo(N, ~MachineOpc, VTs, Ops);
10349 // Reset the NodeID to -1.
10350 New->setNodeId(-1);
10351 if (New != N) {
10352 ReplaceAllUsesWith(N, New);
10354 }
10355 return New;
10356}
10357
10358/// UpdateSDLocOnMergeSDNode - If the opt level is -O0 then it throws away
10359/// the line number information on the merged node since it is not possible to
10360/// preserve the information that operation is associated with multiple lines.
10361/// This will make the debugger working better at -O0, were there is a higher
10362/// probability having other instructions associated with that line.
10363///
10364/// For IROrder, we keep the smaller of the two
10365SDNode *SelectionDAG::UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &OLoc) {
10366 DebugLoc NLoc = N->getDebugLoc();
10367 if (NLoc && OptLevel == CodeGenOptLevel::None && OLoc.getDebugLoc() != NLoc) {
10368 N->setDebugLoc(DebugLoc());
10369 }
10370 unsigned Order = std::min(N->getIROrder(), OLoc.getIROrder());
10371 N->setIROrder(Order);
10372 return N;
10373}
10374
10375/// MorphNodeTo - This *mutates* the specified node to have the specified
10376/// return type, opcode, and operands.
10377///
10378/// Note that MorphNodeTo returns the resultant node. If there is already a
10379/// node of the specified opcode and operands, it returns that node instead of
10380/// the current one. Note that the SDLoc need not be the same.
10381///
10382/// Using MorphNodeTo is faster than creating a new node and swapping it in
10383/// with ReplaceAllUsesWith both because it often avoids allocating a new
10384/// node, and because it doesn't require CSE recalculation for any of
10385/// the node's users.
10386///
10387/// However, note that MorphNodeTo recursively deletes dead nodes from the DAG.
10388/// As a consequence it isn't appropriate to use from within the DAG combiner or
10389/// the legalizer which maintain worklists that would need to be updated when
10390/// deleting things.
10392 SDVTList VTs, ArrayRef<SDValue> Ops) {
10393 // If an identical node already exists, use it.
10394 void *IP = nullptr;
10395 if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
10397 AddNodeIDNode(ID, Opc, VTs, Ops);
10398 if (SDNode *ON = FindNodeOrInsertPos(ID, SDLoc(N), IP))
10399 return UpdateSDLocOnMergeSDNode(ON, SDLoc(N));
10400 }
10401
10402 if (!RemoveNodeFromCSEMaps(N))
10403 IP = nullptr;
10404
10405 // Start the morphing.
10406 N->NodeType = Opc;
10407 N->ValueList = VTs.VTs;
10408 N->NumValues = VTs.NumVTs;
10409
10410 // Clear the operands list, updating used nodes to remove this from their
10411 // use list. Keep track of any operands that become dead as a result.
10412 SmallPtrSet<SDNode*, 16> DeadNodeSet;
10413 for (SDNode::op_iterator I = N->op_begin(), E = N->op_end(); I != E; ) {
10414 SDUse &Use = *I++;
10415 SDNode *Used = Use.getNode();
10416 Use.set(SDValue());
10417 if (Used->use_empty())
10418 DeadNodeSet.insert(Used);
10419 }
10420
10421 // For MachineNode, initialize the memory references information.
10422 if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
10423 MN->clearMemRefs();
10424
10425 // Swap for an appropriately sized array from the recycler.
10426 removeOperands(N);
10427 createOperands(N, Ops);
10428
10429 // Delete any nodes that are still dead after adding the uses for the
10430 // new operands.
10431 if (!DeadNodeSet.empty()) {
10432 SmallVector<SDNode *, 16> DeadNodes;
10433 for (SDNode *N : DeadNodeSet)
10434 if (N->use_empty())
10435 DeadNodes.push_back(N);
10436 RemoveDeadNodes(DeadNodes);
10437 }
10438
10439 if (IP)
10440 CSEMap.InsertNode(N, IP); // Memoize the new node.
10441 return N;
10442}
10443
10445 unsigned OrigOpc = Node->getOpcode();
10446 unsigned NewOpc;
10447 switch (OrigOpc) {
10448 default:
10449 llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!");
10450#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10451 case ISD::STRICT_##DAGN: NewOpc = ISD::DAGN; break;
10452#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
10453 case ISD::STRICT_##DAGN: NewOpc = ISD::SETCC; break;
10454#include "llvm/IR/ConstrainedOps.def"
10455 }
10456
10457 assert(Node->getNumValues() == 2 && "Unexpected number of results!");
10458
10459 // We're taking this node out of the chain, so we need to re-link things.
10460 SDValue InputChain = Node->getOperand(0);
10461 SDValue OutputChain = SDValue(Node, 1);
10462 ReplaceAllUsesOfValueWith(OutputChain, InputChain);
10463
10465 for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i)
10466 Ops.push_back(Node->getOperand(i));
10467
10468 SDVTList VTs = getVTList(Node->getValueType(0));
10469 SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops);
10470
10471 // MorphNodeTo can operate in two ways: if an existing node with the
10472 // specified operands exists, it can just return it. Otherwise, it
10473 // updates the node in place to have the requested operands.
10474 if (Res == Node) {
10475 // If we updated the node in place, reset the node ID. To the isel,
10476 // this should be just like a newly allocated machine node.
10477 Res->setNodeId(-1);
10478 } else {
10479 ReplaceAllUsesWith(Node, Res);
10480 RemoveDeadNode(Node);
10481 }
10482
10483 return Res;
10484}
10485
10486/// getMachineNode - These are used for target selectors to create a new node
10487/// with specified return type(s), MachineInstr opcode, and operands.
10488///
10489/// Note that getMachineNode returns the resultant node. If there is already a
10490/// node of the specified opcode and operands, it returns that node instead of
10491/// the current one.
10493 EVT VT) {
10494 SDVTList VTs = getVTList(VT);
10495 return getMachineNode(Opcode, dl, VTs, std::nullopt);
10496}
10497
10499 EVT VT, SDValue Op1) {
10500 SDVTList VTs = getVTList(VT);
10501 SDValue Ops[] = { Op1 };
10502 return getMachineNode(Opcode, dl, VTs, Ops);
10503}
10504
10506 EVT VT, SDValue Op1, SDValue Op2) {
10507 SDVTList VTs = getVTList(VT);
10508 SDValue Ops[] = { Op1, Op2 };
10509 return getMachineNode(Opcode, dl, VTs, Ops);
10510}
10511
10513 EVT VT, SDValue Op1, SDValue Op2,
10514 SDValue Op3) {
10515 SDVTList VTs = getVTList(VT);
10516 SDValue Ops[] = { Op1, Op2, Op3 };
10517 return getMachineNode(Opcode, dl, VTs, Ops);
10518}
10519
10521 EVT VT, ArrayRef<SDValue> Ops) {
10522 SDVTList VTs = getVTList(VT);
10523 return getMachineNode(Opcode, dl, VTs, Ops);
10524}
10525
10527 EVT VT1, EVT VT2, SDValue Op1,
10528 SDValue Op2) {
10529 SDVTList VTs = getVTList(VT1, VT2);
10530 SDValue Ops[] = { Op1, Op2 };
10531 return getMachineNode(Opcode, dl, VTs, Ops);
10532}
10533
10535 EVT VT1, EVT VT2, SDValue Op1,
10536 SDValue Op2, SDValue Op3) {
10537 SDVTList VTs = getVTList(VT1, VT2);
10538 SDValue Ops[] = { Op1, Op2, Op3 };
10539 return getMachineNode(Opcode, dl, VTs, Ops);
10540}
10541
10543 EVT VT1, EVT VT2,
10544 ArrayRef<SDValue> Ops) {
10545 SDVTList VTs = getVTList(VT1, VT2);
10546 return getMachineNode(Opcode, dl, VTs, Ops);
10547}
10548
10550 EVT VT1, EVT VT2, EVT VT3,
10551 SDValue Op1, SDValue Op2) {
10552 SDVTList VTs = getVTList(VT1, VT2, VT3);
10553 SDValue Ops[] = { Op1, Op2 };
10554 return getMachineNode(Opcode, dl, VTs, Ops);
10555}
10556
10558 EVT VT1, EVT VT2, EVT VT3,
10559 SDValue Op1, SDValue Op2,
10560 SDValue Op3) {
10561 SDVTList VTs = getVTList(VT1, VT2, VT3);
10562 SDValue Ops[] = { Op1, Op2, Op3 };
10563 return getMachineNode(Opcode, dl, VTs, Ops);
10564}
10565
10567 EVT VT1, EVT VT2, EVT VT3,
10568 ArrayRef<SDValue> Ops) {
10569 SDVTList VTs = getVTList(VT1, VT2, VT3);
10570 return getMachineNode(Opcode, dl, VTs, Ops);
10571}
10572
10574 ArrayRef<EVT> ResultTys,
10575 ArrayRef<SDValue> Ops) {
10576 SDVTList VTs = getVTList(ResultTys);
10577 return getMachineNode(Opcode, dl, VTs, Ops);
10578}
10579
10581 SDVTList VTs,
10582 ArrayRef<SDValue> Ops) {
10583 bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
10585 void *IP = nullptr;
10586
10587 if (DoCSE) {
10589 AddNodeIDNode(ID, ~Opcode, VTs, Ops);
10590 IP = nullptr;
10591 if (SDNode *E = FindNodeOrInsertPos(ID, DL, IP)) {
10592 return cast<MachineSDNode>(UpdateSDLocOnMergeSDNode(E, DL));
10593 }
10594 }
10595
10596 // Allocate a new MachineSDNode.
10597 N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
10598 createOperands(N, Ops);
10599
10600 if (DoCSE)
10601 CSEMap.InsertNode(N, IP);
10602
10603 InsertNode(N);
10604 NewSDValueDbgMsg(SDValue(N, 0), "Creating new machine node: ", this);
10605 return N;
10606}
10607
10608/// getTargetExtractSubreg - A convenience function for creating
10609/// TargetOpcode::EXTRACT_SUBREG nodes.
10611 SDValue Operand) {
10612 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10613 SDNode *Subreg = getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL,
10614 VT, Operand, SRIdxVal);
10615 return SDValue(Subreg, 0);
10616}
10617
10618/// getTargetInsertSubreg - A convenience function for creating
10619/// TargetOpcode::INSERT_SUBREG nodes.
10621 SDValue Operand, SDValue Subreg) {
10622 SDValue SRIdxVal = getTargetConstant(SRIdx, DL, MVT::i32);
10623 SDNode *Result = getMachineNode(TargetOpcode::INSERT_SUBREG, DL,
10624 VT, Operand, Subreg, SRIdxVal);
10625 return SDValue(Result, 0);
10626}
10627
10628/// getNodeIfExists - Get the specified node if it's already available, or
10629/// else return NULL.
10631 ArrayRef<SDValue> Ops) {
10632 SDNodeFlags Flags;
10633 if (Inserter)
10634 Flags = Inserter->getFlags();
10635 return getNodeIfExists(Opcode, VTList, Ops, Flags);
10636}
10637
10640 const SDNodeFlags Flags) {
10641 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10643 AddNodeIDNode(ID, Opcode, VTList, Ops);
10644 void *IP = nullptr;
10645 if (SDNode *E = FindNodeOrInsertPos(ID, SDLoc(), IP)) {
10646 E->intersectFlagsWith(Flags);
10647 return E;
10648 }
10649 }
10650 return nullptr;
10651}
10652
10653/// doesNodeExist - Check if a node exists without modifying its flags.
10654bool SelectionDAG::doesNodeExist(unsigned Opcode, SDVTList VTList,
10655 ArrayRef<SDValue> Ops) {
10656 if (VTList.VTs[VTList.NumVTs - 1] != MVT::Glue) {
10658 AddNodeIDNode(ID, Opcode, VTList, Ops);
10659 void *IP = nullptr;
10660 if (FindNodeOrInsertPos(ID, SDLoc(), IP))
10661 return true;
10662 }
10663 return false;
10664}
10665
10666/// getDbgValue - Creates a SDDbgValue node.
10667///
10668/// SDNode
10670 SDNode *N, unsigned R, bool IsIndirect,
10671 const DebugLoc &DL, unsigned O) {
10672 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10673 "Expected inlined-at fields to agree");
10674 return new (DbgInfo->getAlloc())
10675 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromNode(N, R),
10676 {}, IsIndirect, DL, O,
10677 /*IsVariadic=*/false);
10678}
10679
10680/// Constant
10682 DIExpression *Expr,
10683 const Value *C,
10684 const DebugLoc &DL, unsigned O) {
10685 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10686 "Expected inlined-at fields to agree");
10687 return new (DbgInfo->getAlloc())
10688 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromConst(C), {},
10689 /*IsIndirect=*/false, DL, O,
10690 /*IsVariadic=*/false);
10691}
10692
10693/// FrameIndex
10695 DIExpression *Expr, unsigned FI,
10696 bool IsIndirect,
10697 const DebugLoc &DL,
10698 unsigned O) {
10699 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10700 "Expected inlined-at fields to agree");
10701 return getFrameIndexDbgValue(Var, Expr, FI, {}, IsIndirect, DL, O);
10702}
10703
10704/// FrameIndex with dependencies
10706 DIExpression *Expr, unsigned FI,
10707 ArrayRef<SDNode *> Dependencies,
10708 bool IsIndirect,
10709 const DebugLoc &DL,
10710 unsigned O) {
10711 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10712 "Expected inlined-at fields to agree");
10713 return new (DbgInfo->getAlloc())
10714 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromFrameIdx(FI),
10715 Dependencies, IsIndirect, DL, O,
10716 /*IsVariadic=*/false);
10717}
10718
10719/// VReg
10721 unsigned VReg, bool IsIndirect,
10722 const DebugLoc &DL, unsigned O) {
10723 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10724 "Expected inlined-at fields to agree");
10725 return new (DbgInfo->getAlloc())
10726 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, SDDbgOperand::fromVReg(VReg),
10727 {}, IsIndirect, DL, O,
10728 /*IsVariadic=*/false);
10729}
10730
10733 ArrayRef<SDNode *> Dependencies,
10734 bool IsIndirect, const DebugLoc &DL,
10735 unsigned O, bool IsVariadic) {
10736 assert(cast<DILocalVariable>(Var)->isValidLocationForIntrinsic(DL) &&
10737 "Expected inlined-at fields to agree");
10738 return new (DbgInfo->getAlloc())
10739 SDDbgValue(DbgInfo->getAlloc(), Var, Expr, Locs, Dependencies, IsIndirect,
10740 DL, O, IsVariadic);
10741}
10742
10744 unsigned OffsetInBits, unsigned SizeInBits,
10745 bool InvalidateDbg) {
10746 SDNode *FromNode = From.getNode();
10747 SDNode *ToNode = To.getNode();
10748 assert(FromNode && ToNode && "Can't modify dbg values");
10749
10750 // PR35338
10751 // TODO: assert(From != To && "Redundant dbg value transfer");
10752 // TODO: assert(FromNode != ToNode && "Intranode dbg value transfer");
10753 if (From == To || FromNode == ToNode)
10754 return;
10755
10756 if (!FromNode->getHasDebugValue())
10757 return;
10758
10759 SDDbgOperand FromLocOp =
10760 SDDbgOperand::fromNode(From.getNode(), From.getResNo());
10762
10764 for (SDDbgValue *Dbg : GetDbgValues(FromNode)) {
10765 if (Dbg->isInvalidated())
10766 continue;
10767
10768 // TODO: assert(!Dbg->isInvalidated() && "Transfer of invalid dbg value");
10769
10770 // Create a new location ops vector that is equal to the old vector, but
10771 // with each instance of FromLocOp replaced with ToLocOp.
10772 bool Changed = false;
10773 auto NewLocOps = Dbg->copyLocationOps();
10774 std::replace_if(
10775 NewLocOps.begin(), NewLocOps.end(),
10776 [&Changed, FromLocOp](const SDDbgOperand &Op) {
10777 bool Match = Op == FromLocOp;
10778 Changed |= Match;
10779 return Match;
10780 },
10781 ToLocOp);
10782 // Ignore this SDDbgValue if we didn't find a matching location.
10783 if (!Changed)
10784 continue;
10785
10786 DIVariable *Var = Dbg->getVariable();
10787 auto *Expr = Dbg->getExpression();
10788 // If a fragment is requested, update the expression.
10789 if (SizeInBits) {
10790 // When splitting a larger (e.g., sign-extended) value whose
10791 // lower bits are described with an SDDbgValue, do not attempt
10792 // to transfer the SDDbgValue to the upper bits.
10793 if (auto FI = Expr->getFragmentInfo())
10794 if (OffsetInBits + SizeInBits > FI->SizeInBits)
10795 continue;
10796 auto Fragment = DIExpression::createFragmentExpression(Expr, OffsetInBits,
10797 SizeInBits);
10798 if (!Fragment)
10799 continue;
10800 Expr = *Fragment;
10801 }
10802
10803 auto AdditionalDependencies = Dbg->getAdditionalDependencies();
10804 // Clone the SDDbgValue and move it to To.
10805 SDDbgValue *Clone = getDbgValueList(
10806 Var, Expr, NewLocOps, AdditionalDependencies, Dbg->isIndirect(),
10807 Dbg->getDebugLoc(), std::max(ToNode->getIROrder(), Dbg->getOrder()),
10808 Dbg->isVariadic());
10809 ClonedDVs.push_back(Clone);
10810
10811 if (InvalidateDbg) {
10812 // Invalidate value and indicate the SDDbgValue should not be emitted.
10813 Dbg->setIsInvalidated();
10814 Dbg->setIsEmitted();
10815 }
10816 }
10817
10818 for (SDDbgValue *Dbg : ClonedDVs) {
10819 assert(is_contained(Dbg->getSDNodes(), ToNode) &&
10820 "Transferred DbgValues should depend on the new SDNode");
10821 AddDbgValue(Dbg, false);
10822 }
10823}
10824
10826 if (!N.getHasDebugValue())
10827 return;
10828
10830 for (auto *DV : GetDbgValues(&N)) {
10831 if (DV->isInvalidated())
10832 continue;
10833 switch (N.getOpcode()) {
10834 default:
10835 break;
10836 case ISD::ADD: {
10837 SDValue N0 = N.getOperand(0);
10838 SDValue N1 = N.getOperand(1);
10839 if (!isa<ConstantSDNode>(N0)) {
10840 bool RHSConstant = isa<ConstantSDNode>(N1);
10842 if (RHSConstant)
10843 Offset = N.getConstantOperandVal(1);
10844 // We are not allowed to turn indirect debug values variadic, so
10845 // don't salvage those.
10846 if (!RHSConstant && DV->isIndirect())
10847 continue;
10848
10849 // Rewrite an ADD constant node into a DIExpression. Since we are
10850 // performing arithmetic to compute the variable's *value* in the
10851 // DIExpression, we need to mark the expression with a
10852 // DW_OP_stack_value.
10853 auto *DIExpr = DV->getExpression();
10854 auto NewLocOps = DV->copyLocationOps();
10855 bool Changed = false;
10856 size_t OrigLocOpsSize = NewLocOps.size();
10857 for (size_t i = 0; i < OrigLocOpsSize; ++i) {
10858 // We're not given a ResNo to compare against because the whole
10859 // node is going away. We know that any ISD::ADD only has one
10860 // result, so we can assume any node match is using the result.
10861 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10862 NewLocOps[i].getSDNode() != &N)
10863 continue;
10864 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10865 if (RHSConstant) {
10868 DIExpr = DIExpression::appendOpsToArg(DIExpr, ExprOps, i, true);
10869 } else {
10870 // Convert to a variadic expression (if not already).
10871 // convertToVariadicExpression() returns a const pointer, so we use
10872 // a temporary const variable here.
10873 const auto *TmpDIExpr =
10877 ExprOps.push_back(NewLocOps.size());
10878 ExprOps.push_back(dwarf::DW_OP_plus);
10881 NewLocOps.push_back(RHS);
10882 DIExpr = DIExpression::appendOpsToArg(TmpDIExpr, ExprOps, i, true);
10883 }
10884 Changed = true;
10885 }
10886 (void)Changed;
10887 assert(Changed && "Salvage target doesn't use N");
10888
10889 bool IsVariadic =
10890 DV->isVariadic() || OrigLocOpsSize != NewLocOps.size();
10891
10892 auto AdditionalDependencies = DV->getAdditionalDependencies();
10893 SDDbgValue *Clone = getDbgValueList(
10894 DV->getVariable(), DIExpr, NewLocOps, AdditionalDependencies,
10895 DV->isIndirect(), DV->getDebugLoc(), DV->getOrder(), IsVariadic);
10896 ClonedDVs.push_back(Clone);
10897 DV->setIsInvalidated();
10898 DV->setIsEmitted();
10899 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting";
10900 N0.getNode()->dumprFull(this);
10901 dbgs() << " into " << *DIExpr << '\n');
10902 }
10903 break;
10904 }
10905 case ISD::TRUNCATE: {
10906 SDValue N0 = N.getOperand(0);
10907 TypeSize FromSize = N0.getValueSizeInBits();
10908 TypeSize ToSize = N.getValueSizeInBits(0);
10909
10910 DIExpression *DbgExpression = DV->getExpression();
10911 auto ExtOps = DIExpression::getExtOps(FromSize, ToSize, false);
10912 auto NewLocOps = DV->copyLocationOps();
10913 bool Changed = false;
10914 for (size_t i = 0; i < NewLocOps.size(); ++i) {
10915 if (NewLocOps[i].getKind() != SDDbgOperand::SDNODE ||
10916 NewLocOps[i].getSDNode() != &N)
10917 continue;
10918
10919 NewLocOps[i] = SDDbgOperand::fromNode(N0.getNode(), N0.getResNo());
10920 DbgExpression = DIExpression::appendOpsToArg(DbgExpression, ExtOps, i);
10921 Changed = true;
10922 }
10923 assert(Changed && "Salvage target doesn't use N");
10924 (void)Changed;
10925
10926 SDDbgValue *Clone =
10927 getDbgValueList(DV->getVariable(), DbgExpression, NewLocOps,
10928 DV->getAdditionalDependencies(), DV->isIndirect(),
10929 DV->getDebugLoc(), DV->getOrder(), DV->isVariadic());
10930
10931 ClonedDVs.push_back(Clone);
10932 DV->setIsInvalidated();
10933 DV->setIsEmitted();
10934 LLVM_DEBUG(dbgs() << "SALVAGE: Rewriting"; N0.getNode()->dumprFull(this);
10935 dbgs() << " into " << *DbgExpression << '\n');
10936 break;
10937 }
10938 }
10939 }
10940
10941 for (SDDbgValue *Dbg : ClonedDVs) {
10942 assert(!Dbg->getSDNodes().empty() &&
10943 "Salvaged DbgValue should depend on a new SDNode");
10944 AddDbgValue(Dbg, false);
10945 }
10946}
10947
10948/// Creates a SDDbgLabel node.
10950 const DebugLoc &DL, unsigned O) {
10951 assert(cast<DILabel>(Label)->isValidLocationForIntrinsic(DL) &&
10952 "Expected inlined-at fields to agree");
10953 return new (DbgInfo->getAlloc()) SDDbgLabel(Label, DL, O);
10954}
10955
10956namespace {
10957
10958/// RAUWUpdateListener - Helper for ReplaceAllUsesWith - When the node
10959/// pointed to by a use iterator is deleted, increment the use iterator
10960/// so that it doesn't dangle.
10961///
10962class RAUWUpdateListener : public SelectionDAG::DAGUpdateListener {
10965
10966 void NodeDeleted(SDNode *N, SDNode *E) override {
10967 // Increment the iterator as needed.
10968 while (UI != UE && N == *UI)
10969 ++UI;
10970 }
10971
10972public:
10973 RAUWUpdateListener(SelectionDAG &d,
10976 : SelectionDAG::DAGUpdateListener(d), UI(ui), UE(ue) {}
10977};
10978
10979} // end anonymous namespace
10980
10981/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
10982/// This can cause recursive merging of nodes in the DAG.
10983///
10984/// This version assumes From has a single result value.
10985///
10987 SDNode *From = FromN.getNode();
10988 assert(From->getNumValues() == 1 && FromN.getResNo() == 0 &&
10989 "Cannot replace with this method!");
10990 assert(From != To.getNode() && "Cannot replace uses of with self");
10991
10992 // Preserve Debug Values
10993 transferDbgValues(FromN, To);
10994 // Preserve extra info.
10995 copyExtraInfo(From, To.getNode());
10996
10997 // Iterate over all the existing uses of From. New uses will be added
10998 // to the beginning of the use list, which we avoid visiting.
10999 // This specifically avoids visiting uses of From that arise while the
11000 // replacement is happening, because any such uses would be the result
11001 // of CSE: If an existing node looks like From after one of its operands
11002 // is replaced by To, we don't want to replace of all its users with To
11003 // too. See PR3018 for more info.
11004 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11005 RAUWUpdateListener Listener(*this, UI, UE);
11006 while (UI != UE) {
11007 SDNode *User = *UI;
11008
11009 // This node is about to morph, remove its old self from the CSE maps.
11010 RemoveNodeFromCSEMaps(User);
11011
11012 // A user can appear in a use list multiple times, and when this
11013 // happens the uses are usually next to each other in the list.
11014 // To help reduce the number of CSE recomputations, process all
11015 // the uses of this user that we can find this way.
11016 do {
11017 SDUse &Use = UI.getUse();
11018 ++UI;
11019 Use.set(To);
11020 if (To->isDivergent() != From->isDivergent())
11022 } while (UI != UE && *UI == User);
11023 // Now that we have modified User, add it back to the CSE maps. If it
11024 // already exists there, recursively merge the results together.
11025 AddModifiedNodeToCSEMaps(User);
11026 }
11027
11028 // If we just RAUW'd the root, take note.
11029 if (FromN == getRoot())
11030 setRoot(To);
11031}
11032
11033/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11034/// This can cause recursive merging of nodes in the DAG.
11035///
11036/// This version assumes that for each value of From, there is a
11037/// corresponding value in To in the same position with the same type.
11038///
11040#ifndef NDEBUG
11041 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11042 assert((!From->hasAnyUseOfValue(i) ||
11043 From->getValueType(i) == To->getValueType(i)) &&
11044 "Cannot use this version of ReplaceAllUsesWith!");
11045#endif
11046
11047 // Handle the trivial case.
11048 if (From == To)
11049 return;
11050
11051 // Preserve Debug Info. Only do this if there's a use.
11052 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i)
11053 if (From->hasAnyUseOfValue(i)) {
11054 assert((i < To->getNumValues()) && "Invalid To location");
11056 }
11057 // Preserve extra info.
11058 copyExtraInfo(From, To);
11059
11060 // Iterate over just the existing users of From. See the comments in
11061 // the ReplaceAllUsesWith above.
11062 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11063 RAUWUpdateListener Listener(*this, UI, UE);
11064 while (UI != UE) {
11065 SDNode *User = *UI;
11066
11067 // This node is about to morph, remove its old self from the CSE maps.
11068 RemoveNodeFromCSEMaps(User);
11069
11070 // A user can appear in a use list multiple times, and when this
11071 // happens the uses are usually next to each other in the list.
11072 // To help reduce the number of CSE recomputations, process all
11073 // the uses of this user that we can find this way.
11074 do {
11075 SDUse &Use = UI.getUse();
11076 ++UI;
11077 Use.setNode(To);
11078 if (To->isDivergent() != From->isDivergent())
11080 } while (UI != UE && *UI == User);
11081
11082 // Now that we have modified User, add it back to the CSE maps. If it
11083 // already exists there, recursively merge the results together.
11084 AddModifiedNodeToCSEMaps(User);
11085 }
11086
11087 // If we just RAUW'd the root, take note.
11088 if (From == getRoot().getNode())
11089 setRoot(SDValue(To, getRoot().getResNo()));
11090}
11091
11092/// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
11093/// This can cause recursive merging of nodes in the DAG.
11094///
11095/// This version can replace From with any result values. To must match the
11096/// number and types of values returned by From.
11098 if (From->getNumValues() == 1) // Handle the simple case efficiently.
11099 return ReplaceAllUsesWith(SDValue(From, 0), To[0]);
11100
11101 for (unsigned i = 0, e = From->getNumValues(); i != e; ++i) {
11102 // Preserve Debug Info.
11103 transferDbgValues(SDValue(From, i), To[i]);
11104 // Preserve extra info.
11105 copyExtraInfo(From, To[i].getNode());
11106 }
11107
11108 // Iterate over just the existing users of From. See the comments in
11109 // the ReplaceAllUsesWith above.
11110 SDNode::use_iterator UI = From->use_begin(), UE = From->use_end();
11111 RAUWUpdateListener Listener(*this, UI, UE);
11112 while (UI != UE) {
11113 SDNode *User = *UI;
11114
11115 // This node is about to morph, remove its old self from the CSE maps.
11116 RemoveNodeFromCSEMaps(User);
11117
11118 // A user can appear in a use list multiple times, and when this happens the
11119 // uses are usually next to each other in the list. To help reduce the
11120 // number of CSE and divergence recomputations, process all the uses of this
11121 // user that we can find this way.
11122 bool To_IsDivergent = false;
11123 do {
11124 SDUse &Use = UI.getUse();
11125 const SDValue &ToOp = To[Use.getResNo()];
11126 ++UI;
11127 Use.set(ToOp);
11128 To_IsDivergent |= ToOp->isDivergent();
11129 } while (UI != UE && *UI == User);
11130
11131 if (To_IsDivergent != From->isDivergent())
11133
11134 // Now that we have modified User, add it back to the CSE maps. If it
11135 // already exists there, recursively merge the results together.
11136 AddModifiedNodeToCSEMaps(User);
11137 }
11138
11139 // If we just RAUW'd the root, take note.
11140 if (From == getRoot().getNode())
11141 setRoot(SDValue(To[getRoot().getResNo()]));
11142}
11143
11144/// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
11145/// uses of other values produced by From.getNode() alone. The Deleted
11146/// vector is handled the same way as for ReplaceAllUsesWith.
11148 // Handle the really simple, really trivial case efficiently.
11149 if (From == To) return;
11150
11151 // Handle the simple, trivial, case efficiently.
11152 if (From.getNode()->getNumValues() == 1) {
11154 return;
11155 }
11156
11157 // Preserve Debug Info.
11159 copyExtraInfo(From.getNode(), To.getNode());
11160
11161 // Iterate over just the existing users of From. See the comments in
11162 // the ReplaceAllUsesWith above.
11163 SDNode::use_iterator UI = From.getNode()->use_begin(),
11164 UE = From.getNode()->use_end();
11165 RAUWUpdateListener Listener(*this, UI, UE);
11166 while (UI != UE) {
11167 SDNode *User = *UI;
11168 bool UserRemovedFromCSEMaps = false;
11169
11170 // A user can appear in a use list multiple times, and when this
11171 // happens the uses are usually next to each other in the list.
11172 // To help reduce the number of CSE recomputations, process all
11173 // the uses of this user that we can find this way.
11174 do {
11175 SDUse &Use = UI.getUse();
11176
11177 // Skip uses of different values from the same node.
11178 if (Use.getResNo() != From.getResNo()) {
11179 ++UI;
11180 continue;
11181 }
11182
11183 // If this node hasn't been modified yet, it's still in the CSE maps,
11184 // so remove its old self from the CSE maps.
11185 if (!UserRemovedFromCSEMaps) {
11186 RemoveNodeFromCSEMaps(User);
11187 UserRemovedFromCSEMaps = true;
11188 }
11189
11190 ++UI;
11191 Use.set(To);
11192 if (To->isDivergent() != From->isDivergent())
11194 } while (UI != UE && *UI == User);
11195 // We are iterating over all uses of the From node, so if a use
11196 // doesn't use the specific value, no changes are made.
11197 if (!UserRemovedFromCSEMaps)
11198 continue;
11199
11200 // Now that we have modified User, add it back to the CSE maps. If it
11201 // already exists there, recursively merge the results together.
11202 AddModifiedNodeToCSEMaps(User);
11203 }
11204
11205 // If we just RAUW'd the root, take note.
11206 if (From == getRoot())
11207 setRoot(To);
11208}
11209
11210namespace {
11211
11212/// UseMemo - This class is used by SelectionDAG::ReplaceAllUsesOfValuesWith
11213/// to record information about a use.
11214struct UseMemo {
11215 SDNode *User;
11216 unsigned Index;
11217 SDUse *Use;
11218};
11219
11220/// operator< - Sort Memos by User.
11221bool operator<(const UseMemo &L, const UseMemo &R) {
11222 return (intptr_t)L.User < (intptr_t)R.User;
11223}
11224
11225/// RAUOVWUpdateListener - Helper for ReplaceAllUsesOfValuesWith - When the node
11226/// pointed to by a UseMemo is deleted, set the User to nullptr to indicate that
11227/// the node already has been taken care of recursively.
11228class RAUOVWUpdateListener : public SelectionDAG::DAGUpdateListener {
11230
11231 void NodeDeleted(SDNode *N, SDNode *E) override {
11232 for (UseMemo &Memo : Uses)
11233 if (Memo.User == N)
11234 Memo.User = nullptr;
11235 }
11236
11237public:
11238 RAUOVWUpdateListener(SelectionDAG &d, SmallVector<UseMemo, 4> &uses)
11239 : SelectionDAG::DAGUpdateListener(d), Uses(uses) {}
11240};
11241
11242} // end anonymous namespace
11243
11245 if (TLI->isSDNodeAlwaysUniform(N)) {
11246 assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
11247 "Conflicting divergence information!");
11248 return false;
11249 }
11250 if (TLI->isSDNodeSourceOfDivergence(N, FLI, UA))
11251 return true;
11252 for (const auto &Op : N->ops()) {
11253 if (Op.Val.getValueType() != MVT::Other && Op.getNode()->isDivergent())
11254 return true;
11255 }
11256 return false;
11257}
11258
11260 SmallVector<SDNode *, 16> Worklist(1, N);
11261 do {
11262 N = Worklist.pop_back_val();
11263 bool IsDivergent = calculateDivergence(N);
11264 if (N->SDNodeBits.IsDivergent != IsDivergent) {
11265 N->SDNodeBits.IsDivergent = IsDivergent;
11266 llvm::append_range(Worklist, N->uses());
11267 }
11268 } while (!Worklist.empty());
11269}
11270
11271void SelectionDAG::CreateTopologicalOrder(std::vector<SDNode *> &Order) {
11273 Order.reserve(AllNodes.size());
11274 for (auto &N : allnodes()) {
11275 unsigned NOps = N.getNumOperands();
11276 Degree[&N] = NOps;
11277 if (0 == NOps)
11278 Order.push_back(&N);
11279 }
11280 for (size_t I = 0; I != Order.size(); ++I) {
11281 SDNode *N = Order[I];
11282 for (auto *U : N->uses()) {
11283 unsigned &UnsortedOps = Degree[U];
11284 if (0 == --UnsortedOps)
11285 Order.push_back(U);
11286 }
11287 }
11288}
11289
11290#ifndef NDEBUG
11292 std::vector<SDNode *> TopoOrder;
11293 CreateTopologicalOrder(TopoOrder);
11294 for (auto *N : TopoOrder) {
11295 assert(calculateDivergence(N) == N->isDivergent() &&
11296 "Divergence bit inconsistency detected");
11297 }
11298}
11299#endif
11300
11301/// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving
11302/// uses of other values produced by From.getNode() alone. The same value
11303/// may appear in both the From and To list. The Deleted vector is
11304/// handled the same way as for ReplaceAllUsesWith.
11306 const SDValue *To,
11307 unsigned Num){
11308 // Handle the simple, trivial case efficiently.
11309 if (Num == 1)
11310 return ReplaceAllUsesOfValueWith(*From, *To);
11311
11312 transferDbgValues(*From, *To);
11313 copyExtraInfo(From->getNode(), To->getNode());
11314
11315 // Read up all the uses and make records of them. This helps
11316 // processing new uses that are introduced during the
11317 // replacement process.
11319 for (unsigned i = 0; i != Num; ++i) {
11320 unsigned FromResNo = From[i].getResNo();
11321 SDNode *FromNode = From[i].getNode();
11322 for (SDNode::use_iterator UI = FromNode->use_begin(),
11323 E = FromNode->use_end(); UI != E; ++UI) {
11324 SDUse &Use = UI.getUse();
11325 if (Use.getResNo() == FromResNo) {
11326 UseMemo Memo = { *UI, i, &Use };
11327 Uses.push_back(Memo);
11328 }
11329 }
11330 }
11331
11332 // Sort the uses, so that all the uses from a given User are together.
11334 RAUOVWUpdateListener Listener(*this, Uses);
11335
11336 for (unsigned UseIndex = 0, UseIndexEnd = Uses.size();
11337 UseIndex != UseIndexEnd; ) {
11338 // We know that this user uses some value of From. If it is the right
11339 // value, update it.
11340 SDNode *User = Uses[UseIndex].User;
11341 // If the node has been deleted by recursive CSE updates when updating
11342 // another node, then just skip this entry.
11343 if (User == nullptr) {
11344 ++UseIndex;
11345 continue;
11346 }
11347
11348 // This node is about to morph, remove its old self from the CSE maps.
11349 RemoveNodeFromCSEMaps(User);
11350
11351 // The Uses array is sorted, so all the uses for a given User
11352 // are next to each other in the list.
11353 // To help reduce the number of CSE recomputations, process all
11354 // the uses of this user that we can find this way.
11355 do {
11356 unsigned i = Uses[UseIndex].Index;
11357 SDUse &Use = *Uses[UseIndex].Use;
11358 ++UseIndex;
11359
11360 Use.set(To[i]);
11361 } while (UseIndex != UseIndexEnd && Uses[UseIndex].User == User);
11362
11363 // Now that we have modified User, add it back to the CSE maps. If it
11364 // already exists there, recursively merge the results together.
11365 AddModifiedNodeToCSEMaps(User);
11366 }
11367}
11368
11369/// AssignTopologicalOrder - Assign a unique node id for each node in the DAG
11370/// based on their topological order. It returns the maximum id and a vector
11371/// of the SDNodes* in assigned order by reference.
11373 unsigned DAGSize = 0;
11374
11375 // SortedPos tracks the progress of the algorithm. Nodes before it are
11376 // sorted, nodes after it are unsorted. When the algorithm completes
11377 // it is at the end of the list.
11378 allnodes_iterator SortedPos = allnodes_begin();
11379
11380 // Visit all the nodes. Move nodes with no operands to the front of
11381 // the list immediately. Annotate nodes that do have operands with their
11382 // operand count. Before we do this, the Node Id fields of the nodes
11383 // may contain arbitrary values. After, the Node Id fields for nodes
11384 // before SortedPos will contain the topological sort index, and the
11385 // Node Id fields for nodes At SortedPos and after will contain the
11386 // count of outstanding operands.
11388 checkForCycles(&N, this);
11389 unsigned Degree = N.getNumOperands();
11390 if (Degree == 0) {
11391 // A node with no uses, add it to the result array immediately.
11392 N.setNodeId(DAGSize++);
11393 allnodes_iterator Q(&N);
11394 if (Q != SortedPos)
11395 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(Q));
11396 assert(SortedPos != AllNodes.end() && "Overran node list");
11397 ++SortedPos;
11398 } else {
11399 // Temporarily use the Node Id as scratch space for the degree count.
11400 N.setNodeId(Degree);
11401 }
11402 }
11403
11404 // Visit all the nodes. As we iterate, move nodes into sorted order,
11405 // such that by the time the end is reached all nodes will be sorted.
11406 for (SDNode &Node : allnodes()) {
11407 SDNode *N = &Node;
11408 checkForCycles(N, this);
11409 // N is in sorted position, so all its uses have one less operand
11410 // that needs to be sorted.
11411 for (SDNode *P : N->uses()) {
11412 unsigned Degree = P->getNodeId();
11413 assert(Degree != 0 && "Invalid node degree");
11414 --Degree;
11415 if (Degree == 0) {
11416 // All of P's operands are sorted, so P may sorted now.
11417 P->setNodeId(DAGSize++);
11418 if (P->getIterator() != SortedPos)
11419 SortedPos = AllNodes.insert(SortedPos, AllNodes.remove(P));
11420 assert(SortedPos != AllNodes.end() && "Overran node list");
11421 ++SortedPos;
11422 } else {
11423 // Update P's outstanding operand count.
11424 P->setNodeId(Degree);
11425 }
11426 }
11427 if (Node.getIterator() == SortedPos) {
11428#ifndef NDEBUG
11430 SDNode *S = &*++I;
11431 dbgs() << "Overran sorted position:\n";
11432 S->dumprFull(this); dbgs() << "\n";
11433 dbgs() << "Checking if this is due to cycles\n";
11434 checkForCycles(this, true);
11435#endif
11436 llvm_unreachable(nullptr);
11437 }
11438 }
11439
11440 assert(SortedPos == AllNodes.end() &&
11441 "Topological sort incomplete!");
11442 assert(AllNodes.front().getOpcode() == ISD::EntryToken &&
11443 "First node in topological sort is not the entry token!");
11444 assert(AllNodes.front().getNodeId() == 0 &&
11445 "First node in topological sort has non-zero id!");
11446 assert(AllNodes.front().getNumOperands() == 0 &&
11447 "First node in topological sort has operands!");
11448 assert(AllNodes.back().getNodeId() == (int)DAGSize-1 &&
11449 "Last node in topologic sort has unexpected id!");
11450 assert(AllNodes.back().use_empty() &&
11451 "Last node in topologic sort has users!");
11452 assert(DAGSize == allnodes_size() && "Node count mismatch!");
11453 return DAGSize;
11454}
11455
11456/// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
11457/// value is produced by SD.
11458void SelectionDAG::AddDbgValue(SDDbgValue *DB, bool isParameter) {
11459 for (SDNode *SD : DB->getSDNodes()) {
11460 if (!SD)
11461 continue;
11462 assert(DbgInfo->getSDDbgValues(SD).empty() || SD->getHasDebugValue());
11463 SD->setHasDebugValue(true);
11464 }
11465 DbgInfo->add(DB, isParameter);
11466}
11467
11468void SelectionDAG::AddDbgLabel(SDDbgLabel *DB) { DbgInfo->add(DB); }
11469
11471 SDValue NewMemOpChain) {
11472 assert(isa<MemSDNode>(NewMemOpChain) && "Expected a memop node");
11473 assert(NewMemOpChain.getValueType() == MVT::Other && "Expected a token VT");
11474 // The new memory operation must have the same position as the old load in
11475 // terms of memory dependency. Create a TokenFactor for the old load and new
11476 // memory operation and update uses of the old load's output chain to use that
11477 // TokenFactor.
11478 if (OldChain == NewMemOpChain || OldChain.use_empty())
11479 return NewMemOpChain;
11480
11481 SDValue TokenFactor = getNode(ISD::TokenFactor, SDLoc(OldChain), MVT::Other,
11482 OldChain, NewMemOpChain);
11483 ReplaceAllUsesOfValueWith(OldChain, TokenFactor);
11484 UpdateNodeOperands(TokenFactor.getNode(), OldChain, NewMemOpChain);
11485 return TokenFactor;
11486}
11487
11489 SDValue NewMemOp) {
11490 assert(isa<MemSDNode>(NewMemOp.getNode()) && "Expected a memop node");
11491 SDValue OldChain = SDValue(OldLoad, 1);
11492 SDValue NewMemOpChain = NewMemOp.getValue(1);
11493 return makeEquivalentMemoryOrdering(OldChain, NewMemOpChain);
11494}
11495
11497 Function **OutFunction) {
11498 assert(isa<ExternalSymbolSDNode>(Op) && "Node should be an ExternalSymbol");
11499
11500 auto *Symbol = cast<ExternalSymbolSDNode>(Op)->getSymbol();
11501 auto *Module = MF->getFunction().getParent();
11502 auto *Function = Module->getFunction(Symbol);
11503
11504 if (OutFunction != nullptr)
11505 *OutFunction = Function;
11506
11507 if (Function != nullptr) {
11508 auto PtrTy = TLI->getPointerTy(getDataLayout(), Function->getAddressSpace());
11509 return getGlobalAddress(Function, SDLoc(Op), PtrTy);
11510 }
11511
11512 std::string ErrorStr;
11513 raw_string_ostream ErrorFormatter(ErrorStr);
11514 ErrorFormatter << "Undefined external symbol ";
11515 ErrorFormatter << '"' << Symbol << '"';
11516 report_fatal_error(Twine(ErrorFormatter.str()));
11517}
11518
11519//===----------------------------------------------------------------------===//
11520// SDNode Class
11521//===----------------------------------------------------------------------===//
11522
11524 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11525 return Const != nullptr && Const->isZero();
11526}
11527
11529 ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(V);
11530 return Const != nullptr && Const->isZero() && !Const->isNegative();
11531}
11532
11534 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11535 return Const != nullptr && Const->isAllOnes();
11536}
11537
11539 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11540 return Const != nullptr && Const->isOne();
11541}
11542
11544 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(V);
11545 return Const != nullptr && Const->isMinSignedValue();
11546}
11547
11548bool llvm::isNeutralConstant(unsigned Opcode, SDNodeFlags Flags, SDValue V,
11549 unsigned OperandNo) {
11550 // NOTE: The cases should match with IR's ConstantExpr::getBinOpIdentity().
11551 // TODO: Target-specific opcodes could be added.
11552 if (auto *Const = isConstOrConstSplat(V)) {
11553 switch (Opcode) {
11554 case ISD::ADD:
11555 case ISD::OR:
11556 case ISD::XOR:
11557 case ISD::UMAX:
11558 return Const->isZero();
11559 case ISD::MUL:
11560 return Const->isOne();
11561 case ISD::AND:
11562 case ISD::UMIN:
11563 return Const->isAllOnes();
11564 case ISD::SMAX:
11565 return Const->isMinSignedValue();
11566 case ISD::SMIN:
11567 return Const->isMaxSignedValue();
11568 case ISD::SUB:
11569 case ISD::SHL:
11570 case ISD::SRA:
11571 case ISD::SRL:
11572 return OperandNo == 1 && Const->isZero();
11573 case ISD::UDIV:
11574 case ISD::SDIV:
11575 return OperandNo == 1 && Const->isOne();
11576 }
11577 } else if (auto *ConstFP = isConstOrConstSplatFP(V)) {
11578 switch (Opcode) {
11579 case ISD::FADD:
11580 return ConstFP->isZero() &&
11581 (Flags.hasNoSignedZeros() || ConstFP->isNegative());
11582 case ISD::FSUB:
11583 return OperandNo == 1 && ConstFP->isZero() &&
11584 (Flags.hasNoSignedZeros() || !ConstFP->isNegative());
11585 case ISD::FMUL:
11586 return ConstFP->isExactlyValue(1.0);
11587 case ISD::FDIV:
11588 return OperandNo == 1 && ConstFP->isExactlyValue(1.0);
11589 case ISD::FMINNUM:
11590 case ISD::FMAXNUM: {
11591 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
11592 EVT VT = V.getValueType();
11594 APFloat NeutralAF = !Flags.hasNoNaNs()
11595 ? APFloat::getQNaN(Semantics)
11596 : !Flags.hasNoInfs()
11597 ? APFloat::getInf(Semantics)
11598 : APFloat::getLargest(Semantics);
11599 if (Opcode == ISD::FMAXNUM)
11600 NeutralAF.changeSign();
11601
11602 return ConstFP->isExactlyValue(NeutralAF);
11603 }
11604 }
11605 }
11606 return false;
11607}
11608
11610 while (V.getOpcode() == ISD::BITCAST)
11611 V = V.getOperand(0);
11612 return V;
11613}
11614
11616 while (V.getOpcode() == ISD::BITCAST && V.getOperand(0).hasOneUse())
11617 V = V.getOperand(0);
11618 return V;
11619}
11620
11622 while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR)
11623 V = V.getOperand(0);
11624 return V;
11625}
11626
11628 while (V.getOpcode() == ISD::TRUNCATE)
11629 V = V.getOperand(0);
11630 return V;
11631}
11632
11633bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) {
11634 if (V.getOpcode() != ISD::XOR)
11635 return false;
11636 V = peekThroughBitcasts(V.getOperand(1));
11637 unsigned NumBits = V.getScalarValueSizeInBits();
11638 ConstantSDNode *C =
11639 isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true);
11640 return C && (C->getAPIntValue().countr_one() >= NumBits);
11641}
11642
11644 bool AllowTruncation) {
11645 EVT VT = N.getValueType();
11646 APInt DemandedElts = VT.isFixedLengthVector()
11648 : APInt(1, 1);
11649 return isConstOrConstSplat(N, DemandedElts, AllowUndefs, AllowTruncation);
11650}
11651
11653 bool AllowUndefs,
11654 bool AllowTruncation) {
11655 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N))
11656 return CN;
11657
11658 // SplatVectors can truncate their operands. Ignore that case here unless
11659 // AllowTruncation is set.
11660 if (N->getOpcode() == ISD::SPLAT_VECTOR) {
11661 EVT VecEltVT = N->getValueType(0).getVectorElementType();
11662 if (auto *CN = dyn_cast<ConstantSDNode>(N->getOperand(0))) {
11663 EVT CVT = CN->getValueType(0);
11664 assert(CVT.bitsGE(VecEltVT) && "Illegal splat_vector element extension");
11665 if (AllowTruncation || CVT == VecEltVT)
11666 return CN;
11667 }
11668 }
11669
11670 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11671 BitVector UndefElements;
11672 ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements);
11673
11674 // BuildVectors can truncate their operands. Ignore that case here unless
11675 // AllowTruncation is set.
11676 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11677 if (CN && (UndefElements.none() || AllowUndefs)) {
11678 EVT CVT = CN->getValueType(0);
11679 EVT NSVT = N.getValueType().getScalarType();
11680 assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension");
11681 if (AllowTruncation || (CVT == NSVT))
11682 return CN;
11683 }
11684 }
11685
11686 return nullptr;
11687}
11688
11690 EVT VT = N.getValueType();
11691 APInt DemandedElts = VT.isFixedLengthVector()
11693 : APInt(1, 1);
11694 return isConstOrConstSplatFP(N, DemandedElts, AllowUndefs);
11695}
11696
11698 const APInt &DemandedElts,
11699 bool AllowUndefs) {
11700 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N))
11701 return CN;
11702
11703 if (BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(N)) {
11704 BitVector UndefElements;
11705 ConstantFPSDNode *CN =
11706 BV->getConstantFPSplatNode(DemandedElts, &UndefElements);
11707 // TODO: Look into whether we should allow UndefElements in non-DemandedElts
11708 if (CN && (UndefElements.none() || AllowUndefs))
11709 return CN;
11710 }
11711
11712 if (N.getOpcode() == ISD::SPLAT_VECTOR)
11713 if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(N.getOperand(0)))
11714 return CN;
11715
11716 return nullptr;
11717}
11718
11719bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) {
11720 // TODO: may want to use peekThroughBitcast() here.
11721 ConstantSDNode *C =
11722 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation=*/true);
11723 return C && C->isZero();
11724}
11725
11726bool llvm::isOneOrOneSplat(SDValue N, bool AllowUndefs) {
11727 ConstantSDNode *C =
11728 isConstOrConstSplat(N, AllowUndefs, /*AllowTruncation*/ true);
11729 return C && C->isOne();
11730}
11731
11732bool llvm::isAllOnesOrAllOnesSplat(SDValue N, bool AllowUndefs) {
11734 unsigned BitWidth = N.getScalarValueSizeInBits();
11735 ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs);
11736 return C && C->isAllOnes() && C->getValueSizeInBits(0) == BitWidth;
11737}
11738
11740 DropOperands();
11741}
11742
11743GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
11744 const DebugLoc &DL,
11745 const GlobalValue *GA, EVT VT,
11746 int64_t o, unsigned TF)
11747 : SDNode(Opc, Order, DL, getSDVTList(VT)), Offset(o), TargetFlags(TF) {
11748 TheGlobal = GA;
11749}
11750
11752 EVT VT, unsigned SrcAS,
11753 unsigned DestAS)
11754 : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
11755 SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
11756
11757MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
11758 SDVTList VTs, EVT memvt, MachineMemOperand *mmo)
11759 : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) {
11760 MemSDNodeBits.IsVolatile = MMO->isVolatile();
11761 MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal();
11762 MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable();
11763 MemSDNodeBits.IsInvariant = MMO->isInvariant();
11764
11765 // We check here that the size of the memory operand fits within the size of
11766 // the MMO. This is because the MMO might indicate only a possible address
11767 // range instead of specifying the affected memory addresses precisely.
11768 assert(
11769 (!MMO->getType().isValid() ||
11771 "Size mismatch!");
11772}
11773
11774/// Profile - Gather unique data for the node.
11775///
11777 AddNodeIDNode(ID, this);
11778}
11779
11780namespace {
11781
11782 struct EVTArray {
11783 std::vector<EVT> VTs;
11784
11785 EVTArray() {
11786 VTs.reserve(MVT::VALUETYPE_SIZE);
11787 for (unsigned i = 0; i < MVT::VALUETYPE_SIZE; ++i)
11788 VTs.push_back(MVT((MVT::SimpleValueType)i));
11789 }
11790 };
11791
11792} // end anonymous namespace
11793
11794/// getValueTypeList - Return a pointer to the specified value type.
11795///
11796const EVT *SDNode::getValueTypeList(EVT VT) {
11797 static std::set<EVT, EVT::compareRawBits> EVTs;
11798 static EVTArray SimpleVTArray;
11799 static sys::SmartMutex<true> VTMutex;
11800
11801 if (VT.isExtended()) {
11802 sys::SmartScopedLock<true> Lock(VTMutex);
11803 return &(*EVTs.insert(VT).first);
11804 }
11805 assert(VT.getSimpleVT() < MVT::VALUETYPE_SIZE && "Value type out of range!");
11806 return &SimpleVTArray.VTs[VT.getSimpleVT().SimpleTy];
11807}
11808
11809/// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
11810/// indicated value. This method ignores uses of other values defined by this
11811/// operation.
11812bool SDNode::hasNUsesOfValue(unsigned NUses, unsigned Value) const {
11813 assert(Value < getNumValues() && "Bad value!");
11814
11815 // TODO: Only iterate over uses of a given value of the node
11816 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI) {
11817 if (UI.getUse().getResNo() == Value) {
11818 if (NUses == 0)
11819 return false;
11820 --NUses;
11821 }
11822 }
11823
11824 // Found exactly the right number of uses?
11825 return NUses == 0;
11826}
11827
11828/// hasAnyUseOfValue - Return true if there are any use of the indicated
11829/// value. This method ignores uses of other values defined by this operation.
11830bool SDNode::hasAnyUseOfValue(unsigned Value) const {
11831 assert(Value < getNumValues() && "Bad value!");
11832
11833 for (SDNode::use_iterator UI = use_begin(), E = use_end(); UI != E; ++UI)
11834 if (UI.getUse().getResNo() == Value)
11835 return true;
11836
11837 return false;
11838}
11839
11840/// isOnlyUserOf - Return true if this node is the only use of N.
11841bool SDNode::isOnlyUserOf(const SDNode *N) const {
11842 bool Seen = false;
11843 for (const SDNode *User : N->uses()) {
11844 if (User == this)
11845 Seen = true;
11846 else
11847 return false;
11848 }
11849
11850 return Seen;
11851}
11852
11853/// Return true if the only users of N are contained in Nodes.
11855 bool Seen = false;
11856 for (const SDNode *User : N->uses()) {
11857 if (llvm::is_contained(Nodes, User))
11858 Seen = true;
11859 else
11860 return false;
11861 }
11862
11863 return Seen;
11864}
11865
11866/// isOperand - Return true if this node is an operand of N.
11867bool SDValue::isOperandOf(const SDNode *N) const {
11868 return is_contained(N->op_values(), *this);
11869}
11870
11871bool SDNode::isOperandOf(const SDNode *N) const {
11872 return any_of(N->op_values(),
11873 [this](SDValue Op) { return this == Op.getNode(); });
11874}
11875
11876/// reachesChainWithoutSideEffects - Return true if this operand (which must
11877/// be a chain) reaches the specified operand without crossing any
11878/// side-effecting instructions on any chain path. In practice, this looks
11879/// through token factors and non-volatile loads. In order to remain efficient,
11880/// this only looks a couple of nodes in, it does not do an exhaustive search.
11881///
11882/// Note that we only need to examine chains when we're searching for
11883/// side-effects; SelectionDAG requires that all side-effects are represented
11884/// by chains, even if another operand would force a specific ordering. This
11885/// constraint is necessary to allow transformations like splitting loads.
11887 unsigned Depth) const {
11888 if (*this == Dest) return true;
11889
11890 // Don't search too deeply, we just want to be able to see through
11891 // TokenFactor's etc.
11892 if (Depth == 0) return false;
11893
11894 // If this is a token factor, all inputs to the TF happen in parallel.
11895 if (getOpcode() == ISD::TokenFactor) {
11896 // First, try a shallow search.
11897 if (is_contained((*this)->ops(), Dest)) {
11898 // We found the chain we want as an operand of this TokenFactor.
11899 // Essentially, we reach the chain without side-effects if we could
11900 // serialize the TokenFactor into a simple chain of operations with
11901 // Dest as the last operation. This is automatically true if the
11902 // chain has one use: there are no other ordering constraints.
11903 // If the chain has more than one use, we give up: some other
11904 // use of Dest might force a side-effect between Dest and the current
11905 // node.
11906 if (Dest.hasOneUse())
11907 return true;
11908 }
11909 // Next, try a deep search: check whether every operand of the TokenFactor
11910 // reaches Dest.
11911 return llvm::all_of((*this)->ops(), [=](SDValue Op) {
11912 return Op.reachesChainWithoutSideEffects(Dest, Depth - 1);
11913 });
11914 }
11915
11916 // Loads don't have side effects, look through them.
11917 if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(*this)) {
11918 if (Ld->isUnordered())
11919 return Ld->getChain().reachesChainWithoutSideEffects(Dest, Depth-1);
11920 }
11921 return false;
11922}
11923
11924bool SDNode::hasPredecessor(const SDNode *N) const {
11927 Worklist.push_back(this);
11928 return hasPredecessorHelper(N, Visited, Worklist);
11929}
11930
11932 this->Flags.intersectWith(Flags);
11933}
11934
11935SDValue
11937 ArrayRef<ISD::NodeType> CandidateBinOps,
11938 bool AllowPartials) {
11939 // The pattern must end in an extract from index 0.
11940 if (Extract->getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
11941 !isNullConstant(Extract->getOperand(1)))
11942 return SDValue();
11943
11944 // Match against one of the candidate binary ops.
11945 SDValue Op = Extract->getOperand(0);
11946 if (llvm::none_of(CandidateBinOps, [Op](ISD::NodeType BinOp) {
11947 return Op.getOpcode() == unsigned(BinOp);
11948 }))
11949 return SDValue();
11950
11951 // Floating-point reductions may require relaxed constraints on the final step
11952 // of the reduction because they may reorder intermediate operations.
11953 unsigned CandidateBinOp = Op.getOpcode();
11954 if (Op.getValueType().isFloatingPoint()) {
11955 SDNodeFlags Flags = Op->getFlags();
11956 switch (CandidateBinOp) {
11957 case ISD::FADD:
11958 if (!Flags.hasNoSignedZeros() || !Flags.hasAllowReassociation())
11959 return SDValue();
11960 break;
11961 default:
11962 llvm_unreachable("Unhandled FP opcode for binop reduction");
11963 }
11964 }
11965
11966 // Matching failed - attempt to see if we did enough stages that a partial
11967 // reduction from a subvector is possible.
11968 auto PartialReduction = [&](SDValue Op, unsigned NumSubElts) {
11969 if (!AllowPartials || !Op)
11970 return SDValue();
11971 EVT OpVT = Op.getValueType();
11972 EVT OpSVT = OpVT.getScalarType();
11973 EVT SubVT = EVT::getVectorVT(*getContext(), OpSVT, NumSubElts);
11974 if (!TLI->isExtractSubvectorCheap(SubVT, OpVT, 0))
11975 return SDValue();
11976 BinOp = (ISD::NodeType)CandidateBinOp;
11977 return getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Op), SubVT, Op,
11979 };
11980
11981 // At each stage, we're looking for something that looks like:
11982 // %s = shufflevector <8 x i32> %op, <8 x i32> undef,
11983 // <8 x i32> <i32 2, i32 3, i32 undef, i32 undef,
11984 // i32 undef, i32 undef, i32 undef, i32 undef>
11985 // %a = binop <8 x i32> %op, %s
11986 // Where the mask changes according to the stage. E.g. for a 3-stage pyramid,
11987 // we expect something like:
11988 // <4,5,6,7,u,u,u,u>
11989 // <2,3,u,u,u,u,u,u>
11990 // <1,u,u,u,u,u,u,u>
11991 // While a partial reduction match would be:
11992 // <2,3,u,u,u,u,u,u>
11993 // <1,u,u,u,u,u,u,u>
11994 unsigned Stages = Log2_32(Op.getValueType().getVectorNumElements());
11995 SDValue PrevOp;
11996 for (unsigned i = 0; i < Stages; ++i) {
11997 unsigned MaskEnd = (1 << i);
11998
11999 if (Op.getOpcode() != CandidateBinOp)
12000 return PartialReduction(PrevOp, MaskEnd);
12001
12002 SDValue Op0 = Op.getOperand(0);
12003 SDValue Op1 = Op.getOperand(1);
12004
12005 ShuffleVectorSDNode *Shuffle = dyn_cast<ShuffleVectorSDNode>(Op0);
12006 if (Shuffle) {
12007 Op = Op1;
12008 } else {
12009 Shuffle = dyn_cast<ShuffleVectorSDNode>(Op1);
12010 Op = Op0;
12011 }
12012
12013 // The first operand of the shuffle should be the same as the other operand
12014 // of the binop.
12015 if (!Shuffle || Shuffle->getOperand(0) != Op)
12016 return PartialReduction(PrevOp, MaskEnd);
12017
12018 // Verify the shuffle has the expected (at this stage of the pyramid) mask.
12019 for (int Index = 0; Index < (int)MaskEnd; ++Index)
12020 if (Shuffle->getMaskElt(Index) != (int)(MaskEnd + Index))
12021 return PartialReduction(PrevOp, MaskEnd);
12022
12023 PrevOp = Op;
12024 }
12025
12026 // Handle subvector reductions, which tend to appear after the shuffle
12027 // reduction stages.
12028 while (Op.getOpcode() == CandidateBinOp) {
12029 unsigned NumElts = Op.getValueType().getVectorNumElements();
12030 SDValue Op0 = Op.getOperand(0);
12031 SDValue Op1 = Op.getOperand(1);
12032 if (Op0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12034 Op0.getOperand(0) != Op1.getOperand(0))
12035 break;
12036 SDValue Src = Op0.getOperand(0);
12037 unsigned NumSrcElts = Src.getValueType().getVectorNumElements();
12038 if (NumSrcElts != (2 * NumElts))
12039 break;
12040 if (!(Op0.getConstantOperandAPInt(1) == 0 &&
12041 Op1.getConstantOperandAPInt(1) == NumElts) &&
12042 !(Op1.getConstantOperandAPInt(1) == 0 &&
12043 Op0.getConstantOperandAPInt(1) == NumElts))
12044 break;
12045 Op = Src;
12046 }
12047
12048 BinOp = (ISD::NodeType)CandidateBinOp;
12049 return Op;
12050}
12051
12053 EVT VT = N->getValueType(0);
12054 EVT EltVT = VT.getVectorElementType();
12055 unsigned NE = VT.getVectorNumElements();
12056
12057 SDLoc dl(N);
12058
12059 // If ResNE is 0, fully unroll the vector op.
12060 if (ResNE == 0)
12061 ResNE = NE;
12062 else if (NE > ResNE)
12063 NE = ResNE;
12064
12065 if (N->getNumValues() == 2) {
12066 SmallVector<SDValue, 8> Scalars0, Scalars1;
12067 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12068 EVT VT1 = N->getValueType(1);
12069 EVT EltVT1 = VT1.getVectorElementType();
12070
12071 unsigned i;
12072 for (i = 0; i != NE; ++i) {
12073 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12074 SDValue Operand = N->getOperand(j);
12075 EVT OperandVT = Operand.getValueType();
12076
12077 // A vector operand; extract a single element.
12078 EVT OperandEltVT = OperandVT.getVectorElementType();
12079 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12080 Operand, getVectorIdxConstant(i, dl));
12081 }
12082
12083 SDValue EltOp = getNode(N->getOpcode(), dl, {EltVT, EltVT1}, Operands);
12084 Scalars0.push_back(EltOp);
12085 Scalars1.push_back(EltOp.getValue(1));
12086 }
12087
12088 SDValue Vec0 = getBuildVector(VT, dl, Scalars0);
12089 SDValue Vec1 = getBuildVector(VT1, dl, Scalars1);
12090 return getMergeValues({Vec0, Vec1}, dl);
12091 }
12092
12093 assert(N->getNumValues() == 1 &&
12094 "Can't unroll a vector with multiple results!");
12095
12097 SmallVector<SDValue, 4> Operands(N->getNumOperands());
12098
12099 unsigned i;
12100 for (i= 0; i != NE; ++i) {
12101 for (unsigned j = 0, e = N->getNumOperands(); j != e; ++j) {
12102 SDValue Operand = N->getOperand(j);
12103 EVT OperandVT = Operand.getValueType();
12104 if (OperandVT.isVector()) {
12105 // A vector operand; extract a single element.
12106 EVT OperandEltVT = OperandVT.getVectorElementType();
12107 Operands[j] = getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT,
12108 Operand, getVectorIdxConstant(i, dl));
12109 } else {
12110 // A scalar operand; just use it as is.
12111 Operands[j] = Operand;
12112 }
12113 }
12114
12115 switch (N->getOpcode()) {
12116 default: {
12117 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands,
12118 N->getFlags()));
12119 break;
12120 }
12121 case ISD::VSELECT:
12122 Scalars.push_back(getNode(ISD::SELECT, dl, EltVT, Operands));
12123 break;
12124 case ISD::SHL:
12125 case ISD::SRA:
12126 case ISD::SRL:
12127 case ISD::ROTL:
12128 case ISD::ROTR:
12129 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT, Operands[0],
12131 Operands[1])));
12132 break;
12134 EVT ExtVT = cast<VTSDNode>(Operands[1])->getVT().getVectorElementType();
12135 Scalars.push_back(getNode(N->getOpcode(), dl, EltVT,
12136 Operands[0],
12137 getValueType(ExtVT)));
12138 }
12139 }
12140 }
12141
12142 for (; i < ResNE; ++i)
12143 Scalars.push_back(getUNDEF(EltVT));
12144
12145 EVT VecVT = EVT::getVectorVT(*getContext(), EltVT, ResNE);
12146 return getBuildVector(VecVT, dl, Scalars);
12147}
12148
12149std::pair<SDValue, SDValue> SelectionDAG::UnrollVectorOverflowOp(
12150 SDNode *N, unsigned ResNE) {
12151 unsigned Opcode = N->getOpcode();
12152 assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO ||
12153 Opcode == ISD::USUBO || Opcode == ISD::SSUBO ||
12154 Opcode == ISD::UMULO || Opcode == ISD::SMULO) &&
12155 "Expected an overflow opcode");
12156
12157 EVT ResVT = N->getValueType(0);
12158 EVT OvVT = N->getValueType(1);
12159 EVT ResEltVT = ResVT.getVectorElementType();
12160 EVT OvEltVT = OvVT.getVectorElementType();
12161 SDLoc dl(N);
12162
12163 // If ResNE is 0, fully unroll the vector op.
12164 unsigned NE = ResVT.getVectorNumElements();
12165 if (ResNE == 0)
12166 ResNE = NE;
12167 else if (NE > ResNE)
12168 NE = ResNE;
12169
12170 SmallVector<SDValue, 8> LHSScalars;
12171 SmallVector<SDValue, 8> RHSScalars;
12172 ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE);
12173 ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE);
12174
12175 EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT);
12176 SDVTList VTs = getVTList(ResEltVT, SVT);
12177 SmallVector<SDValue, 8> ResScalars;
12178 SmallVector<SDValue, 8> OvScalars;
12179 for (unsigned i = 0; i < NE; ++i) {
12180 SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]);
12181 SDValue Ov =
12182 getSelect(dl, OvEltVT, Res.getValue(1),
12183 getBoolConstant(true, dl, OvEltVT, ResVT),
12184 getConstant(0, dl, OvEltVT));
12185
12186 ResScalars.push_back(Res);
12187 OvScalars.push_back(Ov);
12188 }
12189
12190 ResScalars.append(ResNE - NE, getUNDEF(ResEltVT));
12191 OvScalars.append(ResNE - NE, getUNDEF(OvEltVT));
12192
12193 EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE);
12194 EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE);
12195 return std::make_pair(getBuildVector(NewResVT, dl, ResScalars),
12196 getBuildVector(NewOvVT, dl, OvScalars));
12197}
12198
12201 unsigned Bytes,
12202 int Dist) const {
12203 if (LD->isVolatile() || Base->isVolatile())
12204 return false;
12205 // TODO: probably too restrictive for atomics, revisit
12206 if (!LD->isSimple())
12207 return false;
12208 if (LD->isIndexed() || Base->isIndexed())
12209 return false;
12210 if (LD->getChain() != Base->getChain())
12211 return false;
12212 EVT VT = LD->getMemoryVT();
12213 if (VT.getSizeInBits() / 8 != Bytes)
12214 return false;
12215
12216 auto BaseLocDecomp = BaseIndexOffset::match(Base, *this);
12217 auto LocDecomp = BaseIndexOffset::match(LD, *this);
12218
12219 int64_t Offset = 0;
12220 if (BaseLocDecomp.equalBaseIndex(LocDecomp, *this, Offset))
12221 return (Dist * (int64_t)Bytes == Offset);
12222 return false;
12223}
12224
12225/// InferPtrAlignment - Infer alignment of a load / store address. Return
12226/// std::nullopt if it cannot be inferred.
12228 // If this is a GlobalAddress + cst, return the alignment.
12229 const GlobalValue *GV = nullptr;
12230 int64_t GVOffset = 0;
12231 if (TLI->isGAPlusOffset(Ptr.getNode(), GV, GVOffset)) {
12232 unsigned PtrWidth = getDataLayout().getPointerTypeSizeInBits(GV->getType());
12233 KnownBits Known(PtrWidth);
12235 unsigned AlignBits = Known.countMinTrailingZeros();
12236 if (AlignBits)
12237 return commonAlignment(Align(1ull << std::min(31U, AlignBits)), GVOffset);
12238 }
12239
12240 // If this is a direct reference to a stack slot, use information about the
12241 // stack slot's alignment.
12242 int FrameIdx = INT_MIN;
12243 int64_t FrameOffset = 0;
12244 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Ptr)) {
12245 FrameIdx = FI->getIndex();
12246 } else if (isBaseWithConstantOffset(Ptr) &&
12247 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
12248 // Handle FI+Cst
12249 FrameIdx = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
12250 FrameOffset = Ptr.getConstantOperandVal(1);
12251 }
12252
12253 if (FrameIdx != INT_MIN) {
12255 return commonAlignment(MFI.getObjectAlign(FrameIdx), FrameOffset);
12256 }
12257
12258 return std::nullopt;
12259}
12260
12261/// Split the scalar node with EXTRACT_ELEMENT using the provided
12262/// VTs and return the low/high part.
12263std::pair<SDValue, SDValue> SelectionDAG::SplitScalar(const SDValue &N,
12264 const SDLoc &DL,
12265 const EVT &LoVT,
12266 const EVT &HiVT) {
12267 assert(!LoVT.isVector() && !HiVT.isVector() && !N.getValueType().isVector() &&
12268 "Split node must be a scalar type");
12269 SDValue Lo =
12271 SDValue Hi =
12273 return std::make_pair(Lo, Hi);
12274}
12275
12276/// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
12277/// which is split (or expanded) into two not necessarily identical pieces.
12278std::pair<EVT, EVT> SelectionDAG::GetSplitDestVTs(const EVT &VT) const {
12279 // Currently all types are split in half.
12280 EVT LoVT, HiVT;
12281 if (!VT.isVector())
12282 LoVT = HiVT = TLI->getTypeToTransformTo(*getContext(), VT);
12283 else
12284 LoVT = HiVT = VT.getHalfNumVectorElementsVT(*getContext());
12285
12286 return std::make_pair(LoVT, HiVT);
12287}
12288
12289/// GetDependentSplitDestVTs - Compute the VTs needed for the low/hi parts of a
12290/// type, dependent on an enveloping VT that has been split into two identical
12291/// pieces. Sets the HiIsEmpty flag when hi type has zero storage size.
12292std::pair<EVT, EVT>
12294 bool *HiIsEmpty) const {
12295 EVT EltTp = VT.getVectorElementType();
12296 // Examples:
12297 // custom VL=8 with enveloping VL=8/8 yields 8/0 (hi empty)
12298 // custom VL=9 with enveloping VL=8/8 yields 8/1
12299 // custom VL=10 with enveloping VL=8/8 yields 8/2
12300 // etc.
12301 ElementCount VTNumElts = VT.getVectorElementCount();
12302 ElementCount EnvNumElts = EnvVT.getVectorElementCount();
12303 assert(VTNumElts.isScalable() == EnvNumElts.isScalable() &&
12304 "Mixing fixed width and scalable vectors when enveloping a type");
12305 EVT LoVT, HiVT;
12306 if (VTNumElts.getKnownMinValue() > EnvNumElts.getKnownMinValue()) {
12307 LoVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12308 HiVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts - EnvNumElts);
12309 *HiIsEmpty = false;
12310 } else {
12311 // Flag that hi type has zero storage size, but return split envelop type
12312 // (this would be easier if vector types with zero elements were allowed).
12313 LoVT = EVT::getVectorVT(*getContext(), EltTp, VTNumElts);
12314 HiVT = EVT::getVectorVT(*getContext(), EltTp, EnvNumElts);
12315 *HiIsEmpty = true;
12316 }
12317 return std::make_pair(LoVT, HiVT);
12318}
12319
12320/// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
12321/// low/high part.
12322std::pair<SDValue, SDValue>
12323SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT,
12324 const EVT &HiVT) {
12325 assert(LoVT.isScalableVector() == HiVT.isScalableVector() &&
12326 LoVT.isScalableVector() == N.getValueType().isScalableVector() &&
12327 "Splitting vector with an invalid mixture of fixed and scalable "
12328 "vector types");
12330 N.getValueType().getVectorMinNumElements() &&
12331 "More vector elements requested than available!");
12332 SDValue Lo, Hi;
12333 Lo =
12335 // For scalable vectors it is safe to use LoVT.getVectorMinNumElements()
12336 // (rather than having to use ElementCount), because EXTRACT_SUBVECTOR scales
12337 // IDX with the runtime scaling factor of the result vector type. For
12338 // fixed-width result vectors, that runtime scaling factor is 1.
12341 return std::make_pair(Lo, Hi);
12342}
12343
12344std::pair<SDValue, SDValue> SelectionDAG::SplitEVL(SDValue N, EVT VecVT,
12345 const SDLoc &DL) {
12346 // Split the vector length parameter.
12347 // %evl -> umin(%evl, %halfnumelts) and usubsat(%evl - %halfnumelts).
12348 EVT VT = N.getValueType();
12350 "Expecting the mask to be an evenly-sized vector");
12351 unsigned HalfMinNumElts = VecVT.getVectorMinNumElements() / 2;
12352 SDValue HalfNumElts =
12353 VecVT.isFixedLengthVector()
12354 ? getConstant(HalfMinNumElts, DL, VT)
12355 : getVScale(DL, VT, APInt(VT.getScalarSizeInBits(), HalfMinNumElts));
12356 SDValue Lo = getNode(ISD::UMIN, DL, VT, N, HalfNumElts);
12357 SDValue Hi = getNode(ISD::USUBSAT, DL, VT, N, HalfNumElts);
12358 return std::make_pair(Lo, Hi);
12359}
12360
12361/// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
12363 EVT VT = N.getValueType();
12366 return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N,
12368}
12369
12372 unsigned Start, unsigned Count,
12373 EVT EltVT) {
12374 EVT VT = Op.getValueType();
12375 if (Count == 0)
12376 Count = VT.getVectorNumElements();
12377 if (EltVT == EVT())
12378 EltVT = VT.getVectorElementType();
12379 SDLoc SL(Op);
12380 for (unsigned i = Start, e = Start + Count; i != e; ++i) {
12381 Args.push_back(getNode(ISD::EXTRACT_VECTOR_ELT, SL, EltVT, Op,
12382 getVectorIdxConstant(i, SL)));
12383 }
12384}
12385
12386// getAddressSpace - Return the address space this GlobalAddress belongs to.
12388 return getGlobal()->getType()->getAddressSpace();
12389}
12390
12393 return Val.MachineCPVal->getType();
12394 return Val.ConstVal->getType();
12395}
12396
12397bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
12398 unsigned &SplatBitSize,
12399 bool &HasAnyUndefs,
12400 unsigned MinSplatBits,
12401 bool IsBigEndian) const {
12402 EVT VT = getValueType(0);
12403 assert(VT.isVector() && "Expected a vector type");
12404 unsigned VecWidth = VT.getSizeInBits();
12405 if (MinSplatBits > VecWidth)
12406 return false;
12407
12408 // FIXME: The widths are based on this node's type, but build vectors can
12409 // truncate their operands.
12410 SplatValue = APInt(VecWidth, 0);
12411 SplatUndef = APInt(VecWidth, 0);
12412
12413 // Get the bits. Bits with undefined values (when the corresponding element
12414 // of the vector is an ISD::UNDEF value) are set in SplatUndef and cleared
12415 // in SplatValue. If any of the values are not constant, give up and return
12416 // false.
12417 unsigned int NumOps = getNumOperands();
12418 assert(NumOps > 0 && "isConstantSplat has 0-size build vector");
12419 unsigned EltWidth = VT.getScalarSizeInBits();
12420
12421 for (unsigned j = 0; j < NumOps; ++j) {
12422 unsigned i = IsBigEndian ? NumOps - 1 - j : j;
12423 SDValue OpVal = getOperand(i);
12424 unsigned BitPos = j * EltWidth;
12425
12426 if (OpVal.isUndef())
12427 SplatUndef.setBits(BitPos, BitPos + EltWidth);
12428 else if (auto *CN = dyn_cast<ConstantSDNode>(OpVal))
12429 SplatValue.insertBits(CN->getAPIntValue().zextOrTrunc(EltWidth), BitPos);
12430 else if (auto *CN = dyn_cast<ConstantFPSDNode>(OpVal))
12431 SplatValue.insertBits(CN->getValueAPF().bitcastToAPInt(), BitPos);
12432 else
12433 return false;
12434 }
12435
12436 // The build_vector is all constants or undefs. Find the smallest element
12437 // size that splats the vector.
12438 HasAnyUndefs = (SplatUndef != 0);
12439
12440 // FIXME: This does not work for vectors with elements less than 8 bits.
12441 while (VecWidth > 8) {
12442 // If we can't split in half, stop here.
12443 if (VecWidth & 1)
12444 break;
12445
12446 unsigned HalfSize = VecWidth / 2;
12447 APInt HighValue = SplatValue.extractBits(HalfSize, HalfSize);
12448 APInt LowValue = SplatValue.extractBits(HalfSize, 0);
12449 APInt HighUndef = SplatUndef.extractBits(HalfSize, HalfSize);
12450 APInt LowUndef = SplatUndef.extractBits(HalfSize, 0);
12451
12452 // If the two halves do not match (ignoring undef bits), stop here.
12453 if ((HighValue & ~LowUndef) != (LowValue & ~HighUndef) ||
12454 MinSplatBits > HalfSize)
12455 break;
12456
12457 SplatValue = HighValue | LowValue;
12458 SplatUndef = HighUndef & LowUndef;
12459
12460 VecWidth = HalfSize;
12461 }
12462
12463 // FIXME: The loop above only tries to split in halves. But if the input
12464 // vector for example is <3 x i16> it wouldn't be able to detect a
12465 // SplatBitSize of 16. No idea if that is a design flaw currently limiting
12466 // optimizations. I guess that back in the days when this helper was created
12467 // vectors normally was power-of-2 sized.
12468
12469 SplatBitSize = VecWidth;
12470 return true;
12471}
12472
12474 BitVector *UndefElements) const {
12475 unsigned NumOps = getNumOperands();
12476 if (UndefElements) {
12477 UndefElements->clear();
12478 UndefElements->resize(NumOps);
12479 }
12480 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12481 if (!DemandedElts)
12482 return SDValue();
12483 SDValue Splatted;
12484 for (unsigned i = 0; i != NumOps; ++i) {
12485 if (!DemandedElts[i])
12486 continue;
12487 SDValue Op = getOperand(i);
12488 if (Op.isUndef()) {
12489 if (UndefElements)
12490 (*UndefElements)[i] = true;
12491 } else if (!Splatted) {
12492 Splatted = Op;
12493 } else if (Splatted != Op) {
12494 return SDValue();
12495 }
12496 }
12497
12498 if (!Splatted) {
12499 unsigned FirstDemandedIdx = DemandedElts.countr_zero();
12500 assert(getOperand(FirstDemandedIdx).isUndef() &&
12501 "Can only have a splat without a constant for all undefs.");
12502 return getOperand(FirstDemandedIdx);
12503 }
12504
12505 return Splatted;
12506}
12507
12509 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12510 return getSplatValue(DemandedElts, UndefElements);
12511}
12512
12514 SmallVectorImpl<SDValue> &Sequence,
12515 BitVector *UndefElements) const {
12516 unsigned NumOps = getNumOperands();
12517 Sequence.clear();
12518 if (UndefElements) {
12519 UndefElements->clear();
12520 UndefElements->resize(NumOps);
12521 }
12522 assert(NumOps == DemandedElts.getBitWidth() && "Unexpected vector size");
12523 if (!DemandedElts || NumOps < 2 || !isPowerOf2_32(NumOps))
12524 return false;
12525
12526 // Set the undefs even if we don't find a sequence (like getSplatValue).
12527 if (UndefElements)
12528 for (unsigned I = 0; I != NumOps; ++I)
12529 if (DemandedElts[I] && getOperand(I).isUndef())
12530 (*UndefElements)[I] = true;
12531
12532 // Iteratively widen the sequence length looking for repetitions.
12533 for (unsigned SeqLen = 1; SeqLen < NumOps; SeqLen *= 2) {
12534 Sequence.append(SeqLen, SDValue());
12535 for (unsigned I = 0; I != NumOps; ++I) {
12536 if (!DemandedElts[I])
12537 continue;
12538 SDValue &SeqOp = Sequence[I % SeqLen];
12540 if (Op.isUndef()) {
12541 if (!SeqOp)
12542 SeqOp = Op;
12543 continue;
12544 }
12545 if (SeqOp && !SeqOp.isUndef() && SeqOp != Op) {
12546 Sequence.clear();
12547 break;
12548 }
12549 SeqOp = Op;
12550 }
12551 if (!Sequence.empty())
12552 return true;
12553 }
12554
12555 assert(Sequence.empty() && "Failed to empty non-repeating sequence pattern");
12556 return false;
12557}
12558
12560 BitVector *UndefElements) const {
12561 APInt DemandedElts = APInt::getAllOnes(getNumOperands());
12562 return getRepeatedSequence(DemandedElts, Sequence, UndefElements);
12563}
12564
12567 BitVector *UndefElements) const {
12568 return dyn_cast_or_null<ConstantSDNode>(
12569 getSplatValue(DemandedElts, UndefElements));
12570}
12571
12574 return dyn_cast_or_null<ConstantSDNode>(getSplatValue(UndefElements));
12575}
12576
12579 BitVector *UndefElements) const {
12580 return dyn_cast_or_null<ConstantFPSDNode>(
12581 getSplatValue(DemandedElts, UndefElements));
12582}
12583
12586 return dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements));
12587}
12588
12589int32_t
12591 uint32_t BitWidth) const {
12592 if (ConstantFPSDNode *CN =
12593 dyn_cast_or_null<ConstantFPSDNode>(getSplatValue(UndefElements))) {
12594 bool IsExact;
12595 APSInt IntVal(BitWidth);
12596 const APFloat &APF = CN->getValueAPF();
12597 if (APF.convertToInteger(IntVal, APFloat::rmTowardZero, &IsExact) !=
12598 APFloat::opOK ||
12599 !IsExact)
12600 return -1;
12601
12602 return IntVal.exactLogBase2();
12603 }
12604 return -1;
12605}
12606
12608 bool IsLittleEndian, unsigned DstEltSizeInBits,
12609 SmallVectorImpl<APInt> &RawBitElements, BitVector &UndefElements) const {
12610 // Early-out if this contains anything but Undef/Constant/ConstantFP.
12611 if (!isConstant())
12612 return false;
12613
12614 unsigned NumSrcOps = getNumOperands();
12615 unsigned SrcEltSizeInBits = getValueType(0).getScalarSizeInBits();
12616 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12617 "Invalid bitcast scale");
12618
12619 // Extract raw src bits.
12620 SmallVector<APInt> SrcBitElements(NumSrcOps,
12621 APInt::getZero(SrcEltSizeInBits));
12622 BitVector SrcUndeElements(NumSrcOps, false);
12623
12624 for (unsigned I = 0; I != NumSrcOps; ++I) {
12626 if (Op.isUndef()) {
12627 SrcUndeElements.set(I);
12628 continue;
12629 }
12630 auto *CInt = dyn_cast<ConstantSDNode>(Op);
12631 auto *CFP = dyn_cast<ConstantFPSDNode>(Op);
12632 assert((CInt || CFP) && "Unknown constant");
12633 SrcBitElements[I] = CInt ? CInt->getAPIntValue().trunc(SrcEltSizeInBits)
12634 : CFP->getValueAPF().bitcastToAPInt();
12635 }
12636
12637 // Recast to dst width.
12638 recastRawBits(IsLittleEndian, DstEltSizeInBits, RawBitElements,
12639 SrcBitElements, UndefElements, SrcUndeElements);
12640 return true;
12641}
12642
12643void BuildVectorSDNode::recastRawBits(bool IsLittleEndian,
12644 unsigned DstEltSizeInBits,
12645 SmallVectorImpl<APInt> &DstBitElements,
12646 ArrayRef<APInt> SrcBitElements,
12647 BitVector &DstUndefElements,
12648 const BitVector &SrcUndefElements) {
12649 unsigned NumSrcOps = SrcBitElements.size();
12650 unsigned SrcEltSizeInBits = SrcBitElements[0].getBitWidth();
12651 assert(((NumSrcOps * SrcEltSizeInBits) % DstEltSizeInBits) == 0 &&
12652 "Invalid bitcast scale");
12653 assert(NumSrcOps == SrcUndefElements.size() &&
12654 "Vector size mismatch");
12655
12656 unsigned NumDstOps = (NumSrcOps * SrcEltSizeInBits) / DstEltSizeInBits;
12657 DstUndefElements.clear();
12658 DstUndefElements.resize(NumDstOps, false);
12659 DstBitElements.assign(NumDstOps, APInt::getZero(DstEltSizeInBits));
12660
12661 // Concatenate src elements constant bits together into dst element.
12662 if (SrcEltSizeInBits <= DstEltSizeInBits) {
12663 unsigned Scale = DstEltSizeInBits / SrcEltSizeInBits;
12664 for (unsigned I = 0; I != NumDstOps; ++I) {
12665 DstUndefElements.set(I);
12666 APInt &DstBits = DstBitElements[I];
12667 for (unsigned J = 0; J != Scale; ++J) {
12668 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12669 if (SrcUndefElements[Idx])
12670 continue;
12671 DstUndefElements.reset(I);
12672 const APInt &SrcBits = SrcBitElements[Idx];
12673 assert(SrcBits.getBitWidth() == SrcEltSizeInBits &&
12674 "Illegal constant bitwidths");
12675 DstBits.insertBits(SrcBits, J * SrcEltSizeInBits);
12676 }
12677 }
12678 return;
12679 }
12680
12681 // Split src element constant bits into dst elements.
12682 unsigned Scale = SrcEltSizeInBits / DstEltSizeInBits;
12683 for (unsigned I = 0; I != NumSrcOps; ++I) {
12684 if (SrcUndefElements[I]) {
12685 DstUndefElements.set(I * Scale, (I + 1) * Scale);
12686 continue;
12687 }
12688 const APInt &SrcBits = SrcBitElements[I];
12689 for (unsigned J = 0; J != Scale; ++J) {
12690 unsigned Idx = (I * Scale) + (IsLittleEndian ? J : (Scale - J - 1));
12691 APInt &DstBits = DstBitElements[Idx];
12692 DstBits = SrcBits.extractBits(DstEltSizeInBits, J * DstEltSizeInBits);
12693 }
12694 }
12695}
12696
12698 for (const SDValue &Op : op_values()) {
12699 unsigned Opc = Op.getOpcode();
12700 if (Opc != ISD::UNDEF && Opc != ISD::Constant && Opc != ISD::ConstantFP)
12701 return false;
12702 }
12703 return true;
12704}
12705
12706std::optional<std::pair<APInt, APInt>>
12708 unsigned NumOps = getNumOperands();
12709 if (NumOps < 2)
12710 return std::nullopt;
12711
12712 if (!isa<ConstantSDNode>(getOperand(0)) ||
12713 !isa<ConstantSDNode>(getOperand(1)))
12714 return std::nullopt;
12715
12716 unsigned EltSize = getValueType(0).getScalarSizeInBits();
12717 APInt Start = getConstantOperandAPInt(0).trunc(EltSize);
12718 APInt Stride = getConstantOperandAPInt(1).trunc(EltSize) - Start;
12719
12720 if (Stride.isZero())
12721 return std::nullopt;
12722
12723 for (unsigned i = 2; i < NumOps; ++i) {
12724 if (!isa<ConstantSDNode>(getOperand(i)))
12725 return std::nullopt;
12726
12727 APInt Val = getConstantOperandAPInt(i).trunc(EltSize);
12728 if (Val != (Start + (Stride * i)))
12729 return std::nullopt;
12730 }
12731
12732 return std::make_pair(Start, Stride);
12733}
12734
12735bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) {
12736 // Find the first non-undef value in the shuffle mask.
12737 unsigned i, e;
12738 for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i)
12739 /* search */;
12740
12741 // If all elements are undefined, this shuffle can be considered a splat
12742 // (although it should eventually get simplified away completely).
12743 if (i == e)
12744 return true;
12745
12746 // Make sure all remaining elements are either undef or the same as the first
12747 // non-undef value.
12748 for (int Idx = Mask[i]; i != e; ++i)
12749 if (Mask[i] >= 0 && Mask[i] != Idx)
12750 return false;
12751 return true;
12752}
12753
12754// Returns the SDNode if it is a constant integer BuildVector
12755// or constant integer.
12757 if (isa<ConstantSDNode>(N))
12758 return N.getNode();
12760 return N.getNode();
12761 // Treat a GlobalAddress supporting constant offset folding as a
12762 // constant integer.
12763 if (GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(N))
12764 if (GA->getOpcode() == ISD::GlobalAddress &&
12765 TLI->isOffsetFoldingLegal(GA))
12766 return GA;
12767 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12768 isa<ConstantSDNode>(N.getOperand(0)))
12769 return N.getNode();
12770 return nullptr;
12771}
12772
12773// Returns the SDNode if it is a constant float BuildVector
12774// or constant float.
12776 if (isa<ConstantFPSDNode>(N))
12777 return N.getNode();
12778
12780 return N.getNode();
12781
12782 if ((N.getOpcode() == ISD::SPLAT_VECTOR) &&
12783 isa<ConstantFPSDNode>(N.getOperand(0)))
12784 return N.getNode();
12785
12786 return nullptr;
12787}
12788
12789void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
12790 assert(!Node->OperandList && "Node already has operands");
12792 "too many operands to fit into SDNode");
12793 SDUse *Ops = OperandRecycler.allocate(
12794 ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
12795
12796 bool IsDivergent = false;
12797 for (unsigned I = 0; I != Vals.size(); ++I) {
12798 Ops[I].setUser(Node);
12799 Ops[I].setInitial(Vals[I]);
12800 if (Ops[I].Val.getValueType() != MVT::Other) // Skip Chain. It does not carry divergence.
12801 IsDivergent |= Ops[I].getNode()->isDivergent();
12802 }
12803 Node->NumOperands = Vals.size();
12804 Node->OperandList = Ops;
12805 if (!TLI->isSDNodeAlwaysUniform(Node)) {
12806 IsDivergent |= TLI->isSDNodeSourceOfDivergence(Node, FLI, UA);
12807 Node->SDNodeBits.IsDivergent = IsDivergent;
12808 }
12810}
12811
12814 size_t Limit = SDNode::getMaxNumOperands();
12815 while (Vals.size() > Limit) {
12816 unsigned SliceIdx = Vals.size() - Limit;
12817 auto ExtractedTFs = ArrayRef<SDValue>(Vals).slice(SliceIdx, Limit);
12818 SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs);
12819 Vals.erase(Vals.begin() + SliceIdx, Vals.end());
12820 Vals.emplace_back(NewTF);
12821 }
12822 return getNode(ISD::TokenFactor, DL, MVT::Other, Vals);
12823}
12824
12826 EVT VT, SDNodeFlags Flags) {
12827 switch (Opcode) {
12828 default:
12829 return SDValue();
12830 case ISD::ADD:
12831 case ISD::OR:
12832 case ISD::XOR:
12833 case ISD::UMAX:
12834 return getConstant(0, DL, VT);
12835 case ISD::MUL:
12836 return getConstant(1, DL, VT);
12837 case ISD::AND:
12838 case ISD::UMIN:
12839 return getAllOnesConstant(DL, VT);
12840 case ISD::SMAX:
12842 case ISD::SMIN:
12844 case ISD::FADD:
12845 return getConstantFP(-0.0, DL, VT);
12846 case ISD::FMUL:
12847 return getConstantFP(1.0, DL, VT);
12848 case ISD::FMINNUM:
12849 case ISD::FMAXNUM: {
12850 // Neutral element for fminnum is NaN, Inf or FLT_MAX, depending on FMF.
12851 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12852 APFloat NeutralAF = !Flags.hasNoNaNs() ? APFloat::getQNaN(Semantics) :
12853 !Flags.hasNoInfs() ? APFloat::getInf(Semantics) :
12854 APFloat::getLargest(Semantics);
12855 if (Opcode == ISD::FMAXNUM)
12856 NeutralAF.changeSign();
12857
12858 return getConstantFP(NeutralAF, DL, VT);
12859 }
12860 case ISD::FMINIMUM:
12861 case ISD::FMAXIMUM: {
12862 // Neutral element for fminimum is Inf or FLT_MAX, depending on FMF.
12863 const fltSemantics &Semantics = EVTToAPFloatSemantics(VT);
12864 APFloat NeutralAF = !Flags.hasNoInfs() ? APFloat::getInf(Semantics)
12865 : APFloat::getLargest(Semantics);
12866 if (Opcode == ISD::FMAXIMUM)
12867 NeutralAF.changeSign();
12868
12869 return getConstantFP(NeutralAF, DL, VT);
12870 }
12871
12872 }
12873}
12874
12875/// Helper used to make a call to a library function that has one argument of
12876/// pointer type.
12877///
12878/// Such functions include 'fegetmode', 'fesetenv' and some others, which are
12879/// used to get or set floating-point state. They have one argument of pointer
12880/// type, which points to the memory region containing bits of the
12881/// floating-point state. The value returned by such function is ignored in the
12882/// created call.
12883///
12884/// \param LibFunc Reference to library function (value of RTLIB::Libcall).
12885/// \param Ptr Pointer used to save/load state.
12886/// \param InChain Ingoing token chain.
12887/// \returns Outgoing chain token.
12889 SDValue InChain,
12890 const SDLoc &DLoc) {
12891 assert(InChain.getValueType() == MVT::Other && "Expected token chain");
12894 Entry.Node = Ptr;
12895 Entry.Ty = Ptr.getValueType().getTypeForEVT(*getContext());
12896 Args.push_back(Entry);
12897 RTLIB::Libcall LC = static_cast<RTLIB::Libcall>(LibFunc);
12898 SDValue Callee = getExternalSymbol(TLI->getLibcallName(LC),
12899 TLI->getPointerTy(getDataLayout()));
12901 CLI.setDebugLoc(DLoc).setChain(InChain).setLibCallee(
12902 TLI->getLibcallCallingConv(LC), Type::getVoidTy(*getContext()), Callee,
12903 std::move(Args));
12904 return TLI->LowerCallTo(CLI).second;
12905}
12906
12908 assert(From && To && "Invalid SDNode; empty source SDValue?");
12909 auto I = SDEI.find(From);
12910 if (I == SDEI.end())
12911 return;
12912
12913 // Use of operator[] on the DenseMap may cause an insertion, which invalidates
12914 // the iterator, hence the need to make a copy to prevent a use-after-free.
12915 NodeExtraInfo NEI = I->second;
12916 if (LLVM_LIKELY(!NEI.PCSections)) {
12917 // No deep copy required for the types of extra info set.
12918 //
12919 // FIXME: Investigate if other types of extra info also need deep copy. This
12920 // depends on the types of nodes they can be attached to: if some extra info
12921 // is only ever attached to nodes where a replacement To node is always the
12922 // node where later use and propagation of the extra info has the intended
12923 // semantics, no deep copy is required.
12924 SDEI[To] = std::move(NEI);
12925 return;
12926 }
12927
12928 // We need to copy NodeExtraInfo to all _new_ nodes that are being introduced
12929 // through the replacement of From with To. Otherwise, replacements of a node
12930 // (From) with more complex nodes (To and its operands) may result in lost
12931 // extra info where the root node (To) is insignificant in further propagating
12932 // and using extra info when further lowering to MIR.
12933 //
12934 // In the first step pre-populate the visited set with the nodes reachable
12935 // from the old From node. This avoids copying NodeExtraInfo to parts of the
12936 // DAG that is not new and should be left untouched.
12937 SmallVector<const SDNode *> Leafs{From}; // Leafs reachable with VisitFrom.
12938 DenseSet<const SDNode *> FromReach; // The set of nodes reachable from From.
12939 auto VisitFrom = [&](auto &&Self, const SDNode *N, int MaxDepth) {
12940 if (MaxDepth == 0) {
12941 // Remember this node in case we need to increase MaxDepth and continue
12942 // populating FromReach from this node.
12943 Leafs.emplace_back(N);
12944 return;
12945 }
12946 if (!FromReach.insert(N).second)
12947 return;
12948 for (const SDValue &Op : N->op_values())
12949 Self(Self, Op.getNode(), MaxDepth - 1);
12950 };
12951
12952 // Copy extra info to To and all its transitive operands (that are new).
12954 auto DeepCopyTo = [&](auto &&Self, const SDNode *N) {
12955 if (FromReach.contains(N))
12956 return true;
12957 if (!Visited.insert(N).second)
12958 return true;
12959 if (getEntryNode().getNode() == N)
12960 return false;
12961 for (const SDValue &Op : N->op_values()) {
12962 if (!Self(Self, Op.getNode()))
12963 return false;
12964 }
12965 // Copy only if entry node was not reached.
12966 SDEI[N] = NEI;
12967 return true;
12968 };
12969
12970 // We first try with a lower MaxDepth, assuming that the path to common
12971 // operands between From and To is relatively short. This significantly
12972 // improves performance in the common case. The initial MaxDepth is big
12973 // enough to avoid retry in the common case; the last MaxDepth is large
12974 // enough to avoid having to use the fallback below (and protects from
12975 // potential stack exhaustion from recursion).
12976 for (int PrevDepth = 0, MaxDepth = 16; MaxDepth <= 1024;
12977 PrevDepth = MaxDepth, MaxDepth *= 2, Visited.clear()) {
12978 // StartFrom is the previous (or initial) set of leafs reachable at the
12979 // previous maximum depth.
12981 std::swap(StartFrom, Leafs);
12982 for (const SDNode *N : StartFrom)
12983 VisitFrom(VisitFrom, N, MaxDepth - PrevDepth);
12984 if (LLVM_LIKELY(DeepCopyTo(DeepCopyTo, To)))
12985 return;
12986 // This should happen very rarely (reached the entry node).
12987 LLVM_DEBUG(dbgs() << __func__ << ": MaxDepth=" << MaxDepth << " too low\n");
12988 assert(!Leafs.empty());
12989 }
12990
12991 // This should not happen - but if it did, that means the subgraph reachable
12992 // from From has depth greater or equal to maximum MaxDepth, and VisitFrom()
12993 // could not visit all reachable common operands. Consequently, we were able
12994 // to reach the entry node.
12995 errs() << "warning: incomplete propagation of SelectionDAG::NodeExtraInfo\n";
12996 assert(false && "From subgraph too complex - increase max. MaxDepth?");
12997 // Best-effort fallback if assertions disabled.
12998 SDEI[To] = std::move(NEI);
12999}
13000
13001#ifndef NDEBUG
13002static void checkForCyclesHelper(const SDNode *N,
13005 const llvm::SelectionDAG *DAG) {
13006 // If this node has already been checked, don't check it again.
13007 if (Checked.count(N))
13008 return;
13009
13010 // If a node has already been visited on this depth-first walk, reject it as
13011 // a cycle.
13012 if (!Visited.insert(N).second) {
13013 errs() << "Detected cycle in SelectionDAG\n";
13014 dbgs() << "Offending node:\n";
13015 N->dumprFull(DAG); dbgs() << "\n";
13016 abort();
13017 }
13018
13019 for (const SDValue &Op : N->op_values())
13020 checkForCyclesHelper(Op.getNode(), Visited, Checked, DAG);
13021
13022 Checked.insert(N);
13023 Visited.erase(N);
13024}
13025#endif
13026
13028 const llvm::SelectionDAG *DAG,
13029 bool force) {
13030#ifndef NDEBUG
13031 bool check = force;
13032#ifdef EXPENSIVE_CHECKS
13033 check = true;
13034#endif // EXPENSIVE_CHECKS
13035 if (check) {
13036 assert(N && "Checking nonexistent SDNode");
13039 checkForCyclesHelper(N, visited, checked, DAG);
13040 }
13041#endif // !NDEBUG
13042}
13043
13044void llvm::checkForCycles(const llvm::SelectionDAG *DAG, bool force) {
13045 checkForCycles(DAG->getRoot().getNode(), DAG, force);
13046}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
This file implements the APSInt class, which is a simple class that represents an arbitrary sized int...
This file implements the BitVector class.
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
#define __asan_unpoison_memory_region(p, size)
Definition: Compiler.h:460
#define LLVM_LIKELY(EXPR)
Definition: Compiler.h:240
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Looks at all the uses of the given value Returns the Liveness deduced from the uses of this value Adds all uses that cause the result to be MaybeLive to MaybeLiveRetUses If the result is MaybeLiveUses might be modified but its content should be ignored(since it might not be complete). DeadArgumentEliminationPass
Given that RA is a live propagate it s liveness to any other values it uses(according to Uses). void DeadArgumentEliminationPass
Given that RA is a live value
#define LLVM_DEBUG(X)
Definition: Debug.h:101
This file defines the DenseSet and SmallDenseSet classes.
This file contains constants used for implementing Dwarf debug support.
uint64_t Size
Symbol * Sym
Definition: ELF_riscv.cpp:479
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
This file defines a hash set that can be used to remove duplication of nodes in a graph.
Rewrite Partial Register Uses
#define check(cond)
static const unsigned MaxDepth
static LVOptions Options
Definition: LVOptions.cpp:25
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
Definition: Lint.cpp:531
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
mir Rename Register Operands
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
This file provides utility analysis objects describing memory locations.
This file contains the declarations for metadata subclasses.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
#define P(N)
PowerPC Reduce CR logical Operation
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
static uint64_t umul_ov(uint64_t i, uint64_t j, bool &Overflow)
static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo)
Lower the call to 'memset' intrinsic function into a series of store operations.
static std::optional< APInt > FoldValueWithUndef(unsigned Opcode, const APInt &C1, bool IsUndef1, const APInt &C2, bool IsUndef2)
static SDValue FoldSTEP_VECTOR(const SDLoc &DL, EVT VT, SDValue Step, SelectionDAG &DAG)
static void AddNodeIDNode(FoldingSetNodeID &ID, unsigned OpC, SDVTList VTList, ArrayRef< SDValue > OpList)
static SDValue getMemsetStringVal(EVT VT, const SDLoc &dl, SelectionDAG &DAG, const TargetLowering &TLI, const ConstantDataArraySlice &Slice)
getMemsetStringVal - Similar to getMemsetValue.
static cl::opt< bool > EnableMemCpyDAGOpt("enable-memcpy-dag-opt", cl::Hidden, cl::init(true), cl::desc("Gang up loads and stores generated by inlining of memcpy"))
static bool haveNoCommonBitsSetCommutative(SDValue A, SDValue B)
static void AddNodeIDValueTypes(FoldingSetNodeID &ID, SDVTList VTList)
AddNodeIDValueTypes - Value type lists are intern'd so we can represent them solely with their pointe...
static void commuteShuffle(SDValue &N1, SDValue &N2, MutableArrayRef< int > M)
Swaps the values of N1 and N2.
static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice)
Returns true if memcpy source is constant data.
static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo)
static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA)
static void AddNodeIDOpcode(FoldingSetNodeID &ID, unsigned OpC)
AddNodeIDOpcode - Add the node opcode to the NodeID data.
static ISD::CondCode getSetCCInverseImpl(ISD::CondCode Op, bool isIntegerLike)
static bool doNotCSE(SDNode *N)
doNotCSE - Return true if CSE should not be performed for this node.
static cl::opt< int > MaxLdStGlue("ldstmemcpy-glue-max", cl::desc("Number limit for gluing ld/st of memcpy."), cl::Hidden, cl::init(0))
static void AddNodeIDOperands(FoldingSetNodeID &ID, ArrayRef< SDValue > Ops)
AddNodeIDOperands - Various routines for adding operands to the NodeID data.
static void VerifySDNode(SDNode *N)
VerifySDNode - Check the given SDNode. Aborts if it is invalid.
static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
Try to simplify vector concatenation to an input value, undef, or build vector.
static MachinePointerInfo InferPointerInfo(const MachinePointerInfo &Info, SelectionDAG &DAG, SDValue Ptr, int64_t Offset=0)
InferPointerInfo - If the specified ptr/offset is a frame index, infer a MachinePointerInfo record fr...
static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N)
If this is an SDNode with special info, add this info to the NodeID data.
static void NewSDValueDbgMsg(SDValue V, StringRef Msg, SelectionDAG *G)
static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs)
makeVTList - Return an instance of the SDVTList struct initialized with the specified members.
static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallPtrSetImpl< const SDNode * > &Checked, const llvm::SelectionDAG *DAG)
static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SmallVector< SDValue, 32 > &OutChains, unsigned From, unsigned To, SmallVector< SDValue, 16 > &OutLoadChains, SmallVector< SDValue, 16 > &OutStoreChains)
static int isSignedOp(ISD::CondCode Opcode)
For an integer comparison, return 1 if the comparison is a signed operation and 2 if the result is an...
static std::optional< APInt > FoldValue(unsigned Opcode, const APInt &C1, const APInt &C2)
static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG)
static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, unsigned AS)
static Constant * ConstantFold(Instruction *I, const DataLayout &DL, const SmallDenseMap< Value *, Constant * > &ConstantPool)
Try to fold instruction I into a constant.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR)
Convert ConstantRange OverflowResult into ValueTracking OverflowResult.
Value * RHS
Value * LHS
static unsigned getSize(unsigned Kind)
bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal=false)
Checks whether the given location points to constant memory, or if OrLocal is true whether it points ...
static APFloat getQNaN(const fltSemantics &Sem, bool Negative=false, const APInt *payload=nullptr)
Factory for QNaN values.
Definition: APFloat.h:988
opStatus divide(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1069
void copySign(const APFloat &RHS)
Definition: APFloat.h:1163
opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)
Definition: APFloat.cpp:5196
opStatus subtract(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1051
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
Definition: APFloat.h:1278
opStatus add(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1042
bool isFinite() const
Definition: APFloat.h:1300
bool isNaN() const
Definition: APFloat.h:1293
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
Definition: APFloat.h:1193
opStatus multiply(const APFloat &RHS, roundingMode RM)
Definition: APFloat.h:1060
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
Definition: APFloat.h:1096
bool isZero() const
Definition: APFloat.h:1291
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
Definition: APFloat.h:1006
opStatus convertToInteger(MutableArrayRef< integerPart > Input, unsigned int Width, bool IsSigned, roundingMode RM, bool *IsExact) const
Definition: APFloat.h:1185
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
opStatus mod(const APFloat &RHS)
Definition: APFloat.h:1087
bool isPosZero() const
Definition: APFloat.h:1306
bool isNegZero() const
Definition: APFloat.h:1307
void changeSign()
Definition: APFloat.h:1158
static APFloat getNaN(const fltSemantics &Sem, bool Negative=false, uint64_t payload=0)
Factory for NaN values.
Definition: APFloat.h:977
bool isInfinity() const
Definition: APFloat.h:1292
Class for arbitrary precision integers.
Definition: APInt.h:76
APInt umul_ov(const APInt &RHS, bool &Overflow) const
Definition: APInt.cpp:1977
APInt usub_sat(const APInt &RHS) const
Definition: APInt.cpp:2061
APInt udiv(const APInt &RHS) const
Unsigned division operation.
Definition: APInt.cpp:1579
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:981
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1491
void setHighBits(unsigned hiBits)
Set the top hiBits bits.
Definition: APInt.h:1370
unsigned popcount() const
Count the number of bits set.
Definition: APInt.h:1620
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
Definition: APInt.h:1364
APInt getHiBits(unsigned numBits) const
Compute an APInt containing numBits highbits from this APInt.
Definition: APInt.cpp:608
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition: APInt.cpp:1002
unsigned getActiveBits() const
Compute the number of active bits in the value.
Definition: APInt.h:1463
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:906
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1308
APInt abs() const
Get the absolute value.
Definition: APInt.h:1737
APInt sadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2032
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition: APInt.h:349
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition: APInt.h:358
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition: APInt.cpp:1672
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1439
bool ult(const APInt &RHS) const
Unsigned less than comparison.
Definition: APInt.h:1089
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
bool isNegative() const
Determine sign of this APInt.
Definition: APInt.h:307
APInt sdiv(const APInt &RHS) const
Signed division function for APInt.
Definition: APInt.cpp:1650
void clearAllBits()
Set every bit to 0.
Definition: APInt.h:1375
APInt rotr(unsigned rotateAmt) const
Rotate right by rotateAmt.
Definition: APInt.cpp:1124
APInt reverseBits() const
Definition: APInt.cpp:737
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:812
bool sle(const APInt &RHS) const
Signed less or equal comparison.
Definition: APInt.h:1144
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition: APInt.h:1589
unsigned getNumSignBits() const
Computes the number of leading bits of this APInt that are equal to its sign bit.
Definition: APInt.h:1578
unsigned countl_zero() const
The APInt version of std::countl_zero.
Definition: APInt.h:1548
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
Definition: APInt.cpp:620
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
Definition: APInt.h:197
APInt sshl_sat(const APInt &RHS) const
Definition: APInt.cpp:2092
APInt ushl_sat(const APInt &RHS) const
Definition: APInt.cpp:2106
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
Definition: APInt.cpp:1010
APInt rotl(unsigned rotateAmt) const
Rotate left by rotateAmt.
Definition: APInt.cpp:1111
void insertBits(const APInt &SubBits, unsigned bitPosition)
Insert the bits from a smaller APInt starting at bitPosition.
Definition: APInt.cpp:368
void clearLowBits(unsigned loBits)
Set bottom loBits bits to 0.
Definition: APInt.h:1395
unsigned logBase2() const
Definition: APInt.h:1703
APInt uadd_sat(const APInt &RHS) const
Definition: APInt.cpp:2042
APInt ashr(unsigned ShiftAmt) const
Arithmetic right-shift function.
Definition: APInt.h:805
void setAllBits()
Set every bit to 1.
Definition: APInt.h:1297
APInt srem(const APInt &RHS) const
Function for signed remainder operation.
Definition: APInt.cpp:1742
bool isNonNegative() const
Determine if this APInt Value is non-negative (>= 0)
Definition: APInt.h:312
bool ule(const APInt &RHS) const
Unsigned less or equal comparison.
Definition: APInt.h:1128
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:954
void setBits(unsigned loBit, unsigned hiBit)
Set the bits from loBit (inclusive) to hiBit (exclusive) to 1.
Definition: APInt.h:1345
APInt shl(unsigned shiftAmt) const
Left-shift function.
Definition: APInt.h:851
APInt byteSwap() const
Definition: APInt.cpp:715
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1235
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
Definition: APInt.h:418
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition: APInt.h:284
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition: APInt.h:178
void setLowBits(unsigned loBits)
Set the bottom loBits bits.
Definition: APInt.h:1367
APInt extractBits(unsigned numBits, unsigned bitPosition) const
Return an APInt with the extracted bits [bitPosition,bitPosition+numBits).
Definition: APInt.cpp:453
bool sge(const APInt &RHS) const
Signed greater or equal comparison.
Definition: APInt.h:1215
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition: APInt.h:264
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
void lshrInPlace(unsigned ShiftAmt)
Logical right-shift this APInt by ShiftAmt in place.
Definition: APInt.h:836
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:829
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
Definition: APInt.h:1199
APInt ssub_sat(const APInt &RHS) const
Definition: APInt.cpp:2051
An arbitrary precision integer that knows its signedness.
Definition: APSInt.h:23
AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT, unsigned SrcAS, unsigned DestAS)
Recycle small arrays allocated from a BumpPtrAllocator.
Definition: ArrayRecycler.h:28
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:195
This is an SDNode representing atomic operations.
static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG)
Parses tree in N for base, index, offset addresses.
BitVector & reset()
Definition: BitVector.h:392
void resize(unsigned N, bool t=false)
resize - Grow or shrink the bitvector.
Definition: BitVector.h:341
void clear()
clear - Removes all bits from the bitvector.
Definition: BitVector.h:335
BitVector & set()
Definition: BitVector.h:351
bool none() const
none - Returns true if none of the bits are set.
Definition: BitVector.h:188
size_type size() const
size - Returns the number of bits in this bitvector.
Definition: BitVector.h:159
unsigned getTargetFlags() const
const BlockAddress * getBlockAddress() const
The address of a basic block.
Definition: Constants.h:889
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
A "pseudo-class" with methods for operating on BUILD_VECTORs.
bool getConstantRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &RawBitElements, BitVector &UndefElements) const
Extract the raw bit data from a build vector of Undef, Constant or ConstantFP node elements.
static void recastRawBits(bool IsLittleEndian, unsigned DstEltSizeInBits, SmallVectorImpl< APInt > &DstBitElements, ArrayRef< APInt > SrcBitElements, BitVector &DstUndefElements, const BitVector &SrcUndefElements)
Recast bit data SrcBitElements to DstEltSizeInBits wide elements.
bool getRepeatedSequence(const APInt &DemandedElts, SmallVectorImpl< SDValue > &Sequence, BitVector *UndefElements=nullptr) const
Find the shortest repeating sequence of values in the build vector.
ConstantFPSDNode * getConstantFPSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant FP or null if this is not a constant FP splat.
std::optional< std::pair< APInt, APInt > > isConstantSequence() const
If this BuildVector is constant and represents the numerical series "<a, a+n, a+2n,...
SDValue getSplatValue(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted value or a null value if this is not a splat.
bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
ConstantSDNode * getConstantSplatNode(const APInt &DemandedElts, BitVector *UndefElements=nullptr) const
Returns the demanded splatted constant or null if this is not a constant splat.
int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements, uint32_t BitWidth) const
If this is a constant FP splat and the splatted constant FP is an exact power or 2,...
LLVM_ATTRIBUTE_RETURNS_NONNULL void * Allocate(size_t Size, Align Alignment)
Allocate space at the specified alignment.
Definition: Allocator.h:148
void Reset()
Deallocate all but the current slab and reset the current pointer to the beginning of it,...
Definition: Allocator.h:123
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
bool isExactlyValue(double V) const
We don't rely on operator== working on double values, as it returns true for things that are clearly ...
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:268
This is the shared class of boolean and integer constants.
Definition: Constants.h:80
unsigned getBitWidth() const
getBitWidth - Return the scalar bitwidth of this constant.
Definition: Constants.h:148
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:145
bool isMachineConstantPoolEntry() const
This class represents a range of values.
Definition: ConstantRange.h:47
ConstantRange multiply(const ConstantRange &Other) const
Return a new range representing the possible values resulting from a multiplication of a value in thi...
const APInt * getSingleElement() const
If this set contains a single element, return it, otherwise return null.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const
Return whether unsigned sub of the two ranges always/never overflows.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
KnownBits toKnownBits() const
Return known bits for values in this range.
ConstantRange zeroExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getSignedMin() const
Return the smallest signed value contained in the ConstantRange.
OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const
Return whether unsigned mul of the two ranges always/never overflows.
ConstantRange signExtend(uint32_t BitWidth) const
Return a new range in the specified integer type, which must be strictly larger than the current type...
APInt getUnsignedMax() const
Return the largest unsigned value contained in the ConstantRange.
APInt getSignedMax() const
Return the largest signed value contained in the ConstantRange.
OverflowResult
Represents whether an operation on the given constant range is known to always or never overflow.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
uint32_t getBitWidth() const
Get the bit width of this ConstantRange.
OverflowResult signedSubMayOverflow(const ConstantRange &Other) const
Return whether signed sub of the two ranges always/never overflows.
uint64_t getZExtValue() const
const APInt & getAPIntValue() const
This is an important base class in LLVM.
Definition: Constant.h:41
Constant * getSplatValue(bool AllowUndefs=false) const
If all elements of the vector constant have the same value, return that value.
Definition: Constants.cpp:1699
Constant * getAggregateElement(unsigned Elt) const
For aggregates (struct/array/vector) return the constant that corresponds to the specified element if...
Definition: Constants.cpp:432
DWARF expression.
static ExtOps getExtOps(unsigned FromSize, unsigned ToSize, bool Signed)
Returns the ops for a zero- or sign-extension in a DIExpression.
static void appendOffset(SmallVectorImpl< uint64_t > &Ops, int64_t Offset)
Append Ops with operations to apply the Offset.
static DIExpression * appendOpsToArg(const DIExpression *Expr, ArrayRef< uint64_t > Ops, unsigned ArgNo, bool StackValue=false)
Create a copy of Expr by appending the given list of Ops to each instance of the operand DW_OP_LLVM_a...
static const DIExpression * convertToVariadicExpression(const DIExpression *Expr)
If Expr is a non-variadic expression (i.e.
static std::optional< DIExpression * > createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits)
Create a DIExpression to describe one part of an aggregate variable that is fragmented across multipl...
Base class for variables.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
bool isLittleEndian() const
Layout endianness...
Definition: DataLayout.h:238
IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const
Returns an integer type with size at least as big as that of a pointer in the given address space.
Definition: DataLayout.cpp:878
Align getABITypeAlign(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:865
unsigned getPointerTypeSizeInBits(Type *) const
Layout pointer size, in bits, based on the type.
Definition: DataLayout.cpp:763
Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
Definition: DataLayout.cpp:874
A debug info location.
Definition: DebugLoc.h:33
void reserve(size_type NumEntries)
Grow the densemap so that it can contain at least NumEntries items before resizing again.
Definition: DenseMap.h:103
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
const char * getSymbol() const
FoldingSetNodeID - This class is used to gather all the unique data bits of a node.
Definition: FoldingSet.h:320
MachineBasicBlock * MBB
MBB - The current block.
Data structure describing the variable locations in a function.
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
Definition: Function.h:680
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
Definition: Function.h:677
AttributeList getAttributes() const
Return the attribute list for this Function.
Definition: Function.h:338
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:350
unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
bool isThreadLocal() const
If the value is "Thread Local", its value isn't shared by the threads.
Definition: GlobalValue.h:263
unsigned getAddressSpace() const
Definition: GlobalValue.h:205
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
PointerType * getType() const
Global values are always pointers.
Definition: GlobalValue.h:294
This class is used to form a handle around another node that is persistent and is updated across invo...
static bool compare(const APInt &LHS, const APInt &RHS, ICmpInst::Predicate Pred)
Return result of LHS Pred RHS comparison.
constexpr bool isValid() const
Definition: LowLevelType.h:145
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate the offet and size that ar...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
TypeSize getValue() const
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:40
Metadata node.
Definition: Metadata.h:1067
Machine Value Type.
SimpleValueType SimpleTy
static MVT getIntegerVT(unsigned BitWidth)
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
Abstract base class for all machine specific constantpool value subclasses.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
A description of a memory reference used in the backend.
LocationSize getSize() const
Return the size in bytes of the memory reference.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
Flags getFlags() const
Return the raw flags of the source value,.
An SDNode that represents everything that will be needed to construct a MachineInstr.
This class is used to represent an MGATHER node.
This class is used to represent an MLOAD node.
This class is used to represent an MSCATTER node.
This class is used to represent an MSTORE node.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT memvt, MachineMemOperand *MMO)
MachineMemOperand * MMO
Memory reference information.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
unsigned getRawSubclassData() const
Return the SubclassData value, without HasDebugValue.
EVT getMemoryVT() const
Return the type of the in-memory value.
Representation for a specific memory location.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:191
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
The optimization diagnostic interface.
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:94
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Definition: DerivedTypes.h:679
bool isNull() const
Test if the pointer held in the union is null, regardless of which type it is.
Definition: PointerUnion.h:142
Analysis providing profile information.
void Deallocate(SubClass *E)
Deallocate - Release storage for the pointed-to object.
Keeps track of dbg_value information through SDISel.
Definition: SelectionDAG.h:159
BumpPtrAllocator & getAlloc()
Definition: SelectionDAG.h:188
void add(SDDbgValue *V, bool isParameter)
void erase(const SDNode *Node)
Invalidate all DbgValues attached to the node and remove it from the Node-to-DbgValues map.
ArrayRef< SDDbgValue * > getSDDbgValues(const SDNode *Node) const
Definition: SelectionDAG.h:194
Holds the information from a dbg_label node through SDISel.
Holds the information for a single machine location through SDISel; either an SDNode,...
static SDDbgOperand fromNode(SDNode *Node, unsigned ResNo)
static SDDbgOperand fromFrameIdx(unsigned FrameIdx)
static SDDbgOperand fromVReg(unsigned VReg)
static SDDbgOperand fromConst(const Value *Const)
@ SDNODE
Value is the result of an expression.
Holds the information from a dbg_value node through SDISel.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
const DebugLoc & getDebugLoc() const
unsigned getIROrder() const
This class provides iterator support for SDUse operands that use a specific SDNode.
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
void dumprFull(const SelectionDAG *G=nullptr) const
printrFull to dbgs().
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool isDivergent() const
bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
iterator_range< value_op_iterator > op_values() const
unsigned getIROrder() const
Return the node ordering.
static constexpr size_t getMaxNumOperands()
Return the maximum number of operands that a SDNode can hold.
MemSDNodeBitfields MemSDNodeBits
void Profile(FoldingSetNodeID &ID) const
Gather unique data for the node.
bool getHasDebugValue() const
SDNodeFlags getFlags() const
void setNodeId(int Id)
Set unique node id.
void intersectFlagsWith(const SDNodeFlags Flags)
Clear any flags in this node that aren't also set in Flags.
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
bool use_empty() const
Return true if there are no uses of this node.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
static bool areOnlyUsersOf(ArrayRef< const SDNode * > Nodes, const SDNode *N)
Return true if all the users of N are contained in Nodes.
use_iterator use_begin() const
Provide iteration support to walk over all uses of an SDNode.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
const APInt & getConstantOperandAPInt(unsigned Num) const
Helper method returns the APInt of a ConstantSDNode operand.
bool hasPredecessor(const SDNode *N) const
Return true if N is a predecessor of this node.
bool hasAnyUseOfValue(unsigned Value) const
Return true if there are any use of the indicated value.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Return true if the type of the node type undefined.
bool hasNUsesOfValue(unsigned NUses, unsigned Value) const
Return true if there are exactly NUSES uses of the indicated value.
op_iterator op_end() const
op_iterator op_begin() const
static use_iterator use_end()
void DropOperands()
Release the operands and set this node to have zero operands.
Represents a use of a SDNode.
SDNode * getNode() const
Convenience function for get().getNode().
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node.
bool isOperandOf(const SDNode *N) const
Return true if this node is an operand of N.
bool reachesChainWithoutSideEffects(SDValue Dest, unsigned Depth=2) const
Return true if this operand (which must be a chain) reaches the specified operand without crossing an...
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
bool use_empty() const
Return true if there are no nodes using value ResNo of Node.
const APInt & getConstantOperandAPInt(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
uint64_t getConstantOperandVal(unsigned i) const
unsigned getOpcode() const
virtual SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const
Emit target-specific code that performs a memset.
virtual SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memmove.
virtual SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const
Emit target-specific code that performs a memcpy.
SDNodeFlags getFlags() const
Definition: SelectionDAG.h:379
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
Align getReducedAlign(EVT VT, bool UseABI)
In most cases this function returns the ABI alignment for a given type, except for illegal vector typ...
SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op)
Return the specified value casted to the target's desired shift amount type.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getExtLoadVP(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsExpanding=false)
SDValue getSplatSourceVector(SDValue V, int &SplatIndex)
If V is a splatted value, return the source vector and its splat index.
SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root, MCSymbol *Label)
OverflowKind computeOverflowForUnsignedSub(SDValue N0, SDValue N1) const
Determine if the result of the unsigned sub of 2 nodes can overflow.
unsigned ComputeMaxSignificantBits(SDValue Op, unsigned Depth=0) const
Get the upper bound on bit size for this Value Op as a signed integer.
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getMaskedGather(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, ISD::LoadExtType ExtTy)
bool isKnownNeverSNaN(SDValue Op, unsigned Depth=0) const
SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS, unsigned DestAS)
Return an AddrSpaceCastSDNode.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDValue getStackArgumentTokenFactor(SDValue Chain)
Compute a TokenFactor to force all the incoming stack arguments to be loaded from the stack.
const TargetSubtargetInfo & getSubtarget() const
Definition: SelectionDAG.h:474
const APInt * getValidMaximumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
void updateDivergence(SDNode *N)
SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl)
Constant fold a setcc to true or false.
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getNeutralElement(unsigned Opcode, const SDLoc &DL, EVT VT, SDNodeFlags Flags)
Get the (commutative) neutral element for the given opcode, if it exists.
SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Value, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo)
SDValue getVScale(const SDLoc &DL, EVT VT, APInt MulImm, bool ConstantFold=true)
Return a node that represents the runtime scaling 'MulImm * RuntimeVL'.
SDValue getPseudoProbeNode(const SDLoc &Dl, SDValue Chain, uint64_t Guid, uint64_t Index, uint32_t Attr)
Creates a PseudoProbeSDNode with function GUID Guid and the index of the block Index it is probing,...
SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
SDNode * SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT)
These are used for target selectors to mutate the specified node to have the specified return type,...
SelectionDAG(const TargetMachine &TM, CodeGenOptLevel)
SDValue getBitcastedSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offs=0, bool isT=false, unsigned TargetFlags=0)
SDValue getStridedLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &DL, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDNode * isConstantIntBuildVectorOrConstantInt(SDValue N) const
Test whether the given value is a constant int or similar node.
SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
Gets a node for an atomic cmpxchg op.
SDValue makeEquivalentMemoryOrdering(SDValue OldChain, SDValue NewMemOpChain)
If an existing load has uses of its chain, create a token factor node with that chain and the new mem...
SDDbgValue * getVRegDbgValue(DIVariable *Var, DIExpression *Expr, unsigned VReg, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a VReg SDDbgValue node.
void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To, unsigned Num)
Like ReplaceAllUsesOfValueWith, but for multiple values at once.
SDValue getJumpTableDebugInfo(int JTI, SDValue Chain, const SDLoc &DL)
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getSymbolFunctionGlobalAddress(SDValue Op, Function **TargetFunction=nullptr)
Return a GlobalAddress of the function from the current module with name matching the given ExternalS...
SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
OverflowKind
Used to represent the possible overflow behavior of an operation.
bool isADDLike(SDValue Op) const
Return true if the specified operand is an ISD::OR or ISD::XOR node that can be treated as an ISD::AD...
bool haveNoCommonBitsSet(SDValue A, SDValue B) const
Return true if A and B have no common bits set.
bool calculateDivergence(SDNode *N)
SDValue getElementCount(const SDLoc &DL, EVT VT, ElementCount EC, bool ConstantFold=true)
SDValue getGetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getAssertAlign(const SDLoc &DL, SDValue V, Align A)
Return an AssertAlignSDNode.
SDNode * mutateStrictFPToFP(SDNode *Node)
Mutate the specified strict FP node to its non-strict equivalent, unlinking the node from its chain a...
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getBitcastedZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
SDValue getStepVector(const SDLoc &DL, EVT ResVT, const APInt &StepVal)
Returns a vector of type ResVT whose elements contain the linear sequence <0, Step,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Align getEVTAlign(EVT MemoryVT) const
Compute the default alignment value for the given type.
bool shouldOptForSize() const
SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getVPZExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be an integer vector, to the vector-type VT,...
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
bool isEqualTo(SDValue A, SDValue B) const
Test whether two SDValues are known to compare equal.
static constexpr unsigned MaxRecursionDepth
Definition: SelectionDAG.h:448
SDValue getStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
SDValue expandVACopy(SDNode *Node)
Expand the specified ISD::VACOPY node as the Legalize pass would.
SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
APInt computeVectorKnownZeroElements(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
For each demanded element of a vector, see if it is known to be zero.
void AddDbgValue(SDDbgValue *DB, bool isParameter)
Add a dbg_value SDNode.
bool NewNodesMustHaveLegalTypes
When true, additional steps are taken to ensure that getConstant() and similar functions return DAG n...
Definition: SelectionDAG.h:387
std::pair< EVT, EVT > GetSplitDestVTs(const EVT &VT) const
Compute the VTs needed for the low/hi parts of a type which is split (or expanded) into two not neces...
void salvageDebugInfo(SDNode &N)
To be invoked on an SDNode that is slated to be erased.
SDNode * MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef< SDValue > Ops)
This mutates the specified node to have the specified return type, opcode, and operands.
std::pair< SDValue, SDValue > UnrollVectorOverflowOp(SDNode *N, unsigned ResNE=0)
Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:531
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getGatherVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
Definition: SelectionDAG.h:828
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), AAResults *AA=nullptr)
SDValue getBitcastedAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by first bitcasting (from potentia...
bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
void DeleteNode(SDNode *N)
Remove the specified node from the system.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDDbgValue * getDbgValueList(DIVariable *Var, DIExpression *Expr, ArrayRef< SDDbgOperand > Locs, ArrayRef< SDNode * > Dependencies, bool IsIndirect, const DebugLoc &DL, unsigned O, bool IsVariadic)
Creates a SDDbgValue node from a list of locations.
SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal)
Try to simplify a select/vselect into 1 of its operands or a constant.
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDNode * isConstantFPBuildVectorOrConstantFP(SDValue N) const
Test whether the given value is a constant FP or similar node.
SDValue expandVAArg(SDNode *Node)
Expand the specified ISD::VAARG node as the Legalize pass would.
SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl< SDValue > &Vals)
Creates a new TokenFactor containing Vals.
bool doesNodeExist(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops)
Check if a node exists without modifying its flags.
bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, int Dist) const
Return true if loads are next to each other and can be merged.
SDDbgLabel * getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O)
Creates a SDDbgLabel node.
SDValue getStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
OverflowKind computeOverflowForUnsignedMul(SDValue N0, SDValue N1) const
Determine if the result of the unsigned mul of 2 nodes can overflow.
void copyExtraInfo(SDNode *From, SDNode *To)
Copy extra info associated with one node to another.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())
Returns sum of the base pointer and offset.
SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, bool isTargetGA=false, unsigned TargetFlags=0)
SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
Definition: SelectionDAG.h:659
SDValue getMDNode(const MDNode *MD)
Return an MDNodeSDNode which holds an MDNode.
void clear()
Clear state and free memory necessary to make this SelectionDAG ready to process a new block.
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
const APInt * getValidShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has a constant or splat constant shift amount that is less than the element b...
SDValue makeStateFunctionCall(unsigned LibFunc, SDValue Ptr, SDValue InChain, const SDLoc &DLoc)
Helper used to make a call to a library function that has one argument of pointer type.
bool isGuaranteedNotToBeUndefOrPoison(SDValue Op, bool PoisonOnly=false, unsigned Depth=0) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
SDValue getIndexedLoadVP(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSrcValue(const Value *v)
Construct a node to track a Value* through the backend.
SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op)
Definition: SelectionDAG.h:862
SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
OverflowKind computeOverflowForSignedMul(SDValue N0, SDValue N1) const
Determine if the result of the signed mul of 2 nodes can overflow.
MaybeAlign InferPtrAlign(SDValue Ptr) const
Infer alignment of a load / store address.
bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if '(Op & Mask) == Mask'.
bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
SDValue getRegister(unsigned Reg, EVT VT)
void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE, Pass *PassPtr, const TargetLibraryInfo *LibraryInfo, UniformityInfo *UA, ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin, FunctionVarLocs const *FnVarLocs)
Prepare this SelectionDAG to process code in the given MachineFunction.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
void RemoveDeadNode(SDNode *N)
Remove the specified node from the system.
void AddDbgLabel(SDDbgLabel *DB)
Add a dbg_label SDNode.
bool isConstantValueOfAnyType(SDValue N) const
SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand)
A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
SDValue getBasicBlock(MachineBasicBlock *MBB)
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
bool isKnownToBeAPowerOfTwo(SDValue Val, unsigned Depth=0) const
Test if the given value is known to have exactly one bit set.
SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label)
SDValue getIndexedStoreVP(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
bool isKnownNeverZero(SDValue Op, unsigned Depth=0) const
Test whether the given SDValue is known to contain non-zero value(s).
SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getSetFPEnv(SDValue Chain, const SDLoc &dl, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, bool IsTruncating=false, bool IsCompressing=false)
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
std::pair< SDValue, SDValue > getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT)
Convert Op, which must be a STRICT operation of float type, to the float type VT, by either extending...
std::pair< SDValue, SDValue > SplitEVL(SDValue N, EVT VecVT, const SDLoc &DL)
Split the explicit vector length parameter of a VP operation.
SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either truncating it or perform...
SDValue getVPLogicalNOT(const SDLoc &DL, SDValue Val, SDValue Mask, SDValue EVL, EVT VT)
Create a vector-predicated logical NOT operation as (VP_XOR Val, BooleanOne, Mask,...
SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
iterator_range< allnodes_iterator > allnodes()
Definition: SelectionDAG.h:543
SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, bool isTarget=false, unsigned TargetFlags=0)
SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
bool isKnownNeverZeroFloat(SDValue Op) const
Test whether the given floating point SDValue is known to never be positive or negative zero.
SDValue getLoadVP(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT MemVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, const MDNode *Ranges=nullptr, bool IsExpanding=false)
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDDbgValue * getConstantDbgValue(DIVariable *Var, DIExpression *Expr, const Value *C, const DebugLoc &DL, unsigned O)
Creates a constant SDDbgValue node.
SDValue getScatterVP(SDVTList VTs, EVT VT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType)
SDValue getValueType(EVT)
ArrayRef< SDDbgValue * > GetDbgValues(const SDNode *SD) const
Get the debug values which reference the given SDNode.
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
OverflowKind computeOverflowForSignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the signed addition of 2 nodes can overflow.
SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of float type, to the float type VT, by either extending or rounding (by tr...
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
ilist< SDNode >::size_type allnodes_size() const
Definition: SelectionDAG.h:539
SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Type *SizeTy, unsigned ElemSz, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
bool isKnownNeverNaN(SDValue Op, bool SNaN=false, unsigned Depth=0) const
Test whether the given SDValue (or all elements of it, if it is a vector) is known to never be NaN.
SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTruncStoreVP(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, SDValue Mask, SDValue EVL, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags, const AAMDNodes &AAInfo, bool IsCompressing=false)
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:676
unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
bool MaskedVectorIsZero(SDValue Op, const APInt &DemandedElts, unsigned Depth=0) const
Return true if 'Op' is known to be zero in DemandedElts.
SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT)
Create a true or false constant of type VT using the target's BooleanContent for type OpVT.
SDDbgValue * getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr, unsigned FI, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a FrameIndex SDDbgValue node.
SDValue getExtStridedLoadVP(ISD::LoadExtType ExtType, const SDLoc &DL, EVT VT, SDValue Chain, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT MemVT, MachineMemOperand *MMO, bool IsExpanding=false)
SDValue getJumpTable(int JTI, EVT VT, bool isTarget=false, unsigned TargetFlags=0)
bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
SDValue getVPPtrExtOrTrunc(const SDLoc &DL, EVT VT, SDValue Op, SDValue Mask, SDValue EVL)
Convert a vector-predicated Op, which must be of integer type, to the vector-type integer type VT,...
SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to extend the Op as a pointer value assuming it was the smaller SrcTy ...
bool canCreateUndefOrPoison(SDValue Op, const APInt &DemandedElts, bool PoisonOnly=false, bool ConsiderFlags=true, unsigned Depth=0) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
OverflowKind computeOverflowForUnsignedAdd(SDValue N0, SDValue N1) const
Determine if the result of the unsigned addition of 2 nodes can overflow.
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
Definition: SelectionDAG.h:845
SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
SDValue getTruncStridedStoreVP(SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Stride, SDValue Mask, SDValue EVL, EVT SVT, MachineMemOperand *MMO, bool IsCompressing=false)
void canonicalizeCommutativeBinop(unsigned Opcode, SDValue &N1, SDValue &N2) const
Swap N1 and N2 if Opcode is a commutative binary opcode and the canonical form expects the opposite o...
KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
SDValue getRegisterMask(const uint32_t *RegMask)
SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
SDValue getCondCode(ISD::CondCode Cond)
SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, int FrameIndex, int64_t Size, int64_t Offset=-1)
Creates a LifetimeSDNode that starts (IsStart==true) or ends (IsStart==false) the lifetime of the por...
bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
OverflowKind computeOverflowForSignedSub(SDValue N0, SDValue N1) const
Determine if the result of the signed sub of 2 nodes can overflow.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y, SDNodeFlags Flags)
Try to simplify a floating-point binary operation into 1 of its operands or a constant.
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
Definition: SelectionDAG.h:560
const APInt * getValidMinimumShiftAmountConstant(SDValue V, const APInt &DemandedElts) const
If a SHL/SRA/SRL node V has constant shift amounts that are all less than the element bit-width of th...
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=0, const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
bool isUndef(unsigned Opcode, ArrayRef< SDValue > Ops)
Return true if the result of this operation is always undefined.
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
std::pair< EVT, EVT > GetDependentSplitDestVTs(const EVT &VT, const EVT &EnvVT, bool *HiIsEmpty) const
Compute the VTs needed for the low/hi parts of a type, dependent on an enveloping VT that has been sp...
SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops)
Fold floating-point operations when all operands are constants and/or undefined.
SDNode * getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef< SDValue > Ops, const SDNodeFlags Flags)
Get the specified node if it's already available, or else return NULL.
SDValue FoldSymbolOffset(unsigned Opcode, EVT VT, const GlobalAddressSDNode *GA, const SDNode *N2)
SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base, SDValue Offset, ISD::MemIndexedMode AM)
SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT, SDValue Operand, SDValue Subreg)
A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
SDDbgValue * getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N, unsigned R, bool IsIndirect, const DebugLoc &DL, unsigned O)
Creates a SDDbgValue node.
SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base, SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT, MachineMemOperand *MMO, ISD::MemIndexedMode AM, ISD::LoadExtType, bool IsExpanding=false)
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
Definition: SelectionDAG.h:878
std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp, ArrayRef< ISD::NodeType > CandidateBinOps, bool AllowPartials=false)
Match a binop + shuffle pyramid that represents a horizontal reduction over the elements of a vector ...
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
SDValue simplifyShift(SDValue X, SDValue Y)
Try to simplify a shift into 1 of its operands or a constant.
void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits=0, unsigned SizeInBits=0, bool InvalidateDbg=true)
Transfer debug values from one node to another, while optionally generating fragment expressions for ...
SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a logical NOT operation as (XOR Val, BooleanOne).
SDValue getMaskedScatter(SDVTList VTs, EVT MemVT, const SDLoc &dl, ArrayRef< SDValue > Ops, MachineMemOperand *MMO, ISD::MemIndexType IndexType, bool IsTruncating=false)
ilist< SDNode >::iterator allnodes_iterator
Definition: SelectionDAG.h:534
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
static bool isSplatMask(const int *Mask, EVT VT)
int getMaskElt(unsigned Idx) const
ArrayRef< int > getMask() const
static void commuteMask(MutableArrayRef< int > Mask)
Change values in a shuffle permute mask assuming the two vector operands have swapped position.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void assign(size_type NumElts, ValueParamT Elt)
Definition: SmallVector.h:717
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void reserve(size_type N)
Definition: SmallVector.h:676
iterator erase(const_iterator CI)
Definition: SmallVector.h:750
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Information about stack frame layout on the target.
virtual TargetStackID::Value getStackIDForScalableVectors() const
Returns the StackID that scalable vectors should be associated with.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
Completely target-dependent object reference.
unsigned getTargetFlags() const
Provides information about what library functions are available for the current target.
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const
Return true if it is beneficial to convert a load of a constant to just the constant itself.
virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
const TargetMachine & getTargetMachine() const
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
virtual bool isCommutativeBinOp(unsigned Opcode) const
Returns true if the opcode is a commutative binary operation.
virtual ISD::NodeType getExtendForAtomicOps() const
Returns how the platform's atomic operations are extended (ZERO_EXTEND, SIGN_EXTEND,...
virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const
Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type from this source type with ...
virtual bool shallExtractConstSplatVectorElementToStore(Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const
Return true if the target shall perform extract vector element and store given that the vector is kno...
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const
Return true if the specified condition code is legal on this target.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
BooleanContent
Enum that describes how the target represents true/false values.
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
virtual unsigned getMaxGluedStoresPerMemcpy() const
Get maximum # of store operations to be glued together.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
virtual bool hasVectorBlend() const
Return true if the target has a vector blend instruction.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
MVT getFrameIndexTy(const DataLayout &DL) const
Return the type for frame index, which is determined by the alloca address space specified through th...
virtual bool isLegalStoreImmediate(int64_t Value) const
Return true if the specified immediate is legal for the value input of a store instruction.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const
Vector types are broken down into some number of legal first class types.
static ISD::NodeType getExtendForContent(BooleanContent Content)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual void computeKnownBitsForFrameIndex(int FIOp, KnownBits &Known, const MachineFunction &MF) const
Determine which of the bits of FrameIndex FIOp are known to be 0.
virtual unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
This method can be implemented by targets that want to expose additional information about sign bits ...
virtual bool findOptimalMemOpLowering(std::vector< EVT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes) const
Determines the optimal series of memory ops to replace the memset / memcpy.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN=false, unsigned Depth=0) const
If SNaN is false,.
virtual void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
virtual bool isSDNodeSourceOfDivergence(const SDNode *N, FunctionLoweringInfo *FLI, UniformityInfo *UA) const
virtual bool isSDNodeAlwaysUniform(const SDNode *N) const
virtual bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth=0) const
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
virtual const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const
This method returns the constant pool value that will be loaded by LD.
virtual bool isGAPlusOffset(SDNode *N, const GlobalValue *&GA, int64_t &Offset) const
Returns true (and the GlobalValue and the offset) if the node is a GlobalAddress + offset.
virtual bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
virtual bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const
Return true if Op can create undef or poison from non-undef & non-poison operands.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const
Returns true if a cast between SrcAS and DestAS is a noop.
const Triple & getTargetTriple() const
TargetOptions Options
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const SelectionDAGTargetInfo * getSelectionDAGInfo() const
virtual const TargetRegisterInfo * getRegisterInfo() const
getRegisterInfo - If register information is available, return it.
virtual const TargetFrameLowering * getFrameLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
Definition: Triple.h:542
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:265
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt8Ty(LLVMContext &C)
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
void set(Value *Val)
Definition: Value.h:882
This class is used to represent an VP_GATHER node.
This class is used to represent a VP_LOAD node.
This class is used to represent an VP_SCATTER node.
This class is used to represent a VP_STORE node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_LOAD node.
This class is used to represent an EXPERIMENTAL_VP_STRIDED_STORE node.
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
static constexpr bool isKnownLE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:217
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
constexpr bool isKnownEven() const
A return value of true indicates we know at compile time that the number of elements (vscale * Min) i...
Definition: TypeSize.h:176
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
static constexpr bool isKnownGE(const FixedOrScalableQuantity &LHS, const FixedOrScalableQuantity &RHS)
Definition: TypeSize.h:224
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:660
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:678
SmartMutex - A mutex with a compile time constant parameter that indicates whether this mutex should ...
Definition: Mutex.h:28
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const APInt abdu(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be unsigned.
Definition: APInt.h:2203
APInt avgCeilU(const APInt &C1, const APInt &C2)
Compute the ceil of the unsigned average of C1 and C2.
Definition: APInt.cpp:3120
APInt avgFloorU(const APInt &C1, const APInt &C2)
Compute the floor of the unsigned average of C1 and C2.
Definition: APInt.cpp:3110
const APInt abds(const APInt &A, const APInt &B)
Determine the absolute difference of two APInts considered to be signed.
Definition: APInt.h:2198
APInt ScaleBitMask(const APInt &A, unsigned NewBitWidth, bool MatchAllBits=false)
Splat/Merge neighboring bits to widen/narrow the bitmask represented by.
Definition: APInt.cpp:3011
APInt avgFloorS(const APInt &C1, const APInt &C2)
Compute the floor of the signed average of C1 and C2.
Definition: APInt.cpp:3105
APInt avgCeilS(const APInt &C1, const APInt &C2)
Compute the ceil of the signed average of C1 and C2.
Definition: APInt.cpp:3115
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
CondCode getSetCCInverse(CondCode Operation, bool isIntegerLike)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCAndOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical AND between different comparisons of identical values: ((X op1 Y) & (X...
bool isConstantSplatVectorAllOnes(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0 ...
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ TargetConstantPool
Definition: ISDOpcodes.h:168
@ MDNODE_SDNODE
MDNODE_SDNODE - This is a node that holdes an MDNode*, which is used to reference metadata in the IR.
Definition: ISDOpcodes.h:1171
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ ATOMIC_LOAD_FMAX
Definition: ISDOpcodes.h:1276
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition: ISDOpcodes.h:44
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ FGETSIGN
INT = FGETSIGN(FP) - Return the sign bit of the specified floating point value as an integer 0/1 valu...
Definition: ISDOpcodes.h:497
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ JUMP_TABLE_DEBUG_INFO
JUMP_TABLE_DEBUG_INFO - Jumptable debug info.
Definition: ISDOpcodes.h:1060
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ TargetBlockAddress
Definition: ISDOpcodes.h:170
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:269
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:979
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:820
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1400
@ ATOMIC_LOAD_FADD
Definition: ISDOpcodes.h:1274
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ SRCVALUE
SRCVALUE - This is a node type that holds a Value* that is used to make reference to a value in the L...
Definition: ISDOpcodes.h:1167
@ FrameIndex
Definition: ISDOpcodes.h:80
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1102
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition: ISDOpcodes.h:662
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ TargetExternalSymbol
Definition: ISDOpcodes.h:169
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ TargetJumpTable
Definition: ISDOpcodes.h:167
@ TargetIndex
TargetIndex - Like a constant pool entry, but with completely target-dependent semantics.
Definition: ISDOpcodes.h:177
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:758
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ ATOMIC_LOAD_FSUB
Definition: ISDOpcodes.h:1275
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ STEP_VECTOR
STEP_VECTOR(IMM) - Returns a scalable vector whose lanes are comprised of a linear sequence of unsign...
Definition: ISDOpcodes.h:646
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ RegisterMask
Definition: ISDOpcodes.h:75
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ AssertAlign
AssertAlign - These nodes record if a register contains a value that has a known alignment and the tr...
Definition: ISDOpcodes.h:68
@ ATOMIC_LOAD_FMIN
Definition: ISDOpcodes.h:1277
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:71
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ TargetGlobalAddress
TargetGlobalAddress - Like GlobalAddress, but the DAG does no folding or anything else with this node...
Definition: ISDOpcodes.h:164
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1266
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:978
@ EntryToken
EntryToken - This is the marker used to indicate the start of a region.
Definition: ISDOpcodes.h:47
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ TargetConstantFP
Definition: ISDOpcodes.h:159
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ VSCALE
VSCALE(IMM) - Returns the runtime scaling factor used to calculate the number of elements within a sc...
Definition: ISDOpcodes.h:1329
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1248
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ TargetFrameIndex
Definition: ISDOpcodes.h:166
@ ConstantPool
Definition: ISDOpcodes.h:82
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:809
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ LIFETIME_START
This corresponds to the llvm.lifetime.
Definition: ISDOpcodes.h:1304
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:736
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ HANDLENODE
HANDLENODE node - Used as a handle for various purposes.
Definition: ISDOpcodes.h:1191
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ ATOMIC_LOAD_UDEC_WRAP
Definition: ISDOpcodes.h:1279
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:158
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1007
@ PSEUDO_PROBE
Pseudo probe for AutoFDO, as a place holder in a basic block to improve the sample counts quality.
Definition: ISDOpcodes.h:1324
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition: ISDOpcodes.h:657
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:279
@ SPLAT_VECTOR_PARTS
SPLAT_VECTOR_PARTS(SCALAR1, SCALAR2, ...) - Returns a vector with the scalar values joined together a...
Definition: ISDOpcodes.h:636
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:831
@ ADDRSPACECAST
ADDRSPACECAST - This operator converts between pointers of different address spaces.
Definition: ISDOpcodes.h:907
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ BlockAddress
Definition: ISDOpcodes.h:84
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ ATOMIC_LOAD_UINC_WRAP
Definition: ISDOpcodes.h:1278
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1012
@ SADDO_CARRY
Carry-using overflow-aware nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:313
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ TargetGlobalTLSAddress
Definition: ISDOpcodes.h:165
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
unsigned getVPForBaseOpcode(unsigned Opcode)
Translate this non-VP Opcode to its corresponding VP Opcode.
bool isBuildVectorOfConstantSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantSDNode or undef.
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool matchUnaryPredicate(SDValue Op, std::function< bool(ConstantSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantSDNode predicate.
bool isZEXTLoad(const SDNode *N)
Returns true if the specified node is a ZEXTLOAD.
bool matchUnaryFpPredicate(SDValue Op, std::function< bool(ConstantFPSDNode *)> Match, bool AllowUndefs=false)
Hook for matching ConstantFPSDNode predicate.
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1412
bool isConstantSplatVectorAllZeros(const SDNode *N, bool BuildVectorOnly=false)
Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 o...
bool isVectorShrinkable(const SDNode *N, unsigned NewEltSize, bool Signed)
Returns true if the specified node is a vector where all elements can be truncated to the specified e...
bool matchUnaryPredicateImpl(SDValue Op, std::function< bool(ConstNodeType *)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant BUI...
bool isVPBinaryOp(unsigned Opcode)
Whether this is a vector-predicated binary operation opcode.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
std::optional< unsigned > getBaseOpcodeForVP(unsigned Opcode, bool hasFPExcept)
Translate this VP Opcode to its corresponding non-VP Opcode.
bool isTrueWhenEqual(CondCode Cond)
Return true if the specified condition returns true if the two operands to the condition are equal.
Definition: ISDOpcodes.h:1581
std::optional< unsigned > getVPMaskIdx(unsigned Opcode)
The operand position of the vector mask.
unsigned getUnorderedFlavor(CondCode Cond)
This function returns 0 if the condition is always false if an operand is a NaN, 1 if the condition i...
Definition: ISDOpcodes.h:1586
std::optional< unsigned > getVPExplicitVectorLengthIdx(unsigned Opcode)
The operand position of the explicit vector length parameter.
bool isEXTLoad(const SDNode *N)
Returns true if the specified node is a EXTLOAD.
bool allOperandsUndef(const SDNode *N)
Return true if the node has at least one operand and all operands of the specified node are ISD::UNDE...
bool isFreezeUndef(const SDNode *N)
Return true if the specified node is FREEZE(UNDEF).
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
MemIndexType
MemIndexType enum - This enum defines how to interpret MGATHER/SCATTER's index parameter when calcula...
Definition: ISDOpcodes.h:1485
bool isBuildVectorAllZeros(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are 0 or undef.
bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
bool matchBinaryPredicate(SDValue LHS, SDValue RHS, std::function< bool(ConstantSDNode *, ConstantSDNode *)> Match, bool AllowUndefs=false, bool AllowTypeMismatch=false)
Attempt to match a binary predicate against a pair of scalar/splat constants or every element of a pa...
bool isVPReduction(unsigned Opcode)
Whether this is a vector-predicated reduction opcode.
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1472
bool isBuildVectorOfConstantFPSDNodes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR node of all ConstantFPSDNode or undef.
bool isSEXTLoad(const SDNode *N)
Returns true if the specified node is a SEXTLOAD.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
bool isBuildVectorAllOnes(const SDNode *N)
Return true if the specified node is a BUILD_VECTOR where all of the elements are ~0 or undef.
NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
bool isVPOpcode(unsigned Opcode)
Whether this is a vector-predicated Opcode.
CondCode getSetCCOrOperation(CondCode Op1, CondCode Op2, EVT Type)
Return the result of a logical OR between different comparisons of identical values: ((X op1 Y) | (X ...
BinaryOp_match< LHS, RHS, Instruction::And > m_And(const LHS &L, const RHS &R)
deferredval_ty< Value > m_Deferred(Value *const &V)
Like m_Specific(), but works if the specific value to match is determined as part of the same match()...
Definition: PatternMatch.h:839
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
Definition: PatternMatch.h:92
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
Definition: PatternMatch.h:561
BinaryOp_match< LHS, RHS, Instruction::Sub > m_Sub(const LHS &L, const RHS &R)
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getMEMCPY_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMCPY_ELEMENT_UNORDERED_ATOMIC - Return MEMCPY_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMSET_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMSET_ELEMENT_UNORDERED_ATOMIC - Return MEMSET_ELEMENT_UNORDERED_ATOMIC_* value for the given ele...
Libcall getMEMMOVE_ELEMENT_UNORDERED_ATOMIC(uint64_t ElementSize)
getMEMMOVE_ELEMENT_UNORDERED_ATOMIC - Return MEMMOVE_ELEMENT_UNORDERED_ATOMIC_* value for the given e...
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
@ DW_OP_LLVM_arg
Only used in LLVM metadata.
Definition: Dwarf.h:146
std::lock_guard< SmartMutex< mt_only > > SmartScopedLock
Definition: Mutex.h:69
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
bool operator<(int64_t V1, const APSInt &V2)
Definition: APSInt.h:361
ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred)
getICmpCondCode - Return the ISD condition code corresponding to the given LLVM IR integer condition ...
Definition: Analysis.cpp:233
SDValue peekThroughExtractSubvectors(SDValue V)
Return the non-extracted vector source operand of V if it exists.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1731
bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
bool isUIntN(unsigned N, uint64_t x)
Checks if an unsigned integer fits into the given (dynamic) bit width.
Definition: MathExtras.h:228
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
Definition: Utils.cpp:1457
SDValue getBitwiseNotOperand(SDValue V, SDValue Mask, bool AllowUndefs)
If V is a bitwise not, returns the inverted operand.
SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Definition: STLExtras.h:2415
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
Definition: bit.h:307
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
bool getAlign(const Function &F, unsigned index, unsigned &align)
bool getConstantDataArrayInfo(const Value *V, ConstantDataArraySlice &Slice, unsigned ElementSize, uint64_t Offset=0)
Returns true if the value V is a pointer into a ConstantDataArray.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition: bit.h:317
LLVM_READONLY APFloat maximum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 maximum semantics.
Definition: APFloat.h:1436
void append_range(Container &C, Range &&R)
Wrapper function to append range R to container C.
Definition: STLExtras.h:2082
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
Definition: Utils.cpp:1439
bool isMinSignedConstant(SDValue V)
Returns true if V is a constant min signed integer value.
ConstantFPSDNode * isConstOrConstSplatFP(SDValue N, bool AllowUndefs=false)
Returns the SDNode if it is a constant splat BuildVector or constant float.
ConstantRange getConstantRangeFromMetadata(const MDNode &RangeMD)
Parse out a conservative ConstantRange from !range metadata.
APFloat frexp(const APFloat &X, int &Exp, APFloat::roundingMode RM)
Equivalent of C standard library function.
Definition: APFloat.h:1373
static Error getOffset(const SymbolRef &Sym, SectionRef Sec, uint64_t &Result)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
LLVM_READONLY APFloat maxnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 maximumNumber semantics.
Definition: APFloat.h:1410
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
bool isBitwiseNot(SDValue V, bool AllowUndefs=false)
Returns true if V is a bitwise not operation.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
void checkForCycles(const SelectionDAG *DAG, bool force=false)
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1656
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
SDValue peekThroughTruncates(SDValue V)
Return the non-truncated source operand of V if it exists.
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1745
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
ConstantRange getVScaleRange(const Function *F, unsigned BitWidth)
Determine the possible constant range of vscale with the given bit width, based on the vscale_range f...
SDValue peekThroughOneUseBitcasts(SDValue V)
Return the non-bitcasted and one-use source operand of V if it exists.
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
bool isOneOrOneSplat(SDValue V, bool AllowUndefs=false)
Return true if the value is a constant 1 integer or a splatted vector of a constant 1 integer (with n...
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
@ Other
Any other memory.
LLVM_READONLY APFloat minnum(const APFloat &A, const APFloat &B)
Implements IEEE-754 2019 minimumNumber semantics.
Definition: APFloat.h:1396
@ Mul
Product of integers.
void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...
DWARFExpression::Operation Op
ConstantSDNode * isConstOrConstSplat(SDValue N, bool AllowUndefs=false, bool AllowTruncation=false)
Returns the SDNode if it is a constant splat BuildVector or constant int.
OutputIt copy(R &&Range, OutputIt Out)
Definition: STLExtras.h:1833
constexpr unsigned BitWidth
Definition: BitmaskEnum.h:191
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1888
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
bool isNullFPConstant(SDValue V)
Returns true if V is an FP constant with a value of positive zero.
constexpr int64_t SignExtend64(uint64_t x)
Sign-extend the number in the bottom B bits of X to a 64-bit integer.
Definition: MathExtras.h:452
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:208
void computeKnownBitsFromRangeMetadata(const MDNode &Ranges, KnownBits &Known)
Compute known bits from the range metadata.
LLVM_READONLY APFloat minimum(const APFloat &A, const APFloat &B)
Implements IEEE 754-2019 minimum semantics.
Definition: APFloat.h:1423
bool isNeutralConstant(unsigned Opc, SDNodeFlags Flags, SDValue V, unsigned OperandNo)
Returns true if V is a neutral element of Opc with Flags.
bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition: MathExtras.h:349
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
MDNode * TBAAStruct
The tag for type-based alias analysis (tbaa struct).
Definition: Metadata.h:780
MDNode * TBAA
The tag for type-based alias analysis.
Definition: Metadata.h:777
static const fltSemantics & IEEEsingle() LLVM_READNONE
Definition: APFloat.cpp:249
cmpResult
IEEE-754R 5.11: Floating Point Comparison Relations.
Definition: APFloat.h:220
static constexpr roundingMode rmTowardNegative
Definition: APFloat.h:233
static constexpr roundingMode rmNearestTiesToEven
Definition: APFloat.h:230
static constexpr roundingMode rmTowardZero
Definition: APFloat.h:234
static const fltSemantics & IEEEquad() LLVM_READNONE
Definition: APFloat.cpp:251
static const fltSemantics & IEEEdouble() LLVM_READNONE
Definition: APFloat.cpp:250
static const fltSemantics & IEEEhalf() LLVM_READNONE
Definition: APFloat.cpp:247
static constexpr roundingMode rmTowardPositive
Definition: APFloat.h:232
static const fltSemantics & BFloat() LLVM_READNONE
Definition: APFloat.cpp:248
opStatus
IEEE-754R 7: Default exception handling.
Definition: APFloat.h:246
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Align previous() const
Definition: Alignment.h:88
Represents offset+length into a ConstantDataArray.
uint64_t Length
Length of the slice.
uint64_t Offset
Slice starts at this Offset.
void move(uint64_t Delta)
Moves the Offset and adjusts Length accordingly.
const ConstantDataArray * Array
ConstantDataArray pointer.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
intptr_t getRawBits() const
Definition: ValueTypes.h:495
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
ElementCount getVectorElementCount() const
Definition: ValueTypes.h:340
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
unsigned getVectorMinNumElements() const
Given a vector type, return the minimum number of elements it contains.
Definition: ValueTypes.h:349
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isFixedLengthVector() const
Definition: ValueTypes.h:177
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:246
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isExtended() const
Test if the given EVT is extended (as opposed to being simple).
Definition: ValueTypes.h:141
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition: ValueTypes.h:438
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
static KnownBits makeConstant(const APInt &C)
Create known bits from a known constant.
Definition: KnownBits.h:297
KnownBits sextInReg(unsigned SrcBitWidth) const
Return known bits for a in-register sign extension of the value we're tracking.
Definition: KnownBits.cpp:155
static KnownBits mulhu(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from zero-extended multiply-hi.
Definition: KnownBits.cpp:872
unsigned countMinSignBits() const
Returns the number of times the sign bit is replicated into the other bits.
Definition: KnownBits.h:251
static KnownBits smax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smax(LHS, RHS).
Definition: KnownBits.cpp:208
bool isNonNegative() const
Returns true if this value is known to be non-negative.
Definition: KnownBits.h:104
bool isZero() const
Returns true if value is all zero.
Definition: KnownBits.h:77
void makeNonNegative()
Make this value non-negative.
Definition: KnownBits.h:120
static KnownBits usub_sat(const KnownBits &LHS, const KnownBits &RHS)
Compute knownbits resulting from llvm.usub.sat(LHS, RHS)
Definition: KnownBits.cpp:761
unsigned countMinTrailingZeros() const
Returns the minimum number of trailing zero bits.
Definition: KnownBits.h:238
static KnownBits ashr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for ashr(LHS, RHS).
Definition: KnownBits.cpp:422
static KnownBits urem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for urem(LHS, RHS).
Definition: KnownBits.cpp:1018
bool isUnknown() const
Returns true if we don't know any bits.
Definition: KnownBits.h:63
unsigned countMaxTrailingZeros() const
Returns the maximum number of trailing zero bits possible.
Definition: KnownBits.h:270
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
Definition: KnownBits.cpp:490
void makeNegative()
Make this value negative.
Definition: KnownBits.h:115
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition: KnownBits.h:157
KnownBits byteSwap() const
Definition: KnownBits.h:457
bool hasConflict() const
Returns true if there is conflicting information.
Definition: KnownBits.h:47
unsigned countMaxPopulation() const
Returns the maximum number of bits that could be one.
Definition: KnownBits.h:285
void setAllZero()
Make all bits known to be zero and discard any previous information.
Definition: KnownBits.h:89
KnownBits reverseBits() const
Definition: KnownBits.h:461
KnownBits concat(const KnownBits &Lo) const
Concatenate the bits from Lo onto the bottom of *this.
Definition: KnownBits.h:229
unsigned getBitWidth() const
Get the bit width of this value.
Definition: KnownBits.h:40
static KnownBits umax(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umax(LHS, RHS).
Definition: KnownBits.cpp:184
KnownBits zext(unsigned BitWidth) const
Return known bits for a zero extension of the value we're tracking.
Definition: KnownBits.h:168
void resetAll()
Resets the known state of all bits.
Definition: KnownBits.h:71
KnownBits unionWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for either this or RHS or both.
Definition: KnownBits.h:317
static KnownBits lshr(const KnownBits &LHS, const KnownBits &RHS, bool ShAmtNonZero=false, bool Exact=false)
Compute known bits for lshr(LHS, RHS).
Definition: KnownBits.cpp:364
bool isNonZero() const
Returns true if this value is known to be non-zero.
Definition: KnownBits.h:107
KnownBits extractBits(unsigned NumBits, unsigned BitPosition) const
Return a subset of the known bits from [bitPosition,bitPosition+numBits).
Definition: KnownBits.h:221
KnownBits intersectWith(const KnownBits &RHS) const
Returns KnownBits information that is known to be true for both this and RHS.
Definition: KnownBits.h:307
KnownBits sext(unsigned BitWidth) const
Return known bits for a sign extension of the value we're tracking.
Definition: KnownBits.h:176
static KnownBits computeForSubBorrow(const KnownBits &LHS, KnownBits RHS, const KnownBits &Borrow)
Compute known bits results from subtracting RHS from LHS with 1-bit Borrow.
Definition: KnownBits.cpp:143
KnownBits zextOrTrunc(unsigned BitWidth) const
Return known bits for a zero extension or truncation of the value we're tracking.
Definition: KnownBits.h:192
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
Definition: KnownBits.h:141
static KnownBits smin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for smin(LHS, RHS).
Definition: KnownBits.cpp:221
static KnownBits mulhs(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits from sign-extended multiply-hi.
Definition: KnownBits.cpp:863
static KnownBits srem(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for srem(LHS, RHS).
Definition: KnownBits.cpp:1037
static KnownBits udiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for udiv(LHS, RHS).
Definition: KnownBits.cpp:976
static KnownBits computeForAddSub(bool Add, bool NSW, bool NUW, const KnownBits &LHS, const KnownBits &RHS)
Compute known bits resulting from adding LHS and RHS.
Definition: KnownBits.cpp:57
bool isStrictlyPositive() const
Returns true if this value is known to be positive.
Definition: KnownBits.h:110
static KnownBits sdiv(const KnownBits &LHS, const KnownBits &RHS, bool Exact=false)
Compute known bits for sdiv(LHS, RHS).
Definition: KnownBits.cpp:917
static bool haveNoCommonBitsSet(const KnownBits &LHS, const KnownBits &RHS)
Return true if LHS and RHS have no common bits set.
Definition: KnownBits.h:328
bool isNegative() const
Returns true if this value is known to be negative.
Definition: KnownBits.h:101
static KnownBits computeForAddCarry(const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry)
Compute known bits resulting from adding LHS, RHS and a 1-bit Carry.
Definition: KnownBits.cpp:50
unsigned countMaxLeadingZeros() const
Returns the maximum number of leading zero bits possible.
Definition: KnownBits.h:276
void insertBits(const KnownBits &SubBits, unsigned BitPosition)
Insert the bits from a smaller known bits starting at bitPosition.
Definition: KnownBits.h:215
static KnownBits mul(const KnownBits &LHS, const KnownBits &RHS, bool NoUndefSelfMultiply=false)
Compute known bits resulting from multiplying LHS and RHS.
Definition: KnownBits.cpp:765
KnownBits anyext(unsigned BitWidth) const
Return known bits for an "any" extension of the value we're tracking, where we don't know anything ab...
Definition: KnownBits.h:163
KnownBits abs(bool IntMinIsPoison=false) const
Compute known bits for the absolute value.
Definition: KnownBits.cpp:544
static KnownBits shl(const KnownBits &LHS, const KnownBits &RHS, bool NUW=false, bool NSW=false, bool ShAmtNonZero=false)
Compute known bits for shl(LHS, RHS).
Definition: KnownBits.cpp:279
static KnownBits umin(const KnownBits &LHS, const KnownBits &RHS)
Compute known bits for umin(LHS, RHS).
Definition: KnownBits.cpp:202
This class contains a discriminated union of information about pointers in memory operands,...
bool isDereferenceable(unsigned Size, LLVMContext &C, const DataLayout &DL) const
Return true if memory region [V, V+Offset+Size) is known to be dereferenceable.
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
PointerUnion< const Value *, const PseudoSourceValue * > V
This is the IR pointer value for the access, or it is null if unknown.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)
These are IR-level optimization flags that may be propagated to SDNodes.
void intersectWith(const SDNodeFlags Flags)
Clear any flags in this flag set that aren't also set in Flags.
bool hasNonNeg() const
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
unsigned int NumVTs
Clients of various APIs that cause global effects on the DAG can optionally implement this interface.
Definition: SelectionDAG.h:307
DAGUpdateListener *const Next
Definition: SelectionDAG.h:308
virtual void NodeDeleted(SDNode *N, SDNode *E)
The node N that was deleted and, if E is not null, an equivalent node E that replaced it.
virtual void NodeInserted(SDNode *N)
The node N that was inserted.
virtual void NodeUpdated(SDNode *N)
The node N that was updated.
This structure contains all information that is necessary for lowering calls.
CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)
CallLoweringInfo & setDiscardResult(bool Value=true)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
CallLoweringInfo & setTailCall(bool Value=true)
CallLoweringInfo & setChain(SDValue InChain)