LLVM  12.0.0git
AArch64ISelLowering.h
Go to the documentation of this file.
1 //==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that AArch64 uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #ifndef LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
15 #define LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
16 
17 #include "AArch64.h"
21 #include "llvm/IR/CallingConv.h"
22 #include "llvm/IR/Instruction.h"
23 
24 namespace llvm {
25 
26 namespace AArch64ISD {
27 
28 // For predicated nodes where the result is a vector, the operation is
29 // controlled by a governing predicate and the inactive lanes are explicitly
30 // defined with a value, please stick the following naming convention:
31 //
32 // _MERGE_OP<n> The result value is a vector with inactive lanes equal
33 // to source operand OP<n>.
34 //
35 // _MERGE_ZERO The result value is a vector with inactive lanes
36 // actively zeroed.
37 //
38 // _MERGE_PASSTHRU The result value is a vector with inactive lanes equal
39 // to the last source operand which only purpose is being
40 // a passthru value.
41 //
42 // For other cases where no explicit action is needed to set the inactive lanes,
43 // or when the result is not a vector and it is needed or helpful to
44 // distinguish a node from similar unpredicated nodes, use:
45 //
46 // _PRED
47 //
48 enum NodeType : unsigned {
50  WrapperLarge, // 4-instruction MOVZ/MOVK sequence for 64-bit addresses.
51  CALL, // Function call.
52 
53  // Produces the full sequence of instructions for getting the thread pointer
54  // offset of a variable into X0, using the TLSDesc model.
56  ADRP, // Page address of a TargetGlobalAddress operand.
57  ADR, // ADR
58  ADDlow, // Add the low 12 bits of a TargetGlobalAddress operand.
59  LOADgot, // Load from automatically generated descriptor (e.g. Global
60  // Offset Table, TLS record).
61  RET_FLAG, // Return with a flag operand. Operand 0 is the chain operand.
62  BRCOND, // Conditional branch instruction; "b.cond".
64  FCSEL, // Conditional move instruction.
65  CSINV, // Conditional select invert.
66  CSNEG, // Conditional select negate.
67  CSINC, // Conditional select increment.
68 
69  // Pointer to the thread's local storage area. Materialised from TPIDR_EL0 on
70  // ELF.
72  ADC,
73  SBC, // adc, sbc instructions
74 
75  // Arithmetic instructions
88 
90 
91  // Arithmetic instructions which write flags.
97 
98  // Conditional compares. Operands: left,right,falsecc,cc,flags
102 
103  // Floating point comparison
105 
106  // Scalar extract
108 
109  // Scalar-to-vector duplication
115 
116  // Vector immedate moves
124 
125  // Vector immediate ops
128 
129  // Vector bitwise select: similar to ISD::VSELECT but not all bits within an
130  // element must be identical.
132 
133  // Vector arithmetic negation
135 
136  // Vector shuffles
147 
148  // Vector shift by scalar
152 
153  // Vector shift by scalar (again)
159 
160  // Vector shift by constant and insert
163 
164  // Vector comparisons
173 
174  // Vector zero comparisons
185 
186  // Vector across-lanes addition
187  // Only the lower result lane is defined.
190 
191  // Vector rounding halving addition
194 
195  // Vector across-lanes min/max
196  // Only the lower result lane is defined.
201 
209 
210  // Vector bitwise negation
212 
213  // Vector bitwise insertion
215 
216  // Compare-and-branch
221 
222  // Tail calls
224 
225  // Custom prefetch handling
227 
228  // {s|u}int to FP within a FP register.
231 
232  /// Natural vector cast. ISD::BITCAST is not natural in the big-endian
233  /// world w.r.t vectors; which causes additional REV instructions to be
234  /// generated to compensate for the byte-swapping. But sometimes we do
235  /// need to re-interpret the data in SIMD vector registers in big-endian
236  /// mode without emitting such REV instructions.
238 
241 
242  // Reciprocal estimates and steps.
247 
252 
259 
260  // Floating-point reductions.
267 
271 
274 
276 
285 
286  // Structured loads.
290 
291  // Unsigned gather loads.
299 
300  // Signed gather loads
308 
309  // Unsigned gather loads.
317 
318  // Signed gather loads.
326 
327  // Non-temporal gather loads
331 
332  // Contiguous masked store.
334 
335  // Scatter store
343 
344  // Non-temporal scatter store
347 
348  // Strict (exception-raising) floating point comparison
351 
352  // NEON Load/Store with post-increment base updates
376 
381 
385 };
386 
387 } // end namespace AArch64ISD
388 
389 namespace {
390 
391 // Any instruction that defines a 32-bit result zeros out the high half of the
392 // register. Truncate can be lowered to EXTRACT_SUBREG. CopyFromReg may
393 // be copying from a truncate. But any other 32-bit operation will zero-extend
394 // up to 64 bits.
395 // FIXME: X86 also checks for CMOV here. Do we need something similar?
396 static inline bool isDef32(const SDNode &N) {
397  unsigned Opc = N.getOpcode();
398  return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG &&
399  Opc != ISD::CopyFromReg;
400 }
401 
402 } // end anonymous namespace
403 
404 class AArch64Subtarget;
406 
408 public:
409  explicit AArch64TargetLowering(const TargetMachine &TM,
410  const AArch64Subtarget &STI);
411 
412  /// Selects the correct CCAssignFn for a given CallingConvention value.
413  CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const;
414 
415  /// Selects the correct CCAssignFn for a given CallingConvention value.
416  CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC) const;
417 
418  /// Determine which of the bits specified in Mask are known to be either zero
419  /// or one and return them in the KnownZero/KnownOne bitsets.
420  void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known,
421  const APInt &DemandedElts,
422  const SelectionDAG &DAG,
423  unsigned Depth = 0) const override;
424 
425  MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override {
426  // Returning i64 unconditionally here (i.e. even for ILP32) means that the
427  // *DAG* representation of pointers will always be 64-bits. They will be
428  // truncated and extended when transferred to memory, but the 64-bit DAG
429  // allows us to use AArch64's addressing modes much more easily.
430  return MVT::getIntegerVT(64);
431  }
432 
433  bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits,
434  const APInt &DemandedElts,
435  TargetLoweringOpt &TLO) const override;
436 
437  MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override;
438 
439  /// Returns true if the target allows unaligned memory accesses of the
440  /// specified type.
441  bool allowsMisalignedMemoryAccesses(
442  EVT VT, unsigned AddrSpace = 0, unsigned Align = 1,
444  bool *Fast = nullptr) const override;
445  /// LLT variant.
446  bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace,
447  Align Alignment,
449  bool *Fast = nullptr) const override;
450 
451  /// Provide custom lowering hooks for some operations.
452  SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
453 
454  const char *getTargetNodeName(unsigned Opcode) const override;
455 
456  SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
457 
458  /// Returns true if a cast between SrcAS and DestAS is a noop.
459  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override {
460  // Addrspacecasts are always noops.
461  return true;
462  }
463 
464  /// This method returns a target specific FastISel object, or null if the
465  /// target does not support "fast" ISel.
467  const TargetLibraryInfo *libInfo) const override;
468 
469  bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
470 
471  bool isFPImmLegal(const APFloat &Imm, EVT VT,
472  bool ForCodeSize) const override;
473 
474  /// Return true if the given shuffle mask can be codegen'd directly, or if it
475  /// should be stack expanded.
476  bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
477 
478  /// Return the ISD::SETCC ValueType.
479  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
480  EVT VT) const override;
481 
482  SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
483 
484  MachineBasicBlock *EmitF128CSEL(MachineInstr &MI,
485  MachineBasicBlock *BB) const;
486 
487  MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
488  MachineBasicBlock *BB) const;
489 
491  EmitInstrWithCustomInserter(MachineInstr &MI,
492  MachineBasicBlock *MBB) const override;
493 
494  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
495  MachineFunction &MF,
496  unsigned Intrinsic) const override;
497 
498  bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy,
499  EVT NewVT) const override;
500 
501  bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
502  bool isTruncateFree(EVT VT1, EVT VT2) const override;
503 
504  bool isProfitableToHoist(Instruction *I) const override;
505 
506  bool isZExtFree(Type *Ty1, Type *Ty2) const override;
507  bool isZExtFree(EVT VT1, EVT VT2) const override;
508  bool isZExtFree(SDValue Val, EVT VT2) const override;
509 
510  bool shouldSinkOperands(Instruction *I,
511  SmallVectorImpl<Use *> &Ops) const override;
512 
513  bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override;
514 
515  unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
516 
517  bool lowerInterleavedLoad(LoadInst *LI,
519  ArrayRef<unsigned> Indices,
520  unsigned Factor) const override;
521  bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
522  unsigned Factor) const override;
523 
524  bool isLegalAddImmediate(int64_t) const override;
525  bool isLegalICmpImmediate(int64_t) const override;
526 
527  bool shouldConsiderGEPOffsetSplit() const override;
528 
529  EVT getOptimalMemOpType(const MemOp &Op,
530  const AttributeList &FuncAttributes) const override;
531 
532  LLT getOptimalMemOpLLT(const MemOp &Op,
533  const AttributeList &FuncAttributes) const override;
534 
535  /// Return true if the addressing mode represented by AM is legal for this
536  /// target, for a load/store of the specified type.
537  bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
538  unsigned AS,
539  Instruction *I = nullptr) const override;
540 
541  /// Return the cost of the scaling factor used in the addressing
542  /// mode represented by AM for this target, for a load/store
543  /// of the specified type.
544  /// If the AM is supported, the return value must be >= 0.
545  /// If the AM is not supported, it returns a negative value.
546  int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty,
547  unsigned AS) const override;
548 
549  /// Return true if an FMA operation is faster than a pair of fmul and fadd
550  /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
551  /// returns true, otherwise fmuladd is expanded to fmul + fadd.
552  bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
553  EVT VT) const override;
554  bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
555 
556  const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
557 
558  /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.
559  bool isDesirableToCommuteWithShift(const SDNode *N,
560  CombineLevel Level) const override;
561 
562  /// Returns true if it is beneficial to convert a load of a constant
563  /// to just the constant itself.
564  bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
565  Type *Ty) const override;
566 
567  /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
568  /// with this index.
569  bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
570  unsigned Index) const override;
571 
572  bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
573  bool MathUsed) const override {
574  // Using overflow ops for overflow checks only should beneficial on
575  // AArch64.
576  return TargetLowering::shouldFormOverflowOp(Opcode, VT, true);
577  }
578 
579  Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
580  AtomicOrdering Ord) const override;
581  Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val,
582  Value *Addr, AtomicOrdering Ord) const override;
583 
584  void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const override;
585 
587  shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
588  bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
590  shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
591 
593  shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override;
594 
595  bool useLoadStackGuardNode() const override;
597  getPreferredVectorAction(MVT VT) const override;
598 
599  /// If the target has a standard location for the stack protector cookie,
600  /// returns the address of that location. Otherwise, returns nullptr.
601  Value *getIRStackGuard(IRBuilder<> &IRB) const override;
602 
603  void insertSSPDeclarations(Module &M) const override;
604  Value *getSDagStackGuard(const Module &M) const override;
605  Function *getSSPStackGuardCheck(const Module &M) const override;
606 
607  /// If the target has a standard location for the unsafe stack pointer,
608  /// returns the address of that location. Otherwise, returns nullptr.
609  Value *getSafeStackPointerLocation(IRBuilder<> &IRB) const override;
610 
611  /// If a physical register, this returns the register that receives the
612  /// exception address on entry to an EH pad.
613  Register
614  getExceptionPointerRegister(const Constant *PersonalityFn) const override {
615  // FIXME: This is a guess. Has this been defined yet?
616  return AArch64::X0;
617  }
618 
619  /// If a physical register, this returns the register that receives the
620  /// exception typeid on entry to a landing pad.
621  Register
622  getExceptionSelectorRegister(const Constant *PersonalityFn) const override {
623  // FIXME: This is a guess. Has this been defined yet?
624  return AArch64::X1;
625  }
626 
627  bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
628 
629  bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
630  const SelectionDAG &DAG) const override {
631  // Do not merge to float value size (128 bytes) if no implicit
632  // float attribute is set.
633 
634  bool NoFloat = DAG.getMachineFunction().getFunction().hasFnAttribute(
635  Attribute::NoImplicitFloat);
636 
637  if (NoFloat)
638  return (MemVT.getSizeInBits() <= 64);
639  return true;
640  }
641 
642  bool isCheapToSpeculateCttz() const override {
643  return true;
644  }
645 
646  bool isCheapToSpeculateCtlz() const override {
647  return true;
648  }
649 
650  bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
651 
652  bool hasAndNotCompare(SDValue V) const override {
653  // We can use bics for any scalar.
654  return V.getValueType().isScalarInteger();
655  }
656 
657  bool hasAndNot(SDValue Y) const override {
658  EVT VT = Y.getValueType();
659 
660  if (!VT.isVector())
661  return hasAndNotCompare(Y);
662 
663  return VT.getSizeInBits() >= 64; // vector 'bic'
664  }
665 
666  bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
668  unsigned OldShiftOpcode, unsigned NewShiftOpcode,
669  SelectionDAG &DAG) const override;
670 
671  bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const override;
672 
674  unsigned KeptBits) const override {
675  // For vectors, we don't have a preference..
676  if (XVT.isVector())
677  return false;
678 
679  auto VTIsOk = [](EVT VT) -> bool {
680  return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
681  VT == MVT::i64;
682  };
683 
684  // We are ok with KeptBitsVT being byte/word/dword, what SXT supports.
685  // XVT will be larger than KeptBitsVT.
686  MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
687  return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
688  }
689 
690  bool preferIncOfAddToSubOfNot(EVT VT) const override;
691 
692  bool hasBitPreservingFPLogic(EVT VT) const override {
693  // FIXME: Is this always true? It should be true for vectors at least.
694  return VT == MVT::f32 || VT == MVT::f64;
695  }
696 
697  bool supportSplitCSR(MachineFunction *MF) const override {
699  MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
700  }
701  void initializeSplitCSR(MachineBasicBlock *Entry) const override;
702  void insertCopiesSplitCSR(
703  MachineBasicBlock *Entry,
704  const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
705 
706  bool supportSwiftError() const override {
707  return true;
708  }
709 
710  /// Enable aggressive FMA fusion on targets that want it.
711  bool enableAggressiveFMAFusion(EVT VT) const override;
712 
713  /// Returns the size of the platform's va_list object.
714  unsigned getVaListSizeInBits(const DataLayout &DL) const override;
715 
716  /// Returns true if \p VecTy is a legal interleaved access type. This
717  /// function checks the vector element type and the overall width of the
718  /// vector.
719  bool isLegalInterleavedAccessType(VectorType *VecTy,
720  const DataLayout &DL) const;
721 
722  /// Returns the number of interleaved accesses that will be generated when
723  /// lowering accesses of the given type.
724  unsigned getNumInterleavedAccesses(VectorType *VecTy,
725  const DataLayout &DL) const;
726 
727  MachineMemOperand::Flags getTargetMMOFlags(
728  const Instruction &I) const override;
729 
730  bool functionArgumentNeedsConsecutiveRegisters(Type *Ty,
731  CallingConv::ID CallConv,
732  bool isVarArg) const override;
733  /// Used for exception handling on Win64.
734  bool needsFixedCatchObjects() const override;
735 
736  bool fallBackToDAGISel(const Instruction &Inst) const override;
737 
738  /// SVE code generation for fixed length vectors does not custom lower
739  /// BUILD_VECTOR. This makes BUILD_VECTOR legalisation a source of stores to
740  /// merge. However, merging them creates a BUILD_VECTOR that is just as
741  /// illegal as the original, thus leading to an infinite legalisation loop.
742  /// NOTE: Once BUILD_VECTOR is legal or can be custom lowered for all legal
743  /// vector types this override can be removed.
744  bool mergeStoresAfterLegalization(EVT VT) const override {
745  return !useSVEForFixedLengthVectors();
746  }
747 
748 private:
749  /// Keep a pointer to the AArch64Subtarget around so that we can
750  /// make the right decision when generating code for different targets.
751  const AArch64Subtarget *Subtarget;
752 
753  bool isExtFreeImpl(const Instruction *Ext) const override;
754 
755  void addTypeForNEON(MVT VT, MVT PromotedBitwiseVT);
756  void addTypeForFixedLengthSVE(MVT VT);
757  void addDRTypeForNEON(MVT VT);
758  void addQRTypeForNEON(MVT VT);
759 
760  SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
761  bool isVarArg,
763  const SDLoc &DL, SelectionDAG &DAG,
764  SmallVectorImpl<SDValue> &InVals) const override;
765 
766  SDValue LowerCall(CallLoweringInfo & /*CLI*/,
767  SmallVectorImpl<SDValue> &InVals) const override;
768 
769  SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
770  CallingConv::ID CallConv, bool isVarArg,
772  const SDLoc &DL, SelectionDAG &DAG,
773  SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
774  SDValue ThisVal) const;
775 
776  SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
777 
778  SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
779 
780  bool isEligibleForTailCallOptimization(
781  SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
783  const SmallVectorImpl<SDValue> &OutVals,
784  const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
785 
786  /// Finds the incoming stack arguments which overlap the given fixed stack
787  /// object and incorporates their load into the current chain. This prevents
788  /// an upcoming store from clobbering the stack argument before it's used.
789  SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,
790  MachineFrameInfo &MFI, int ClobberedFI) const;
791 
792  bool DoesCalleeRestoreStack(CallingConv::ID CallCC, bool TailCallOpt) const;
793 
794  void saveVarArgRegisters(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &DL,
795  SDValue &Chain) const;
796 
797  bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
798  bool isVarArg,
800  LLVMContext &Context) const override;
801 
802  SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
804  const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
805  SelectionDAG &DAG) const override;
806 
808  unsigned Flag) const;
810  unsigned Flag) const;
812  unsigned Flag) const;
814  unsigned Flag) const;
815  template <class NodeTy>
816  SDValue getGOT(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
817  template <class NodeTy>
818  SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
819  template <class NodeTy>
820  SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
821  template <class NodeTy>
822  SDValue getAddrTiny(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
823  SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
824  SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
825  SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
826  SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
827  SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
828  SDValue LowerELFTLSLocalExec(const GlobalValue *GV, SDValue ThreadBase,
829  const SDLoc &DL, SelectionDAG &DAG) const;
830  SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
831  SelectionDAG &DAG) const;
832  SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
833  SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
834  SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
835  SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
836  SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
838  SDValue TVal, SDValue FVal, const SDLoc &dl,
839  SelectionDAG &DAG) const;
840  SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
841  SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
842  SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
843  SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
844  SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;
845  SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;
846  SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
847  SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
848  SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
849  SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
850  SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
851  SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
853  SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
854  SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
857  SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
859  SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
860  SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
861  SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,
862  unsigned NewOp) const;
865  SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
866  SDValue LowerShiftLeftParts(SDValue Op, SelectionDAG &DAG) const;
867  SDValue LowerShiftRightParts(SDValue Op, SelectionDAG &DAG) const;
868  SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
869  SDValue LowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
870  SDValue LowerF128Call(SDValue Op, SelectionDAG &DAG,
871  RTLIB::Libcall Call) const;
872  SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
873  SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
874  SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
876  SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
877  SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
878  SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
880  SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
881  SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;
882  SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
883  SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
884  SDValue LowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
885  SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
887  SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SDValue Chain,
888  SDValue &Size,
889  SelectionDAG &DAG) const;
890  SDValue LowerSVEStructLoad(unsigned Intrinsic, ArrayRef<SDValue> LoadOps,
891  EVT VT, SelectionDAG &DAG, const SDLoc &DL) const;
892 
893  SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
894  SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
895  SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
896  SelectionDAG &DAG) const;
897 
898  SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
899  SmallVectorImpl<SDNode *> &Created) const override;
900  SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
901  int &ExtraSteps, bool &UseOneConst,
902  bool Reciprocal) const override;
903  SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
904  int &ExtraSteps) const override;
905  unsigned combineRepeatedFPDivisors() const override;
906 
907  ConstraintType getConstraintType(StringRef Constraint) const override;
908  Register getRegisterByName(const char* RegName, LLT VT,
909  const MachineFunction &MF) const override;
910 
911  /// Examine constraint string and operand type and determine a weight value.
912  /// The operand object must already have been set up with the operand type.
914  getSingleConstraintMatchWeight(AsmOperandInfo &info,
915  const char *constraint) const override;
916 
917  std::pair<unsigned, const TargetRegisterClass *>
918  getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
919  StringRef Constraint, MVT VT) const override;
920 
921  const char *LowerXConstraint(EVT ConstraintVT) const override;
922 
923  void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint,
924  std::vector<SDValue> &Ops,
925  SelectionDAG &DAG) const override;
926 
927  unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
928  if (ConstraintCode == "Q")
930  // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are
931  // followed by llvm_unreachable so we'll leave them unimplemented in
932  // the backend for now.
933  return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
934  }
935 
936  bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
937  bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
938  bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
939  bool getIndexedAddressParts(SDNode *Op, SDValue &Base, SDValue &Offset,
940  ISD::MemIndexedMode &AM, bool &IsInc,
941  SelectionDAG &DAG) const;
942  bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
944  SelectionDAG &DAG) const override;
945  bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
946  SDValue &Offset, ISD::MemIndexedMode &AM,
947  SelectionDAG &DAG) const override;
948 
949  void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
950  SelectionDAG &DAG) const override;
951  void ReplaceExtractSubVectorResults(SDNode *N,
952  SmallVectorImpl<SDValue> &Results,
953  SelectionDAG &DAG) const;
954 
955  bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override;
956 
957  void finalizeLowering(MachineFunction &MF) const override;
958 
959  bool shouldLocalize(const MachineInstr &MI,
960  const TargetTransformInfo *TTI) const override;
961 
962  bool useSVEForFixedLengthVectors() const;
963  bool useSVEForFixedLengthVectorVT(EVT VT) const;
964 };
965 
966 namespace AArch64 {
968  const TargetLibraryInfo *libInfo);
969 } // end namespace AArch64
970 
971 } // end namespace llvm
972 
973 #endif
static MVT getIntegerVT(unsigned BitWidth)
static SDValue LowerCallResult(SDValue Chain, SDValue InFlag, const SmallVectorImpl< CCValAssign > &RVLocs, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals)
LowerCallResult - Lower the result values of a call into the appropriate copies out of appropriate ph...
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1111
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
static SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
EVT getValueType() const
Return the ValueType of the referenced return value.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
LLVMContext & Context
bool supportSplitCSR(MachineFunction *MF) const override
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
This class represents lattice values for constants.
Definition: AllocatorList.h:23
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:67
An instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:521
static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:151
This class represents a function call, abstracting a target machine&#39;s calling convention.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change...
Function Alias Analysis Results
This instruction constructs a fixed permutation of two input vectors.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:330
unsigned const TargetRegisterInfo * TRI
F(f)
An instruction for reading from memory.
Definition: Instructions.h:173
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:701
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
bool hasAndNotCompare(SDValue V) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) != Y —> (~X & Y) ...
Function & getFunction()
Return the LLVM function that this machine code represents.
MachineBasicBlock & MBB
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG, const SparcTargetLowering &TLI, const SparcSubtarget *Subtarget)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
Definition: ISDOpcodes.h:1117
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override
Returns true if a cast between SrcAS and DestAS is a noop.
This is a fast-path instruction selection class that generates poor code and doesn&#39;t support illegal ...
Definition: FastISel.h:65
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:421
static SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG)
This contains information for each constraint that we are lowering.
An instruction for storing to memory.
Definition: Instructions.h:302
Natural vector cast.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:1204
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
Analysis containing CSE Info
Definition: CSEInfo.cpp:25
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:138
static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *ST)
Machine Value Type.
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
This is an important base class in LLVM.
Definition: Constant.h:41
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1184
CombineLevel
Definition: DAGCombine.h:15
static SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG, const SparcSubtarget *Subtarget)
static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG, const SparcSubtarget *Subtarget)
lazy value info
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
assume Assume Builder
Extended Value Type.
Definition: ValueTypes.h:35
static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:315
This structure contains all information that is necessary for lowering calls.
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
static Value * LowerCTPOP(LLVMContext &Context, Value *V, Instruction *IP)
Emit the code to lower ctpop of V before the specified instruction IP.
virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const
Try to convert math with an overflow comparison into the corresponding DAG node operation.
CCState - This class holds information needed while lowering arguments and return values...
bool mergeStoresAfterLegalization(EVT VT) const override
SVE code generation for fixed length vectors does not custom lower BUILD_VECTOR.
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:219
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:223
static SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG, const ARMSubtarget *Subtarget)
Provides information about what library functions are available for the current target.
AddressSpace
Definition: NVPTXBaseInfo.h:21
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1123
Represents one node in the SelectionDAG.
static bool Enabled
Definition: Statistic.cpp:50
bool hasBitPreservingFPLogic(EVT VT) const override
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
Base class of all SIMD vector types.
Definition: DerivedTypes.h:390
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
Class for arbitrary precision integers.
Definition: APInt.h:69
bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override
Try to convert math with an overflow comparison into the corresponding DAG node operation.
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const SelectionDAG &DAG) const override
Returns if it&#39;s reasonable to merge stores to MemVT size.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG)
static SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG, const SparcTargetLowering &TLI, bool hasHardQuad)
Flags
Flags values. These may be or&#39;d together.
static SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG, const SparcTargetLowering &TLI, bool hasHardQuad)
Representation of each machine instruction.
Definition: MachineInstr.h:62
static unsigned getScalingFactorCost(const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F, const Loop &L)
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:156
#define I(x, y, z)
Definition: MD5.cpp:59
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
uint32_t Size
Definition: Profile.cpp:46
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
Definition: ISDOpcodes.h:192
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
LLVM Value Representation.
Definition: Value.h:74
static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG)
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
AtomicOrdering
Atomic ordering for LLVM&#39;s memory model.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
bool isCheapToSpeculateCttz() const override
Return true if it is cheap to speculate a call to intrinsic cttz.
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:653
bool isCheapToSpeculateCtlz() const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
This file describes how to lower LLVM code to machine code.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Level
Definition: Debugify.cpp:33
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1153