LLVM 19.0.0git
X86ISelLowering.h
Go to the documentation of this file.
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
15#define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
16
19
20namespace llvm {
21 class X86Subtarget;
22 class X86TargetMachine;
23
24 namespace X86ISD {
25 // X86 Specific DAG Nodes
26 enum NodeType : unsigned {
27 // Start the numbering where the builtin ops leave off.
29
30 /// Bit scan forward.
32 /// Bit scan reverse.
34
35 /// X86 funnel/double shift i16 instructions. These correspond to
36 /// X86::SHLDW and X86::SHRDW instructions which have different amt
37 /// modulo rules to generic funnel shifts.
38 /// NOTE: The operand order matches ISD::FSHL/FSHR not SHLD/SHRD.
41
42 /// Bitwise logical AND of floating point values. This corresponds
43 /// to X86::ANDPS or X86::ANDPD.
45
46 /// Bitwise logical OR of floating point values. This corresponds
47 /// to X86::ORPS or X86::ORPD.
49
50 /// Bitwise logical XOR of floating point values. This corresponds
51 /// to X86::XORPS or X86::XORPD.
53
54 /// Bitwise logical ANDNOT of floating point values. This
55 /// corresponds to X86::ANDNPS or X86::ANDNPD.
57
58 /// These operations represent an abstract X86 call
59 /// instruction, which includes a bunch of information. In particular the
60 /// operands of these node are:
61 ///
62 /// #0 - The incoming token chain
63 /// #1 - The callee
64 /// #2 - The number of arg bytes the caller pushes on the stack.
65 /// #3 - The number of arg bytes the callee pops off the stack.
66 /// #4 - The value to pass in AL/AX/EAX (optional)
67 /// #5 - The value to pass in DL/DX/EDX (optional)
68 ///
69 /// The result values of these nodes are:
70 ///
71 /// #0 - The outgoing token chain
72 /// #1 - The first register result value (optional)
73 /// #2 - The second register result value (optional)
74 ///
76
77 /// Same as call except it adds the NoTrack prefix.
79
80 // Pseudo for a OBJC call that gets emitted together with a special
81 // marker instruction.
83
84 /// X86 compare and logical compare instructions.
89
90 /// X86 bit-test instructions.
92
93 /// X86 SetCC. Operand 0 is condition code, and operand 1 is the EFLAGS
94 /// operand, usually produced by a CMP instruction.
96
97 /// X86 Select
99
100 // Same as SETCC except it's materialized with a sbb and the value is all
101 // one's or all zero's.
102 SETCC_CARRY, // R = carry_bit ? ~0 : 0
103
104 /// X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
105 /// Operands are two FP values to compare; result is a mask of
106 /// 0s or 1s. Generally DTRT for C/C++ with NaNs.
108
109 /// X86 FP SETCC, similar to above, but with output as an i1 mask and
110 /// and a version with SAE.
113
114 /// X86 conditional moves. Operand 0 and operand 1 are the two values
115 /// to select from. Operand 2 is the condition code, and operand 3 is the
116 /// flag operand produced by a CMP or TEST instruction.
118
119 /// X86 conditional branches. Operand 0 is the chain operand, operand 1
120 /// is the block to branch if condition is true, operand 2 is the
121 /// condition code, and operand 3 is the flag operand produced by a CMP
122 /// or TEST instruction.
124
125 /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and
126 /// operand 1 is the target address.
128
129 /// Return with a glue operand. Operand 0 is the chain operand, operand
130 /// 1 is the number of bytes of stack to pop.
132
133 /// Return from interrupt. Operand 0 is the number of bytes to pop.
135
136 /// Repeat fill, corresponds to X86::REP_STOSx.
138
139 /// Repeat move, corresponds to X86::REP_MOVSx.
141
142 /// On Darwin, this node represents the result of the popl
143 /// at function entry, used for PIC code.
145
146 /// A wrapper node for TargetConstantPool, TargetJumpTable,
147 /// TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress,
148 /// MCSymbol and TargetBlockAddress.
150
151 /// Special wrapper used under X86-64 PIC mode for RIP
152 /// relative displacements.
154
155 /// Copies a 64-bit value from an MMX vector to the low word
156 /// of an XMM vector, with the high word zero filled.
158
159 /// Copies a 64-bit value from the low word of an XMM vector
160 /// to an MMX vector.
162
163 /// Copies a 32-bit value from the low word of a MMX
164 /// vector to a GPR.
166
167 /// Copies a GPR into the low 32-bit word of a MMX vector
168 /// and zero out the high word.
170
171 /// Extract an 8-bit value from a vector and zero extend it to
172 /// i32, corresponds to X86::PEXTRB.
174
175 /// Extract a 16-bit value from a vector and zero extend it to
176 /// i32, corresponds to X86::PEXTRW.
178
179 /// Insert any element of a 4 x float vector into any element
180 /// of a destination 4 x floatvector.
182
183 /// Insert the lower 8-bits of a 32-bit value to a vector,
184 /// corresponds to X86::PINSRB.
186
187 /// Insert the lower 16-bits of a 32-bit value to a vector,
188 /// corresponds to X86::PINSRW.
190
191 /// Shuffle 16 8-bit values within a vector.
193
194 /// Compute Sum of Absolute Differences.
196 /// Compute Double Block Packed Sum-Absolute-Differences
198
199 /// Bitwise Logical AND NOT of Packed FP values.
201
202 /// Blend where the selector is an immediate.
204
205 /// Dynamic (non-constant condition) vector blend where only the sign bits
206 /// of the condition elements are used. This is used to enforce that the
207 /// condition mask is not valid for generic VSELECT optimizations. This
208 /// is also used to implement the intrinsics.
209 /// Operands are in VSELECT order: MASK, TRUE, FALSE
211
212 /// Combined add and sub on an FP vector.
214
215 // FP vector ops with rounding mode.
235
236 // FP vector get exponent.
241 // Extract Normalized Mantissas.
246 // FP Scale.
251
252 /// Integer horizontal add/sub.
255
256 /// Floating point horizontal add/sub.
259
260 // Detect Conflicts Within a Vector
262
263 /// Floating point max and min.
266
267 /// Commutative FMIN and FMAX.
270
271 /// Scalar intrinsic floating point max and min.
274
275 /// Floating point reciprocal-sqrt and reciprocal approximation.
276 /// Note that these typically require refinement
277 /// in order to obtain suitable precision.
280
281 // AVX-512 reciprocal approximations with a little more precision.
286
287 // Thread Local Storage.
289
290 // Thread Local Storage. A call to get the start address
291 // of the TLS block for the current module.
293
294 // Thread Local Storage. When calling to an OS provided
295 // thunk at the address from an earlier relocation.
297
298 // Exception Handling helpers.
300
301 // SjLj exception handling setjmp.
303
304 // SjLj exception handling longjmp.
306
307 // SjLj exception handling dispatch.
309
310 /// Tail call return. See X86TargetLowering::LowerCall for
311 /// the list of operands.
313
314 // Vector move to low scalar and zero higher vector elements.
316
317 // Vector integer truncate.
319 // Vector integer truncate with unsigned/signed saturation.
322
323 // Masked version of the above. Used when less than a 128-bit result is
324 // produced since the mask only applies to the lower elements and can't
325 // be represented by a select.
326 // SRC, PASSTHRU, MASK
330
331 // Vector FP extend.
336
337 // Vector FP round.
342
343 // Masked version of above. Used for v2f64->v4f32.
344 // SRC, PASSTHRU, MASK
346
347 // 128-bit vector logical left / right shift
350
351 // Vector shift elements
355
356 // Vector variable shift
360
361 // Vector shift elements by immediate
365
366 // Shifts of mask registers.
369
370 // Bit rotate by immediate
373
374 // Vector packed double/float comparison.
376
377 // Vector integer comparisons.
380
381 // v8i16 Horizontal minimum and position.
383
385
386 /// Vector comparison generating mask bits for fp and
387 /// integer signed and unsigned data types.
389 // Vector mask comparison generating mask bits for FP values.
391 // Vector mask comparison with SAE for FP values.
393
394 // Arithmetic operations with FLAGS results.
404
405 // Bit field extract.
408
409 // Zero High Bits Starting with Specified Bit Position.
411
412 // Parallel extract and deposit.
415
416 // X86-specific multiply by immediate.
418
419 // Vector sign bit extraction.
421
422 // Vector bitwise comparisons.
424
425 // Vector packed fp sign bitwise comparisons.
427
428 // OR/AND test for masks.
431
432 // ADD for masks.
434
435 // Several flavors of instructions with vector shuffle behaviors.
436 // Saturated signed/unnsigned packing.
439 // Intra-lane alignr.
441 // AVX512 inter-lane alignr.
447 // VBMI2 Concat & Shift.
452 // Shuffle Packed Values at 128-bit granularity.
468
469 // Variable Permute (VPERM).
470 // Res = VPERMV MaskV, V0
472
473 // 3-op Variable Permute (VPERMT2).
474 // Res = VPERMV3 V0, MaskV, V1
476
477 // Bitwise ternary logic.
479 // Fix Up Special Packed Float32/64 values.
484 // Range Restriction Calculation For Packed Pairs of Float32/64 values.
489 // Reduce - Perform Reduction Transformation on scalar\packed FP.
494 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
495 // Also used by the legacy (V)ROUND intrinsics where we mask out the
496 // scaling part of the immediate.
501 // Tests Types Of a FP Values for packed types.
503 // Tests Types Of a FP Values for scalar types.
505
506 // Broadcast (splat) scalar or element 0 of a vector. If the operand is
507 // a vector, this node may change the vector length as part of the splat.
509 // Broadcast mask to vector.
511
512 /// SSE4A Extraction and Insertion.
515
516 // XOP arithmetic/logical shifts.
519 // XOP signed/unsigned integer comparisons.
522 // XOP packed permute bytes.
524 // XOP two source permutation.
526
527 // Vector multiply packed unsigned doubleword integers.
529 // Vector multiply packed signed doubleword integers.
531 // Vector Multiply Packed UnsignedIntegers with Round and Scale.
533
534 // Multiply and Add Packed Integers.
537
538 // AVX512IFMA multiply and add.
539 // NOTE: These are different than the instruction and perform
540 // op0 x op1 + op2.
543
544 // VNNI
549
550 // FMA nodes.
551 // We use the target independent ISD::FMA for the non-inverted case.
557
558 // FMA with rounding mode.
565
566 // AVX512-FP16 complex addition and multiplication.
571
576
581
586
593
594 // Compress and expand.
597
598 // Bits shuffle
600
601 // Convert Unsigned/Integer to Floating-Point Value with rounding mode.
608
609 // Vector float/double to signed/unsigned integer.
614 // Scalar float/double to signed/unsigned integer.
619
620 // Vector float/double to signed/unsigned integer with truncation.
625 // Scalar float/double to signed/unsigned integer with truncation.
630
631 // Vector signed/unsigned integer to float/double.
634
635 // Masked versions of above. Used for v2f64->v4f32.
636 // SRC, PASSTHRU, MASK
643
644 // Vector float to bfloat16.
645 // Convert TWO packed single data to one packed BF16 data
647 // Convert packed single data to packed BF16 data
649 // Masked version of above.
650 // SRC, PASSTHRU, MASK
652
653 // Dot product of BF16 pairs to accumulated into
654 // packed single precision.
656
657 // A stack checking function call. On Windows it's _chkstk call.
659
660 // For allocating variable amounts of stack space when using
661 // segmented stacks. Check if the current stacklet has enough space, and
662 // falls back to heap allocation if not.
664
665 // For allocating stack space when using stack clash protector.
666 // Allocation is performed by block, and each block is probed.
668
669 // Memory barriers.
671
672 // Get a random integer and indicate whether it is valid in CF.
674
675 // Get a NIST SP800-90B & C compliant random integer and
676 // indicate whether it is valid in CF.
678
679 // Protection keys
680 // RDPKRU - Operand 0 is chain. Operand 1 is value for ECX.
681 // WRPKRU - Operand 0 is chain. Operand 1 is value for EDX. Operand 2 is
682 // value for ECX.
685
686 // SSE42 string comparisons.
687 // These nodes produce 3 results, index, mask, and flags. X86ISelDAGToDAG
688 // will emit one or two instructions based on which results are used. If
689 // flags and index/mask this allows us to use a single instruction since
690 // we won't have to pick and opcode for flags. Instead we can rely on the
691 // DAG to CSE everything and decide at isel.
694
695 // Test if in transactional execution.
697
698 // ERI instructions.
709
710 // Conversions between float and half-float.
715
716 // Masked version of above.
717 // SRC, RND, PASSTHRU, MASK
720
721 // Galois Field Arithmetic Instructions
725
726 // LWP insert record.
728
729 // User level wait
732
733 // Enqueue Stores Instructions
736
737 // For avx512-vp2intersect
739
740 // User level interrupts - testui
742
743 // Perform an FP80 add after changing precision control in FPCW.
745
746 /// X86 strict FP compare instructions.
749
750 // Vector packed double/float comparison.
752
753 /// Vector comparison generating mask bits for fp and
754 /// integer signed and unsigned data types.
756
757 // Vector float/double to signed/unsigned integer with truncation.
760
761 // Vector FP extend.
763
764 // Vector FP round.
766
767 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
768 // Also used by the legacy (V)ROUND intrinsics where we mask out the
769 // scaling part of the immediate.
771
772 // Vector signed/unsigned integer to float/double.
775
776 // Strict FMA nodes.
780
781 // Conversions between float and half-float.
784
785 // Perform an FP80 add after changing precision control in FPCW.
787
788 // WARNING: Only add nodes here if they are strict FP nodes. Non-memory and
789 // non-strict FP nodes should be above FIRST_TARGET_STRICTFP_OPCODE.
790
791 // Compare and swap.
796
797 /// LOCK-prefixed arithmetic read-modify-write instructions.
798 /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)
810
811 /// RAO arithmetic instructions.
812 /// OUTCHAIN = AADD(INCHAIN, PTR, RHS)
817
818 // Load, scalar_to_vector, and zero extend.
820
821 // extract_vector_elt, store.
823
824 // scalar broadcast from memory.
826
827 // subvector broadcast from memory.
829
830 // Store FP control word into i16 memory.
832
833 // Load FP control word from i16 memory.
835
836 // Store x87 FPU environment into memory.
838
839 // Load x87 FPU environment from memory.
841
842 /// This instruction implements FP_TO_SINT with the
843 /// integer destination in memory and a FP reg source. This corresponds
844 /// to the X86::FIST*m instructions and the rounding mode change stuff. It
845 /// has two inputs (token chain and address) and two outputs (int value
846 /// and token chain). Memory VT specifies the type to store to.
848
849 /// This instruction implements SINT_TO_FP with the
850 /// integer source in memory and FP reg result. This corresponds to the
851 /// X86::FILD*m instructions. It has two inputs (token chain and address)
852 /// and two outputs (FP value and token chain). The integer source type is
853 /// specified by the memory VT.
855
856 /// This instruction implements a fp->int store from FP stack
857 /// slots. This corresponds to the fist instruction. It takes a
858 /// chain operand, value to store, address, and glue. The memory VT
859 /// specifies the type to store as.
861
862 /// This instruction implements an extending load to FP stack slots.
863 /// This corresponds to the X86::FLD32m / X86::FLD64m. It takes a chain
864 /// operand, and ptr to load from. The memory VT specifies the type to
865 /// load from.
867
868 /// This instruction implements a truncating store from FP stack
869 /// slots. This corresponds to the X86::FST32m / X86::FST64m. It takes a
870 /// chain operand, value to store, address, and glue. The memory VT
871 /// specifies the type to store as.
873
874 /// These instructions grab the address of the next argument
875 /// from a va_list. (reads and modifies the va_list in memory)
878
879 // Vector truncating store with unsigned/signed saturation
882 // Vector truncating masked store with unsigned/signed saturation
885
886 // X86 specific gather and scatter
889
890 // Key locker nodes that produce flags.
899
900 /// Compare and Add if Condition is Met. Compare value in operand 2 with
901 /// value in memory of operand 1. If condition of operand 4 is met, add
902 /// value operand 3 to m32 and write new value in operand 1. Operand 2 is
903 /// always updated with the original value from operand 1.
905
906 // Save xmm argument registers to the stack, according to %al. An operator
907 // is needed so that this can be expanded with control flow.
909
910 // WARNING: Do not add anything in the end unless you want the node to
911 // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
912 // opcodes will be thought as target memory ops!
913 };
914 } // end namespace X86ISD
915
916 namespace X86 {
917 /// Current rounding mode is represented in bits 11:10 of FPSR. These
918 /// values are same as corresponding constants for rounding mode used
919 /// in glibc.
921 rmToNearest = 0, // FE_TONEAREST
922 rmDownward = 1 << 10, // FE_DOWNWARD
923 rmUpward = 2 << 10, // FE_UPWARD
924 rmTowardZero = 3 << 10, // FE_TOWARDZERO
925 rmMask = 3 << 10 // Bit mask selecting rounding mode
926 };
927 }
928
929 /// Define some predicates that are used for node matching.
930 namespace X86 {
931 /// Returns true if Elt is a constant zero or floating point constant +0.0.
932 bool isZeroNode(SDValue Elt);
933
934 /// Returns true of the given offset can be
935 /// fit into displacement field of the instruction.
937 bool hasSymbolicDisplacement);
938
939 /// Determines whether the callee is required to pop its
940 /// own arguments. Callee pop is necessary to support tail calls.
941 bool isCalleePop(CallingConv::ID CallingConv,
942 bool is64Bit, bool IsVarArg, bool GuaranteeTCO);
943
944 /// If Op is a constant whose elements are all the same constant or
945 /// undefined, return true and return the constant value in \p SplatVal.
946 /// If we have undef bits that don't cover an entire element, we treat these
947 /// as zero if AllowPartialUndefs is set, else we fail and return false.
948 bool isConstantSplat(SDValue Op, APInt &SplatVal,
949 bool AllowPartialUndefs = true);
950
951 /// Check if Op is a load operation that could be folded into some other x86
952 /// instruction as a memory operand. Example: vpaddd (%rdi), %xmm0, %xmm0.
953 bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
954 bool AssumeSingleUse = false);
955
956 /// Check if Op is a load operation that could be folded into a vector splat
957 /// instruction as a memory operand. Example: vbroadcastss 16(%rdi), %xmm2.
958 bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
959 const X86Subtarget &Subtarget,
960 bool AssumeSingleUse = false);
961
962 /// Check if Op is a value that could be used to fold a store into some
963 /// other x86 instruction as a memory operand. Ex: pextrb $0, %xmm0, (%rdi).
964 bool mayFoldIntoStore(SDValue Op);
965
966 /// Check if Op is an operation that could be folded into a zero extend x86
967 /// instruction.
968 bool mayFoldIntoZeroExtend(SDValue Op);
969
970 /// True if the target supports the extended frame for async Swift
971 /// functions.
972 bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget,
973 const MachineFunction &MF);
974 } // end namespace X86
975
976 //===--------------------------------------------------------------------===//
977 // X86 Implementation of the TargetLowering interface
978 class X86TargetLowering final : public TargetLowering {
979 public:
980 explicit X86TargetLowering(const X86TargetMachine &TM,
981 const X86Subtarget &STI);
982
983 unsigned getJumpTableEncoding() const override;
984 bool useSoftFloat() const override;
985
986 void markLibCallAttributes(MachineFunction *MF, unsigned CC,
987 ArgListTy &Args) const override;
988
989 MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {
990 return MVT::i8;
991 }
992
993 const MCExpr *
995 const MachineBasicBlock *MBB, unsigned uid,
996 MCContext &Ctx) const override;
997
998 /// Returns relocation base for the given PIC jumptable.
1000 SelectionDAG &DAG) const override;
1001 const MCExpr *
1003 unsigned JTI, MCContext &Ctx) const override;
1004
1005 /// Return the desired alignment for ByVal aggregate
1006 /// function arguments in the caller parameter area. For X86, aggregates
1007 /// that contains are placed at 16-byte boundaries while the rest are at
1008 /// 4-byte boundaries.
1010 const DataLayout &DL) const override;
1011
1013 const AttributeList &FuncAttributes) const override;
1014
1015 /// Returns true if it's safe to use load / store of the
1016 /// specified type to expand memcpy / memset inline. This is mostly true
1017 /// for all types except for some special cases. For example, on X86
1018 /// targets without SSE2 f64 load / store are done with fldl / fstpl which
1019 /// also does type conversion. Note the specified type doesn't have to be
1020 /// legal as the hook is used before type legalization.
1021 bool isSafeMemOpType(MVT VT) const override;
1022
1023 bool isMemoryAccessFast(EVT VT, Align Alignment) const;
1024
1025 /// Returns true if the target allows unaligned memory accesses of the
1026 /// specified type. Returns whether it is "fast" in the last argument.
1027 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
1029 unsigned *Fast) const override;
1030
1031 /// This function returns true if the memory access is aligned or if the
1032 /// target allows this specific unaligned memory access. If the access is
1033 /// allowed, the optional final parameter returns a relative speed of the
1034 /// access (as defined by the target).
1035 bool allowsMemoryAccess(
1036 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1037 Align Alignment,
1039 unsigned *Fast = nullptr) const override;
1040
1042 const MachineMemOperand &MMO,
1043 unsigned *Fast) const {
1044 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
1045 MMO.getAlign(), MMO.getFlags(), Fast);
1046 }
1047
1048 /// Provide custom lowering hooks for some operations.
1049 ///
1050 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
1051
1052 /// Replace the results of node with an illegal result
1053 /// type with new values built out of custom code.
1054 ///
1056 SelectionDAG &DAG) const override;
1057
1058 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
1059
1060 bool preferABDSToABSWithNSW(EVT VT) const override;
1061
1062 bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT,
1063 EVT ExtVT) const override;
1064
1066 EVT VT) const override;
1067
1068 /// Return true if the target has native support for
1069 /// the specified value type and it is 'desirable' to use the type for the
1070 /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
1071 /// instruction encodings are longer and some i16 instructions are slow.
1072 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
1073
1074 /// Return true if the target has native support for the
1075 /// specified value type and it is 'desirable' to use the type. e.g. On x86
1076 /// i16 is legal, but undesirable since i16 instruction encodings are longer
1077 /// and some i16 instructions are slow.
1078 bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
1079
1080 /// Return prefered fold type, Abs if this is a vector, AddAnd if its an
1081 /// integer, None otherwise.
1084 const SDNode *SETCC0,
1085 const SDNode *SETCC1) const override;
1086
1087 /// Return the newly negated expression if the cost is not expensive and
1088 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
1089 /// do the negation.
1091 bool LegalOperations, bool ForCodeSize,
1093 unsigned Depth) const override;
1094
1097 MachineBasicBlock *MBB) const override;
1098
1099 /// This method returns the name of a target specific DAG node.
1100 const char *getTargetNodeName(unsigned Opcode) const override;
1101
1102 /// Do not merge vector stores after legalization because that may conflict
1103 /// with x86-specific store splitting optimizations.
1104 bool mergeStoresAfterLegalization(EVT MemVT) const override {
1105 return !MemVT.isVector();
1106 }
1107
1108 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
1109 const MachineFunction &MF) const override;
1110
1111 bool isCheapToSpeculateCttz(Type *Ty) const override;
1112
1113 bool isCheapToSpeculateCtlz(Type *Ty) const override;
1114
1115 bool isCtlzFast() const override;
1116
1117 bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {
1118 // If the pair to store is a mixture of float and int values, we will
1119 // save two bitwise instructions and one float-to-int instruction and
1120 // increase one store instruction. There is potentially a more
1121 // significant benefit because it avoids the float->int domain switch
1122 // for input value. So It is more likely a win.
1123 if ((LTy.isFloatingPoint() && HTy.isInteger()) ||
1124 (LTy.isInteger() && HTy.isFloatingPoint()))
1125 return true;
1126 // If the pair only contains int values, we will save two bitwise
1127 // instructions and increase one store instruction (costing one more
1128 // store buffer). Since the benefit is more blurred so we leave
1129 // such pair out until we get testcase to prove it is a win.
1130 return false;
1131 }
1132
1133 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
1134
1135 bool hasAndNotCompare(SDValue Y) const override;
1136
1137 bool hasAndNot(SDValue Y) const override;
1138
1139 bool hasBitTest(SDValue X, SDValue Y) const override;
1140
1143 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
1144 SelectionDAG &DAG) const override;
1145
1147 EVT VT, unsigned ShiftOpc, bool MayTransformRotate,
1148 const APInt &ShiftOrRotateAmt,
1149 const std::optional<APInt> &AndMask) const override;
1150
1151 bool preferScalarizeSplat(SDNode *N) const override;
1152
1154 CombineLevel Level) const override;
1155
1156 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;
1157
1158 bool
1160 unsigned KeptBits) const override {
1161 // For vectors, we don't have a preference..
1162 if (XVT.isVector())
1163 return false;
1164
1165 auto VTIsOk = [](EVT VT) -> bool {
1166 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
1167 VT == MVT::i64;
1168 };
1169
1170 // We are ok with KeptBitsVT being byte/word/dword, what MOVS supports.
1171 // XVT will be larger than KeptBitsVT.
1172 MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
1173 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
1174 }
1175
1178 unsigned ExpansionFactor) const override;
1179
1180 bool shouldSplatInsEltVarIndex(EVT VT) const override;
1181
1182 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {
1183 // Converting to sat variants holds little benefit on X86 as we will just
1184 // need to saturate the value back using fp arithmatic.
1186 }
1187
1188 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
1189 return VT.isScalarInteger();
1190 }
1191
1192 /// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
1193 MVT hasFastEqualityCompare(unsigned NumBits) const override;
1194
1195 /// Return the value type to use for ISD::SETCC.
1197 EVT VT) const override;
1198
1200 const APInt &DemandedElts,
1201 TargetLoweringOpt &TLO) const override;
1202
1203 /// Determine which of the bits specified in Mask are known to be either
1204 /// zero or one and return them in the KnownZero/KnownOne bitsets.
1206 KnownBits &Known,
1207 const APInt &DemandedElts,
1208 const SelectionDAG &DAG,
1209 unsigned Depth = 0) const override;
1210
1211 /// Determine the number of bits in the operation that are sign bits.
1213 const APInt &DemandedElts,
1214 const SelectionDAG &DAG,
1215 unsigned Depth) const override;
1216
1218 const APInt &DemandedElts,
1219 APInt &KnownUndef,
1220 APInt &KnownZero,
1221 TargetLoweringOpt &TLO,
1222 unsigned Depth) const override;
1223
1225 const APInt &DemandedElts,
1226 unsigned MaskIndex,
1227 TargetLoweringOpt &TLO,
1228 unsigned Depth) const;
1229
1231 const APInt &DemandedBits,
1232 const APInt &DemandedElts,
1233 KnownBits &Known,
1234 TargetLoweringOpt &TLO,
1235 unsigned Depth) const override;
1236
1238 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1239 SelectionDAG &DAG, unsigned Depth) const override;
1240
1242 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1243 bool PoisonOnly, unsigned Depth) const override;
1244
1246 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1247 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override;
1248
1249 bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
1250 APInt &UndefElts, const SelectionDAG &DAG,
1251 unsigned Depth) const override;
1252
1254 // Peek through bitcasts/extracts/inserts to see if we have a broadcast
1255 // vector from memory.
1256 while (Op.getOpcode() == ISD::BITCAST ||
1257 Op.getOpcode() == ISD::EXTRACT_SUBVECTOR ||
1258 (Op.getOpcode() == ISD::INSERT_SUBVECTOR &&
1259 Op.getOperand(0).isUndef()))
1260 Op = Op.getOperand(Op.getOpcode() == ISD::INSERT_SUBVECTOR ? 1 : 0);
1261
1262 return Op.getOpcode() == X86ISD::VBROADCAST_LOAD ||
1264 }
1265
1266 const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
1267
1268 SDValue unwrapAddress(SDValue N) const override;
1269
1271
1272 bool ExpandInlineAsm(CallInst *CI) const override;
1273
1274 ConstraintType getConstraintType(StringRef Constraint) const override;
1275
1276 /// Examine constraint string and operand type and determine a weight value.
1277 /// The operand object must already have been set up with the operand type.
1279 getSingleConstraintMatchWeight(AsmOperandInfo &Info,
1280 const char *Constraint) const override;
1281
1282 const char *LowerXConstraint(EVT ConstraintVT) const override;
1283
1284 /// Lower the specified operand into the Ops vector. If it is invalid, don't
1285 /// add anything to Ops. If hasMemory is true it means one of the asm
1286 /// constraint of the inline asm instruction being processed is 'm'.
1288 std::vector<SDValue> &Ops,
1289 SelectionDAG &DAG) const override;
1290
1292 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
1293 if (ConstraintCode == "v")
1295 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
1296 }
1297
1298 /// Handle Lowering flag assembly outputs.
1300 const SDLoc &DL,
1301 const AsmOperandInfo &Constraint,
1302 SelectionDAG &DAG) const override;
1303
1304 /// Given a physical register constraint
1305 /// (e.g. {edx}), return the register number and the register class for the
1306 /// register. This should only be used for C_Register constraints. On
1307 /// error, this returns a register number of 0.
1308 std::pair<unsigned, const TargetRegisterClass *>
1310 StringRef Constraint, MVT VT) const override;
1311
1312 /// Return true if the addressing mode represented
1313 /// by AM is legal for this target, for a load/store of the specified type.
1314 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
1315 Type *Ty, unsigned AS,
1316 Instruction *I = nullptr) const override;
1317
1318 /// Return true if the specified immediate is legal
1319 /// icmp immediate, that is the target has icmp instructions which can
1320 /// compare a register against the immediate without having to materialize
1321 /// the immediate into a register.
1322 bool isLegalICmpImmediate(int64_t Imm) const override;
1323
1324 /// Return true if the specified immediate is legal
1325 /// add immediate, that is the target has add instructions which can
1326 /// add a register and the immediate without having to materialize
1327 /// the immediate into a register.
1328 bool isLegalAddImmediate(int64_t Imm) const override;
1329
1330 bool isLegalStoreImmediate(int64_t Imm) const override;
1331
1332 /// This is used to enable splatted operand transforms for vector shifts
1333 /// and vector funnel shifts.
1334 bool isVectorShiftByScalarCheap(Type *Ty) const override;
1335
1336 /// Add x86-specific opcodes to the default list.
1337 bool isBinOp(unsigned Opcode) const override;
1338
1339 /// Returns true if the opcode is a commutative binary operation.
1340 bool isCommutativeBinOp(unsigned Opcode) const override;
1341
1342 /// Return true if it's free to truncate a value of
1343 /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
1344 /// register EAX to i16 by referencing its sub-register AX.
1345 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
1346 bool isTruncateFree(EVT VT1, EVT VT2) const override;
1347
1348 bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
1349
1350 /// Return true if any actual instruction that defines a
1351 /// value of type Ty1 implicit zero-extends the value to Ty2 in the result
1352 /// register. This does not necessarily include registers defined in
1353 /// unknown ways, such as incoming arguments, or copies from unknown
1354 /// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this
1355 /// does not necessarily apply to truncate instructions. e.g. on x86-64,
1356 /// all instructions that define 32-bit values implicit zero-extend the
1357 /// result out to 64 bits.
1358 bool isZExtFree(Type *Ty1, Type *Ty2) const override;
1359 bool isZExtFree(EVT VT1, EVT VT2) const override;
1360 bool isZExtFree(SDValue Val, EVT VT2) const override;
1361
1363 SmallVectorImpl<Use *> &Ops) const override;
1364 bool shouldConvertPhiType(Type *From, Type *To) const override;
1365
1366 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
1367 /// extend node) is profitable.
1368 bool isVectorLoadExtDesirable(SDValue) const override;
1369
1370 /// Return true if an FMA operation is faster than a pair of fmul and fadd
1371 /// instructions. fmuladd intrinsics will be expanded to FMAs when this
1372 /// method returns true, otherwise fmuladd is expanded to fmul + fadd.
1374 EVT VT) const override;
1375
1376 /// Return true if it's profitable to narrow operations of type SrcVT to
1377 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not
1378 /// from i32 to i16.
1379 bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override;
1380
1381 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
1382 EVT VT) const override;
1383
1384 /// Given an intrinsic, checks if on the target the intrinsic will need to map
1385 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
1386 /// true and stores the intrinsic information into the IntrinsicInfo that was
1387 /// passed to the function.
1388 bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
1389 MachineFunction &MF,
1390 unsigned Intrinsic) const override;
1391
1392 /// Returns true if the target can instruction select the
1393 /// specified FP immediate natively. If false, the legalizer will
1394 /// materialize the FP immediate as a load from a constant pool.
1395 bool isFPImmLegal(const APFloat &Imm, EVT VT,
1396 bool ForCodeSize) const override;
1397
1398 /// Targets can use this to indicate that they only support *some*
1399 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
1400 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to
1401 /// be legal.
1402 bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
1403
1404 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
1405 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
1406 /// constant pool entry.
1407 bool isVectorClearMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
1408
1409 /// Returns true if lowering to a jump table is allowed.
1410 bool areJTsAllowed(const Function *Fn) const override;
1411
1413 EVT ConditionVT) const override;
1414
1415 /// If true, then instruction selection should
1416 /// seek to shrink the FP constant of the specified type to a smaller type
1417 /// in order to save space and / or reduce runtime.
1418 bool ShouldShrinkFPConstant(EVT VT) const override;
1419
1420 /// Return true if we believe it is correct and profitable to reduce the
1421 /// load node to a smaller type.
1423 EVT NewVT) const override;
1424
1425 /// Return true if the specified scalar FP type is computed in an SSE
1426 /// register, not on the X87 floating point stack.
1427 bool isScalarFPTypeInSSEReg(EVT VT) const;
1428
1429 /// Returns true if it is beneficial to convert a load of a constant
1430 /// to just the constant itself.
1432 Type *Ty) const override;
1433
1434 bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override;
1435
1436 bool convertSelectOfConstantsToMath(EVT VT) const override;
1437
1438 bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
1439 SDValue C) const override;
1440
1441 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
1442 /// with this index.
1443 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
1444 unsigned Index) const override;
1445
1446 /// Scalar ops always have equal or better analysis/performance/power than
1447 /// the vector equivalent, so this always makes sense if the scalar op is
1448 /// supported.
1449 bool shouldScalarizeBinop(SDValue) const override;
1450
1451 /// Extract of a scalar FP value from index 0 of a vector is free.
1452 bool isExtractVecEltCheap(EVT VT, unsigned Index) const override {
1453 EVT EltVT = VT.getScalarType();
1454 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
1455 }
1456
1457 /// Overflow nodes should get combined/lowered to optimal instructions
1458 /// (they should allow eliminating explicit compares by getting flags from
1459 /// math ops).
1460 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
1461 bool MathUsed) const override;
1462
1463 bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
1464 unsigned AddrSpace) const override {
1465 // If we can replace more than 2 scalar stores, there will be a reduction
1466 // in instructions even after we add a vector constant load.
1467 return IsZero || NumElem > 2;
1468 }
1469
1470 bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
1471 const SelectionDAG &DAG,
1472 const MachineMemOperand &MMO) const override;
1473
1474 /// Intel processors have a unified instruction and data cache
1475 const char * getClearCacheBuiltinName() const override {
1476 return nullptr; // nothing to do, move along.
1477 }
1478
1479 Register getRegisterByName(const char* RegName, LLT VT,
1480 const MachineFunction &MF) const override;
1481
1482 /// If a physical register, this returns the register that receives the
1483 /// exception address on entry to an EH pad.
1484 Register
1485 getExceptionPointerRegister(const Constant *PersonalityFn) const override;
1486
1487 /// If a physical register, this returns the register that receives the
1488 /// exception typeid on entry to a landing pad.
1489 Register
1490 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
1491
1492 bool needsFixedCatchObjects() const override;
1493
1494 /// This method returns a target specific FastISel object,
1495 /// or null if the target does not support "fast" ISel.
1497 const TargetLibraryInfo *libInfo) const override;
1498
1499 /// If the target has a standard location for the stack protector cookie,
1500 /// returns the address of that location. Otherwise, returns nullptr.
1501 Value *getIRStackGuard(IRBuilderBase &IRB) const override;
1502
1503 bool useLoadStackGuardNode() const override;
1504 bool useStackGuardXorFP() const override;
1505 void insertSSPDeclarations(Module &M) const override;
1506 Value *getSDagStackGuard(const Module &M) const override;
1507 Function *getSSPStackGuardCheck(const Module &M) const override;
1509 const SDLoc &DL) const override;
1510
1511
1512 /// Return true if the target stores SafeStack pointer at a fixed offset in
1513 /// some non-standard address space, and populates the address space and
1514 /// offset as appropriate.
1515 Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override;
1516
1517 std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,
1518 SDValue Chain, SDValue Pointer,
1519 MachinePointerInfo PtrInfo,
1520 Align Alignment,
1521 SelectionDAG &DAG) const;
1522
1523 /// Customize the preferred legalization strategy for certain types.
1525
1526 bool softPromoteHalfType() const override { return true; }
1527
1529 EVT VT) const override;
1530
1533 EVT VT) const override;
1534
1536 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1537 unsigned &NumIntermediates, MVT &RegisterVT) const override;
1538
1539 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
1540
1541 bool supportSwiftError() const override;
1542
1543 bool supportKCFIBundles() const override { return true; }
1544
1547 const TargetInstrInfo *TII) const override;
1548
1549 bool hasStackProbeSymbol(const MachineFunction &MF) const override;
1550 bool hasInlineStackProbe(const MachineFunction &MF) const override;
1551 StringRef getStackProbeSymbolName(const MachineFunction &MF) const override;
1552
1553 unsigned getStackProbeSize(const MachineFunction &MF) const;
1554
1555 bool hasVectorBlend() const override { return true; }
1556
1557 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
1558
1560 unsigned OpNo) const override;
1561
1562 /// Lower interleaved load(s) into target specific
1563 /// instructions/intrinsics.
1566 ArrayRef<unsigned> Indices,
1567 unsigned Factor) const override;
1568
1569 /// Lower interleaved store(s) into target specific
1570 /// instructions/intrinsics.
1572 unsigned Factor) const override;
1573
1575 int JTI, SelectionDAG &DAG) const override;
1576
1577 Align getPrefLoopAlignment(MachineLoop *ML) const override;
1578
1579 EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {
1580 if (VT == MVT::f80)
1581 return EVT::getIntegerVT(Context, 96);
1583 }
1584
1585 protected:
1586 std::pair<const TargetRegisterClass *, uint8_t>
1588 MVT VT) const override;
1589
1590 private:
1591 /// Keep a reference to the X86Subtarget around so that we can
1592 /// make the right decision when generating code for different targets.
1593 const X86Subtarget &Subtarget;
1594
1595 /// A list of legal FP immediates.
1596 std::vector<APFloat> LegalFPImmediates;
1597
1598 /// Indicate that this x86 target can instruction
1599 /// select the specified FP immediate natively.
1600 void addLegalFPImmediate(const APFloat& Imm) {
1601 LegalFPImmediates.push_back(Imm);
1602 }
1603
1604 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
1605 CallingConv::ID CallConv, bool isVarArg,
1606 const SmallVectorImpl<ISD::InputArg> &Ins,
1607 const SDLoc &dl, SelectionDAG &DAG,
1608 SmallVectorImpl<SDValue> &InVals,
1609 uint32_t *RegMask) const;
1610 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1611 const SmallVectorImpl<ISD::InputArg> &ArgInfo,
1612 const SDLoc &dl, SelectionDAG &DAG,
1613 const CCValAssign &VA, MachineFrameInfo &MFI,
1614 unsigned i) const;
1615 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1616 const SDLoc &dl, SelectionDAG &DAG,
1617 const CCValAssign &VA,
1618 ISD::ArgFlagsTy Flags, bool isByval) const;
1619
1620 // Call lowering helpers.
1621
1622 /// Check whether the call is eligible for tail call optimization. Targets
1623 /// that want to do tail call optimization should implement this function.
1624 bool IsEligibleForTailCallOptimization(
1625 SDValue Callee, CallingConv::ID CalleeCC, bool IsCalleeStackStructRet,
1626 bool isVarArg, Type *RetTy, const SmallVectorImpl<ISD::OutputArg> &Outs,
1627 const SmallVectorImpl<SDValue> &OutVals,
1628 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
1629 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
1630 SDValue Chain, bool IsTailCall,
1631 bool Is64Bit, int FPDiff,
1632 const SDLoc &dl) const;
1633
1634 unsigned GetAlignedArgumentStackSize(unsigned StackSize,
1635 SelectionDAG &DAG) const;
1636
1637 unsigned getAddressSpace() const;
1638
1639 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned,
1640 SDValue &Chain) const;
1641 SDValue LRINT_LLRINTHelper(SDNode *N, SelectionDAG &DAG) const;
1642
1643 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1644 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
1645 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1646 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1647
1648 unsigned getGlobalWrapperKind(const GlobalValue *GV,
1649 const unsigned char OpFlags) const;
1650 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1651 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1652 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1653 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1654 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
1655
1656 /// Creates target global address or external symbol nodes for calls or
1657 /// other uses.
1658 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1659 bool ForCall) const;
1660
1661 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1662 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1663 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1664 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1665 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
1666 SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;
1667 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1668 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1669 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1670 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
1671 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1672 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1673 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1674 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1675 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1676 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1677 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1678 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1679 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
1680 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1681 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1682 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
1683 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1684 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1685 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1686 SDValue LowerGET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
1687 SDValue LowerSET_FPENV_MEM(SDValue Op, SelectionDAG &DAG) const;
1688 SDValue LowerRESET_FPENV(SDValue Op, SelectionDAG &DAG) const;
1689 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
1690 SDValue LowerWin64_FP_TO_INT128(SDValue Op, SelectionDAG &DAG,
1691 SDValue &Chain) const;
1692 SDValue LowerWin64_INT128_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1693 SDValue LowerGC_TRANSITION(SDValue Op, SelectionDAG &DAG) const;
1694 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1695 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
1696 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1697 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
1698 SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
1699
1700 SDValue
1701 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1702 const SmallVectorImpl<ISD::InputArg> &Ins,
1703 const SDLoc &dl, SelectionDAG &DAG,
1704 SmallVectorImpl<SDValue> &InVals) const override;
1705 SDValue LowerCall(CallLoweringInfo &CLI,
1706 SmallVectorImpl<SDValue> &InVals) const override;
1707
1708 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1709 const SmallVectorImpl<ISD::OutputArg> &Outs,
1710 const SmallVectorImpl<SDValue> &OutVals,
1711 const SDLoc &dl, SelectionDAG &DAG) const override;
1712
1713 bool supportSplitCSR(MachineFunction *MF) const override {
1714 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
1715 MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
1716 }
1717 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
1718 void insertCopiesSplitCSR(
1719 MachineBasicBlock *Entry,
1720 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
1721
1722 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
1723
1724 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
1725
1726 EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
1727 ISD::NodeType ExtendKind) const override;
1728
1729 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
1730 bool isVarArg,
1731 const SmallVectorImpl<ISD::OutputArg> &Outs,
1732 LLVMContext &Context) const override;
1733
1734 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
1735 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;
1736
1738 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
1740 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
1742 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
1744 shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const;
1745 void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1746 void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1747
1748 LoadInst *
1749 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
1750
1751 bool needsCmpXchgNb(Type *MemType) const;
1752
1753 void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
1754 MachineBasicBlock *DispatchBB, int FI) const;
1755
1756 // Utility function to emit the low-level va_arg code for X86-64.
1757 MachineBasicBlock *
1758 EmitVAARGWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
1759
1760 /// Utility function to emit the xmm reg save portion of va_start.
1761 MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,
1762 MachineInstr &MI2,
1763 MachineBasicBlock *BB) const;
1764
1765 MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,
1766 MachineBasicBlock *BB) const;
1767
1768 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
1769 MachineBasicBlock *BB) const;
1770
1771 MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,
1772 MachineBasicBlock *BB) const;
1773
1774 MachineBasicBlock *EmitLoweredProbedAlloca(MachineInstr &MI,
1775 MachineBasicBlock *BB) const;
1776
1777 MachineBasicBlock *EmitLoweredTLSAddr(MachineInstr &MI,
1778 MachineBasicBlock *BB) const;
1779
1780 MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
1781 MachineBasicBlock *BB) const;
1782
1783 MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
1784 MachineBasicBlock *BB) const;
1785
1786 MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
1787 MachineBasicBlock *MBB) const;
1788
1789 void emitSetJmpShadowStackFix(MachineInstr &MI,
1790 MachineBasicBlock *MBB) const;
1791
1792 MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
1793 MachineBasicBlock *MBB) const;
1794
1795 MachineBasicBlock *emitLongJmpShadowStackFix(MachineInstr &MI,
1796 MachineBasicBlock *MBB) const;
1797
1798 MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,
1799 MachineBasicBlock *MBB) const;
1800
1801 /// Emit flags for the given setcc condition and operands. Also returns the
1802 /// corresponding X86 condition code constant in X86CC.
1803 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1, ISD::CondCode CC,
1804 const SDLoc &dl, SelectionDAG &DAG,
1805 SDValue &X86CC) const;
1806
1807 bool optimizeFMulOrFDivAsShiftAddBitcast(SDNode *N, SDValue FPConst,
1808 SDValue IntPow2) const override;
1809
1810 /// Check if replacement of SQRT with RSQRT should be disabled.
1811 bool isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const override;
1812
1813 /// Use rsqrt* to speed up sqrt calculations.
1814 SDValue getSqrtEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
1815 int &RefinementSteps, bool &UseOneConstNR,
1816 bool Reciprocal) const override;
1817
1818 /// Use rcp* to speed up fdiv calculations.
1819 SDValue getRecipEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
1820 int &RefinementSteps) const override;
1821
1822 /// Reassociate floating point divisions into multiply by reciprocal.
1823 unsigned combineRepeatedFPDivisors() const override;
1824
1825 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1826 SmallVectorImpl<SDNode *> &Created) const override;
1827
1828 SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
1829 SDValue V2) const;
1830 };
1831
1832 namespace X86 {
1833 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
1834 const TargetLibraryInfo *libInfo);
1835 } // end namespace X86
1836
1837 // X86 specific Gather/Scatter nodes.
1838 // The class has the same order of operands as MaskedGatherScatterSDNode for
1839 // convenience.
1841 public:
1842 // This is a intended as a utility and should never be directly created.
1845
1846 const SDValue &getBasePtr() const { return getOperand(3); }
1847 const SDValue &getIndex() const { return getOperand(4); }
1848 const SDValue &getMask() const { return getOperand(2); }
1849 const SDValue &getScale() const { return getOperand(5); }
1850
1851 static bool classof(const SDNode *N) {
1852 return N->getOpcode() == X86ISD::MGATHER ||
1853 N->getOpcode() == X86ISD::MSCATTER;
1854 }
1855 };
1856
1858 public:
1859 const SDValue &getPassThru() const { return getOperand(1); }
1860
1861 static bool classof(const SDNode *N) {
1862 return N->getOpcode() == X86ISD::MGATHER;
1863 }
1864 };
1865
1867 public:
1868 const SDValue &getValue() const { return getOperand(1); }
1869
1870 static bool classof(const SDNode *N) {
1871 return N->getOpcode() == X86ISD::MSCATTER;
1872 }
1873 };
1874
1875 /// Generate unpacklo/unpackhi shuffle mask.
1876 void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
1877 bool Unary);
1878
1879 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
1880 /// imposed by AVX and specific to the unary pattern. Example:
1881 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
1882 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
1883 void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);
1884
1885} // end namespace llvm
1886
1887#endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
BlockVerifier::State From
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
return RetTy
uint64_t Addr
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define RegName(no)
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
This file describes how to lower LLVM code to machine code.
static bool is64Bit(const char *name)
Class for arbitrary precision integers.
Definition: APInt.h:76
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:178
This class is used to represent ISD::LOAD nodes.
Context object for machine code objects.
Definition: MCContext.h:76
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
Machine Value Type.
static MVT getIntegerVT(unsigned BitWidth)
Instructions::iterator instr_iterator
Representation of each machine instruction.
Definition: MachineInstr.h:68
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const SDValue & getOperand(unsigned Num) const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
An instruction for storing to memory.
Definition: Instructions.h:302
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
NegatibleCost
Enum that specifies when a float negation is beneficial.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
const SDValue & getPassThru() const
static bool classof(const SDNode *N)
const SDValue & getBasePtr() const
const SDValue & getScale() const
static bool classof(const SDNode *N)
const SDValue & getIndex() const
const SDValue & getValue() const
static bool classof(const SDNode *N)
bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override
Overflow nodes should get combined/lowered to optimal instructions (they should allow eliminating exp...
Align getPrefLoopAlignment(MachineLoop *ML) const override
Return the preferred loop alignment.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool preferSextInRegOfTruncate(EVT TruncVT, EVT VT, EVT ExtVT) const override
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const override
Returns relocation base for the given PIC jumptable.
bool preferABDSToABSWithNSW(EVT VT) const override
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
std::pair< SDValue, SDValue > BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer, MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool isMemoryAccessFast(EVT VT, Align Alignment) const
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, const SDLoc &DL, const AsmOperandInfo &Constraint, SelectionDAG &DAG) const override
Handle Lowering flag assembly outputs.
bool supportKCFIBundles() const override
Return true if the target supports kcfi operand bundles.
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const override
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth) const override
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
bool convertSelectOfConstantsToMath(EVT VT) const override
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint letter, return the type of constraint for this target.
bool hasVectorBlend() const override
Return true if the target has a vector blend instruction.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
bool useSoftFloat() const override
bool isVectorShiftByScalarCheap(Type *Ty) const override
This is used to enable splatted operand transforms for vector shifts and vector funnel shifts.
InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const override
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool isLegalStoreImmediate(int64_t Imm) const override
Return true if the specified immediate is legal for the value input of a store instruction.
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const override
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
bool isCtlzFast() const override
Return true if ctlz instruction is fast.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool isSafeMemOpType(MVT VT) const override
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const override
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool shouldSplatInsEltVarIndex(EVT VT) const override
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Return true if sinking I's operands to the same basic block as I is profitable, e....
bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const override
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
MVT hasFastEqualityCompare(unsigned NumBits) const override
Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
It returns EVT::Other if the type should be determined using generic target-independent logic.
bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op, const APInt &DemandedElts, unsigned MaskIndex, TargetLoweringOpt &TLO, unsigned Depth) const
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
unsigned preferedOpcodeForCmpEqPiecesOfOperand(EVT VT, unsigned ShiftOpc, bool MayTransformRotate, const APInt &ShiftOrRotateAmt, const std::optional< APInt > &AndMask) const override
bool isXAndYEqZeroPreferableToXAndYEqY(ISD::CondCode Cond, EVT VT) const override
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if we believe it is correct and profitable to reduce the load node to a smaller type.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, const MachineMemOperand &MMO, unsigned *Fast) const
bool preferScalarizeSplat(SDNode *N) const override
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
Returns true if the target allows unaligned memory accesses of the specified type.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const override
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower interleaved load(s) into target specific instructions/intrinsics.
StringRef getStackProbeSymbolName(const MachineFunction &MF) const override
Returns the name of the symbol used to emit stack probes or the empty string if not applicable.
uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override
Return the desired alignment for ByVal aggregate function arguments in the caller parameter area.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
const char * getClearCacheBuiltinName() const override
Intel processors have a unified instruction and data cache.
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isShuffleMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
bool useStackGuardXorFP() const override
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
bool shouldScalarizeBinop(SDValue) const override
Scalar ops always have equal or better analysis/performance/power than the vector equivalent,...
void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const override
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type Ty1 to type Ty2.
Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const override
Return true if the target stores SafeStack pointer at a fixed offset in some non-standard address spa...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool areJTsAllowed(const Function *Fn) const override
Returns true if lowering to a jump table is allowed.
bool isCommutativeBinOp(unsigned Opcode) const override
Returns true if the opcode is a commutative binary operation.
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const override
Returns preferred type for switch condition.
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower interleaved store(s) into target specific instructions/intrinsics.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool convertSetCCLogicToBitwiseLogic(EVT VT) const override
Use bitwise logic to make pairs of compares more efficient.
bool isVectorClearMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Similar to isShuffleMaskLegal.
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &Info, const char *Constraint) const override
Examine constraint string and operand type and determine a weight value.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Customize the preferred legalization strategy for certain types.
bool shouldConvertPhiType(Type *From, Type *To) const override
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
bool hasStackProbeSymbol(const MachineFunction &MF) const override
Returns true if stack probing through a function call is requested.
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type Ty1 implicit zero-extends the valu...
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isTargetCanonicalConstantNode(SDValue Op) const override
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool softPromoteHalfType() const override
SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const override
bool mergeStoresAfterLegalization(EVT MemVT) const override
Do not merge vector stores after legalization because that may conflict with x86-specific store split...
TargetLowering::AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const override
Return prefered fold type, Abs if this is a vector, AddAnd if its an integer, None otherwise.
bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, int JTI, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expansion.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isBinOp(unsigned Opcode) const override
Add x86-specific opcodes to the default list.
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDValue unwrapAddress(SDValue N) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the value type to use for ISD::SETCC.
bool isVectorLoadExtDesirable(SDValue) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override
For types supported by the target, this is an identity function.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
unsigned getStackProbeSize(const MachineFunction &MF) const
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
Replace the results of node with an illegal result type with new values built out of custom code.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool needsFixedCatchObjects() const override
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Extract of a scalar FP value from index 0 of a vector is free.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ CXX_FAST_TLS
Used for access functions.
Definition: CallingConv.h:72
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1389
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1401
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
Definition: ISDOpcodes.h:1395
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1512
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1492
@ X86
Windows x64, Windows Itanium (IA-64)
@ FST
This instruction implements a truncating store from FP stack slots.
@ REP_MOVS
Repeat move, corresponds to X86::REP_MOVSx.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ FMAX
Floating point max and min.
@ BT
X86 bit-test instructions.
@ HADD
Integer horizontal add/sub.
@ MOVQ2DQ
Copies a 64-bit value from an MMX vector to the low word of an XMM vector, with the high word zero fi...
@ BLENDI
Blend where the selector is an immediate.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ ADDSUB
Combined add and sub on an FP vector.
@ RET_GLUE
Return with a glue operand.
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ FHADD
Floating point horizontal add/sub.
@ FMAXS
Scalar intrinsic floating point max and min.
@ BSR
Bit scan reverse.
@ IRET
Return from interrupt. Operand 0 is the number of bytes to pop.
@ SETCC
X86 SetCC.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ SELECTS
X86 Select.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ PEXTRB
Extract an 8-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRB.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ FSETCC
X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
@ PINSRB
Insert the lower 8-bits of a 32-bit value to a vector, corresponds to X86::PINSRB.
@ REP_STOS
Repeat fill, corresponds to X86::REP_STOSx.
@ INSERTPS
Insert any element of a 4 x float vector into any element of a destination 4 x floatvector.
@ PSHUFB
Shuffle 16 8-bit values within a vector.
@ PEXTRW
Extract a 16-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRW.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ AADD
RAO arithmetic instructions.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FMAXC
Commutative FMIN and FMAX.
@ EXTRQI
SSE4A Extraction and Insertion.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ PSADBW
Compute Sum of Absolute Differences.
@ FOR
Bitwise logical OR of floating point values.
@ FIST
This instruction implements a fp->int store from FP stack slots.
@ FP_TO_INT_IN_MEM
This instruction implements FP_TO_SINT with the integer destination in memory and a FP reg source.
@ LADD
LOCK-prefixed arithmetic read-modify-write instructions.
@ DBPSADBW
Compute Double Block Packed Sum-Absolute-Differences.
@ MMX_MOVW2D
Copies a GPR into the low 32-bit word of a MMX vector and zero out the high word.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ PINSRW
Insert the lower 16-bits of a 32-bit value to a vector, corresponds to X86::PINSRW.
@ CMPCCXADD
Compare and Add if Condition is Met.
@ NT_CALL
Same as call except it adds the NoTrack prefix.
@ MMX_MOVD2W
Copies a 32-bit value from the low word of a MMX vector to a GPR.
@ FILD
This instruction implements SINT_TO_FP with the integer source in memory and FP reg result.
@ MOVDQ2Q
Copies a 64-bit value from the low word of an XMM vector to an MMX vector.
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ BSF
Bit scan forward.
@ VAARG_64
These instructions grab the address of the next argument from a va_list.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
@ FSHL
X86 funnel/double shift i16 instructions.
@ FRSQRT
Floating point reciprocal-sqrt and reciprocal approximation.
RoundingMode
Current rounding mode is represented in bits 11:10 of FPSR.
bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into a vector splat instruction as a memory oper...
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
bool mayFoldIntoZeroExtend(SDValue Op)
Check if Op is an operation that could be folded into a zero extend x86 instruction.
bool mayFoldIntoStore(SDValue Op)
Check if Op is a value that could be used to fold a store into some other x86 instruction as a memory...
bool isExtendedSwiftAsyncFrameSupported(const X86Subtarget &Subtarget, const MachineFunction &MF)
True if the target supports the extended frame for async Swift functions.
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool GuaranteeTCO)
Determines whether the callee is required to pop its own arguments.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
AddressSpace
Definition: NVPTXBaseInfo.h:21
void createUnpackShuffleMask(EVT VT, SmallVectorImpl< int > &Mask, bool Lo, bool Unary)
Generate unpacklo/unpackhi shuffle mask.
void createSplat2ShuffleMask(MVT VT, SmallVectorImpl< int > &Mask, bool Lo)
Similar to unpacklo/unpackhi, but without the 128-bit lane limitation imposed by AVX and specific to ...
CombineLevel
Definition: DAGCombine.h:15
DWARFExpression::Operation Op
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:139
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
This class contains a discriminated union of information about pointers in memory operands,...